Skip to content
This repository was archived by the owner on Oct 9, 2026. It is now read-only.

Commit 86e740d

Browse files
authored
feat(koplugin): bulk sync annotations (#84)
### New Features Bulk Annotation Sync: The "Synchronize data" button now syncs all annotations from all books in your reading history at once (previously only the currently open book). Sync on suspend still syncs statistics and annotations for the currently open book only (keeping suspend snappy). ### Bug Fixes Fixed Book Duplicates: Some users saw the same book twice in KoInsight, one with statistics, one with annotations. We now match books using their unique MD5 checksum instead of title. If you have duplicates, either: 1. Delete your database and re-sync (recommended - clean start) 2. Manually delete duplicate books in the web interface (the faulty one) New syncs won't create duplicates. If you still see duplicates, you most likely have duplicates in your KoReader statistics database and KoInsight makes those visible.
1 parent 1eb7dcd commit 86e740d

9 files changed

Lines changed: 603 additions & 137 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Changelog
2+
3+
## [Unreleased]
4+
5+
### New Features
6+
7+
Bulk Annotation Sync: The "Synchronize data" button now syncs all annotations from all books in your reading history at once (previously only the currently open book). Sync on suspend still syncs statistics and annotations for the currently open book only (keeping suspend snappy).
8+
9+
### Bug Fixes
10+
11+
Fixed Book Duplicates: Some users saw the same book twice in KoInsight, one with statistics, one with annotations. We now match books using their unique MD5 checksum instead of title.
12+
13+
If you have duplicates, either:
14+
15+
1. Delete your database and re-sync (recommended - clean start)
16+
2. Manually delete duplicate books in the web interface (the faulty one)
17+
18+
New syncs won't create duplicates. If you still see duplicates, you most likely have duplicates in your KoReader statistics database and KoInsight makes those visible.
19+
20+
### Breaking Changes
21+
22+
Plugin version 0.3.0 required. Update it before syncing.
23+
24+
---
25+
26+
## [0.2.2] - 2026-01-11
27+
28+
### Added
29+
30+
- Annotation sync support for currently open book
31+
- Mark deleted annotations in the database
32+
33+
### Fixed
34+
35+
- Annotations now properly marked as deleted when removed in KoReader
36+
- Docker build issues
37+
38+
## [0.2.0] - 2026-01-11
39+
40+
### Added
41+
42+
- Plugin versioning system
43+
- Server validates plugin version before accepting data
44+
45+
### Changed
46+
47+
- **BREAKING:** Server now requires specific plugin version
48+
49+
---
50+
51+
## Earlier Versions
52+
53+
See git history for changes prior to v0.2.0.
54+
55+
[Unreleased]: https://github.com/GeorgeSG/koinsight/compare/v0.2.2...HEAD
56+
[0.2.2]: https://github.com/GeorgeSG/koinsight/compare/v0.2.0...v0.2.2
57+
[0.2.0]: https://github.com/GeorgeSG/koinsight/releases/tag/v0.2.0

‎apps/server/src/koplugin/koplugin-router.test.ts‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,18 @@ import request from 'supertest';
33
import { createDevice } from '../db/factories/device-factory';
44
import { fakeKoReaderAnnotation } from '../db/factories/koreader-annotation-factory';
55
import { db } from '../knex';
6-
import { kopluginRouter } from './koplugin-router';
6+
import { kopluginRouter, REQUIRED_PLUGIN_VERSION } from './koplugin-router';
77

88
describe('koplugin-router', () => {
99
const app = express();
1010
app.use(express.json());
1111
app.use('/koplugin', kopluginRouter);
1212

13-
const PLUGIN_VERSION = '0.2.0';
14-
1513
describe('POST /koplugin/device', () => {
1614
it('registers a device', async () => {
1715
const response = await request(app)
1816
.post('/koplugin/device')
19-
.send({ id: 'device-123', model: 'Kindle Paperwhite', version: PLUGIN_VERSION });
17+
.send({ id: 'device-123', model: 'Kindle Paperwhite', version: REQUIRED_PLUGIN_VERSION });
2018

2119
expect(response.status).toBe(200);
2220
expect(response.body).toEqual({ message: 'Device registered successfully' });
@@ -33,7 +31,7 @@ describe('koplugin-router', () => {
3331
it('returns 400 when device ID is missing', async () => {
3432
const response = await request(app)
3533
.post('/koplugin/device')
36-
.send({ model: 'Kindle', version: PLUGIN_VERSION });
34+
.send({ model: 'Kindle', version: REQUIRED_PLUGIN_VERSION });
3735

3836
expect(response.status).toBe(400);
3937
expect(response.body).toEqual({ error: 'Missing device ID or model' });
@@ -42,7 +40,7 @@ describe('koplugin-router', () => {
4240
it('returns 400 when model is missing', async () => {
4341
const response = await request(app)
4442
.post('/koplugin/device')
45-
.send({ id: 'device-123', version: PLUGIN_VERSION });
43+
.send({ id: 'device-123', version: REQUIRED_PLUGIN_VERSION });
4644

4745
expect(response.status).toBe(400);
4846
expect(response.body).toEqual({ error: 'Missing device ID or model' });
@@ -75,7 +73,7 @@ describe('koplugin-router', () => {
7573
const response = await request(app)
7674
.post('/koplugin/import')
7775
.send({
78-
version: PLUGIN_VERSION,
76+
version: REQUIRED_PLUGIN_VERSION,
7977
books: [
8078
{
8179
md5: bookMd5,
@@ -130,7 +128,7 @@ describe('koplugin-router', () => {
130128
const response = await request(app)
131129
.post('/koplugin/import')
132130
.send({
133-
version: PLUGIN_VERSION,
131+
version: REQUIRED_PLUGIN_VERSION,
134132
books: [
135133
{
136134
md5: bookMd5,
@@ -222,7 +220,7 @@ describe('koplugin-router', () => {
222220
it('returns health status', async () => {
223221
const response = await request(app)
224222
.get('/koplugin/health')
225-
.send({ version: PLUGIN_VERSION });
223+
.send({ version: REQUIRED_PLUGIN_VERSION });
226224

227225
expect(response.status).toBe(200);
228226
expect(response.body).toEqual({ message: 'Plugin is healthy' });

‎apps/server/src/koplugin/koplugin-router.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import { UploadService } from '../upload/upload-service';
1111
// Router for KoInsight koreader plugin
1212
const router = Router();
1313

14-
const REQUIRED_PLUGIN_VERSION = '0.2.0';
14+
export const REQUIRED_PLUGIN_VERSION = '0.3.0';
1515

1616
const rejectOldPluginVersion = (req: Request, res: Response, next: NextFunction) => {
1717
const { version } = req.body;
@@ -53,6 +53,7 @@ router.post('/import', rejectOldPluginVersion, async (req, res) => {
5353
const koreaderBooks: KoReaderBook[] = req.body.books;
5454
const newPageStats: PageStat[] = req.body.stats;
5555
const annotations: Record<string, KoReaderAnnotation[]> = req.body.annotations || {};
56+
const deviceId: string | undefined = req.body.device_id; // For annotation sync path
5657

5758
try {
5859
console.debug('Importing books:', koreaderBooks);
@@ -63,7 +64,7 @@ router.post('/import', rejectOldPluginVersion, async (req, res) => {
6364
'books with annotations'
6465
);
6566

66-
await UploadService.uploadStatisticData(koreaderBooks, newPageStats, annotations);
67+
await UploadService.uploadStatisticData(koreaderBooks, newPageStats, annotations, deviceId);
6768
res.status(200).json({ message: 'Upload successful' });
6869
} catch (err) {
6970
console.error(err);

‎apps/server/src/upload/upload-service.ts‎

Lines changed: 42 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ export class UploadService {
4141
static uploadStatisticData(
4242
booksToImport: KoReaderBook[],
4343
newPageStats: PageStat[],
44-
annotationsByBook?: Record<string, KoReaderAnnotation[]>
44+
annotationsByBook?: Record<string, KoReaderAnnotation[]>,
45+
deviceIdOverride?: string // For annotation sync path without stats
4546
) {
4647
return db.transaction(async (trx) => {
4748
// Insert books
@@ -58,8 +59,13 @@ export class UploadService {
5859
newBooks.map(({ id, ...book }) => trx<Book>('book').insert(book).onConflict('md5').ignore())
5960
);
6061

61-
const hasUnknownDevices =
62-
newPageStats.length > 0 && newPageStats[0].device_id === this.UNKNOWN_DEVICE_ID;
62+
// Determine device ID: from stats, override, or fall back to unknown device
63+
const deviceId =
64+
newPageStats.length > 0
65+
? newPageStats[0].device_id
66+
: deviceIdOverride || this.UNKNOWN_DEVICE_ID;
67+
68+
const hasUnknownDevices = deviceId === this.UNKNOWN_DEVICE_ID;
6369

6470
if (hasUnknownDevices) {
6571
let unknownDevice = await trx<Device>('device')
@@ -76,47 +82,54 @@ export class UploadService {
7682
}
7783

7884
const newBookDevices: Omit<BookDevice, 'id'>[] = booksToImport.map((book) => ({
79-
device_id: newPageStats[0].device_id,
85+
device_id: deviceId,
8086
book_md5: book.md5,
8187
last_open: book.last_open,
8288
pages: book.pages,
8389
notes: book.notes,
8490
highlights: book.highlights,
85-
total_read_pages: book.total_read_pages,
86-
total_read_time: book.total_read_time,
91+
total_read_pages: book.total_read_pages ?? 0,
92+
total_read_time: book.total_read_time ?? 0,
8793
}));
8894

8995
await Promise.all(
90-
newBookDevices.map((bookDevice) =>
91-
trx<BookDevice>('book_device')
96+
newBookDevices.map((bookDevice) => {
97+
const { book_md5, device_id, total_read_time, total_read_pages, ...otherFields } =
98+
bookDevice;
99+
100+
// Always merge these fields
101+
const fieldsToMerge: (keyof BookDevice)[] = ['last_open', 'pages', 'notes', 'highlights'];
102+
103+
// Only merge statistics fields if they have actual values (if on statistics.db sync path)
104+
// This prevents annotation-only syncs from overwriting with zeros
105+
if (total_read_time !== undefined && total_read_time > 0) {
106+
fieldsToMerge.push('total_read_time');
107+
}
108+
if (total_read_pages !== undefined && total_read_pages > 0) {
109+
fieldsToMerge.push('total_read_pages');
110+
}
111+
112+
return trx<BookDevice>('book_device')
92113
.insert(bookDevice)
93114
.onConflict(['book_md5', 'device_id'])
94-
.merge([
95-
'last_open',
96-
'pages',
97-
'notes',
98-
'highlights',
99-
'total_read_time',
100-
'total_read_pages',
101-
])
102-
)
115+
.merge(fieldsToMerge);
116+
})
103117
);
104118

105-
// Insert page stats
106-
await Promise.all(
107-
newPageStats.map((pageStat) =>
108-
trx<PageStat>('page_stat')
109-
.insert(pageStat)
110-
.onConflict(['device_id', 'book_md5', 'page', 'start_time'])
111-
.merge(['duration', 'total_pages'])
112-
)
113-
);
119+
// Insert page stats (only on stats sync path! there are non for annotation sync path)
120+
if (newPageStats.length > 0) {
121+
await Promise.all(
122+
newPageStats.map((pageStat) =>
123+
trx<PageStat>('page_stat')
124+
.insert(pageStat)
125+
.onConflict(['device_id', 'book_md5', 'page', 'start_time'])
126+
.merge(['duration', 'total_pages'])
127+
)
128+
);
129+
}
114130

115131
// Insert annotations if provided
116132
if (annotationsByBook) {
117-
const deviceId =
118-
newPageStats.length > 0 ? newPageStats[0].device_id : this.UNKNOWN_DEVICE_ID;
119-
120133
await Promise.all(
121134
Object.entries(annotationsByBook).map(([bookMd5, annotations]) =>
122135
AnnotationsRepository.bulkInsert(bookMd5, deviceId, annotations, trx)

‎packages/common/types/book.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ export type KoReaderBook = {
99
pages: number;
1010
series: string;
1111
language: string;
12-
total_read_time: number;
13-
total_read_pages: number;
12+
// These fields only come from statistics.db sync, not annotation sync
13+
total_read_time?: number;
14+
total_read_pages?: number;
1415
};
1516

1617
export type DbBook = {

0 commit comments

Comments
 (0)