Ka-Note/ka-note/server/src/lib/ai-export-service.ts

513 lines
20 KiB
TypeScript

import { randomUUID } from 'crypto';
import { db, sqlite } from '../db/connection.js';
import { aiLocks, contexts, topics, historyEntries, ratings, imageBlobs, pages, notebooks, pageNotebooks } from '../db/schema.js';
import { eq, and, sql } from 'drizzle-orm';
import { zipSync, unzipSync, strToU8, strFromU8 } from 'fflate';
import type { AgendaContext, Topic, HistoryEntry, Rating, ImageBlob, Page, Notebook, PageNotebook } from '@ka-note/shared';
import { generateAiReadme } from './ai-agent-readme.js';
const LOCK_EXPIRY_HOURS = Number(process.env.AI_LOCK_EXPIRY_HOURS ?? 24);
const stmtFtsHistoryUpsert = sqlite.prepare(`INSERT OR REPLACE INTO fts_history(rowid, id, user_id, text, date, topic_id) SELECT rowid, id, user_id, text, date, topic_id FROM history_entries WHERE id = ? AND user_id = ?`);
const stmtFtsHistoryDelete = sqlite.prepare(`DELETE FROM fts_history WHERE id = ? AND user_id = ?`);
const stmtFtsPagesUpsert = sqlite.prepare(`INSERT OR REPLACE INTO fts_pages(rowid, id, user_id, title, body) SELECT rowid, id, user_id, title, body FROM pages WHERE id = ? AND user_id = ?`);
const stmtFtsPagesDelete = sqlite.prepare(`DELETE FROM fts_pages WHERE id = ? AND user_id = ?`);
type AiTableDef = typeof contexts | typeof topics | typeof historyEntries | typeof ratings | typeof imageBlobs | typeof pages | typeof notebooks | typeof pageNotebooks;
async function applyOps(ops: Array<{ action: 'insert' | 'update'; table: AiTableDef; row: Record<string, unknown> }>, userId: string): Promise<number> {
let accepted = 0;
for (const op of ops) {
if (op.action === 'insert') {
await db.insert(op.table).values(op.row as never);
} else {
await db.update(op.table).set(op.row as never)
.where(and(sql`${op.table.id} = ${op.row.id}`, sql`${op.table.userId} = ${userId}`));
}
accepted++;
if (op.table === historyEntries) {
if (op.row.deletedAt) stmtFtsHistoryDelete.run(op.row.id, userId);
else stmtFtsHistoryUpsert.run(op.row.id, userId);
} else if (op.table === pages) {
if (op.row.deletedAt) stmtFtsPagesDelete.run(op.row.id, userId);
else stmtFtsPagesUpsert.run(op.row.id, userId);
}
}
return accepted;
}
function now(): string {
return new Date().toISOString();
}
function expiresAt(): string {
const d = new Date();
d.setHours(d.getHours() + LOCK_EXPIRY_HOURS);
return d.toISOString();
}
// --- Lock management ---
export interface AiLock {
userId: string;
lockedAt: string;
token: string;
expiresAt: string;
}
export async function isLocked(userId: string): Promise<AiLock | null> {
const row = await db.select().from(aiLocks).where(eq(aiLocks.userId, userId)).get();
if (!row) return null;
if (new Date(row.expiresAt) < new Date()) {
await db.delete(aiLocks).where(eq(aiLocks.userId, userId));
return null;
}
return row;
}
export async function acquireLock(userId: string): Promise<AiLock> {
const existing = await isLocked(userId);
if (existing) return existing;
const lock: AiLock = {
userId,
lockedAt: now(),
token: randomUUID(),
expiresAt: expiresAt(),
};
await db.insert(aiLocks).values(lock).onConflictDoUpdate({
target: aiLocks.userId,
set: { lockedAt: lock.lockedAt, token: lock.token, expiresAt: lock.expiresAt },
});
return lock;
}
export async function validateLock(userId: string, token: string): Promise<AiLock | null> {
const lock = await isLocked(userId);
if (!lock || lock.token !== token) return null;
return lock;
}
export async function releaseLock(userId: string): Promise<void> {
await db.delete(aiLocks).where(eq(aiLocks.userId, userId));
}
// --- ZIP Export ---
function mimeToExt(mimeType: string): string {
const map: Record<string, string> = {
'image/jpeg': 'jpg',
'image/jpg': 'jpg',
'image/png': 'png',
'image/gif': 'gif',
'image/webp': 'webp',
};
return map[mimeType] ?? 'bin';
}
async function assembleZipFiles(userId: string, manifest: Record<string, unknown>): Promise<Uint8Array> {
const [ctxRows, topicRows, heRows, ratingRows, blobRows, pageRows, notebookRows, pageNotebookRows] = await Promise.all([
db.select().from(contexts).where(eq(contexts.userId, userId)),
db.select().from(topics).where(eq(topics.userId, userId)),
db.select().from(historyEntries).where(eq(historyEntries.userId, userId)),
db.select().from(ratings).where(eq(ratings.userId, userId)),
db.select().from(imageBlobs).where(eq(imageBlobs.userId, userId)),
db.select().from(pages).where(eq(pages.userId, userId)),
db.select().from(notebooks).where(eq(notebooks.userId, userId)),
db.select().from(pageNotebooks).where(eq(pageNotebooks.userId, userId)),
]);
const mappedContexts: AgendaContext[] = ctxRows.map((r) => ({
id: r.id, name: r.name, type: r.type, sortOrder: r.sortOrder,
meta: r.meta ? JSON.parse(r.meta) : null,
archivedAt: r.archivedAt, isFavorite: r.isFavorite,
updatedAt: r.updatedAt, deletedAt: r.deletedAt, purgedAt: r.purgedAt ?? null, version: r.version,
}));
const mappedTopics: Topic[] = topicRows.map((r) => ({
id: r.id, contextId: r.contextId, title: r.title, status: r.status,
snoozeUntil: r.snoozeUntil, sortOrder: r.sortOrder, isNew: r.isNew,
updatedAt: r.updatedAt, deletedAt: r.deletedAt, purgedAt: r.purgedAt ?? null, version: r.version,
}));
const historyMeta: Omit<HistoryEntry, 'text'>[] = heRows.map((r) => ({
id: r.id, topicId: r.topicId, date: r.date, sortOrder: r.sortOrder,
linkedContextId: r.linkedContextId, doneAt: r.doneAt,
wiedervorlageDate: r.wiedervorlageDate ?? null,
wiedervorlageResolvedAt: r.wiedervorlageResolvedAt ?? null,
updatedAt: r.updatedAt, deletedAt: r.deletedAt, purgedAt: r.purgedAt ?? null, version: r.version,
}));
const mappedRatings: Rating[] = ratingRows.map((r) => ({
id: r.id, topicId: r.topicId, historyEntryId: r.historyEntryId,
personName: r.personName, value: r.value as 1 | 2 | 3 | 4,
comment: r.comment, updatedAt: r.updatedAt, deletedAt: r.deletedAt, purgedAt: r.purgedAt ?? null, version: r.version,
}));
const mappedNotebooks: Notebook[] = notebookRows.map((r) => ({
id: r.id, name: r.name, contextId: r.contextId, isPrivate: r.isPrivate, isFavorite: r.isFavorite,
sortOrder: r.sortOrder, updatedAt: r.updatedAt, deletedAt: r.deletedAt,
purgedAt: r.purgedAt ?? null, version: r.version,
}));
const mappedPageNotebooks: PageNotebook[] = pageNotebookRows.map((r) => ({
id: r.id, pageId: r.pageId, notebookId: r.notebookId, sortOrder: r.sortOrder,
updatedAt: r.updatedAt, deletedAt: r.deletedAt, purgedAt: r.purgedAt ?? null, version: r.version,
}));
const files: Record<string, Uint8Array> = {};
files['manifest.json'] = strToU8(JSON.stringify(manifest, null, 2));
if ('lockToken' in manifest) {
files['README.md'] = strToU8(generateAiReadme(manifest as unknown as Parameters<typeof generateAiReadme>[0]));
}
files['contexts.json'] = strToU8(JSON.stringify(mappedContexts, null, 2));
files['topics.json'] = strToU8(JSON.stringify(mappedTopics, null, 2));
files['ratings.json'] = strToU8(JSON.stringify(mappedRatings, null, 2));
for (const meta of historyMeta) {
files[`history/${meta.id}.meta.json`] = strToU8(JSON.stringify(meta, null, 2));
}
for (const r of heRows) {
files[`history/${r.id}.md`] = strToU8(r.text);
}
for (const b of blobRows) {
const ext = mimeToExt(b.mimeType);
files[`images/${b.id}.${ext}`] = new Uint8Array(b.data as Buffer);
}
files['notebooks.json'] = strToU8(JSON.stringify(mappedNotebooks, null, 2));
files['page_notebooks.json'] = strToU8(JSON.stringify(mappedPageNotebooks, null, 2));
for (const pg of pageRows) {
const meta: Omit<Page, 'body'> = {
id: pg.id, title: pg.title, isPrivate: pg.isPrivate, sortOrder: pg.sortOrder,
updatedAt: pg.updatedAt, deletedAt: pg.deletedAt, purgedAt: pg.purgedAt ?? null, version: pg.version,
};
files[`wiki/${pg.id}.meta.json`] = strToU8(JSON.stringify(meta, null, 2));
files[`wiki/${pg.id}.md`] = strToU8(pg.body);
}
return zipSync(files, { level: 1 });
}
export async function exportWorkspaceZip(userId: string): Promise<{ zip: Uint8Array; lock: AiLock }> {
const lock = await acquireLock(userId);
const manifest = {
userId: lock.userId,
exportedAt: lock.lockedAt,
lockToken: lock.token,
expiresAt: lock.expiresAt,
exportVersion: 3,
};
const zip = await assembleZipFiles(userId, manifest);
return { zip, lock };
}
export async function buildBackupZip(userId: string): Promise<Uint8Array> {
const manifest = {
userId,
exportedAt: now(),
exportVersion: 3,
type: 'backup',
};
return assembleZipFiles(userId, manifest);
}
// --- ZIP Upload ---
export interface AiUploadResult {
accepted: number;
skipped: number;
conflicts: Array<{ entityType: string; entityId: string; clientVersion: number; serverVersion: number }>;
}
type TableDef = AiTableDef;
async function checkConflict(
table: TableDef,
entity: Record<string, unknown>,
userId: string,
): Promise<'insert' | 'update' | 'skip' | { serverVersion: number }> {
const id = entity.id as string;
const clientVersion = entity.version as number;
const existing = await db.select().from(table)
.where(and(sql`${table.id} = ${id}`, sql`${table.userId} = ${userId}`))
.get();
if (!existing) return 'insert';
const sv = (existing as { version: number }).version;
if (clientVersion > sv) return 'update';
if (clientVersion === sv) return 'skip';
return { serverVersion: sv };
}
export async function applyUploadFromZip(
zipBuffer: Buffer,
userId: string,
force = false,
): Promise<{ result: AiUploadResult; conflict: boolean }> {
const rawUnzipped = unzipSync(new Uint8Array(zipBuffer));
// Normalize backslash paths (Windows Compress-Archive creates entries with backslashes)
const unzipped: typeof rawUnzipped = {};
for (const [key, val] of Object.entries(rawUnzipped)) {
unzipped[key.replace(/\\/g, '/')] = val;
}
const readJson = <T>(path: string, fallback: T): T => {
const file = unzipped[path];
if (!file) return fallback;
try { return JSON.parse(strFromU8(file)) as T; } catch { return fallback; }
};
const manifest = readJson<{ lockToken?: string }>('manifest.json', {});
if (!manifest.lockToken) {
throw new Error('manifest.json missing or no lockToken');
}
const ctxs: AgendaContext[] = readJson('contexts.json', []);
const tpcs: Topic[] = readJson('topics.json', []);
const rats: Rating[] = readJson('ratings.json', []);
const nbs: Notebook[] = readJson('notebooks.json', []);
const pns: PageNotebook[] = readJson('page_notebooks.json', []);
// Reconstruct historyEntries from history/*.meta.json + history/*.md
const hes: HistoryEntry[] = [];
for (const path of Object.keys(unzipped)) {
if (!path.startsWith('history/') || !path.endsWith('.meta.json')) continue;
const meta = readJson<Omit<HistoryEntry, 'text'>>(path, null as never);
if (!meta?.id) continue;
const mdPath = `history/${meta.id}.md`;
const text = unzipped[mdPath] ? strFromU8(unzipped[mdPath]) : '';
hes.push({ ...meta, text });
}
// Reconstruct pages from wiki/*.meta.json + wiki/*.md
const pgs: Page[] = [];
for (const path of Object.keys(unzipped)) {
if (!path.startsWith('wiki/') || !path.endsWith('.meta.json')) continue;
const meta = readJson<Omit<Page, 'body'>>(path, null as never);
if (!meta?.id) continue;
const mdPath = `wiki/${meta.id}.md`;
const body = unzipped[mdPath] ? strFromU8(unzipped[mdPath]) : '';
pgs.push({ ...meta, body });
}
// Reconstruct imageBlobs from images/*
const blobs: Array<{
id: string; mimeType: string; contentHash: string;
data: Buffer; updatedAt: string; deletedAt: string | null; version: number;
}> = [];
for (const path of Object.keys(unzipped)) {
if (!path.startsWith('images/')) continue;
const filename = path.replace('images/', '');
const dotIdx = filename.lastIndexOf('.');
if (dotIdx < 0) continue;
const id = filename.slice(0, dotIdx);
const ext = filename.slice(dotIdx + 1);
const extMime: Record<string, string> = { jpg: 'image/jpeg', png: 'image/png', gif: 'image/gif', webp: 'image/webp' };
const mimeType = extMime[ext] ?? 'application/octet-stream';
const data = Buffer.from(unzipped[path]);
// Compute SHA-256 for contentHash
const { createHash } = await import('crypto');
const contentHash = createHash('sha256').update(data).digest('hex');
blobs.push({ id, mimeType, contentHash, data, updatedAt: now(), deletedAt: null, version: 1 });
}
type ConflictEntry = { entityType: string; entityId: string; clientVersion: number; serverVersion: number };
const conflicts: ConflictEntry[] = [];
type Op = { table: TableDef; action: 'insert' | 'update'; row: Record<string, unknown> };
const ops: Op[] = [];
let skipped = 0;
const check = async (
table: TableDef,
entity: Record<string, unknown>,
entityType: string,
row: Record<string, unknown>,
) => {
const result = await checkConflict(table, entity, userId);
if (result === 'insert' || result === 'update') {
ops.push({ table, action: result, row });
} else if (result === 'skip') {
skipped++;
} else {
conflicts.push({
entityType,
entityId: entity.id as string,
clientVersion: entity.version as number,
serverVersion: result.serverVersion,
});
}
};
for (const ctx of ctxs) {
await check(contexts, ctx as unknown as Record<string, unknown>, 'context', {
id: ctx.id, userId, name: ctx.name, type: ctx.type, sortOrder: ctx.sortOrder,
meta: ctx.meta ? JSON.stringify(ctx.meta) : null,
archivedAt: ctx.archivedAt, isFavorite: ctx.isFavorite,
updatedAt: ctx.updatedAt, deletedAt: ctx.deletedAt, version: ctx.version,
});
}
for (const topic of tpcs) {
await check(topics, topic as unknown as Record<string, unknown>, 'topic', {
id: topic.id, userId, contextId: topic.contextId, title: topic.title,
status: topic.status, snoozeUntil: topic.snoozeUntil, sortOrder: topic.sortOrder,
isNew: topic.isNew, updatedAt: topic.updatedAt, deletedAt: topic.deletedAt, version: topic.version,
});
}
for (const he of hes) {
await check(historyEntries, he as unknown as Record<string, unknown>, 'historyEntry', {
id: he.id, userId, topicId: he.topicId, date: he.date, text: he.text,
sortOrder: he.sortOrder, linkedContextId: he.linkedContextId, doneAt: he.doneAt,
wiedervorlageDate: he.wiedervorlageDate, wiedervorlageResolvedAt: he.wiedervorlageResolvedAt,
updatedAt: he.updatedAt, deletedAt: he.deletedAt, version: he.version,
});
}
for (const rat of rats) {
await check(ratings, rat as unknown as Record<string, unknown>, 'rating', {
id: rat.id, userId, topicId: rat.topicId, historyEntryId: rat.historyEntryId,
personName: rat.personName, value: rat.value, comment: rat.comment,
updatedAt: rat.updatedAt, deletedAt: rat.deletedAt, purgedAt: rat.purgedAt ?? null, version: rat.version,
});
}
for (const pg of pgs) {
await check(pages, pg as unknown as Record<string, unknown>, 'page', {
id: pg.id, userId, title: pg.title, body: pg.body, isPrivate: pg.isPrivate,
sortOrder: pg.sortOrder, updatedAt: pg.updatedAt, deletedAt: pg.deletedAt, purgedAt: pg.purgedAt ?? null, version: pg.version,
});
}
for (const nb of nbs) {
await check(notebooks, nb as unknown as Record<string, unknown>, 'notebook', {
id: nb.id, userId, name: nb.name, contextId: nb.contextId, isPrivate: nb.isPrivate,
sortOrder: nb.sortOrder, updatedAt: nb.updatedAt, deletedAt: nb.deletedAt, purgedAt: nb.purgedAt ?? null, version: nb.version,
});
}
for (const pn of pns) {
await check(pageNotebooks, pn as unknown as Record<string, unknown>, 'pageNotebook', {
id: pn.id, userId, pageId: pn.pageId, notebookId: pn.notebookId, sortOrder: pn.sortOrder,
updatedAt: pn.updatedAt, deletedAt: pn.deletedAt, purgedAt: pn.purgedAt ?? null, version: pn.version,
});
}
// Images: deduplicate by contentHash — skip if same hash already exists
for (const b of blobs) {
const existingByHash = await db.select().from(imageBlobs)
.where(and(eq(imageBlobs.userId, userId), sql`${imageBlobs.contentHash} = ${b.contentHash}`))
.get();
if (existingByHash && existingByHash.id !== b.id) {
skipped++;
continue;
}
await check(imageBlobs, b as unknown as Record<string, unknown>, 'imageBlob', {
id: b.id, userId, mimeType: b.mimeType, contentHash: b.contentHash,
data: b.data, updatedAt: b.updatedAt, deletedAt: b.deletedAt, version: b.version,
});
}
if (conflicts.length > 0 && !force) {
return { result: { accepted: 0, skipped, conflicts }, conflict: true };
}
const accepted = await applyOps(ops, userId);
return { result: { accepted, skipped, conflicts }, conflict: false };
}
// --- Legacy JSON upload (kept for backwards compat during transition) ---
export interface AiUploadRequest {
lockToken: string;
force?: boolean;
contexts?: AgendaContext[];
topics?: Topic[];
historyEntries?: HistoryEntry[];
ratings?: Rating[];
}
export async function applyUpload(
request: AiUploadRequest,
userId: string,
): Promise<{ result: AiUploadResult; conflict: boolean }> {
const force = request.force ?? false;
const ctxs = request.contexts ?? [];
const tpcs = request.topics ?? [];
const hes = request.historyEntries ?? [];
const rats = request.ratings ?? [];
type ConflictEntry = { entityType: string; entityId: string; clientVersion: number; serverVersion: number };
const conflicts: ConflictEntry[] = [];
type Op = { table: TableDef; action: 'insert' | 'update'; row: Record<string, unknown> };
const ops: Op[] = [];
let skipped = 0;
const check = async (
table: TableDef,
entity: Record<string, unknown>,
entityType: string,
row: Record<string, unknown>,
) => {
const result = await checkConflict(table, entity, userId);
if (result === 'insert' || result === 'update') {
ops.push({ table, action: result, row });
} else if (result === 'skip') {
skipped++;
} else {
conflicts.push({
entityType,
entityId: entity.id as string,
clientVersion: entity.version as number,
serverVersion: result.serverVersion,
});
}
};
for (const ctx of ctxs) {
await check(contexts, ctx as unknown as Record<string, unknown>, 'context', {
id: ctx.id, userId, name: ctx.name, type: ctx.type, sortOrder: ctx.sortOrder,
meta: ctx.meta ? JSON.stringify(ctx.meta) : null,
archivedAt: ctx.archivedAt, isFavorite: ctx.isFavorite,
updatedAt: ctx.updatedAt, deletedAt: ctx.deletedAt, version: ctx.version,
});
}
for (const topic of tpcs) {
await check(topics, topic as unknown as Record<string, unknown>, 'topic', {
id: topic.id, userId, contextId: topic.contextId, title: topic.title,
status: topic.status, snoozeUntil: topic.snoozeUntil, sortOrder: topic.sortOrder,
isNew: topic.isNew, updatedAt: topic.updatedAt, deletedAt: topic.deletedAt, version: topic.version,
});
}
for (const he of hes) {
await check(historyEntries, he as unknown as Record<string, unknown>, 'historyEntry', {
id: he.id, userId, topicId: he.topicId, date: he.date, text: he.text,
sortOrder: he.sortOrder, linkedContextId: he.linkedContextId, doneAt: he.doneAt,
wiedervorlageDate: he.wiedervorlageDate, wiedervorlageResolvedAt: he.wiedervorlageResolvedAt,
updatedAt: he.updatedAt, deletedAt: he.deletedAt, version: he.version,
});
}
for (const rat of rats) {
await check(ratings, rat as unknown as Record<string, unknown>, 'rating', {
id: rat.id, userId, topicId: rat.topicId, historyEntryId: rat.historyEntryId,
personName: rat.personName, value: rat.value, comment: rat.comment,
updatedAt: rat.updatedAt, deletedAt: rat.deletedAt, purgedAt: rat.purgedAt ?? null, version: rat.version,
});
}
if (conflicts.length > 0 && !force) {
return { result: { accepted: 0, skipped, conflicts }, conflict: true };
}
const accepted = await applyOps(ops, userId);
return { result: { accepted, skipped, conflicts }, conflict: false };
}