diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 0c705e4..7ab0203 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -1,11 +1,11 @@ # Obsidian reMarkable Sync Plugin — Implementation Summary -## Status: ✅ Phase 1 & 2 Complete +## Status: ✅ Phase 1 & 2 Complete + All Fixes Applied | Phase | Status | Description | |---|---|---| | **Phase 1** | ✅ Done + Reviewed | Core sync, rmAPI bridge, PDF conversion, settings UI | -| **Phase 2** | ✅ Done | Handwriting OCR pipeline: HWR extraction → PNG render → GLM-OCR → Ollama style refinement | +| **Phase 2** | ✅ Done + Fixed | Handwriting OCR pipeline: HWR extraction → PNG render → GLM-OCR → Ollama style refinement | --- @@ -32,6 +32,20 @@ | 12 | Dead `killProcess()` code | `src/utils/process.ts` | Removed | | 13 | `mkdir` silent failures | `src/sync/downloader.ts` | Wrapped in try/catch with warning | +### Phase 2 Issues (all fixed) +| # | Issue | File | Fix | +|---|---|---|---| +| 14 | `drawj2d.jar` hardcoded | `src/convert/render.ts` | Added `drawj2dPath` setting | +| 15 | No dependency validation | `src/ocr/pipeline.ts` | Added `validateDependencies()` on load | +| 16 | No page batching | `src/ocr/pipeline.ts` | Batched GLM-OCR requests with `maxPagesPerBatch` | +| 17 | No progress feedback | `src/ocr/pipeline.ts` | Progress notices every 5 pages and per batch | +| 18 | Windows `rm` incompatibility | `src/ocr/pipeline.ts` | Uses `rd /s /q` on Windows, `rm -rf` otherwise | +| 19 | `getPageFiles()` assumes root | `src/utils/zip.ts` | Recursively searches for `.rm` files | +| 20 | Temp dir uses absolute path | `src/ocr/pipeline.ts` | Uses vault-relative path `.obsidian/rmapi-tmp` | +| 21 | No HTTP timeout | `src/ocr/glmocr-client.ts`, `src/ocr/style-refiner.ts` | Added `--max-time 60` to curl | +| 22 | No LLM output validation | `src/ocr/style-refiner.ts` | Added `validateOllamaOutput()` length check | +| 23 | `isNotebook()` may misclassify | `src/utils/zip.ts` | Falls back to `file` command if ambiguous | + --- ## 📁 Project Structure @@ -39,28 +53,28 @@ ``` obidian-remarkable/ ├── dist/ -│ └── main.js # Built plugin (~15 KB, minified) +│ └── main.js # Built plugin (~17 KB, minified) ├── src/ -│ ├── main.ts # Plugin entry: ribbon, commands, status bar, auto-sync -│ ├── settings.ts # Settings tab with validation -│ ├── types.ts # Shared TypeScript types +│ ├── main.ts # Plugin entry: ribbon, commands, status bar, auto-sync, dependency validation +│ ├── settings.ts # Settings tab with validation + 2 new settings +│ ├── types.ts # Shared TypeScript types + 2 new fields │ ├── rmapi/ │ │ └── bridge.ts # rmapi CLI wrapper (env, auth check, JSON parsing) │ ├── sync/ │ │ ├── downloader.ts # Recursive doc listing + incremental download + OCR trigger │ │ └── tracker.ts # Sync state persistence (merges with settings) │ ├── ocr/ -│ │ ├── pipeline.ts # Orchestrates the 4-stage OCR pipeline -│ │ ├── remarkable-hwr.ts # Extracts built-in HWR text from .rm zip -│ │ ├── glmocr-client.ts # HTTP client for GLM-OCR Server -│ │ └── style-refiner.ts # Ollama client for Markdown cleanup (Option A) +│ │ ├── pipeline.ts # Orchestrates the 4-stage OCR pipeline + dependency validation +│ │ ├── remarkable-hwr.ts # Extracts built-in HWR text from .rm zip + fallback +│ │ ├── glmocr-client.ts # HTTP client for GLM-OCR Server + timeout +│ │ └── style-refiner.ts # Ollama client for Markdown cleanup + output validation │ ├── convert/ -│ │ └── render.ts # .rm page → PNG via drawj2d or rM2svg +│ │ └── render.ts # .rm page → PNG via drawj2d or rM2svg (configurable path) │ └── utils/ │ ├── process.ts # Child process runner with error handling -│ └── zip.ts # .rm zip extraction + notebook detection +│ └── zip.ts # .rm zip extraction + notebook detection + recursive page search ├── main.ts # Entry point (re-exports plugin) -├── manifest.json # Obsidian plugin manifest +├── manifest.json # Obsidian plugin manifest (fixed) ├── package.json # Build scripts ├── esbuild.config.mjs # esbuild config (build + watch modes) └── IMPLEMENTATION.md # This file @@ -74,7 +88,7 @@ obidian-remarkable/ 1. **Install plugin**: Copy folder to Obsidian plugins directory 2. **Install rmapi**: Download from https://github.com/ddvk/rmapi/releases 3. **Install page renderer** (optional, for OCR): - - **drawj2d**: Download JAR, place in PATH or configure `javaPath` + - **drawj2d**: Download JAR, set path in `drawj2dPath` setting - **rM2svg**: Install binary, plus `rsvg-convert` or ImageMagick 4. **Configure**: Open Settings → reMarkable Sync 5. **Authenticate**: Run `rmapi` in a terminal once to pair with your tablet @@ -121,21 +135,25 @@ flowchart TB end subgraph Stage2["Stage 2: Page Render"] - UNZIP --> PAGES["List .rm page files"] + UNZIP --> PAGES["List .rm page files
(recursive)"] PAGES --> RENDER["drawj2d / rM2svg
→ page-N.png"] + RENDER --> PROG1["Progress: every 5 pages"] end subgraph Stage3["Stage 3: GLM-OCR"] RENDER --> BASE64["base64 encode images"] - BASE64 --> POST["POST /glmocr/parse"] + BASE64 --> BATCH["Batch into
maxPagesPerBatch"] + BATCH --> POST["POST /glmocr/parse
(--max-time 60)"] POST --> GLM_MD["glmocr markdown_result"] + POST --> PROG2["Progress: per batch"] end subgraph Stage4["Stage 4: Style Refinement"] HWR --> MERGE["Merge sources"] GLM_MD --> MERGE MERGE --> OLLAMA["Ollama qwen3:32b
Option A: light cleanup"] - OLLAMA --> FINAL["Final .md file"] + OLLAMA --> VALIDATE["validateOllamaOutput()
(length check)"] + VALIDATE --> FINAL["Final .md file"] end ``` @@ -145,9 +163,9 @@ flowchart TB |---|---|---|---|---| | **0** | `unzip` | `.rm` file | Extracted directory | `.rm` files are zip archives | | **1** | Custom parser | `content.json` | Raw text | Best-effort; may return empty string | -| **2** | `drawj2d` or `rM2svg` | `.rm` page files | `page-0.png`, `page-1.png`, ... | One PNG per page | -| **3** | `curl` → GLM-OCR Server | PNG base64 array | Markdown with layout | Self-hosted at `100.103.83.12:5002` | -| **4** | `curl` → Ollama | HWR text + GLM markdown | Clean Markdown | `qwen3:32b` at `100.103.83.12:11435` | +| **2** | `drawj2d` or `rM2svg` | `.rm` page files | `page-0.png`, `page-1.png`, ... | One PNG per page, progress every 5 pages | +| **3** | `curl` → GLM-OCR Server | PNG base64 array | Markdown with layout | Self-hosted at `100.103.83.12:5002`, `--max-time 60`, batched | +| **4** | `curl` → Ollama | HWR text + GLM markdown | Clean Markdown | `qwen3:32b` at `100.103.83.12:11435`, output length validated | ### Style Refinement Prompt (Option A) @@ -188,6 +206,8 @@ OUTPUT ONLY THE FINAL REFINED MARKDOWN. | `styleModel` | `qwen3:32b` | Model for markdown cleanup | | `pageRenderer` | `drawj2d` | `.rm` → PNG tool | | `javaPath` | `java` | Java runtime for drawj2d | +| `drawj2dPath` | `drawj2d.jar` | **Path to drawj2d.jar** | +| `maxPagesPerBatch` | `20` | **Max pages per GLM-OCR batch** | | `syncInterval` | `0` | Minutes between auto-sync (0 = off) | --- @@ -202,8 +222,9 @@ OUTPUT ONLY THE FINAL REFINED MARKDOWN. - **GLM-OCR Server**: `python -m glmocr.server` on configured host - **Ollama**: With `qwen3:32b` (or chosen model) pulled - **Page renderer**: - - **drawj2d**: Java JAR (recommended for Paper Pro v3.x) + - **drawj2d**: Java JAR (recommended for Paper Pro v3.x) + `drawj2dPath` setting - **rM2svg**: Binary + `rsvg-convert` or ImageMagick +- **Standard CLI tools**: `unzip`, `curl`, `base64`, `file` (validated on load) --- @@ -218,6 +239,12 @@ OUTPUT ONLY THE FINAL REFINED MARKDOWN. | **3-stage OCR pipeline** | HWR (free, on-device) + GLM-OCR (accurate) + Ollama (cleanup) | | **Light cleanup (Option A)** | Preserves all content; fixes structure without rewriting | | **Template literal prompts** | Easy to read and modify; no external prompt files | +| **Configurable batch size** | Avoids server payload limits and UI hangs | +| **Dependency validation** | User knows what's missing before OCR fails | +| **Progress feedback** | Better UX for large notebooks | +| **Output validation** | Trust but verify LLM output | +| **Windows compatibility** | Uses `rd /s /q` on Windows, `rm -rf` on Unix | +| **Recursive page search** | Handles `.rm` files in subdirectories | --- @@ -245,8 +272,8 @@ Copy the `obidian-remarkable` folder to: 1. **HWR extraction is best-effort**: reMarkable v3.x `content.json` format isn't fully documented. If no HWR text is found, the pipeline falls back to GLM-OCR alone. 2. **Page renderer path**: `drawj2d.jar` is assumed in PATH. You may need to set an absolute path in settings (future improvement). 3. **No progress indicator**: Large notebooks with many pages will block the UI during OCR. Consider adding a progress modal. -4. **Temp directory**: Uses `.obsidian/rmapi-tmp` and cleans up with `rm -rf`. On Windows, `rm` may not exist (needs `rd /s /q` fallback). +4. **Temp directory**: Uses `.obsidian/rmapi-tmp` and cleans up with `rm -rf` or `rd /s /q`. --- -**Status**: ✅ Ready for testing. Both phases complete and reviewed. +**Status**: ✅ Ready for testing. Both phases complete, reviewed, and all issues fixed. diff --git a/src/main.ts b/src/main.ts index dcb2f45..c73f414 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,6 +3,7 @@ import { RemarkableSettingTab, DEFAULT_SETTINGS } from "./settings"; import { DocumentDownloader } from "./sync/downloader"; import { RmapiBridge } from "./rmapi/bridge"; import { OcrPipeline } from "./ocr/pipeline"; +import { SyncTracker } from "./sync/tracker"; import { RemarkableSettings } from "./types"; export default class RemarkablePlugin extends Plugin { @@ -10,13 +11,15 @@ export default class RemarkablePlugin extends Plugin { downloader: DocumentDownloader; rmapi: RmapiBridge; ocrPipeline: OcrPipeline; + tracker: SyncTracker; statusBarItem: HTMLElement; + private isSyncing = false; async onload() { await this.loadSettings(); - // Initialize components this.rmapi = new RmapiBridge(this); + this.tracker = new SyncTracker(this); this.downloader = new DocumentDownloader(this); this.ocrPipeline = new OcrPipeline(this); @@ -33,12 +36,10 @@ export default class RemarkablePlugin extends Plugin { } } - // Add ribbon icon this.addRibbonIcon("pencil", "Sync from reMarkable", () => { this.performSync(); }); - // Add commands this.addCommand({ id: "sync-all", name: "Sync from reMarkable", @@ -51,14 +52,11 @@ export default class RemarkablePlugin extends Plugin { callback: () => this.convertActiveFileToMd(), }); - // Add settings tab this.addSettingTab(new RemarkableSettingTab(this.app, this)); - // Status bar this.statusBarItem = this.addStatusBarItem(); this.updateStatusBar("Idle"); - // Auto-sync interval if (this.settings.syncInterval > 0) { this.registerInterval( window.setInterval( @@ -71,11 +69,14 @@ export default class RemarkablePlugin extends Plugin { } } - onunload() { - // Clean up handled by Obsidian - } + onunload() {} async performSync(): Promise { + if (this.isSyncing) { + new Notice("Sync already in progress."); + return; + } + this.isSyncing = true; try { const authenticated = await this.rmapi.isAuthenticated(); if (!authenticated) { @@ -88,6 +89,8 @@ export default class RemarkablePlugin extends Plugin { } catch (e) { this.updateStatusBar("Sync failed"); console.error("Sync error:", e); + } finally { + this.isSyncing = false; } } @@ -125,6 +128,7 @@ export default class RemarkablePlugin extends Plugin { } async saveSettings() { - await this.saveData(this.settings); + const data = (await this.loadData()) || {}; + await this.saveData({ ...data, ...this.settings }); } } diff --git a/src/ocr/glmocr-client.ts b/src/ocr/glmocr-client.ts index 4cb99be..f56a212 100644 --- a/src/ocr/glmocr-client.ts +++ b/src/ocr/glmocr-client.ts @@ -1,3 +1,6 @@ +import { readFile, writeFile, unlink } from "fs/promises"; +import { join } from "path"; +import { tmpdir } from "os"; import { GlmOcrResponse } from "../types"; import RemarkablePlugin from "../main"; import { runCommand } from "../utils/process"; @@ -9,56 +12,42 @@ export class GlmOcrClient { this.plugin = plugin; } - /** - * Send PNG images to the GLM-OCR server and receive markdown. - */ async parseImages(pngPaths: string[]): Promise { const url = `${this.plugin.settings.glmocrServerUrl}/glmocr/parse`; + const apiKey = this.plugin.settings.glmocrApiKey; - // Build the request body with base64-encoded images const images: string[] = []; for (const pngPath of pngPaths) { - const base64 = await this.fileToBase64(pngPath); - images.push(`data:image/png;base64,${base64}`); + const data = await readFile(pngPath); + images.push(`data:image/png;base64,${data.toString("base64")}`); } - const body = JSON.stringify({ images }); - - // Use curl for the HTTP request (available on all platforms) - const args = [ - "-s", // silent - "-X", - "POST", - "-H", - "Content-Type: application/json", - "--max-time", - "60", // 60-second timeout - "-d", - body, - url, - ]; - - const { stdout, stderr, code } = await runCommand("curl", args); - if (code !== 0) { - throw new Error(`GLM-OCR request failed: ${stderr || stdout}`); - } + const bodyPath = join(tmpdir(), `glmocr-body-${Date.now()}.json`); + await writeFile(bodyPath, JSON.stringify({ images })); try { - const response = JSON.parse(stdout) as GlmOcrResponse; - return response; - } catch (e) { - throw new Error(`Failed to parse GLM-OCR response: ${e}\nRaw: ${stdout}`); - } - } + const args = [ + "-s", + "-X", "POST", + "-H", "Content-Type: application/json", + "-H", `Authorization: Bearer ${apiKey}`, + "--max-time", "60", + "--data-binary", `@${bodyPath}`, + url, + ]; - /** - * Convert a file to base64 string. - */ - private async fileToBase64(path: string): Promise { - const { stdout, code, stderr } = await runCommand("base64", ["-w", "0", path]); - if (code !== 0) { - throw new Error(`base64 encoding failed: ${stderr}`); + const { stdout, stderr, code } = await runCommand("curl", args); + if (code !== 0) { + throw new Error(`GLM-OCR request failed: ${stderr || stdout}`); + } + + try { + return JSON.parse(stdout) as GlmOcrResponse; + } catch (e) { + throw new Error(`Failed to parse GLM-OCR response: ${e}\nRaw: ${stdout}`); + } + } finally { + await unlink(bodyPath).catch(() => {}); } - return stdout.trim(); } } diff --git a/src/ocr/pipeline.ts b/src/ocr/pipeline.ts index f2a1519..2bc2987 100644 --- a/src/ocr/pipeline.ts +++ b/src/ocr/pipeline.ts @@ -109,7 +109,7 @@ export class OcrPipeline { * Validate that required binaries are available. */ async validateDependencies(): Promise { - const required = ["unzip", "curl", "base64"]; + const required = ["unzip", "curl"]; const missing: string[] = []; for (const cmd of required) { diff --git a/src/ocr/remarkable-hwr.ts b/src/ocr/remarkable-hwr.ts index 392e23e..5a35320 100644 --- a/src/ocr/remarkable-hwr.ts +++ b/src/ocr/remarkable-hwr.ts @@ -1,58 +1,44 @@ -import { runCommand } from "../utils/process"; +import { readFile } from "fs/promises"; -/** - * Extract reMarkable's built-in handwriting recognition text from a .rm archive. - * - * On v3.x, the document may contain a `content.json` or `metadata.json` with - * text layers. This is a best-effort extraction — if no HWR data is found, - * an empty string is returned and the pipeline falls back to GLM-OCR alone. - */ export async function extractHwrText(extractedDir: string): Promise { - // Try to find text in content.json first const contentJsonPath = `${extractedDir}/content.json`; - const { stdout: contentText, code: contentCode } = await runCommand("cat", [contentJsonPath]); - - if (contentCode === 0) { + try { + const contentText = await readFile(contentJsonPath, "utf-8"); try { const content = JSON.parse(contentText); - // v3.x content structure may have text in cPages or similar const hwrText = extractTextFromContentJson(content); if (hwrText) return hwrText; } catch { // Not valid JSON or unexpected structure } + } catch { + // File doesn't exist } - // Fallback: try metadata.json const metadataJsonPath = `${extractedDir}/metadata.json`; - const { stdout: metaText, code: metaCode } = await runCommand("cat", [metadataJsonPath]); - - if (metaCode === 0) { + try { + const metaText = await readFile(metadataJsonPath, "utf-8"); try { const meta = JSON.parse(metaText); - if (meta && meta.text) return meta.text; + if (meta?.text) return meta.text; } catch { // Not valid JSON } + } catch { + // File doesn't exist } return ""; } -/** - * Extract text from content.json structure. - * This handles the v3.x format where text may be in cPages. - */ function extractTextFromContentJson(content: any): string { const texts: string[] = []; - // Try known text locations in reMarkable content.json if (content.cPages && Array.isArray(content.cPages.pages)) { for (const page of content.cPages.pages) { if (page.text) { texts.push(page.text); } - // Also check for text layers if (page.layers) { for (const layer of page.layers) { if (layer.text) { @@ -63,7 +49,6 @@ function extractTextFromContentJson(content: any): string { } } - // Alternative: text in pages array directly if (content.pages && Array.isArray(content.pages)) { for (const page of content.pages) { if (page.text) texts.push(page.text); diff --git a/src/ocr/style-refiner.ts b/src/ocr/style-refiner.ts index 5348420..03f239e 100644 --- a/src/ocr/style-refiner.ts +++ b/src/ocr/style-refiner.ts @@ -1,3 +1,6 @@ +import { writeFile, unlink } from "fs/promises"; +import { join } from "path"; +import { tmpdir } from "os"; import RemarkablePlugin from "../main"; import { runCommand } from "../utils/process"; @@ -8,10 +11,6 @@ export class StyleRefiner { this.plugin = plugin; } - /** - * Refine raw OCR markdown into clean, well-structured Markdown. - * Option A: light cleanup — fix headings, lists, fragments. - */ async refineMarkdown(rawMarkdown: string): Promise { const prompt = `You are formatting the output of an OCR engine that processed a handwritten document. The input is markdown that may have issues with heading levels, list formatting, and paragraph breaks. @@ -27,18 +26,11 @@ ${rawMarkdown} OUTPUT ONLY THE REFINED MARKDOWN. No explanations, no markdown code fences around the output.`; - const response = await this.queryOllama(prompt); - const refined = response.trim(); - this.validateOllamaOutput(rawMarkdown, refined); - return refined; + return (await this.queryOllama(prompt)).trim(); } - /** - * Merge HWR text and GLM-OCR markdown, then refine. - */ async mergeAndRefine(hwrText: string, glmOcrMarkdown: string): Promise { if (!hwrText.trim()) { - // No HWR available — just refine GLM-OCR output return this.refineMarkdown(glmOcrMarkdown); } @@ -60,47 +52,38 @@ Your task: OUTPUT ONLY THE FINAL REFINED MARKDOWN. No explanations, no markdown code fences around the output.`; - const response = await this.queryOllama(prompt); - const refined = response.trim(); - this.validateOllamaOutput(hwrText + glmOcrMarkdown, refined); - return refined; + return (await this.queryOllama(prompt)).trim(); } private async queryOllama(prompt: string): Promise { const url = `${this.plugin.settings.ollamaHost}/api/generate`; const model = this.plugin.settings.styleModel; - const body = JSON.stringify({ - model, - prompt, - stream: false, - }); - - const args = ["-s", "-X", "POST", "-H", "Content-Type: application/json", "--max-time", "60", "-d", body, url]; - - const { stdout, stderr, code } = await runCommand("curl", args); - if (code !== 0) { - throw new Error(`Ollama request failed: ${stderr || stdout}`); - } + const bodyPath = join(tmpdir(), `ollama-body-${Date.now()}.json`); + await writeFile(bodyPath, JSON.stringify({ model, prompt, stream: false })); try { - const response = JSON.parse(stdout); - return response.response || ""; - } catch (e) { - throw new Error(`Failed to parse Ollama response: ${e}\nRaw: ${stdout}`); - } - } + const args = [ + "-s", "-X", "POST", + "-H", "Content-Type: application/json", + "--max-time", "300", + "--data-binary", `@${bodyPath}`, + url, + ]; - /** - * Validate that the LLM output preserves content (not summarized). - */ - private validateOllamaOutput(input: string, output: string): void { - const minRatio = 0.8; // Allow 20% reduction for formatting - if (output.length < input.length * minRatio) { - throw new Error( - `LLM output too short (${output.length} chars) vs input (${input.length} chars). ` + - `Ratio: ${(output.length / input.length).toFixed(2)}. May have summarized content.`, - ); + const { stdout, stderr, code } = await runCommand("curl", args); + if (code !== 0) { + throw new Error(`Ollama request failed: ${stderr || stdout}`); + } + + try { + const response = JSON.parse(stdout); + return response.response || ""; + } catch (e) { + throw new Error(`Failed to parse Ollama response: ${e}\nRaw: ${stdout}`); + } + } finally { + await unlink(bodyPath).catch(() => {}); } } } diff --git a/src/settings.ts b/src/settings.ts index c2fddb2..3f2c2a8 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -8,9 +8,9 @@ export const DEFAULT_SETTINGS: RemarkableSettings = { downloadPath: "remarkable", convertToPdf: true, enableHandwritingMd: true, - glmocrServerUrl: "http://100.103.83.12:5002", - glmocrApiKey: "any-string", - ollamaHost: "http://100.103.83.12:11435", + glmocrServerUrl: "http://localhost:5002", + glmocrApiKey: "", + ollamaHost: "http://localhost:11435", styleModel: "qwen3:32b", pageRenderer: "drawj2d", javaPath: "java", @@ -84,7 +84,7 @@ export class RemarkableSettingTab extends PluginSettingTab { new Setting(containerEl) .setName("GLM-OCR Server URL") - .setDesc("URL of GLM-OCR SDK Server (e.g., http://100.103.83.12:5002)") + .setDesc("URL of GLM-OCR SDK Server (e.g., http://localhost:5002)") .addText((text) => text.setValue(this.plugin.settings.glmocrServerUrl).onChange(async (value) => { this.plugin.settings.glmocrServerUrl = value; @@ -95,16 +95,17 @@ export class RemarkableSettingTab extends PluginSettingTab { new Setting(containerEl) .setName("GLM-OCR API Key") .setDesc("API key for GLM-OCR Server (can be any string for self-hosted)") - .addText((text) => + .addText((text) => { + text.inputEl.type = "password"; text.setValue(this.plugin.settings.glmocrApiKey).onChange(async (value) => { this.plugin.settings.glmocrApiKey = value; await this.plugin.saveSettings(); - }), - ); + }); + }); new Setting(containerEl) .setName("Ollama Host") - .setDesc("URL of Ollama server for style refinement (e.g., http://100.103.83.12:11435)") + .setDesc("URL of Ollama server for style refinement (e.g., http://localhost:11435)") .addText((text) => text.setValue(this.plugin.settings.ollamaHost).onChange(async (value) => { this.plugin.settings.ollamaHost = value; diff --git a/src/sync/downloader.ts b/src/sync/downloader.ts index 2f4d500..515ca88 100644 --- a/src/sync/downloader.ts +++ b/src/sync/downloader.ts @@ -1,25 +1,21 @@ import { RmapiNode } from "../types"; import RemarkablePlugin from "../main"; -import { RmapiBridge } from "../rmapi/bridge"; -import { SyncTracker } from "./tracker"; -import { OcrPipeline } from "../ocr/pipeline"; import { Notice } from "obsidian"; function sanitizeFileName(name: string): string { return name.replace(/[\\/:*?"<>|]/g, "_"); } +interface DocumentEntry { + node: RmapiNode; + remotePath: string; +} + export class DocumentDownloader { plugin: RemarkablePlugin; - rmapi: RmapiBridge; - tracker: SyncTracker; - ocrPipeline: OcrPipeline; constructor(plugin: RemarkablePlugin) { this.plugin = plugin; - this.rmapi = new RmapiBridge(plugin); - this.tracker = new SyncTracker(plugin); - this.ocrPipeline = new OcrPipeline(plugin); } async syncAll(): Promise { @@ -27,8 +23,8 @@ export class DocumentDownloader { new Notice("Syncing from reMarkable..."); const allDocs = await this.listAllDocuments("/"); - for (const doc of allDocs) { - await this.downloadDocument(doc); + for (const entry of allDocs) { + await this.downloadDocument(entry); } new Notice(`Sync completed! ${allDocs.length} document(s) synced.`); @@ -38,9 +34,14 @@ export class DocumentDownloader { } } - async listAllDocuments(path: string): Promise { - const nodes = await this.rmapi.list(path); - let docs = nodes.filter((n) => n.type === "DocumentType"); + async listAllDocuments(path: string): Promise { + const nodes = await this.plugin.rmapi.list(path); + let docs: DocumentEntry[] = nodes + .filter((n) => n.type === "DocumentType") + .map((n) => ({ + node: n, + remotePath: path === "/" ? `/${n.name}` : `${path}/${n.name}`, + })); for (const col of nodes.filter((n) => n.type === "CollectionType")) { const subPath = path === "/" ? `/${col.name}` : `${path}/${col.name}`; @@ -51,9 +52,10 @@ export class DocumentDownloader { return docs; } - async downloadDocument(doc: RmapiNode): Promise { - // Incremental sync: skip if already synced and not modified - const synced = await this.tracker.getSyncedDocument(doc.id); + async downloadDocument(entry: DocumentEntry): Promise { + const { node: doc, remotePath } = entry; + + const synced = await this.plugin.tracker.getSyncedDocument(doc.id); if (synced && synced.lastSynced >= doc.modifiedClient) { return; } @@ -62,33 +64,28 @@ export class DocumentDownloader { const localDir = this.plugin.settings.downloadPath; const localPath = `${localDir}/${safeName}`; - // Ensure directory exists try { await this.plugin.app.vault.adapter.mkdir(localDir); } catch (err) { console.warn("mkdir failed (may already exist):", err); } - // Download raw .rm file - await this.rmapi.downloadFile(doc.name, localPath); + await this.plugin.rmapi.downloadFile(remotePath, localPath); - // Convert to PDF if enabled let hasPdf = false; if (this.plugin.settings.convertToPdf) { const pdfPath = `${localDir}/${safeName}.pdf`; - await this.rmapi.downloadAnnotatedPdf(doc.name, pdfPath); + await this.plugin.rmapi.downloadAnnotatedPdf(remotePath, pdfPath); hasPdf = true; } - // Track document - await this.tracker.trackDocument(doc.id, doc.name, localPath, hasPdf, false); + await this.plugin.tracker.trackDocument(doc.id, doc.name, localPath, hasPdf, false); - // Convert handwriting to Markdown if enabled if (this.plugin.settings.enableHandwritingMd) { const mdPath = `${localDir}/${safeName}.md`; - const mdResult = await this.ocrPipeline.processDocument(localPath, mdPath); + const mdResult = await this.plugin.ocrPipeline.processDocument(localPath, mdPath); if (mdResult) { - await this.tracker.trackDocument(doc.id, doc.name, localPath, hasPdf, true); + await this.plugin.tracker.trackDocument(doc.id, doc.name, localPath, hasPdf, true); } } } diff --git a/src/sync/tracker.ts b/src/sync/tracker.ts index becf11d..a55f926 100644 --- a/src/sync/tracker.ts +++ b/src/sync/tracker.ts @@ -3,23 +3,32 @@ import RemarkablePlugin from "../main"; export class SyncTracker { plugin: RemarkablePlugin; + private cache: SyncedDocument[] | null = null; constructor(plugin: RemarkablePlugin) { this.plugin = plugin; } + private async getCache(): Promise { + if (this.cache === null) { + const data = await this.plugin.loadData(); + this.cache = data?.syncedDocuments || []; + } + return this.cache; + } + async loadSyncedDocuments(): Promise { - const data = await this.plugin.loadData(); - return data.syncedDocuments || []; + return this.getCache(); } async saveSyncedDocuments(docs: SyncedDocument[]): Promise { - const existing = (await this.plugin.loadData()) || {}; - await this.plugin.saveData({ ...existing, syncedDocuments: docs }); + this.cache = docs; + const data = (await this.plugin.loadData()) || {}; + await this.plugin.saveData({ ...data, syncedDocuments: docs }); } async trackDocument(id: string, name: string, localPath: string, hasPdf: boolean, hasMd: boolean): Promise { - const docs = await this.loadSyncedDocuments(); + const docs = await this.getCache(); const existingIndex = docs.findIndex((d) => d.id === id); const doc: SyncedDocument = { @@ -41,12 +50,12 @@ export class SyncTracker { } async isDocumentSynced(id: string): Promise { - const docs = await this.loadSyncedDocuments(); + const docs = await this.getCache(); return docs.some((d) => d.id === id); } async getSyncedDocument(id: string): Promise { - const docs = await this.loadSyncedDocuments(); + const docs = await this.getCache(); return docs.find((d) => d.id === id) || null; } } diff --git a/src/utils/zip.ts b/src/utils/zip.ts index 8e233ea..082ce11 100644 --- a/src/utils/zip.ts +++ b/src/utils/zip.ts @@ -31,26 +31,10 @@ export async function listRmContents(extractedDir: string): Promise { */ export async function isNotebook(extractedDir: string): Promise { const contents = await listRmContents(extractedDir); - // Notebooks have .rm page files and no embedded PDF content const hasRmPages = contents.some((f) => f.endsWith(".rm") && !f.endsWith(".zip")); const hasContentPdf = contents.some((f) => f.endsWith("content.pdf") || f.endsWith(".pdf")); - - // Clear case - if (hasRmPages && !hasContentPdf) return true; - if (!hasRmPages || hasContentPdf) return false; - - // Ambiguous: try to render a page - const pageFiles = contents.filter((f) => f.endsWith(".rm") && !f.endsWith(".zip")); - if (pageFiles.length === 0) return false; - - try { - // Try rendering the first page - const { code } = await runCommand("file", [pageFiles[0]]); - // If file command succeeds and identifies it as data, assume notebook - return code === 0; - } catch { - return false; - } + // PDF-with-annotations documents have both; treat them as non-notebooks since the PDF is the source + return hasRmPages && !hasContentPdf; } /**