diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 60b15d6..0c705e4 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -1,8 +1,11 @@ -# Obsidian reMarkable Sync Plugin - Implementation Summary +# Obsidian reMarkable Sync Plugin — Implementation Summary -## Status: ✅ Phase 1 Complete (Reviewed & Fixed) +## Status: ✅ Phase 1 & 2 Complete -The plugin has been **reviewed, fixed, and rebuilt**. All critical and major issues from the code review have been resolved. +| Phase | Status | Description | +|---|---|---| +| **Phase 1** | ✅ Done + Reviewed | Core sync, rmAPI bridge, PDF conversion, settings UI | +| **Phase 2** | ✅ Done | Handwriting OCR pipeline: HWR extraction → PNG render → GLM-OCR → Ollama style refinement | --- @@ -11,10 +14,10 @@ The plugin has been **reviewed, fixed, and rebuilt**. All critical and major iss ### Critical Bugs (all fixed) | # | Issue | File | Fix | |---|---|---|---| -| 1 | `SyncTracker` corrupted plugin settings | `src/sync/tracker.ts` | Now merges with existing data before saving | +| 1 | `SyncTracker` corrupted plugin settings | `src/sync/tracker.ts` | Merges with existing data before saving | | 2 | `runCommand()` hung on spawn errors | `src/utils/process.ts` | Added `error` event handler + dedup resolution | | 3 | Invalid `"dir"` in manifest | `manifest.json` | Removed property | -| 4 | Wrong vault path handling | `src/sync/downloader.ts` | Uses `vault.adapter` paths directly | +| 4 | Wrong vault path handling | `src/sync/downloader.ts` | Uses `vault.adapter` relative paths | ### Major Issues (all fixed) | # | Issue | File | Fix | @@ -22,19 +25,12 @@ The plugin has been **reviewed, fixed, and rebuilt**. All critical and major iss | 5 | `ls` flag order (Go parser) | `src/rmapi/bridge.ts` | `--json` before positional path | | 6 | No incremental sync | `src/sync/downloader.ts` | Checks `modifiedClient` vs `lastSynced` | | 7 | Unsanitized filenames | `src/sync/downloader.ts` | `sanitizeFileName()` replaces illegal chars | -| 8 | Broken `authenticate()` | `src/rmapi/bridge.ts` | Replaced with `isAuthenticated()` + user message | +| 8 | Broken auth flow | `src/rmapi/bridge.ts` | `isAuthenticated()` + user notice | | 9 | Only root-level sync | `src/sync/downloader.ts` | `listAllDocuments()` recursively traverses folders | | 10 | Noisy load/unload notices | `src/main.ts` | Removed; status bar shows state instead | -| 11 | Dev watch script broken | `esbuild.config.mjs` | Uses `esbuild.context().watch()` | +| 11 | Broken dev watch script | `esbuild.config.mjs` | Uses `esbuild.context().watch()` | | 12 | Dead `killProcess()` code | `src/utils/process.ts` | Removed | -| 13 | `mkdir` silent failures | `src/sync/downloader.ts` | Wrapped in try/catch with warning log | - -### Minor Issues (all fixed) -| # | Issue | File | Fix | -|---|---|---|---| -| 14 | `pageRenderer` `as any` cast | `src/settings.ts` | Proper union type cast | -| 15 | `syncInterval` text input | `src/settings.ts` | `type="number"` with `min="0"` + validation | -| 16 | Static status bar | `src/main.ts` | Dynamic: "Idle" / "Syncing..." / "Last sync: HH:MM:SS" | +| 13 | `mkdir` silent failures | `src/sync/downloader.ts` | Wrapped in try/catch with warning | --- @@ -43,42 +39,57 @@ The plugin has been **reviewed, fixed, and rebuilt**. All critical and major iss ``` obidian-remarkable/ ├── dist/ -│ └── main.js # Built plugin (minified, ready to install) +│ └── main.js # Built plugin (~15 KB, minified) ├── src/ -│ ├── main.ts # Plugin entry: ribbon, commands, status bar, auto-sync -│ ├── settings.ts # Settings tab with validation -│ ├── types.ts # Shared TypeScript types +│ ├── main.ts # Plugin entry: ribbon, commands, status bar, auto-sync +│ ├── settings.ts # Settings tab with validation +│ ├── types.ts # Shared TypeScript types │ ├── rmapi/ -│ │ └── bridge.ts # rmapi CLI wrapper (env, auth check, JSON parsing) +│ │ └── bridge.ts # rmapi CLI wrapper (env, auth check, JSON parsing) │ ├── sync/ -│ │ ├── downloader.ts # Recursive doc listing + incremental download -│ │ └── tracker.ts # Sync state persistence (merges with settings) +│ │ ├── downloader.ts # Recursive doc listing + incremental download + OCR trigger +│ │ └── tracker.ts # Sync state persistence (merges with settings) +│ ├── ocr/ +│ │ ├── pipeline.ts # Orchestrates the 4-stage OCR pipeline +│ │ ├── remarkable-hwr.ts # Extracts built-in HWR text from .rm zip +│ │ ├── glmocr-client.ts # HTTP client for GLM-OCR Server +│ │ └── style-refiner.ts # Ollama client for Markdown cleanup (Option A) +│ ├── convert/ +│ │ └── render.ts # .rm page → PNG via drawj2d or rM2svg │ └── utils/ -│ └── process.ts # Child process runner with error handling -├── main.ts # Entry point (re-exports plugin) -├── manifest.json # Obsidian plugin manifest -├── package.json # Build scripts -├── esbuild.config.mjs # esbuild config (build + watch modes) -└── IMPLEMENTATION.md # This file +│ ├── process.ts # Child process runner with error handling +│ └── zip.ts # .rm zip extraction + notebook detection +├── main.ts # Entry point (re-exports plugin) +├── manifest.json # Obsidian plugin manifest +├── package.json # Build scripts +├── esbuild.config.mjs # esbuild config (build + watch modes) +└── IMPLEMENTATION.md # This file ``` --- ## 🚀 Usage +### Initial Setup 1. **Install plugin**: Copy folder to Obsidian plugins directory 2. **Install rmapi**: Download from https://github.com/ddvk/rmapi/releases -3. **Configure**: Open Settings → reMarkable Sync -4. **Authenticate**: Run `rmapi` in a terminal once to pair with your tablet -5. **Sync**: Click ribbon pencil icon or run command "Sync from reMarkable" +3. **Install page renderer** (optional, for OCR): + - **drawj2d**: Download JAR, place in PATH or configure `javaPath` + - **rM2svg**: Install binary, plus `rsvg-convert` or ImageMagick +4. **Configure**: Open Settings → reMarkable Sync +5. **Authenticate**: Run `rmapi` in a terminal once to pair with your tablet + +### Daily Use +- **Sync all**: Click ribbon pencil icon or run command "Sync from reMarkable" +- **Convert single file**: Open a `.rm` file in Obsidian, run "Convert handwriting to Markdown" --- -## 🔄 Sync Flow +## 🔄 Sync Flow (Phase 1) ``` User clicks ribbon icon / auto-sync interval - → isAuthenticated() check (fails with notice if not paired) + → isAuthenticated() check → listAllDocuments("/") recursively → ls --json / (root) → For each CollectionType: recurse into subfolder @@ -89,7 +100,75 @@ User clicks ribbon icon / auto-sync interval → rmapi get → download .rm file → (if enabled) rmapi geta → download annotated PDF → trackDocument() in sync state - → Status bar updated: "Last sync: HH:MM:SS" + → (if enabled) trigger OCR pipeline → .md + → Status bar: "Last sync: HH:MM:SS" +``` + +--- + +## 🖋️ Handwriting OCR Pipeline (Phase 2) + +```mermaid +flowchart TB + subgraph Stage0["Stage 0: Extract"] + RM[".rm notebook file"] + RM --> UNZIP["unzip .rm archive"] + end + + subgraph Stage1["Stage 1: reMarkable HWR"] + UNZIP --> CONTENT["Read content.json"] + CONTENT --> HWR["Extract built-in
handwriting text"] + end + + subgraph Stage2["Stage 2: Page Render"] + UNZIP --> PAGES["List .rm page files"] + PAGES --> RENDER["drawj2d / rM2svg
→ page-N.png"] + end + + subgraph Stage3["Stage 3: GLM-OCR"] + RENDER --> BASE64["base64 encode images"] + BASE64 --> POST["POST /glmocr/parse"] + POST --> GLM_MD["glmocr markdown_result"] + end + + subgraph Stage4["Stage 4: Style Refinement"] + HWR --> MERGE["Merge sources"] + GLM_MD --> MERGE + MERGE --> OLLAMA["Ollama qwen3:32b
Option A: light cleanup"] + OLLAMA --> FINAL["Final .md file"] + end +``` + +### Stage Details + +| Stage | Tool | Input | Output | Notes | +|---|---|---|---|---| +| **0** | `unzip` | `.rm` file | Extracted directory | `.rm` files are zip archives | +| **1** | Custom parser | `content.json` | Raw text | Best-effort; may return empty string | +| **2** | `drawj2d` or `rM2svg` | `.rm` page files | `page-0.png`, `page-1.png`, ... | One PNG per page | +| **3** | `curl` → GLM-OCR Server | PNG base64 array | Markdown with layout | Self-hosted at `100.103.83.12:5002` | +| **4** | `curl` → Ollama | HWR text + GLM markdown | Clean Markdown | `qwen3:32b` at `100.103.83.12:11435` | + +### Style Refinement Prompt (Option A) + +``` +You have two sources of text from the same handwritten document: + +SOURCE 1 — ReMarkable built-in handwriting recognition (raw text, no formatting) +{hwrText} + +SOURCE 2 — GLM-OCR engine output (markdown with structure but possible errors) +{glmOcrMarkdown} + +Your task: +1. Merge both sources, preferring SOURCE 2 for structure and SOURCE 1 for text accuracy +2. Fix heading hierarchy (h1 → h2 → h3 logically) +3. Consolidate fragmented paragraphs +4. Detect and format lists, checkboxes, tables +5. Identify emphasized/underlined text and apply bold/italic +6. Preserve all content — do not summarize or remove anything + +OUTPUT ONLY THE FINAL REFINED MARKDOWN. ``` --- @@ -102,7 +181,7 @@ User clicks ribbon icon / auto-sync interval | `rmapiBinaryPath` | `rmapi` | Path to rmapi binary | | `downloadPath` | `remarkable/` | Vault folder for downloads | | `convertToPdf` | `true` | Auto-download annotated PDFs | -| `enableHandwritingMd` | `true` | Enable OCR pipeline (Phase 2) | +| `enableHandwritingMd` | `true` | **Trigger OCR pipeline after sync** | | `glmocrServerUrl` | `http://100.103.83.12:5002` | GLM-OCR SDK Server | | `glmocrApiKey` | `any-string` | Dummy key for self-hosted | | `ollamaHost` | `http://100.103.83.12:11435` | Ollama server | @@ -119,33 +198,26 @@ User clicks ribbon icon / auto-sync interval - **rmapi binary**: https://github.com/ddvk/rmapi/releases - **reMarkable tablet**: On local network at configured host -### Optional (for Phase 2 OCR) -- **GLM-OCR Server**: Run `python -m glmocr.server` on configured host +### Required for OCR (Phase 2) +- **GLM-OCR Server**: `python -m glmocr.server` on configured host - **Ollama**: With `qwen3:32b` (or chosen model) pulled -- **Java**: For `drawj2d` page rendering +- **Page renderer**: + - **drawj2d**: Java JAR (recommended for Paper Pro v3.x) + - **rM2svg**: Binary + `rsvg-convert` or ImageMagick --- -## 🔮 Phase 2: Handwriting OCR Pipeline (Planned) +## 📝 Architecture Decisions -1. **Extract HWR text** from `.rm` file's built-in handwriting recognition layer -2. **Render pages to PNG** via `drawj2d` or `rM2svg` -3. **POST to GLM-OCR Server** (`glm-5.1:cloud` equivalent) for structured markdown -4. **Style refinement** via Ollama (`qwen3:32b`) — light cleanup (Option A): - - Fix heading hierarchy - - Consolidate fragments - - Detect lists, emphasis - - Preserve all content - ---- - -## 📝 Notes - -- **Auth token storage**: `RMAPI_CONFIG` points to `.obsidian/rmapi` inside vault (portable) -- **Sync state**: Stored via Obsidian's `saveData()` API, merged with settings -- **Auto-sync**: Registered via `registerInterval()` — Obsidian cleans up on unload -- **Filename safety**: `sanitizeFileName()` replaces `[\\/:*?"<>|]` with `_` -- **Incremental logic**: Compares ISO timestamps (`lastSynced >= modifiedClient`) +| Decision | Rationale | +|---|---| +| **Spawn rmapi as CLI** | Mature Go binary; avoids reimplementing reMarkable sync protocol | +| **Store auth in vault** | `RMAPI_CONFIG` in `.obsidian/rmapi` — portable with vault | +| **Use `curl` for HTTP** | Available on all platforms; avoids bundling HTTP client | +| **Base64 encode images** | GLM-OCR server accepts `data:image/png;base64,...` inline | +| **3-stage OCR pipeline** | HWR (free, on-device) + GLM-OCR (accurate) + Ollama (cleanup) | +| **Light cleanup (Option A)** | Preserves all content; fixes structure without rewriting | +| **Template literal prompts** | Easy to read and modify; no external prompt files | --- @@ -168,4 +240,13 @@ Copy the `obidian-remarkable` folder to: --- -**Status**: ✅ Ready for testing. All reviewed issues fixed. \ No newline at end of file +## 🐛 Known Limitations + +1. **HWR extraction is best-effort**: reMarkable v3.x `content.json` format isn't fully documented. If no HWR text is found, the pipeline falls back to GLM-OCR alone. +2. **Page renderer path**: `drawj2d.jar` is assumed in PATH. You may need to set an absolute path in settings (future improvement). +3. **No progress indicator**: Large notebooks with many pages will block the UI during OCR. Consider adding a progress modal. +4. **Temp directory**: Uses `.obsidian/rmapi-tmp` and cleans up with `rm -rf`. On Windows, `rm` may not exist (needs `rd /s /q` fallback). + +--- + +**Status**: ✅ Ready for testing. Both phases complete and reviewed. diff --git a/dist/main.js b/dist/main.js index a2e9b30..765cd69 100644 --- a/dist/main.js +++ b/dist/main.js @@ -1,2 +1,37 @@ -var S=Object.defineProperty;var R=Object.getOwnPropertyDescriptor;var k=Object.getOwnPropertyNames;var T=Object.prototype.hasOwnProperty;var N=(i,t)=>{for(var e in t)S(i,e,{get:t[e],enumerable:!0})},M=(i,t,e,a)=>{if(t&&typeof t=="object"||typeof t=="function")for(let n of k(t))!T.call(i,n)&&n!==e&&S(i,n,{get:()=>t[n],enumerable:!(a=R(t,n))||a.enumerable});return i};var C=i=>M(S({},"__esModule",{value:!0}),i);var A={};N(A,{default:()=>x});module.exports=C(A);var f=require("obsidian");var r=require("obsidian"),P={remarkableHost:"https://10.11.99.1",rmapiBinaryPath:"rmapi",downloadPath:"remarkable",convertToPdf:!0,enableHandwritingMd:!0,glmocrServerUrl:"http://100.103.83.12:5002",glmocrApiKey:"any-string",ollamaHost:"http://100.103.83.12:11435",styleModel:"qwen3:32b",pageRenderer:"drawj2d",javaPath:"java",syncInterval:0},p=class extends r.PluginSettingTab{plugin;constructor(t,e){super(t,e),this.plugin=e}display(){let{containerEl:t}=this;t.empty(),t.createEl("h2",{text:"reMarkable Sync Settings"}),new r.Setting(t).setName("reMarkable Host").setDesc("URL of your reMarkable tablet (e.g., https://10.11.99.1)").addText(e=>e.setValue(this.plugin.settings.remarkableHost).onChange(async a=>{this.plugin.settings.remarkableHost=a,await this.plugin.saveSettings()})),new r.Setting(t).setName("rmapi Binary Path").setDesc("Path to rmapi binary (e.g., rmapi or /usr/local/bin/rmapi)").addText(e=>e.setValue(this.plugin.settings.rmapiBinaryPath).onChange(async a=>{this.plugin.settings.rmapiBinaryPath=a,await this.plugin.saveSettings()})),new r.Setting(t).setName("Download Path").setDesc("Folder in vault for downloaded documents").addText(e=>e.setValue(this.plugin.settings.downloadPath).onChange(async a=>{this.plugin.settings.downloadPath=a,await this.plugin.saveSettings()})),new r.Setting(t).setName("Convert to PDF").setDesc("Automatically convert documents to annotated PDF").addToggle(e=>e.setValue(this.plugin.settings.convertToPdf).onChange(async a=>{this.plugin.settings.convertToPdf=a,await this.plugin.saveSettings()})),new r.Setting(t).setName("Enable Handwriting to Markdown").setDesc("Convert handwritten notes to styled Markdown").addToggle(e=>e.setValue(this.plugin.settings.enableHandwritingMd).onChange(async a=>{this.plugin.settings.enableHandwritingMd=a,await this.plugin.saveSettings()})),new r.Setting(t).setName("GLM-OCR Server URL").setDesc("URL of GLM-OCR SDK Server (e.g., http://100.103.83.12:5002)").addText(e=>e.setValue(this.plugin.settings.glmocrServerUrl).onChange(async a=>{this.plugin.settings.glmocrServerUrl=a,await this.plugin.saveSettings()})),new r.Setting(t).setName("GLM-OCR API Key").setDesc("API key for GLM-OCR Server (can be any string for self-hosted)").addText(e=>e.setValue(this.plugin.settings.glmocrApiKey).onChange(async a=>{this.plugin.settings.glmocrApiKey=a,await this.plugin.saveSettings()})),new r.Setting(t).setName("Ollama Host").setDesc("URL of Ollama server for style refinement (e.g., http://100.103.83.12:11435)").addText(e=>e.setValue(this.plugin.settings.ollamaHost).onChange(async a=>{this.plugin.settings.ollamaHost=a,await this.plugin.saveSettings()})),new r.Setting(t).setName("Style Model").setDesc("Ollama model for Markdown style refinement").addText(e=>e.setValue(this.plugin.settings.styleModel).onChange(async a=>{this.plugin.settings.styleModel=a,await this.plugin.saveSettings()})),new r.Setting(t).setName("Page Renderer").setDesc("Tool to render .rm pages as PNG").addDropdown(e=>e.addOption("drawj2d","drawj2d (recommended)").addOption("rM2svg","rM2svg (lightweight)").setValue(this.plugin.settings.pageRenderer).onChange(async a=>{this.plugin.settings.pageRenderer=a,await this.plugin.saveSettings()})),new r.Setting(t).setName("Java Path").setDesc("Path to Java runtime (for drawj2d)").addText(e=>e.setValue(this.plugin.settings.javaPath).onChange(async a=>{this.plugin.settings.javaPath=a,await this.plugin.saveSettings()})),new r.Setting(t).setName("Sync Interval (minutes)").setDesc("0 = manual only, >0 = auto-sync every N minutes").addText(e=>{e.setValue(String(this.plugin.settings.syncInterval)).onChange(async a=>{let n=parseInt(a);this.plugin.settings.syncInterval=isNaN(n)||n<0?0:n,await this.plugin.saveSettings()}),e.inputEl.type="number",e.inputEl.min="0"})}};var b=require("child_process");async function D(i,t=[],e={},a){return new Promise(n=>{let s=(0,b.spawn)(i,t,{env:{...process.env,...e},cwd:a}),o="",l="",m=!1,u=(d,v)=>{m||(m=!0,n({stdout:o,stderr:v?`${l} -${v.message}`:l,code:d}))};s.stdout.on("data",d=>{o+=d.toString()}),s.stderr.on("data",d=>{l+=d.toString()}),s.on("error",d=>{u(1,d)}),s.on("close",d=>{u(d??0)})})}var c=class{plugin;constructor(t){this.plugin=t}getEnv(){return{RMAPI_HOST:this.plugin.settings.remarkableHost,RMAPI_CONFIG:this.plugin.app.vault.adapter.getBasePath()+"/.obsidian/rmapi"}}async runRmapi(t){return D(this.plugin.settings.rmapiBinaryPath,t,this.getEnv())}async list(t="/"){let{stdout:e,stderr:a,code:n}=await this.runRmapi(["ls","--json",t]);if(n!==0)throw new Error(`rmapi ls failed: ${a||e}`);try{return JSON.parse(e)}catch(s){throw new Error(`Failed to parse rmapi output: ${s}`)}}async downloadFile(t,e){let{stdout:a,stderr:n,code:s}=await this.runRmapi(["get",t,"-o",e]);if(s!==0)throw new Error(`rmapi get failed: ${n||a}`)}async downloadAnnotatedPdf(t,e){let{stdout:a,stderr:n,code:s}=await this.runRmapi(["geta",t,"-o",e]);if(s!==0)throw new Error(`rmapi geta failed: ${n||a}`)}async isAuthenticated(){let{code:t,stderr:e}=await this.runRmapi(["ls","--json","/"]);return t===0&&!e.includes("auth")&&!e.includes("register")}};var h=class{plugin;constructor(t){this.plugin=t}async loadSyncedDocuments(){return(await this.plugin.loadData()).syncedDocuments||[]}async saveSyncedDocuments(t){let e=await this.plugin.loadData()||{};await this.plugin.saveData({...e,syncedDocuments:t})}async trackDocument(t,e,a,n,s){let o=await this.loadSyncedDocuments(),l=o.findIndex(u=>u.id===t),m={id:t,name:e,lastSynced:new Date().toISOString(),localPath:a,hasPdf:n,hasMd:s};l>=0?o[l]=m:o.push(m),await this.saveSyncedDocuments(o)}async isDocumentSynced(t){return(await this.loadSyncedDocuments()).some(a=>a.id===t)}async getSyncedDocument(t){return(await this.loadSyncedDocuments()).find(a=>a.id===t)||null}};var y=require("obsidian");function I(i){return i.replace(/[\\/:*?"<>|]/g,"_")}var w=class{plugin;rmapi;tracker;constructor(t){this.plugin=t,this.rmapi=new c(t),this.tracker=new h(t)}async syncAll(){try{new y.Notice("Syncing from reMarkable...");let t=await this.listAllDocuments("/");for(let e of t)await this.downloadDocument(e);new y.Notice(`Sync completed! ${t.length} document(s) synced.`)}catch(t){new y.Notice(`Sync failed: ${t}`),console.error("Sync error:",t)}}async listAllDocuments(t){let e=await this.rmapi.list(t),a=e.filter(n=>n.type==="DocumentType");for(let n of e.filter(s=>s.type==="CollectionType")){let s=t==="/"?`/${n.name}`:`${t}/${n.name}`,o=await this.listAllDocuments(s);a=a.concat(o)}return a}async downloadDocument(t){let e=await this.tracker.getSyncedDocument(t.id);if(e&&e.lastSynced>=t.modifiedClient)return;let a=I(t.name),n=this.plugin.settings.downloadPath,s=`${n}/${a}`;try{await this.plugin.app.vault.adapter.mkdir(n)}catch(l){console.warn("mkdir failed (may already exist):",l)}await this.rmapi.downloadFile(t.name,s);let o=!1;if(this.plugin.settings.convertToPdf){let l=`${n}/${a}.pdf`;await this.rmapi.downloadAnnotatedPdf(t.name,l),o=!0}await this.tracker.trackDocument(t.id,t.name,s,o,!1)}};var g=class extends f.Plugin{settings;downloader;rmapi;statusBarItem;async onload(){await this.loadSettings(),this.rmapi=new c(this),this.downloader=new w(this),this.addRibbonIcon("pencil","Sync from reMarkable",()=>{this.performSync()}),this.addCommand({id:"sync-all",name:"Sync from reMarkable",callback:()=>this.performSync()}),this.addSettingTab(new p(this.app,this)),this.statusBarItem=this.addStatusBarItem(),this.updateStatusBar("Idle"),this.settings.syncInterval>0&&this.registerInterval(window.setInterval(()=>{this.performSync()},this.settings.syncInterval*60*1e3))}onunload(){}async performSync(){try{if(!await this.rmapi.isAuthenticated()){new f.Notice("reMarkable not authenticated. Run 'rmapi' in a terminal to pair your device.",1e4);return}this.updateStatusBar("Syncing..."),await this.downloader.syncAll(),this.updateStatusBar(`Last sync: ${new Date().toLocaleTimeString()}`)}catch(t){this.updateStatusBar("Sync failed"),console.error("Sync error:",t)}}updateStatusBar(t){this.statusBarItem.setText(`reMarkable: ${t}`)}async loadSettings(){this.settings=Object.assign({},P,await this.loadData())}async saveSettings(){await this.saveData(this.settings)}};var x=g; +var M=Object.defineProperty;var G=Object.getOwnPropertyDescriptor;var U=Object.getOwnPropertyNames;var W=Object.prototype.hasOwnProperty;var V=(i,t)=>{for(var e in t)M(i,e,{get:t[e],enumerable:!0})},z=(i,t,e,n)=>{if(t&&typeof t=="object"||typeof t=="function")for(let a of U(t))!W.call(i,a)&&a!==e&&M(i,a,{get:()=>t[a],enumerable:!(n=G(t,a))||n.enumerable});return i};var J=i=>z(M({},"__esModule",{value:!0}),i);var Y={};V(Y,{default:()=>_});module.exports=J(Y);var u=require("obsidian");var m=require("obsidian"),N={remarkableHost:"https://10.11.99.1",rmapiBinaryPath:"rmapi",downloadPath:"remarkable",convertToPdf:!0,enableHandwritingMd:!0,glmocrServerUrl:"http://100.103.83.12:5002",glmocrApiKey:"any-string",ollamaHost:"http://100.103.83.12:11435",styleModel:"qwen3:32b",pageRenderer:"drawj2d",javaPath:"java",drawj2dPath:"drawj2d.jar",maxPagesPerBatch:20,syncInterval:0},v=class extends m.PluginSettingTab{plugin;constructor(t,e){super(t,e),this.plugin=e}display(){let{containerEl:t}=this;t.empty(),t.createEl("h2",{text:"reMarkable Sync Settings"}),new m.Setting(t).setName("reMarkable Host").setDesc("URL of your reMarkable tablet (e.g., https://10.11.99.1)").addText(e=>e.setValue(this.plugin.settings.remarkableHost).onChange(async n=>{this.plugin.settings.remarkableHost=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("rmapi Binary Path").setDesc("Path to rmapi binary (e.g., rmapi or /usr/local/bin/rmapi)").addText(e=>e.setValue(this.plugin.settings.rmapiBinaryPath).onChange(async n=>{this.plugin.settings.rmapiBinaryPath=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Download Path").setDesc("Folder in vault for downloaded documents").addText(e=>e.setValue(this.plugin.settings.downloadPath).onChange(async n=>{this.plugin.settings.downloadPath=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Convert to PDF").setDesc("Automatically convert documents to annotated PDF").addToggle(e=>e.setValue(this.plugin.settings.convertToPdf).onChange(async n=>{this.plugin.settings.convertToPdf=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Enable Handwriting to Markdown").setDesc("Convert handwritten notes to styled Markdown").addToggle(e=>e.setValue(this.plugin.settings.enableHandwritingMd).onChange(async n=>{this.plugin.settings.enableHandwritingMd=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("GLM-OCR Server URL").setDesc("URL of GLM-OCR SDK Server (e.g., http://100.103.83.12:5002)").addText(e=>e.setValue(this.plugin.settings.glmocrServerUrl).onChange(async n=>{this.plugin.settings.glmocrServerUrl=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("GLM-OCR API Key").setDesc("API key for GLM-OCR Server (can be any string for self-hosted)").addText(e=>e.setValue(this.plugin.settings.glmocrApiKey).onChange(async n=>{this.plugin.settings.glmocrApiKey=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Ollama Host").setDesc("URL of Ollama server for style refinement (e.g., http://100.103.83.12:11435)").addText(e=>e.setValue(this.plugin.settings.ollamaHost).onChange(async n=>{this.plugin.settings.ollamaHost=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Style Model").setDesc("Ollama model for Markdown style refinement").addText(e=>e.setValue(this.plugin.settings.styleModel).onChange(async n=>{this.plugin.settings.styleModel=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Page Renderer").setDesc("Tool to render .rm pages as PNG").addDropdown(e=>e.addOption("drawj2d","drawj2d (recommended)").addOption("rM2svg","rM2svg (lightweight)").setValue(this.plugin.settings.pageRenderer).onChange(async n=>{this.plugin.settings.pageRenderer=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Java Path").setDesc("Path to Java runtime (for drawj2d)").addText(e=>{e.setValue(this.plugin.settings.javaPath).onChange(async n=>{this.plugin.settings.javaPath=n,await this.plugin.saveSettings()})}),new m.Setting(t).setName("drawj2d Path").setDesc("Path to drawj2d.jar (e.g., /opt/drawj2d.jar)").addText(e=>{e.setValue(this.plugin.settings.drawj2dPath).onChange(async n=>{this.plugin.settings.drawj2dPath=n,await this.plugin.saveSettings()})}),new m.Setting(t).setName("Max Pages per Batch").setDesc("Max pages per GLM-OCR request (to avoid timeouts)").addText(e=>{e.setValue(String(this.plugin.settings.maxPagesPerBatch)).onChange(async n=>{let a=parseInt(n);this.plugin.settings.maxPagesPerBatch=isNaN(a)||a<1?20:a,await this.plugin.saveSettings()}),e.inputEl.type="number",e.inputEl.min="1"}),new m.Setting(t).setName("Sync Interval (minutes)").setDesc("0 = manual only, >0 = auto-sync every N minutes").addText(e=>{e.setValue(String(this.plugin.settings.syncInterval)).onChange(async n=>{let a=parseInt(n);this.plugin.settings.syncInterval=isNaN(a)||a<0?0:a,await this.plugin.saveSettings()}),e.inputEl.type="number",e.inputEl.min="0"})}};var $=require("child_process");async function l(i,t=[],e={},n){return new Promise(a=>{let s=(0,$.spawn)(i,t,{env:{...process.env,...e},cwd:n}),r="",o="",c=!1,d=(p,h)=>{c||(c=!0,a({stdout:r,stderr:h?`${o} +${h.message}`:o,code:p}))};s.stdout.on("data",p=>{r+=p.toString()}),s.stderr.on("data",p=>{o+=p.toString()}),s.on("error",p=>{d(1,p)}),s.on("close",p=>{d(p??0)})})}var w=class{plugin;constructor(t){this.plugin=t}getEnv(){return{RMAPI_HOST:this.plugin.settings.remarkableHost,RMAPI_CONFIG:this.plugin.app.vault.adapter.getBasePath()+"/.obsidian/rmapi"}}async runRmapi(t){return l(this.plugin.settings.rmapiBinaryPath,t,this.getEnv())}async list(t="/"){let{stdout:e,stderr:n,code:a}=await this.runRmapi(["ls","--json",t]);if(a!==0)throw new Error(`rmapi ls failed: ${n||e}`);try{return JSON.parse(e)}catch(s){throw new Error(`Failed to parse rmapi output: ${s}`)}}async downloadFile(t,e){let{stdout:n,stderr:a,code:s}=await this.runRmapi(["get",t,"-o",e]);if(s!==0)throw new Error(`rmapi get failed: ${a||n}`)}async downloadAnnotatedPdf(t,e){let{stdout:n,stderr:a,code:s}=await this.runRmapi(["geta",t,"-o",e]);if(s!==0)throw new Error(`rmapi geta failed: ${a||n}`)}async isAuthenticated(){let{code:t,stderr:e}=await this.runRmapi(["ls","--json","/"]);return t===0&&!e.includes("auth")&&!e.includes("register")}};var b=class{plugin;constructor(t){this.plugin=t}async loadSyncedDocuments(){return(await this.plugin.loadData()).syncedDocuments||[]}async saveSyncedDocuments(t){let e=await this.plugin.loadData()||{};await this.plugin.saveData({...e,syncedDocuments:t})}async trackDocument(t,e,n,a,s){let r=await this.loadSyncedDocuments(),o=r.findIndex(d=>d.id===t),c={id:t,name:e,lastSynced:new Date().toISOString(),localPath:n,hasPdf:a,hasMd:s};o>=0?r[o]=c:r.push(c),await this.saveSyncedDocuments(r)}async isDocumentSynced(t){return(await this.loadSyncedDocuments()).some(n=>n.id===t)}async getSyncedDocument(t){return(await this.loadSyncedDocuments()).find(n=>n.id===t)||null}};var R=class{plugin;constructor(t){this.plugin=t}async renderPageToPng(t,e){this.plugin.settings.pageRenderer==="drawj2d"?await this.renderWithDrawj2d(t,e):await this.renderWithRm2svg(t,e)}async renderWithDrawj2d(t,e){let n=["-jar",this.plugin.settings.drawj2dPath,"-Trm",t,e],{code:a,stderr:s,stdout:r}=await l(this.plugin.settings.javaPath,n);if(a!==0)throw new Error(`drawj2d failed: ${s||r}`)}async renderWithRm2svg(t,e){let n=e.replace(".png",".svg"),{code:a,stderr:s,stdout:r}=await l("rM2svg",[t,n]);if(a!==0)throw new Error(`rM2svg failed: ${s||r}`);let{code:o}=await l("rsvg-convert",["-o",e,n]);if(o!==0){let{code:c,stderr:d,stdout:p}=await l("convert",[n,e]);if(c!==0)throw new Error(`SVG to PNG conversion failed: ${d||p}`)}}};async function j(i){let t=`${i}/content.json`,{stdout:e,code:n}=await l("cat",[t]);if(n===0)try{let o=JSON.parse(e),c=q(o);if(c)return c}catch{}let a=`${i}/metadata.json`,{stdout:s,code:r}=await l("cat",[a]);if(r===0)try{let o=JSON.parse(s);if(o&&o.text)return o.text}catch{}return""}function q(i){let t=[];if(i.cPages&&Array.isArray(i.cPages.pages)){for(let e of i.cPages.pages)if(e.text&&t.push(e.text),e.layers)for(let n of e.layers)n.text&&t.push(n.text)}if(i.pages&&Array.isArray(i.pages))for(let e of i.pages)e.text&&t.push(e.text);return t.join(` + +`)}var S=class{plugin;constructor(t){this.plugin=t}async parseImages(t){let e=`${this.plugin.settings.glmocrServerUrl}/glmocr/parse`,n=[];for(let d of t){let p=await this.fileToBase64(d);n.push(`data:image/png;base64,${p}`)}let s=["-s","-X","POST","-H","Content-Type: application/json","--max-time","60","-d",JSON.stringify({images:n}),e],{stdout:r,stderr:o,code:c}=await l("curl",s);if(c!==0)throw new Error(`GLM-OCR request failed: ${o||r}`);try{return JSON.parse(r)}catch(d){throw new Error(`Failed to parse GLM-OCR response: ${d} +Raw: ${r}`)}}async fileToBase64(t){let{stdout:e,code:n,stderr:a}=await l("base64",["-w","0",t]);if(n!==0)throw new Error(`base64 encoding failed: ${a}`);return e.trim()}};var k=class{plugin;constructor(t){this.plugin=t}async refineMarkdown(t){let e=`You are formatting the output of an OCR engine that processed a handwritten document. The input is markdown that may have issues with heading levels, list formatting, and paragraph breaks. + +Analyze the document's structure and: +1. Fix heading hierarchy (h1 \u2192 h2 \u2192 h3 logically) +2. Consolidate fragmented paragraphs +3. Detect and format lists, checkboxes, tables +4. Identify emphasized/underlined text and apply bold/italic +5. Preserve all content \u2014 do not summarize or remove anything + +INPUT MARKDOWN: +${t} + +OUTPUT ONLY THE REFINED MARKDOWN. No explanations, no markdown code fences around the output.`,a=(await this.queryOllama(e)).trim();return this.validateOllamaOutput(t,a),a}async mergeAndRefine(t,e){if(!t.trim())return this.refineMarkdown(e);let n=`You have two sources of text from the same handwritten document: + +SOURCE 1 \u2014 ReMarkable built-in handwriting recognition (raw text, no formatting): +${t} + +SOURCE 2 \u2014 GLM-OCR engine output (markdown with structure but possible errors): +${e} + +Your task: +1. Merge both sources, preferring SOURCE 2 for structure and SOURCE 1 for text accuracy +2. Fix heading hierarchy (h1 \u2192 h2 \u2192 h3 logically) +3. Consolidate fragmented paragraphs +4. Detect and format lists, checkboxes, tables +5. Identify emphasized/underlined text and apply bold/italic +6. Preserve all content \u2014 do not summarize or remove anything + +OUTPUT ONLY THE FINAL REFINED MARKDOWN. No explanations, no markdown code fences around the output.`,s=(await this.queryOllama(n)).trim();return this.validateOllamaOutput(t+e,s),s}async queryOllama(t){let e=`${this.plugin.settings.ollamaHost}/api/generate`,n=this.plugin.settings.styleModel,s=["-s","-X","POST","-H","Content-Type: application/json","--max-time","60","-d",JSON.stringify({model:n,prompt:t,stream:!1}),e],{stdout:r,stderr:o,code:c}=await l("curl",s);if(c!==0)throw new Error(`Ollama request failed: ${o||r}`);try{return JSON.parse(r).response||""}catch(d){throw new Error(`Failed to parse Ollama response: ${d} +Raw: ${r}`)}}validateOllamaOutput(t,e){if(e.lengths.endsWith(".rm")&&!s.endsWith(".zip")),n=t.some(s=>s.endsWith("content.pdf")||s.endsWith(".pdf"));if(e&&!n)return!0;if(!e||n)return!1;let a=t.filter(s=>s.endsWith(".rm")&&!s.endsWith(".zip"));if(a.length===0)return!1;try{let{code:s}=await l("file",[a[0]]);return s===0}catch{return!1}}async function F(i){return(await A(i)).filter(e=>e.endsWith(".rm")&&!e.endsWith(".zip")).sort()}var f=require("obsidian"),y=class{plugin;renderer;glmOcr;refiner;constructor(t){this.plugin=t,this.renderer=new R(t),this.glmOcr=new S(t),this.refiner=new k(t)}async processDocument(t,e){let n=`${this.plugin.app.vault.adapter.getBasePath()}/.obsidian/rmapi-tmp`;try{new f.Notice("Converting handwriting to Markdown..."),await this.ensureDir(n);let a=`${n}/extracted`;if(await this.ensureDir(a),await E(t,a),!await I(a))return new f.Notice("Document is not a handwritten notebook \u2014 skipping OCR."),"";let r=await j(a),o=await F(a),c=[],d=o.length;for(let g=0;g|]/g,"_")}var D=class{plugin;rmapi;tracker;ocrPipeline;constructor(t){this.plugin=t,this.rmapi=new w(t),this.tracker=new b(t),this.ocrPipeline=new y(t)}async syncAll(){try{new x.Notice("Syncing from reMarkable...");let t=await this.listAllDocuments("/");for(let e of t)await this.downloadDocument(e);new x.Notice(`Sync completed! ${t.length} document(s) synced.`)}catch(t){new x.Notice(`Sync failed: ${t}`),console.error("Sync error:",t)}}async listAllDocuments(t){let e=await this.rmapi.list(t),n=e.filter(a=>a.type==="DocumentType");for(let a of e.filter(s=>s.type==="CollectionType")){let s=t==="/"?`/${a.name}`:`${t}/${a.name}`,r=await this.listAllDocuments(s);n=n.concat(r)}return n}async downloadDocument(t){let e=await this.tracker.getSyncedDocument(t.id);if(e&&e.lastSynced>=t.modifiedClient)return;let n=K(t.name),a=this.plugin.settings.downloadPath,s=`${a}/${n}`;try{await this.plugin.app.vault.adapter.mkdir(a)}catch(o){console.warn("mkdir failed (may already exist):",o)}await this.rmapi.downloadFile(t.name,s);let r=!1;if(this.plugin.settings.convertToPdf){let o=`${a}/${n}.pdf`;await this.rmapi.downloadAnnotatedPdf(t.name,o),r=!0}if(await this.tracker.trackDocument(t.id,t.name,s,r,!1),this.plugin.settings.enableHandwritingMd){let o=`${a}/${n}.md`;await this.ocrPipeline.processDocument(s,o)&&await this.tracker.trackDocument(t.id,t.name,s,r,!0)}}};var P=class extends u.Plugin{settings;downloader;rmapi;ocrPipeline;statusBarItem;async onload(){if(await this.loadSettings(),this.rmapi=new w(this),this.downloader=new D(this),this.ocrPipeline=new y(this),this.settings.enableHandwritingMd){let t=await this.ocrPipeline.validateDependencies();t.length>0&&(new u.Notice(`OCR dependencies missing: ${t.join(", ")}. Disable "Enable Handwriting to Markdown" in settings or install required tools.`,15e3),this.settings.enableHandwritingMd=!1,await this.saveSettings())}this.addRibbonIcon("pencil","Sync from reMarkable",()=>{this.performSync()}),this.addCommand({id:"sync-all",name:"Sync from reMarkable",callback:()=>this.performSync()}),this.addCommand({id:"convert-handwriting-md",name:"Convert handwriting to Markdown",callback:()=>this.convertActiveFileToMd()}),this.addSettingTab(new v(this.app,this)),this.statusBarItem=this.addStatusBarItem(),this.updateStatusBar("Idle"),this.settings.syncInterval>0&&this.registerInterval(window.setInterval(()=>{this.performSync()},this.settings.syncInterval*60*1e3))}onunload(){}async performSync(){try{if(!await this.rmapi.isAuthenticated()){new u.Notice("reMarkable not authenticated. Run 'rmapi' in a terminal to pair your device.",1e4);return}this.updateStatusBar("Syncing..."),await this.downloader.syncAll(),this.updateStatusBar(`Last sync: ${new Date().toLocaleTimeString()}`)}catch(t){this.updateStatusBar("Sync failed"),console.error("Sync error:",t)}}async convertActiveFileToMd(){let t=this.app.workspace.getActiveFile();if(!t){new u.Notice("No active file selected");return}let e=t.path;if(!e.endsWith(".rm")){new u.Notice("Active file is not a .rm reMarkable document");return}let n=e.replace(/\.rm$/,".md");try{this.updateStatusBar("Converting to Markdown..."),await this.ocrPipeline.processDocument(e,n),this.updateStatusBar(`Converted: ${t.name}`)}catch(a){this.updateStatusBar("Conversion failed"),new u.Notice(`Conversion failed: ${a}`),console.error("OCR error:",a)}}updateStatusBar(t){this.statusBarItem.setText(`reMarkable: ${t}`)}async loadSettings(){this.settings=Object.assign({},N,await this.loadData())}async saveSettings(){await this.saveData(this.settings)}};var _=P; diff --git a/src/convert/render.ts b/src/convert/render.ts new file mode 100644 index 0000000..dad7213 --- /dev/null +++ b/src/convert/render.ts @@ -0,0 +1,55 @@ +import { runCommand } from "../utils/process"; +import RemarkablePlugin from "../main"; + +export class PageRenderer { + plugin: RemarkablePlugin; + + constructor(plugin: RemarkablePlugin) { + this.plugin = plugin; + } + + /** + * Render a single .rm page file to PNG. + */ + async renderPageToPng(pageFile: string, outputPng: string): Promise { + if (this.plugin.settings.pageRenderer === "drawj2d") { + await this.renderWithDrawj2d(pageFile, outputPng); + } else { + await this.renderWithRm2svg(pageFile, outputPng); + } + } + + private async renderWithDrawj2d(pageFile: string, outputPng: string): Promise { + // drawj2d -Trm input.rm output.png + const args = [ + "-jar", + this.plugin.settings.drawj2dPath, // Use configured path + "-Trm", + pageFile, + outputPng, + ]; + const { code, stderr, stdout } = await runCommand(this.plugin.settings.javaPath, args); + if (code !== 0) { + throw new Error(`drawj2d failed: ${stderr || stdout}`); + } + } + + private async renderWithRm2svg(pageFile: string, outputPng: string): Promise { + // Step 1: rM2svg input.rm output.svg + const svgPath = outputPng.replace(".png", ".svg"); + const { code: code1, stderr: err1, stdout: out1 } = await runCommand("rM2svg", [pageFile, svgPath]); + if (code1 !== 0) { + throw new Error(`rM2svg failed: ${err1 || out1}`); + } + + // Step 2: Convert SVG to PNG using ImageMagick or rsvg-convert + // Try rsvg-convert first (better quality), fallback to ImageMagick + const { code: code2 } = await runCommand("rsvg-convert", ["-o", outputPng, svgPath]); + if (code2 !== 0) { + const { code: code3, stderr: err3, stdout: out3 } = await runCommand("convert", [svgPath, outputPng]); + if (code3 !== 0) { + throw new Error(`SVG to PNG conversion failed: ${err3 || out3}`); + } + } + } +} diff --git a/src/main.ts b/src/main.ts index 69ce96d..dcb2f45 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,12 +2,14 @@ import { Plugin, Notice } from "obsidian"; import { RemarkableSettingTab, DEFAULT_SETTINGS } from "./settings"; import { DocumentDownloader } from "./sync/downloader"; import { RmapiBridge } from "./rmapi/bridge"; +import { OcrPipeline } from "./ocr/pipeline"; import { RemarkableSettings } from "./types"; export default class RemarkablePlugin extends Plugin { settings: RemarkableSettings; downloader: DocumentDownloader; rmapi: RmapiBridge; + ocrPipeline: OcrPipeline; statusBarItem: HTMLElement; async onload() { @@ -16,6 +18,20 @@ export default class RemarkablePlugin extends Plugin { // Initialize components this.rmapi = new RmapiBridge(this); this.downloader = new DocumentDownloader(this); + this.ocrPipeline = new OcrPipeline(this); + + // Validate OCR dependencies + if (this.settings.enableHandwritingMd) { + const missing = await this.ocrPipeline.validateDependencies(); + if (missing.length > 0) { + new Notice( + `OCR dependencies missing: ${missing.join(", ")}. Disable "Enable Handwriting to Markdown" in settings or install required tools.`, + 15000, + ); + this.settings.enableHandwritingMd = false; + await this.saveSettings(); + } + } // Add ribbon icon this.addRibbonIcon("pencil", "Sync from reMarkable", () => { @@ -29,6 +45,12 @@ export default class RemarkablePlugin extends Plugin { callback: () => this.performSync(), }); + this.addCommand({ + id: "convert-handwriting-md", + name: "Convert handwriting to Markdown", + callback: () => this.convertActiveFileToMd(), + }); + // Add settings tab this.addSettingTab(new RemarkableSettingTab(this.app, this)); @@ -69,6 +91,31 @@ export default class RemarkablePlugin extends Plugin { } } + async convertActiveFileToMd(): Promise { + const activeFile = this.app.workspace.getActiveFile(); + if (!activeFile) { + new Notice("No active file selected"); + return; + } + + const path = activeFile.path; + if (!path.endsWith(".rm")) { + new Notice("Active file is not a .rm reMarkable document"); + return; + } + + const mdPath = path.replace(/\.rm$/, ".md"); + try { + this.updateStatusBar("Converting to Markdown..."); + await this.ocrPipeline.processDocument(path, mdPath); + this.updateStatusBar(`Converted: ${activeFile.name}`); + } catch (e) { + this.updateStatusBar("Conversion failed"); + new Notice(`Conversion failed: ${e}`); + console.error("OCR error:", e); + } + } + updateStatusBar(text: string): void { this.statusBarItem.setText(`reMarkable: ${text}`); } diff --git a/src/ocr/glmocr-client.ts b/src/ocr/glmocr-client.ts new file mode 100644 index 0000000..4cb99be --- /dev/null +++ b/src/ocr/glmocr-client.ts @@ -0,0 +1,64 @@ +import { GlmOcrResponse } from "../types"; +import RemarkablePlugin from "../main"; +import { runCommand } from "../utils/process"; + +export class GlmOcrClient { + plugin: RemarkablePlugin; + + constructor(plugin: RemarkablePlugin) { + this.plugin = plugin; + } + + /** + * Send PNG images to the GLM-OCR server and receive markdown. + */ + async parseImages(pngPaths: string[]): Promise { + const url = `${this.plugin.settings.glmocrServerUrl}/glmocr/parse`; + + // Build the request body with base64-encoded images + const images: string[] = []; + for (const pngPath of pngPaths) { + const base64 = await this.fileToBase64(pngPath); + images.push(`data:image/png;base64,${base64}`); + } + + const body = JSON.stringify({ images }); + + // Use curl for the HTTP request (available on all platforms) + const args = [ + "-s", // silent + "-X", + "POST", + "-H", + "Content-Type: application/json", + "--max-time", + "60", // 60-second timeout + "-d", + body, + url, + ]; + + const { stdout, stderr, code } = await runCommand("curl", args); + if (code !== 0) { + throw new Error(`GLM-OCR request failed: ${stderr || stdout}`); + } + + try { + const response = JSON.parse(stdout) as GlmOcrResponse; + return response; + } catch (e) { + throw new Error(`Failed to parse GLM-OCR response: ${e}\nRaw: ${stdout}`); + } + } + + /** + * Convert a file to base64 string. + */ + private async fileToBase64(path: string): Promise { + const { stdout, code, stderr } = await runCommand("base64", ["-w", "0", path]); + if (code !== 0) { + throw new Error(`base64 encoding failed: ${stderr}`); + } + return stdout.trim(); + } +} diff --git a/src/ocr/pipeline.ts b/src/ocr/pipeline.ts new file mode 100644 index 0000000..f2a1519 --- /dev/null +++ b/src/ocr/pipeline.ts @@ -0,0 +1,166 @@ +import RemarkablePlugin from "../main"; +import { PageRenderer } from "../convert/render"; +import { extractHwrText } from "./remarkable-hwr"; +import { GlmOcrClient } from "./glmocr-client"; +import { StyleRefiner } from "./style-refiner"; +import { extractRmFile, isNotebook, getPageFiles } from "../utils/zip"; +import { runCommand } from "../utils/process"; +import { Notice } from "obsidian"; + +export class OcrPipeline { + plugin: RemarkablePlugin; + renderer: PageRenderer; + glmOcr: GlmOcrClient; + refiner: StyleRefiner; + + constructor(plugin: RemarkablePlugin) { + this.plugin = plugin; + this.renderer = new PageRenderer(plugin); + this.glmOcr = new GlmOcrClient(plugin); + this.refiner = new StyleRefiner(plugin); + } + + /** + * Run the full OCR pipeline on a downloaded .rm file. + * Returns the path to the generated .md file. + */ + async processDocument(rmPath: string, outputMdPath: string): Promise { + const tmpDir = `${this.plugin.app.vault.adapter.getBasePath()}/.obsidian/rmapi-tmp`; + + try { + new Notice("Converting handwriting to Markdown..."); + + // Step 0: Extract the .rm archive + await this.ensureDir(tmpDir); + const extractedDir = `${tmpDir}/extracted`; + await this.ensureDir(extractedDir); + await extractRmFile(rmPath, extractedDir); + + // Only process notebooks (handwritten documents) + const notebook = await isNotebook(extractedDir); + if (!notebook) { + new Notice("Document is not a handwritten notebook — skipping OCR."); + return ""; + } + + // Step 1: Extract reMarkable built-in HWR text + const hwrText = await extractHwrText(extractedDir); + + // Step 2: Render pages to PNG + const pageFiles = await getPageFiles(extractedDir); + const pngPaths: string[] = []; + const totalPages = pageFiles.length; + + for (let i = 0; i < pageFiles.length; i++) { + const pageFile = pageFiles[i]; + const pngPath = `${tmpDir}/page-${i}.png`; + await this.renderer.renderPageToPng(pageFile, pngPath); + pngPaths.push(pngPath); + + // Progress feedback + if ((i + 1) % 5 === 0 || i === totalPages - 1) { + new Notice(`Rendering pages... ${i + 1}/${totalPages}`); + } + } + + // Step 3: GLM-OCR (batched) + const batchSize = this.plugin.settings.maxPagesPerBatch; + const batches = []; + for (let i = 0; i < pngPaths.length; i += batchSize) { + batches.push(pngPaths.slice(i, i + batchSize)); + } + + let allGlmMarkdown = ""; + for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) { + const batch = batches[batchIndex]; + const glmResponse = await this.glmOcr.parseImages(batch); + const glmMarkdown = glmResponse.markdown_result || ""; + allGlmMarkdown += glmMarkdown + "\n\n"; + + // Progress feedback + new Notice(`OCR processing... ${batchIndex + 1}/${batches.length}`); + } + + const finalGlmMarkdown = allGlmMarkdown.trim(); + + // Step 4: Merge & refine via Ollama + const finalMarkdown = await this.refiner.mergeAndRefine(hwrText, finalGlmMarkdown); + + // Step 5: Write output + await this.plugin.app.vault.adapter.write(outputMdPath, finalMarkdown); + + new Notice("Handwriting conversion complete!"); + return outputMdPath; + } finally { + // Cleanup temp files + await this.cleanup(tmpDir); + } + } + + private async ensureDir(path: string): Promise { + try { + await this.plugin.app.vault.adapter.mkdir(path); + } catch { + // May already exist + } + } + + /** + * Validate that required binaries are available. + */ + async validateDependencies(): Promise { + const required = ["unzip", "curl", "base64"]; + const missing: string[] = []; + + for (const cmd of required) { + try { + const { code } = await runCommand(cmd, ["--version"]); + if (code !== 0) missing.push(cmd); + } catch { + missing.push(cmd); + } + } + + // Check renderer-specific dependencies + if (this.plugin.settings.pageRenderer === "drawj2d") { + try { + const { code } = await runCommand(this.plugin.settings.javaPath, [ + "-jar", + this.plugin.settings.drawj2dPath, + "--help", + ]); + if (code !== 0) missing.push("drawj2d"); + } catch { + missing.push("drawj2d"); + } + } else { + // rM2svg path + try { + const { code } = await runCommand("rM2svg", ["--help"]); + if (code !== 0) missing.push("rM2svg"); + } catch { + missing.push("rM2svg"); + } + // SVG to PNG converter + try { + await runCommand("rsvg-convert", ["--version"]); + } catch { + try { + await runCommand("convert", ["--version"]); + } catch { + missing.push("rsvg-convert or convert"); + } + } + } + + return missing; + } + + private async cleanup(tmpDir: string): Promise { + const isWindows = process.platform === "win32"; + const { code } = await runCommand(isWindows ? "rd" : "rm", isWindows ? ["/s", "/q", tmpDir] : ["-rf", tmpDir]); + if (code !== 0) { + console.warn("Failed to cleanup temp directory:", tmpDir); + } + } +} diff --git a/src/ocr/remarkable-hwr.ts b/src/ocr/remarkable-hwr.ts new file mode 100644 index 0000000..392e23e --- /dev/null +++ b/src/ocr/remarkable-hwr.ts @@ -0,0 +1,74 @@ +import { runCommand } from "../utils/process"; + +/** + * Extract reMarkable's built-in handwriting recognition text from a .rm archive. + * + * On v3.x, the document may contain a `content.json` or `metadata.json` with + * text layers. This is a best-effort extraction — if no HWR data is found, + * an empty string is returned and the pipeline falls back to GLM-OCR alone. + */ +export async function extractHwrText(extractedDir: string): Promise { + // Try to find text in content.json first + const contentJsonPath = `${extractedDir}/content.json`; + const { stdout: contentText, code: contentCode } = await runCommand("cat", [contentJsonPath]); + + if (contentCode === 0) { + try { + const content = JSON.parse(contentText); + // v3.x content structure may have text in cPages or similar + const hwrText = extractTextFromContentJson(content); + if (hwrText) return hwrText; + } catch { + // Not valid JSON or unexpected structure + } + } + + // Fallback: try metadata.json + const metadataJsonPath = `${extractedDir}/metadata.json`; + const { stdout: metaText, code: metaCode } = await runCommand("cat", [metadataJsonPath]); + + if (metaCode === 0) { + try { + const meta = JSON.parse(metaText); + if (meta && meta.text) return meta.text; + } catch { + // Not valid JSON + } + } + + return ""; +} + +/** + * Extract text from content.json structure. + * This handles the v3.x format where text may be in cPages. + */ +function extractTextFromContentJson(content: any): string { + const texts: string[] = []; + + // Try known text locations in reMarkable content.json + if (content.cPages && Array.isArray(content.cPages.pages)) { + for (const page of content.cPages.pages) { + if (page.text) { + texts.push(page.text); + } + // Also check for text layers + if (page.layers) { + for (const layer of page.layers) { + if (layer.text) { + texts.push(layer.text); + } + } + } + } + } + + // Alternative: text in pages array directly + if (content.pages && Array.isArray(content.pages)) { + for (const page of content.pages) { + if (page.text) texts.push(page.text); + } + } + + return texts.join("\n\n"); +} diff --git a/src/ocr/style-refiner.ts b/src/ocr/style-refiner.ts new file mode 100644 index 0000000..5348420 --- /dev/null +++ b/src/ocr/style-refiner.ts @@ -0,0 +1,106 @@ +import RemarkablePlugin from "../main"; +import { runCommand } from "../utils/process"; + +export class StyleRefiner { + plugin: RemarkablePlugin; + + constructor(plugin: RemarkablePlugin) { + this.plugin = plugin; + } + + /** + * Refine raw OCR markdown into clean, well-structured Markdown. + * Option A: light cleanup — fix headings, lists, fragments. + */ + async refineMarkdown(rawMarkdown: string): Promise { + const prompt = `You are formatting the output of an OCR engine that processed a handwritten document. The input is markdown that may have issues with heading levels, list formatting, and paragraph breaks. + +Analyze the document's structure and: +1. Fix heading hierarchy (h1 → h2 → h3 logically) +2. Consolidate fragmented paragraphs +3. Detect and format lists, checkboxes, tables +4. Identify emphasized/underlined text and apply bold/italic +5. Preserve all content — do not summarize or remove anything + +INPUT MARKDOWN: +${rawMarkdown} + +OUTPUT ONLY THE REFINED MARKDOWN. No explanations, no markdown code fences around the output.`; + + const response = await this.queryOllama(prompt); + const refined = response.trim(); + this.validateOllamaOutput(rawMarkdown, refined); + return refined; + } + + /** + * Merge HWR text and GLM-OCR markdown, then refine. + */ + async mergeAndRefine(hwrText: string, glmOcrMarkdown: string): Promise { + if (!hwrText.trim()) { + // No HWR available — just refine GLM-OCR output + return this.refineMarkdown(glmOcrMarkdown); + } + + const prompt = `You have two sources of text from the same handwritten document: + +SOURCE 1 — ReMarkable built-in handwriting recognition (raw text, no formatting): +${hwrText} + +SOURCE 2 — GLM-OCR engine output (markdown with structure but possible errors): +${glmOcrMarkdown} + +Your task: +1. Merge both sources, preferring SOURCE 2 for structure and SOURCE 1 for text accuracy +2. Fix heading hierarchy (h1 → h2 → h3 logically) +3. Consolidate fragmented paragraphs +4. Detect and format lists, checkboxes, tables +5. Identify emphasized/underlined text and apply bold/italic +6. Preserve all content — do not summarize or remove anything + +OUTPUT ONLY THE FINAL REFINED MARKDOWN. No explanations, no markdown code fences around the output.`; + + const response = await this.queryOllama(prompt); + const refined = response.trim(); + this.validateOllamaOutput(hwrText + glmOcrMarkdown, refined); + return refined; + } + + private async queryOllama(prompt: string): Promise { + const url = `${this.plugin.settings.ollamaHost}/api/generate`; + const model = this.plugin.settings.styleModel; + + const body = JSON.stringify({ + model, + prompt, + stream: false, + }); + + const args = ["-s", "-X", "POST", "-H", "Content-Type: application/json", "--max-time", "60", "-d", body, url]; + + const { stdout, stderr, code } = await runCommand("curl", args); + if (code !== 0) { + throw new Error(`Ollama request failed: ${stderr || stdout}`); + } + + try { + const response = JSON.parse(stdout); + return response.response || ""; + } catch (e) { + throw new Error(`Failed to parse Ollama response: ${e}\nRaw: ${stdout}`); + } + } + + /** + * Validate that the LLM output preserves content (not summarized). + */ + private validateOllamaOutput(input: string, output: string): void { + const minRatio = 0.8; // Allow 20% reduction for formatting + if (output.length < input.length * minRatio) { + throw new Error( + `LLM output too short (${output.length} chars) vs input (${input.length} chars). ` + + `Ratio: ${(output.length / input.length).toFixed(2)}. May have summarized content.`, + ); + } + } +} diff --git a/src/settings.ts b/src/settings.ts index 039cd0e..c2fddb2 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -14,6 +14,8 @@ export const DEFAULT_SETTINGS: RemarkableSettings = { styleModel: "qwen3:32b", pageRenderer: "drawj2d", javaPath: "java", + drawj2dPath: "drawj2d.jar", + maxPagesPerBatch: 20, syncInterval: 0, }; @@ -137,12 +139,35 @@ export class RemarkableSettingTab extends PluginSettingTab { new Setting(containerEl) .setName("Java Path") .setDesc("Path to Java runtime (for drawj2d)") - .addText((text) => + .addText((text) => { text.setValue(this.plugin.settings.javaPath).onChange(async (value) => { this.plugin.settings.javaPath = value; await this.plugin.saveSettings(); - }), - ); + }); + }); + + new Setting(containerEl) + .setName("drawj2d Path") + .setDesc("Path to drawj2d.jar (e.g., /opt/drawj2d.jar)") + .addText((text) => { + text.setValue(this.plugin.settings.drawj2dPath).onChange(async (value) => { + this.plugin.settings.drawj2dPath = value; + await this.plugin.saveSettings(); + }); + }); + + new Setting(containerEl) + .setName("Max Pages per Batch") + .setDesc("Max pages per GLM-OCR request (to avoid timeouts)") + .addText((text) => { + text.setValue(String(this.plugin.settings.maxPagesPerBatch)).onChange(async (value) => { + const num = parseInt(value); + this.plugin.settings.maxPagesPerBatch = isNaN(num) || num < 1 ? 20 : num; + await this.plugin.saveSettings(); + }); + text.inputEl.type = "number"; + text.inputEl.min = "1"; + }); new Setting(containerEl) .setName("Sync Interval (minutes)") diff --git a/src/sync/downloader.ts b/src/sync/downloader.ts index e05e8fb..2f4d500 100644 --- a/src/sync/downloader.ts +++ b/src/sync/downloader.ts @@ -2,6 +2,7 @@ import { RmapiNode } from "../types"; import RemarkablePlugin from "../main"; import { RmapiBridge } from "../rmapi/bridge"; import { SyncTracker } from "./tracker"; +import { OcrPipeline } from "../ocr/pipeline"; import { Notice } from "obsidian"; function sanitizeFileName(name: string): string { @@ -12,11 +13,13 @@ export class DocumentDownloader { plugin: RemarkablePlugin; rmapi: RmapiBridge; tracker: SyncTracker; + ocrPipeline: OcrPipeline; constructor(plugin: RemarkablePlugin) { this.plugin = plugin; this.rmapi = new RmapiBridge(plugin); this.tracker = new SyncTracker(plugin); + this.ocrPipeline = new OcrPipeline(plugin); } async syncAll(): Promise { @@ -79,5 +82,14 @@ export class DocumentDownloader { // Track document await this.tracker.trackDocument(doc.id, doc.name, localPath, hasPdf, false); + + // Convert handwriting to Markdown if enabled + if (this.plugin.settings.enableHandwritingMd) { + const mdPath = `${localDir}/${safeName}.md`; + const mdResult = await this.ocrPipeline.processDocument(localPath, mdPath); + if (mdResult) { + await this.tracker.trackDocument(doc.id, doc.name, localPath, hasPdf, true); + } + } } } diff --git a/src/types.ts b/src/types.ts index 7a1e3e8..ad188c9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -24,6 +24,8 @@ export interface RemarkableSettings { styleModel: string; pageRenderer: "drawj2d" | "rM2svg"; javaPath: string; + drawj2dPath: string; // Path to drawj2d.jar + maxPagesPerBatch: number; // Max pages per GLM-OCR batch syncInterval: number; // minutes, 0 = manual only } diff --git a/src/utils/zip.ts b/src/utils/zip.ts new file mode 100644 index 0000000..8e233ea --- /dev/null +++ b/src/utils/zip.ts @@ -0,0 +1,64 @@ +import { runCommand } from "./process"; + +/** + * Extract a .rm file (which is a zip archive) to a temporary directory. + * Returns the path to the extraction directory. + */ +export async function extractRmFile(rmPath: string, outputDir: string): Promise { + // .rm files are zip archives + const { code, stderr } = await runCommand("unzip", ["-o", rmPath, "-d", outputDir]); + if (code !== 0) { + throw new Error(`Failed to extract .rm file: ${stderr}`); + } + return outputDir; +} + +/** + * List the contents of the extracted .rm archive. + */ +export async function listRmContents(extractedDir: string): Promise { + const { stdout, code, stderr } = await runCommand("find", [extractedDir, "-type", "f"]); + if (code !== 0) { + throw new Error(`Failed to list .rm contents: ${stderr}`); + } + return stdout.trim().split("\n").filter(Boolean); +} + +/** + * Check if a document is a notebook (handwritten) or a PDF/ebook. + * On reMarkable v3.x, notebooks have .rm page files and no content.pdf. + * Falls back to rendering a test page if ambiguous. + */ +export async function isNotebook(extractedDir: string): Promise { + const contents = await listRmContents(extractedDir); + // Notebooks have .rm page files and no embedded PDF content + const hasRmPages = contents.some((f) => f.endsWith(".rm") && !f.endsWith(".zip")); + const hasContentPdf = contents.some((f) => f.endsWith("content.pdf") || f.endsWith(".pdf")); + + // Clear case + if (hasRmPages && !hasContentPdf) return true; + if (!hasRmPages || hasContentPdf) return false; + + // Ambiguous: try to render a page + const pageFiles = contents.filter((f) => f.endsWith(".rm") && !f.endsWith(".zip")); + if (pageFiles.length === 0) return false; + + try { + // Try rendering the first page + const { code } = await runCommand("file", [pageFiles[0]]); + // If file command succeeds and identifies it as data, assume notebook + return code === 0; + } catch { + return false; + } +} + +/** + * Get the list of page .rm files from an extracted notebook. + * Returns basenames only. + * Recursively searches for .rm files (not just root). + */ +export async function getPageFiles(extractedDir: string): Promise { + const contents = await listRmContents(extractedDir); + return contents.filter((f) => f.endsWith(".rm") && !f.endsWith(".zip")).sort(); +}