fegger
0211cf33f3
Add handwriting OCR pipeline and update documentation - Introduce full OCR workflow: HWR extraction → PNG rendering → GLM-OCR → Ollama refinement - Add new modules for page rendering, GLM-OCR client, style refinement, and ZIP utilities - Update settings with drawj2d path and batch size controls - Document Phase 2 completion and known limitations in IMPLEMENTATION.md - Validate dependencies and handle cross-platform temp directory cleanup ```
38 lines
17 KiB
JavaScript
38 lines
17 KiB
JavaScript
var M=Object.defineProperty;var G=Object.getOwnPropertyDescriptor;var U=Object.getOwnPropertyNames;var W=Object.prototype.hasOwnProperty;var V=(i,t)=>{for(var e in t)M(i,e,{get:t[e],enumerable:!0})},z=(i,t,e,n)=>{if(t&&typeof t=="object"||typeof t=="function")for(let a of U(t))!W.call(i,a)&&a!==e&&M(i,a,{get:()=>t[a],enumerable:!(n=G(t,a))||n.enumerable});return i};var J=i=>z(M({},"__esModule",{value:!0}),i);var Y={};V(Y,{default:()=>_});module.exports=J(Y);var u=require("obsidian");var m=require("obsidian"),N={remarkableHost:"https://10.11.99.1",rmapiBinaryPath:"rmapi",downloadPath:"remarkable",convertToPdf:!0,enableHandwritingMd:!0,glmocrServerUrl:"http://100.103.83.12:5002",glmocrApiKey:"any-string",ollamaHost:"http://100.103.83.12:11435",styleModel:"qwen3:32b",pageRenderer:"drawj2d",javaPath:"java",drawj2dPath:"drawj2d.jar",maxPagesPerBatch:20,syncInterval:0},v=class extends m.PluginSettingTab{plugin;constructor(t,e){super(t,e),this.plugin=e}display(){let{containerEl:t}=this;t.empty(),t.createEl("h2",{text:"reMarkable Sync Settings"}),new m.Setting(t).setName("reMarkable Host").setDesc("URL of your reMarkable tablet (e.g., https://10.11.99.1)").addText(e=>e.setValue(this.plugin.settings.remarkableHost).onChange(async n=>{this.plugin.settings.remarkableHost=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("rmapi Binary Path").setDesc("Path to rmapi binary (e.g., rmapi or /usr/local/bin/rmapi)").addText(e=>e.setValue(this.plugin.settings.rmapiBinaryPath).onChange(async n=>{this.plugin.settings.rmapiBinaryPath=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Download Path").setDesc("Folder in vault for downloaded documents").addText(e=>e.setValue(this.plugin.settings.downloadPath).onChange(async n=>{this.plugin.settings.downloadPath=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Convert to PDF").setDesc("Automatically convert documents to annotated PDF").addToggle(e=>e.setValue(this.plugin.settings.convertToPdf).onChange(async n=>{this.plugin.settings.convertToPdf=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Enable Handwriting to Markdown").setDesc("Convert handwritten notes to styled Markdown").addToggle(e=>e.setValue(this.plugin.settings.enableHandwritingMd).onChange(async n=>{this.plugin.settings.enableHandwritingMd=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("GLM-OCR Server URL").setDesc("URL of GLM-OCR SDK Server (e.g., http://100.103.83.12:5002)").addText(e=>e.setValue(this.plugin.settings.glmocrServerUrl).onChange(async n=>{this.plugin.settings.glmocrServerUrl=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("GLM-OCR API Key").setDesc("API key for GLM-OCR Server (can be any string for self-hosted)").addText(e=>e.setValue(this.plugin.settings.glmocrApiKey).onChange(async n=>{this.plugin.settings.glmocrApiKey=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Ollama Host").setDesc("URL of Ollama server for style refinement (e.g., http://100.103.83.12:11435)").addText(e=>e.setValue(this.plugin.settings.ollamaHost).onChange(async n=>{this.plugin.settings.ollamaHost=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Style Model").setDesc("Ollama model for Markdown style refinement").addText(e=>e.setValue(this.plugin.settings.styleModel).onChange(async n=>{this.plugin.settings.styleModel=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Page Renderer").setDesc("Tool to render .rm pages as PNG").addDropdown(e=>e.addOption("drawj2d","drawj2d (recommended)").addOption("rM2svg","rM2svg (lightweight)").setValue(this.plugin.settings.pageRenderer).onChange(async n=>{this.plugin.settings.pageRenderer=n,await this.plugin.saveSettings()})),new m.Setting(t).setName("Java Path").setDesc("Path to Java runtime (for drawj2d)").addText(e=>{e.setValue(this.plugin.settings.javaPath).onChange(async n=>{this.plugin.settings.javaPath=n,await this.plugin.saveSettings()})}),new m.Setting(t).setName("drawj2d Path").setDesc("Path to drawj2d.jar (e.g., /opt/drawj2d.jar)").addText(e=>{e.setValue(this.plugin.settings.drawj2dPath).onChange(async n=>{this.plugin.settings.drawj2dPath=n,await this.plugin.saveSettings()})}),new m.Setting(t).setName("Max Pages per Batch").setDesc("Max pages per GLM-OCR request (to avoid timeouts)").addText(e=>{e.setValue(String(this.plugin.settings.maxPagesPerBatch)).onChange(async n=>{let a=parseInt(n);this.plugin.settings.maxPagesPerBatch=isNaN(a)||a<1?20:a,await this.plugin.saveSettings()}),e.inputEl.type="number",e.inputEl.min="1"}),new m.Setting(t).setName("Sync Interval (minutes)").setDesc("0 = manual only, >0 = auto-sync every N minutes").addText(e=>{e.setValue(String(this.plugin.settings.syncInterval)).onChange(async n=>{let a=parseInt(n);this.plugin.settings.syncInterval=isNaN(a)||a<0?0:a,await this.plugin.saveSettings()}),e.inputEl.type="number",e.inputEl.min="0"})}};var $=require("child_process");async function l(i,t=[],e={},n){return new Promise(a=>{let s=(0,$.spawn)(i,t,{env:{...process.env,...e},cwd:n}),r="",o="",c=!1,d=(p,h)=>{c||(c=!0,a({stdout:r,stderr:h?`${o}
|
|
${h.message}`:o,code:p}))};s.stdout.on("data",p=>{r+=p.toString()}),s.stderr.on("data",p=>{o+=p.toString()}),s.on("error",p=>{d(1,p)}),s.on("close",p=>{d(p??0)})})}var w=class{plugin;constructor(t){this.plugin=t}getEnv(){return{RMAPI_HOST:this.plugin.settings.remarkableHost,RMAPI_CONFIG:this.plugin.app.vault.adapter.getBasePath()+"/.obsidian/rmapi"}}async runRmapi(t){return l(this.plugin.settings.rmapiBinaryPath,t,this.getEnv())}async list(t="/"){let{stdout:e,stderr:n,code:a}=await this.runRmapi(["ls","--json",t]);if(a!==0)throw new Error(`rmapi ls failed: ${n||e}`);try{return JSON.parse(e)}catch(s){throw new Error(`Failed to parse rmapi output: ${s}`)}}async downloadFile(t,e){let{stdout:n,stderr:a,code:s}=await this.runRmapi(["get",t,"-o",e]);if(s!==0)throw new Error(`rmapi get failed: ${a||n}`)}async downloadAnnotatedPdf(t,e){let{stdout:n,stderr:a,code:s}=await this.runRmapi(["geta",t,"-o",e]);if(s!==0)throw new Error(`rmapi geta failed: ${a||n}`)}async isAuthenticated(){let{code:t,stderr:e}=await this.runRmapi(["ls","--json","/"]);return t===0&&!e.includes("auth")&&!e.includes("register")}};var b=class{plugin;constructor(t){this.plugin=t}async loadSyncedDocuments(){return(await this.plugin.loadData()).syncedDocuments||[]}async saveSyncedDocuments(t){let e=await this.plugin.loadData()||{};await this.plugin.saveData({...e,syncedDocuments:t})}async trackDocument(t,e,n,a,s){let r=await this.loadSyncedDocuments(),o=r.findIndex(d=>d.id===t),c={id:t,name:e,lastSynced:new Date().toISOString(),localPath:n,hasPdf:a,hasMd:s};o>=0?r[o]=c:r.push(c),await this.saveSyncedDocuments(r)}async isDocumentSynced(t){return(await this.loadSyncedDocuments()).some(n=>n.id===t)}async getSyncedDocument(t){return(await this.loadSyncedDocuments()).find(n=>n.id===t)||null}};var R=class{plugin;constructor(t){this.plugin=t}async renderPageToPng(t,e){this.plugin.settings.pageRenderer==="drawj2d"?await this.renderWithDrawj2d(t,e):await this.renderWithRm2svg(t,e)}async renderWithDrawj2d(t,e){let n=["-jar",this.plugin.settings.drawj2dPath,"-Trm",t,e],{code:a,stderr:s,stdout:r}=await l(this.plugin.settings.javaPath,n);if(a!==0)throw new Error(`drawj2d failed: ${s||r}`)}async renderWithRm2svg(t,e){let n=e.replace(".png",".svg"),{code:a,stderr:s,stdout:r}=await l("rM2svg",[t,n]);if(a!==0)throw new Error(`rM2svg failed: ${s||r}`);let{code:o}=await l("rsvg-convert",["-o",e,n]);if(o!==0){let{code:c,stderr:d,stdout:p}=await l("convert",[n,e]);if(c!==0)throw new Error(`SVG to PNG conversion failed: ${d||p}`)}}};async function j(i){let t=`${i}/content.json`,{stdout:e,code:n}=await l("cat",[t]);if(n===0)try{let o=JSON.parse(e),c=q(o);if(c)return c}catch{}let a=`${i}/metadata.json`,{stdout:s,code:r}=await l("cat",[a]);if(r===0)try{let o=JSON.parse(s);if(o&&o.text)return o.text}catch{}return""}function q(i){let t=[];if(i.cPages&&Array.isArray(i.cPages.pages)){for(let e of i.cPages.pages)if(e.text&&t.push(e.text),e.layers)for(let n of e.layers)n.text&&t.push(n.text)}if(i.pages&&Array.isArray(i.pages))for(let e of i.pages)e.text&&t.push(e.text);return t.join(`
|
|
|
|
`)}var S=class{plugin;constructor(t){this.plugin=t}async parseImages(t){let e=`${this.plugin.settings.glmocrServerUrl}/glmocr/parse`,n=[];for(let d of t){let p=await this.fileToBase64(d);n.push(`data:image/png;base64,${p}`)}let s=["-s","-X","POST","-H","Content-Type: application/json","--max-time","60","-d",JSON.stringify({images:n}),e],{stdout:r,stderr:o,code:c}=await l("curl",s);if(c!==0)throw new Error(`GLM-OCR request failed: ${o||r}`);try{return JSON.parse(r)}catch(d){throw new Error(`Failed to parse GLM-OCR response: ${d}
|
|
Raw: ${r}`)}}async fileToBase64(t){let{stdout:e,code:n,stderr:a}=await l("base64",["-w","0",t]);if(n!==0)throw new Error(`base64 encoding failed: ${a}`);return e.trim()}};var k=class{plugin;constructor(t){this.plugin=t}async refineMarkdown(t){let e=`You are formatting the output of an OCR engine that processed a handwritten document. The input is markdown that may have issues with heading levels, list formatting, and paragraph breaks.
|
|
|
|
Analyze the document's structure and:
|
|
1. Fix heading hierarchy (h1 \u2192 h2 \u2192 h3 logically)
|
|
2. Consolidate fragmented paragraphs
|
|
3. Detect and format lists, checkboxes, tables
|
|
4. Identify emphasized/underlined text and apply bold/italic
|
|
5. Preserve all content \u2014 do not summarize or remove anything
|
|
|
|
INPUT MARKDOWN:
|
|
${t}
|
|
|
|
OUTPUT ONLY THE REFINED MARKDOWN. No explanations, no markdown code fences around the output.`,a=(await this.queryOllama(e)).trim();return this.validateOllamaOutput(t,a),a}async mergeAndRefine(t,e){if(!t.trim())return this.refineMarkdown(e);let n=`You have two sources of text from the same handwritten document:
|
|
|
|
SOURCE 1 \u2014 ReMarkable built-in handwriting recognition (raw text, no formatting):
|
|
${t}
|
|
|
|
SOURCE 2 \u2014 GLM-OCR engine output (markdown with structure but possible errors):
|
|
${e}
|
|
|
|
Your task:
|
|
1. Merge both sources, preferring SOURCE 2 for structure and SOURCE 1 for text accuracy
|
|
2. Fix heading hierarchy (h1 \u2192 h2 \u2192 h3 logically)
|
|
3. Consolidate fragmented paragraphs
|
|
4. Detect and format lists, checkboxes, tables
|
|
5. Identify emphasized/underlined text and apply bold/italic
|
|
6. Preserve all content \u2014 do not summarize or remove anything
|
|
|
|
OUTPUT ONLY THE FINAL REFINED MARKDOWN. No explanations, no markdown code fences around the output.`,s=(await this.queryOllama(n)).trim();return this.validateOllamaOutput(t+e,s),s}async queryOllama(t){let e=`${this.plugin.settings.ollamaHost}/api/generate`,n=this.plugin.settings.styleModel,s=["-s","-X","POST","-H","Content-Type: application/json","--max-time","60","-d",JSON.stringify({model:n,prompt:t,stream:!1}),e],{stdout:r,stderr:o,code:c}=await l("curl",s);if(c!==0)throw new Error(`Ollama request failed: ${o||r}`);try{return JSON.parse(r).response||""}catch(d){throw new Error(`Failed to parse Ollama response: ${d}
|
|
Raw: ${r}`)}}validateOllamaOutput(t,e){if(e.length<t.length*.8)throw new Error(`LLM output too short (${e.length} chars) vs input (${t.length} chars). Ratio: ${(e.length/t.length).toFixed(2)}. May have summarized content.`)}};async function E(i,t){let{code:e,stderr:n}=await l("unzip",["-o",i,"-d",t]);if(e!==0)throw new Error(`Failed to extract .rm file: ${n}`);return t}async function A(i){let{stdout:t,code:e,stderr:n}=await l("find",[i,"-type","f"]);if(e!==0)throw new Error(`Failed to list .rm contents: ${n}`);return t.trim().split(`
|
|
`).filter(Boolean)}async function I(i){let t=await A(i),e=t.some(s=>s.endsWith(".rm")&&!s.endsWith(".zip")),n=t.some(s=>s.endsWith("content.pdf")||s.endsWith(".pdf"));if(e&&!n)return!0;if(!e||n)return!1;let a=t.filter(s=>s.endsWith(".rm")&&!s.endsWith(".zip"));if(a.length===0)return!1;try{let{code:s}=await l("file",[a[0]]);return s===0}catch{return!1}}async function F(i){return(await A(i)).filter(e=>e.endsWith(".rm")&&!e.endsWith(".zip")).sort()}var f=require("obsidian"),y=class{plugin;renderer;glmOcr;refiner;constructor(t){this.plugin=t,this.renderer=new R(t),this.glmOcr=new S(t),this.refiner=new k(t)}async processDocument(t,e){let n=`${this.plugin.app.vault.adapter.getBasePath()}/.obsidian/rmapi-tmp`;try{new f.Notice("Converting handwriting to Markdown..."),await this.ensureDir(n);let a=`${n}/extracted`;if(await this.ensureDir(a),await E(t,a),!await I(a))return new f.Notice("Document is not a handwritten notebook \u2014 skipping OCR."),"";let r=await j(a),o=await F(a),c=[],d=o.length;for(let g=0;g<o.length;g++){let C=o[g],O=`${n}/page-${g}.png`;await this.renderer.renderPageToPng(C,O),c.push(O),((g+1)%5===0||g===d-1)&&new f.Notice(`Rendering pages... ${g+1}/${d}`)}let p=this.plugin.settings.maxPagesPerBatch,h=[];for(let g=0;g<c.length;g+=p)h.push(c.slice(g,g+p));let T="";for(let g=0;g<h.length;g++){let C=h[g],L=(await this.glmOcr.parseImages(C)).markdown_result||"";T+=L+`
|
|
|
|
`,new f.Notice(`OCR processing... ${g+1}/${h.length}`)}let B=T.trim(),H=await this.refiner.mergeAndRefine(r,B);return await this.plugin.app.vault.adapter.write(e,H),new f.Notice("Handwriting conversion complete!"),e}finally{await this.cleanup(n)}}async ensureDir(t){try{await this.plugin.app.vault.adapter.mkdir(t)}catch{}}async validateDependencies(){let t=["unzip","curl","base64"],e=[];for(let n of t)try{let{code:a}=await l(n,["--version"]);a!==0&&e.push(n)}catch{e.push(n)}if(this.plugin.settings.pageRenderer==="drawj2d")try{let{code:n}=await l(this.plugin.settings.javaPath,["-jar",this.plugin.settings.drawj2dPath,"--help"]);n!==0&&e.push("drawj2d")}catch{e.push("drawj2d")}else{try{let{code:n}=await l("rM2svg",["--help"]);n!==0&&e.push("rM2svg")}catch{e.push("rM2svg")}try{await l("rsvg-convert",["--version"])}catch{try{await l("convert",["--version"])}catch{e.push("rsvg-convert or convert")}}}return e}async cleanup(t){let e=process.platform==="win32",{code:n}=await l(e?"rd":"rm",e?["/s","/q",t]:["-rf",t]);n!==0&&console.warn("Failed to cleanup temp directory:",t)}};var x=require("obsidian");function K(i){return i.replace(/[\\/:*?"<>|]/g,"_")}var D=class{plugin;rmapi;tracker;ocrPipeline;constructor(t){this.plugin=t,this.rmapi=new w(t),this.tracker=new b(t),this.ocrPipeline=new y(t)}async syncAll(){try{new x.Notice("Syncing from reMarkable...");let t=await this.listAllDocuments("/");for(let e of t)await this.downloadDocument(e);new x.Notice(`Sync completed! ${t.length} document(s) synced.`)}catch(t){new x.Notice(`Sync failed: ${t}`),console.error("Sync error:",t)}}async listAllDocuments(t){let e=await this.rmapi.list(t),n=e.filter(a=>a.type==="DocumentType");for(let a of e.filter(s=>s.type==="CollectionType")){let s=t==="/"?`/${a.name}`:`${t}/${a.name}`,r=await this.listAllDocuments(s);n=n.concat(r)}return n}async downloadDocument(t){let e=await this.tracker.getSyncedDocument(t.id);if(e&&e.lastSynced>=t.modifiedClient)return;let n=K(t.name),a=this.plugin.settings.downloadPath,s=`${a}/${n}`;try{await this.plugin.app.vault.adapter.mkdir(a)}catch(o){console.warn("mkdir failed (may already exist):",o)}await this.rmapi.downloadFile(t.name,s);let r=!1;if(this.plugin.settings.convertToPdf){let o=`${a}/${n}.pdf`;await this.rmapi.downloadAnnotatedPdf(t.name,o),r=!0}if(await this.tracker.trackDocument(t.id,t.name,s,r,!1),this.plugin.settings.enableHandwritingMd){let o=`${a}/${n}.md`;await this.ocrPipeline.processDocument(s,o)&&await this.tracker.trackDocument(t.id,t.name,s,r,!0)}}};var P=class extends u.Plugin{settings;downloader;rmapi;ocrPipeline;statusBarItem;async onload(){if(await this.loadSettings(),this.rmapi=new w(this),this.downloader=new D(this),this.ocrPipeline=new y(this),this.settings.enableHandwritingMd){let t=await this.ocrPipeline.validateDependencies();t.length>0&&(new u.Notice(`OCR dependencies missing: ${t.join(", ")}. Disable "Enable Handwriting to Markdown" in settings or install required tools.`,15e3),this.settings.enableHandwritingMd=!1,await this.saveSettings())}this.addRibbonIcon("pencil","Sync from reMarkable",()=>{this.performSync()}),this.addCommand({id:"sync-all",name:"Sync from reMarkable",callback:()=>this.performSync()}),this.addCommand({id:"convert-handwriting-md",name:"Convert handwriting to Markdown",callback:()=>this.convertActiveFileToMd()}),this.addSettingTab(new v(this.app,this)),this.statusBarItem=this.addStatusBarItem(),this.updateStatusBar("Idle"),this.settings.syncInterval>0&&this.registerInterval(window.setInterval(()=>{this.performSync()},this.settings.syncInterval*60*1e3))}onunload(){}async performSync(){try{if(!await this.rmapi.isAuthenticated()){new u.Notice("reMarkable not authenticated. Run 'rmapi' in a terminal to pair your device.",1e4);return}this.updateStatusBar("Syncing..."),await this.downloader.syncAll(),this.updateStatusBar(`Last sync: ${new Date().toLocaleTimeString()}`)}catch(t){this.updateStatusBar("Sync failed"),console.error("Sync error:",t)}}async convertActiveFileToMd(){let t=this.app.workspace.getActiveFile();if(!t){new u.Notice("No active file selected");return}let e=t.path;if(!e.endsWith(".rm")){new u.Notice("Active file is not a .rm reMarkable document");return}let n=e.replace(/\.rm$/,".md");try{this.updateStatusBar("Converting to Markdown..."),await this.ocrPipeline.processDocument(e,n),this.updateStatusBar(`Converted: ${t.name}`)}catch(a){this.updateStatusBar("Conversion failed"),new u.Notice(`Conversion failed: ${a}`),console.error("OCR error:",a)}}updateStatusBar(t){this.statusBarItem.setText(`reMarkable: ${t}`)}async loadSettings(){this.settings=Object.assign({},N,await this.loadData())}async saveSettings(){await this.saveData(this.settings)}};var _=P;
|