obsidian_ollama/src/indexing-pipeline/extraction.ts

// src/indexing-pipeline/extraction.ts

// VaultFile interface is defined locally since it's not exported from types
interface VaultFile {
  basename: string;
  path: string;
}

export interface Frontmatter {
  title?: string;
  tags?: string;
  [key: string]: unknown;
}

export interface ExtractedContent {
  basename: string;
  path: string;
  content: string;
  frontmatter: Frontmatter;
  headings: string[];
  embeddedCodeBlocks: string[];
  firstParagraph?: string;
}

/**
 * Extracts raw content from a vault file including:
 * - Markdown content
 * - YAML frontmatter
 * - Headings
 * - Embedded code blocks
 * - First paragraph
 */
export class ContentExtractor {
  extractFromFile(file: VaultFile, content: string): ExtractedContent {
    const frontmatter: Frontmatter = {};
    const headings: string[] = [];
    const embeddedCodeBlocks: string[] = [];
    let firstParagraph: string | undefined;

    // Extract frontmatter
    const frontmatterMatch = content.match(/^---(.*?)---/s);
    if (frontmatterMatch) {
      try {
        const frontmatterContent = frontmatterMatch[1];
        const lines = frontmatterContent.trim().split('\n');
        for (const line of lines) {
          const [key, ...valueParts] = line.split(':');
          if (!key) continue;
          const value = valueParts.join(':').trim();
          if (key.trim() === 'title') {
            if (value) {
              frontmatter.title = value;
            }
          } else if (key.trim() === 'tags') {
            if (value) {
              frontmatter.tags = value;
            }
          } else {
            // Store other frontmatter fields as-is
            frontmatter[key.trim()] = value;
          }
        }
      } catch {
        // If frontmatter parsing fails, continue with empty frontmatter
      }
    }

    // Extract headings
    const headingMatches = content.match(/^#{1,6} (.*?)$/gm);
    if (headingMatches) {
      headings.push(...headingMatches.map((h: string) => h.replace(/^#{1,6} /, '')));
    }

    // Extract embedded code blocks
    const codeBlockMatches = content.match(/```([\s\S]*?)```/g);
    if (codeBlockMatches) {
      embeddedCodeBlocks.push(...codeBlockMatches);
    }

    // Extract first paragraph
    const paragraphMatch = content.match(/^([^#]*?)(?=#|\n\n|$)/s);
    if (paragraphMatch) {
      firstParagraph = paragraphMatch[1].trim();
    }

    return {
      basename: file.basename,
      path: file.path,
      content,
      frontmatter,
      headings,
      embeddedCodeBlocks,
      firstParagraph,
    };
  }

  /**
   * Extracts just the raw text content without headers, frontmatter, etc.
   */
  extractRawText(content: string): string {
    return content
      .replace(/^---.*?---/s, '')
      .replace(/^#.*?$/gm, '')
      .replace(/```.*?```/gs, '')
      .replace(/`.*?`/g, '')
      .replace(/\[.*?\]\(.*?\)/g, '')
      .trim();
  }
}