Refactor error handling, client, and tests for Ollama integration

debug_scoring.js

@@ -0,0 +1,92 @@
+// Debug the scoring logic
+const file1Content = "# Algorithm Design\n\nThis discusses design patterns";
+const file2Content = "This file mentions algorithm somewhere in the body text";
+const query = "algorithm";
+
+function stemToken(token) {
+  if (token.endsWith('s')) return token.slice(0, -1);
+  if (token.endsWith('ed')) return token.slice(0, -2);
+  if (token.endsWith('ing')) return token.slice(0, -3);
+  return token;
+}
+
+function tokenize(text) {
+  const stopWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'in', 'on', 'at', 'to', 'of', 'for', 'with', 'as', 'by', 'it', 'its', 'that', 'this', 'these', 'those']);
+  return text
+    .toLowerCase()
+    .split(/\W+/)
+    .filter((token) => token.length > 1 && !stopWords.has(token));
+}
+
+function exactMatch(content, token) {
+  const stemmed = stemToken(token);
+  const contentTokens = tokenize(content);
+  return contentTokens.some((ct) => stemToken(ct) === stemmed);
+}
+
+function extractHeadings(content) {
+  const headingMatches = content.match(/^# (.*?)$/gm);
+  if (headingMatches) {
+    return headingMatches.map((h) => h.replace(/^# /, ''));
+  }
+  return [];
+}
+
+function extractContentTokens(content) {
+  const allText = content
+    .replace(/^---.*?---/s, '')
+    .replace(/^#.*?$/gm, '')
+    .replace(/```.*?```/gs, '')
+    .replace(/`.*?`/g, '')
+    .replace(/\[.*?\]\(.*?\)/g, '');
+  return tokenize(allText);
+}
+
+const queryTokens = tokenize(query);
+const file1Headings = extractHeadings(file1Content);
+const file1Tokens = extractContentTokens(file1Content);
+const file2Headings = extractHeadings(file2Content);
+const file2Tokens = extractContentTokens(file2Content);
+
+console.log("Query tokens:", queryTokens);
+console.log("File 1 headings:", file1Headings);
+console.log("File 1 content tokens:", file1Tokens);
+console.log("File 2 headings:", file2Headings);
+console.log("File 2 content tokens:", file2Tokens);
+
+// Calculate scores
+function calculateScore(headings, contentTokens, queryTokens) {
+  let totalScore = 0;
+  
+  for (const queryToken of queryTokens) {
+    let tokenScore = 0;
+    const stemmed = stemToken(queryToken);
+    let matched = false;
+    
+    // Weight 2: Heading check
+    if (headings.some((heading) => heading.toLowerCase().includes(stemmed))) {
+      tokenScore += 2;
+      matched = true;
+    }
+    
+    // Weight 1: Content token check
+    const contentMatch = contentTokens.includes(stemmed);
+    if (contentMatch) {
+      tokenScore += 1;
+      matched = true;
+    }
+    
+    if (matched) {
+      totalScore += tokenScore;
+    }
+  }
+  
+  return totalScore;
+}
+
+const score1 = calculateScore(file1Headings, file1Tokens, queryTokens);
+const score2 = calculateScore(file2Headings, file2Tokens, queryTokens);
+
+console.log("File 1 score:", score1);
+console.log("File 2 score:", score2);
+console.log("File 1 should be first:", score1 > score2);