// Debug the scoring logic
const file1Content = "# Algorithm Design\n\nThis discusses design patterns";
const file2Content = "This file mentions algorithm somewhere in the body text";
const query = "algorithm";

function stemToken(token) {
  if (token.endsWith('s')) return token.slice(0, -1);
  if (token.endsWith('ed')) return token.slice(0, -2);
  if (token.endsWith('ing')) return token.slice(0, -3);
  return token;
}

function tokenize(text) {
  const stopWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'in', 'on', 'at', 'to', 'of', 'for', 'with', 'as', 'by', 'it', 'its', 'that', 'this', 'these', 'those']);
  return text
    .toLowerCase()
    .split(/\W+/)
    .filter((token) => token.length > 1 && !stopWords.has(token));
}

function exactMatch(content, token) {
  const stemmed = stemToken(token);
  const contentTokens = tokenize(content);
  return contentTokens.some((ct) => stemToken(ct) === stemmed);
}

function extractHeadings(content) {
  const headingMatches = content.match(/^# (.*?)$/gm);
  if (headingMatches) {
    return headingMatches.map((h) => h.replace(/^# /, ''));
  }
  return [];
}

function extractContentTokens(content) {
  const allText = content
    .replace(/^---.*?---/s, '')
    .replace(/^#.*?$/gm, '')
    .replace(/```.*?```/gs, '')
    .replace(/`.*?`/g, '')
    .replace(/\[.*?\]\(.*?\)/g, '');
  return tokenize(allText);
}

const queryTokens = tokenize(query);
const file1Headings = extractHeadings(file1Content);
const file1Tokens = extractContentTokens(file1Content);
const file2Headings = extractHeadings(file2Content);
const file2Tokens = extractContentTokens(file2Content);

console.log("Query tokens:", queryTokens);
console.log("File 1 headings:", file1Headings);
console.log("File 1 content tokens:", file1Tokens);
console.log("File 2 headings:", file2Headings);
console.log("File 2 content tokens:", file2Tokens);

// Calculate scores
function calculateScore(headings, contentTokens, queryTokens) {
  let totalScore = 0;
  
  for (const queryToken of queryTokens) {
    let tokenScore = 0;
    const stemmed = stemToken(queryToken);
    let matched = false;
    
    // Weight 2: Heading check
    if (headings.some((heading) => heading.toLowerCase().includes(stemmed))) {
      tokenScore += 2;
      matched = true;
    }
    
    // Weight 1: Content token check
    const contentMatch = contentTokens.includes(stemmed);
    if (contentMatch) {
      tokenScore += 1;
      matched = true;
    }
    
    if (matched) {
      totalScore += tokenScore;
    }
  }
  
  return totalScore;
}

const score1 = calculateScore(file1Headings, file1Tokens, queryTokens);
const score2 = calculateScore(file2Headings, file2Tokens, queryTokens);

console.log("File 1 score:", score1);
console.log("File 2 score:", score2);
console.log("File 1 should be first:", score1 > score2);