// Debug the heading matching
const heading = "Algorithm Design";
const content = "This file mentions algorithm somewhere in the body text";
const query = "algorithm";

function stemToken(token) {
  if (token.endsWith('s')) return token.slice(0, -1);
  if (token.endsWith('ed')) return token.slice(0, -2);
  if (token.endsWith('ing')) return token.slice(0, -3);
  return token;
}

function tokenize(text) {
  const stopWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'in', 'on', 'at', 'to', 'of', 'for', 'with', 'as', 'by', 'it', 'its', 'that', 'this', 'these', 'those']);
  return text
    .toLowerCase()
    .split(/\W+/)
    .filter((token) => token.length > 1 && !stopWords.has(token));
}

const queryTokens = tokenize(query);
const headingTokens = tokenize(heading);
const contentTokens = tokenize(content);

console.log("Query:", query);
console.log("Query tokens:", queryTokens);
console.log("Heading:", heading);
console.log("Heading tokens:", headingTokens);
console.log("Content:", content);
console.log("Content tokens:", contentTokens);

const queryStemmed = queryTokens.map(t => stemToken(t));
const headingStemmed = headingTokens.map(t => stemToken(t));
const contentStemmed = contentTokens.map(t => stemToken(t));

console.log("Query stemmed:", queryStemmed);
console.log("Heading stemmed:", headingStemmed);
console.log("Content stemmed:", contentStemmed);

// Check heading match
const headingMatch = headingStemmed.some(h => h.includes(stemToken(queryStemmed[0])));
console.log("Heading match:", headingMatch);

// Check content match  
const contentMatch = contentStemmed.includes(stemToken(queryStemmed[0]));
console.log("Content match:", contentMatch);