47 lines
1.7 KiB
JavaScript
47 lines
1.7 KiB
JavaScript
// Debug the heading matching
|
|
const heading = "Algorithm Design";
|
|
const content = "This file mentions algorithm somewhere in the body text";
|
|
const query = "algorithm";
|
|
|
|
function stemToken(token) {
|
|
if (token.endsWith('s')) return token.slice(0, -1);
|
|
if (token.endsWith('ed')) return token.slice(0, -2);
|
|
if (token.endsWith('ing')) return token.slice(0, -3);
|
|
return token;
|
|
}
|
|
|
|
function tokenize(text) {
|
|
const stopWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'in', 'on', 'at', 'to', 'of', 'for', 'with', 'as', 'by', 'it', 'its', 'that', 'this', 'these', 'those']);
|
|
return text
|
|
.toLowerCase()
|
|
.split(/\W+/)
|
|
.filter((token) => token.length > 1 && !stopWords.has(token));
|
|
}
|
|
|
|
const queryTokens = tokenize(query);
|
|
const headingTokens = tokenize(heading);
|
|
const contentTokens = tokenize(content);
|
|
|
|
console.log("Query:", query);
|
|
console.log("Query tokens:", queryTokens);
|
|
console.log("Heading:", heading);
|
|
console.log("Heading tokens:", headingTokens);
|
|
console.log("Content:", content);
|
|
console.log("Content tokens:", contentTokens);
|
|
|
|
const queryStemmed = queryTokens.map(t => stemToken(t));
|
|
const headingStemmed = headingTokens.map(t => stemToken(t));
|
|
const contentStemmed = contentTokens.map(t => stemToken(t));
|
|
|
|
console.log("Query stemmed:", queryStemmed);
|
|
console.log("Heading stemmed:", headingStemmed);
|
|
console.log("Content stemmed:", contentStemmed);
|
|
|
|
// Check heading match
|
|
const headingMatch = headingStemmed.some(h => h.includes(stemToken(queryStemmed[0])));
|
|
console.log("Heading match:", headingMatch);
|
|
|
|
// Check content match
|
|
const contentMatch = contentStemmed.includes(stemToken(queryStemmed[0]));
|
|
console.log("Content match:", contentMatch);
|