93 lines
2.8 KiB
JavaScript
93 lines
2.8 KiB
JavaScript
// Debug the scoring logic
|
|
const file1Content = "# Algorithm Design\n\nThis discusses design patterns";
|
|
const file2Content = "This file mentions algorithm somewhere in the body text";
|
|
const query = "algorithm";
|
|
|
|
function stemToken(token) {
|
|
if (token.endsWith('s')) return token.slice(0, -1);
|
|
if (token.endsWith('ed')) return token.slice(0, -2);
|
|
if (token.endsWith('ing')) return token.slice(0, -3);
|
|
return token;
|
|
}
|
|
|
|
function tokenize(text) {
|
|
const stopWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'in', 'on', 'at', 'to', 'of', 'for', 'with', 'as', 'by', 'it', 'its', 'that', 'this', 'these', 'those']);
|
|
return text
|
|
.toLowerCase()
|
|
.split(/\W+/)
|
|
.filter((token) => token.length > 1 && !stopWords.has(token));
|
|
}
|
|
|
|
function exactMatch(content, token) {
|
|
const stemmed = stemToken(token);
|
|
const contentTokens = tokenize(content);
|
|
return contentTokens.some((ct) => stemToken(ct) === stemmed);
|
|
}
|
|
|
|
function extractHeadings(content) {
|
|
const headingMatches = content.match(/^# (.*?)$/gm);
|
|
if (headingMatches) {
|
|
return headingMatches.map((h) => h.replace(/^# /, ''));
|
|
}
|
|
return [];
|
|
}
|
|
|
|
function extractContentTokens(content) {
|
|
const allText = content
|
|
.replace(/^---.*?---/s, '')
|
|
.replace(/^#.*?$/gm, '')
|
|
.replace(/```.*?```/gs, '')
|
|
.replace(/`.*?`/g, '')
|
|
.replace(/\[.*?\]\(.*?\)/g, '');
|
|
return tokenize(allText);
|
|
}
|
|
|
|
const queryTokens = tokenize(query);
|
|
const file1Headings = extractHeadings(file1Content);
|
|
const file1Tokens = extractContentTokens(file1Content);
|
|
const file2Headings = extractHeadings(file2Content);
|
|
const file2Tokens = extractContentTokens(file2Content);
|
|
|
|
console.log("Query tokens:", queryTokens);
|
|
console.log("File 1 headings:", file1Headings);
|
|
console.log("File 1 content tokens:", file1Tokens);
|
|
console.log("File 2 headings:", file2Headings);
|
|
console.log("File 2 content tokens:", file2Tokens);
|
|
|
|
// Calculate scores
|
|
function calculateScore(headings, contentTokens, queryTokens) {
|
|
let totalScore = 0;
|
|
|
|
for (const queryToken of queryTokens) {
|
|
let tokenScore = 0;
|
|
const stemmed = stemToken(queryToken);
|
|
let matched = false;
|
|
|
|
// Weight 2: Heading check
|
|
if (headings.some((heading) => heading.toLowerCase().includes(stemmed))) {
|
|
tokenScore += 2;
|
|
matched = true;
|
|
}
|
|
|
|
// Weight 1: Content token check
|
|
const contentMatch = contentTokens.includes(stemmed);
|
|
if (contentMatch) {
|
|
tokenScore += 1;
|
|
matched = true;
|
|
}
|
|
|
|
if (matched) {
|
|
totalScore += tokenScore;
|
|
}
|
|
}
|
|
|
|
return totalScore;
|
|
}
|
|
|
|
const score1 = calculateScore(file1Headings, file1Tokens, queryTokens);
|
|
const score2 = calculateScore(file2Headings, file2Tokens, queryTokens);
|
|
|
|
console.log("File 1 score:", score1);
|
|
console.log("File 2 score:", score2);
|
|
console.log("File 1 should be first:", score1 > score2);
|