/* eslint-disable max-depth */
/*
DESC: YAKE!
CITE: Campos, R., Mangaravite, V., Pasquali, A., Jorge, A., Nunes, C.,
& Jatowt, A. (2020). YAKE! Keyword extraction from single documents
using multiple local features. _Information Sciences_, 509, 257-289.
NOTE: Instead of "term" and "frequency" we use "word" and "count". This
wording makes NLP easier to understand (at the expence of precision).
*/
import {isStopword} from "../_words/stopwords.ts";
import {
mean,
median,
splitChunkIntoTokens,
splitSentenceIntoChunks,
splitTextIntoSentences,
stdev,
tag,
TagTypes,
} from "./util.ts";
export type AnnotatedText = {sentences: AnnotatedSentence[]};
export type AnnotatedSentence = {chunks: AnnotatedChunk[]};
export type AnnotatedChunk = {tokens: AnnotatedToken[]};
export type AnnotatedToken = {
isStopword: boolean;
lowercase: string;
tag: TagTypes;
};
export type Words = Record<string, Term>;
export type Coocurrances = Record<string, Record<string, number>>;
export class Term {
termType: TagTypes; // Tag
lowercase: string;
isStopword: boolean;
offsetsSentences: number[] = []; // Position in Sentence
// Features
// INFO: Explanation in the `computeFeatures` function.
termFrequency = 0;
normalizedTermFrequency = 0;
acronymTermFrequency = 0; // Tf_a
uppercaseTermFrequency = 0; // TFCase
termCase = 0;
termPosition = 0;
termRelatedness = 0; // TRel
termSentences = 0; // TSent
constructor(token: AnnotatedToken) {
this.lowercase = token.lowercase;
this.termType = token.tag;
this.isStopword = token.isStopword;
}
}
/**
* Stage 1: Pre-processing
* @see Campos et al., 2020
* @param text The text to pre-process. It can contain multiple sentences.
* @returns An array of sentences, each containing an array of chunks. Finally
* each chunk includes an array of tokens.
*/
export function preprocessText(text: string): AnnotatedSentence[] {
const annotatedSentences: AnnotatedSentence[] = [];
const sentences = splitTextIntoSentences(text);
for (const sentence of sentences) {
const annotatedSentence: AnnotatedSentence = {chunks: []};
annotatedSentences.push(annotatedSentence);
let tokenCountInSentence = 0;
const chunks = splitSentenceIntoChunks(sentence);
for (const chunk of chunks) {
const annotatedChunk: AnnotatedChunk = {tokens: []};
annotatedSentence.chunks.push(annotatedChunk);
const tokens = splitChunkIntoTokens(chunk);
for (const token of tokens) {
const isFirstTokenInSentence = tokenCountInSentence === 0;
annotatedChunk.tokens.push({
isStopword: isStopword(token),
lowercase: token.toLowerCase(),
tag: tag(token, isFirstTokenInSentence),
});
tokenCountInSentence++;
}
}
}
return annotatedSentences;
}
/**
* Stage 2: Feature Extraction, Score Calculation
* @see Campos et al., 2020
* @param annotatedText The pre-processed text.
*/
export function computeTermStatistics(
annotatedText: AnnotatedText,
windowSize: number,
): {terms: Words; cooccur: Coocurrances} {
const terms: Words = {};
const cooccur: Coocurrances = {};
let sentenceIndex = 0;
for (const sentence of annotatedText.sentences) {
for (const chunk of sentence.chunks) {
for (const [index, token] of chunk.tokens.entries()) {
const termId = token.lowercase;
terms[termId] ||= new Term(token);
terms[termId].termFrequency++;
terms[termId].offsetsSentences.push(sentenceIndex);
if (token.tag === TagTypes.Acronym) {
terms[termId].acronymTermFrequency++;
}
if (token.tag === TagTypes.Uppercase) {
terms[termId].uppercaseTermFrequency++;
}
// This implementation differs from the original YAKE implementation.
// Rather than create an asymmetrical cooccurance matrix we make it
// symmetrical but it might be redundant since the top and bottom half
// of a cooccurance matrix are usually the same.
//
// The YAKE Implementation:
//
// ```
// if (index - distance >= 0) {
// const leftTermId = chunk.tokens[index - distance].lowercase;
// cooccur[leftTermId] ||= {};
// cooccur[leftTermId][termId] ||= 0;
// cooccur[leftTermId][termId]++;
// }
//
// if (index + distance < chunk.tokens.length) {
// const rightTermId = chunk.tokens[index + distance].lowercase;
// cooccur[termId] ||= {};
// cooccur[termId][rightTermId] ||= 0;
// cooccur[termId][rightTermId]++;
// }
// ```
for (let distance = 1; distance <= windowSize; distance++) {
const leftIndex = index - distance;
if (leftIndex < 0) break;
const leftToken = chunk.tokens[leftIndex];
const leftTermId = leftToken.lowercase;
// Make sure both sides exist
cooccur[leftTermId] ||= {};
cooccur[termId] ||= {};
// Increment leftTermId.termId
cooccur[leftTermId][termId] ||= 0;
cooccur[leftTermId][termId]++;
// Also increment termId.leftTermId (to keep it symmetric)
cooccur[termId][leftTermId] ||= 0;
cooccur[termId][leftTermId]++;
}
}
}
sentenceIndex++;
}
return {terms, cooccur};
}
// Features are the most important part of an NLP program. A feature is anything
// important that you want to count. For example, how many words start with Q?
// This program looks at the words around a capitalized word.
//
// First, we count how often each word appears. Next, we count up the important
// words. Words like "the" and "to" aren't important. Finally, we count the
// average number of times all words appear.
//
// @param terms The terms to collect information about.
function computeFeatures(
terms: Term[],
cooccur: Coocurrances,
numberOfSentences: number,
): Term[] {
const termFrequencies: number[] = []; // All TFs
const validTermFrequencies: number[] = []; // Valid TFs
for (const term of terms) {
termFrequencies.push(term.termFrequency);
if (!term.isStopword) {
validTermFrequencies.push(term.termFrequency);
}
}
const averageTermFrequency = mean(validTermFrequencies);
const standardDeviation = stdev(validTermFrequencies);
const normalizedTermFrequency = averageTermFrequency + standardDeviation;
const maximumTermFrequency = Math.max(...termFrequencies);
for (const term of terms) {
term.termCase =
Math.max(term.acronymTermFrequency, term.uppercaseTermFrequency) /
(1 + Math.log(term.termFrequency));
term.termPosition = Math.log(3 + median(term.offsetsSentences));
term.normalizedTermFrequency = term.termFrequency / normalizedTermFrequency;
let uniqueLeftNeighbors = 0;
let sumOfUniqueLeftNeighbors = 0;
for (const leftTerm of Object.keys(cooccur)) {
const uniqueNeighborCount = cooccur[leftTerm][term.lowercase] || 0;
sumOfUniqueLeftNeighbors += uniqueNeighborCount;
if (uniqueNeighborCount > 0) {
uniqueLeftNeighbors++;
}
}
const leftDispersion =
sumOfUniqueLeftNeighbors > 0
? uniqueLeftNeighbors / sumOfUniqueLeftNeighbors
: 0;
let uniqueRightNeighbors = 0;
let sumOfUniqueRightNeighbors = 0;
if (cooccur[term.lowercase]) {
for (const rightTerm of Object.keys(cooccur[term.lowercase])) {
const uniqueNeighborCount = cooccur[term.lowercase][rightTerm] || 0;
sumOfUniqueRightNeighbors += uniqueNeighborCount;
if (uniqueNeighborCount > 0) {
uniqueRightNeighbors++;
}
}
}
const rightDispersion =
sumOfUniqueRightNeighbors > 0
? uniqueRightNeighbors / sumOfUniqueRightNeighbors
: 0;
term.termRelatedness =
1 +
(leftDispersion + rightDispersion) *
(term.termFrequency / maximumTermFrequency);
term.termSentences = term.offsetsSentences.length / numberOfSentences;
}
return terms;
}
// Extract n-grams from a set of sentences.
// (Sentences are arrays of terms)
// export function getNgrams(
// sentences: Term[][],
// minTerms = 2,
// maxTerms = 5,
// ): Ngram[] {
// const ngrams: Ngram[] = [];
// for (const sentence of sentences) {
// for (let start = 0; start < sentence.length; start++) {
// for (let length = minTerms; length <= maxTerms; length++) {
// const slice = sentence.slice(start, start + length);
// if (slice.length < minTerms || slice.length > maxTerms) {
// continue;
// }
// // Create an Ngram only if it passes all checks
// const ngram = new Ngram(slice);
// if (ngram.valueOf().length > 0) {
// ngrams.push(ngram);
// }
// }
// }
// }
// return ngrams;
// }
import {test, expect, describe} from "bun:test";
import {
computeTermStatistics,
preprocessText,
Term,
type AnnotatedText,
type AnnotatedToken,
} from "./code.ts";
import {
splitAndTrim,
splitSentenceIntoChunks,
splitTextIntoSentences,
tag,
TagTypes,
} from "./util.ts";
class TestTerm extends Term {
setTermFrequency(value: number) {
this.termFrequency = value;
return this;
}
setAcronymTermFrequency(value: number) {
this.acronymTermFrequency = value;
return this;
}
setUppercaseTermFrequency(value: number) {
this.uppercaseTermFrequency = value;
return this;
}
}
describe("Splitting", () => {
test("Split", () => {
const result = splitAndTrim(" foo bar , baz quux ", /,/);
expect(result.length).toBe(2);
expect(result[0]).toBe("foo bar");
expect(result[1]).toBe("baz quux");
});
test("Sentence", () => {
const result = splitTextIntoSentences("Lorem ipsum. Foo bar?");
expect(result.length).toBe(2);
expect(result[0]).toBe("Lorem ipsum");
expect(result[1]).toBe("Foo bar");
});
test("Chunk", () => {
const result = splitSentenceIntoChunks("Lorem McRib; Dolor sit (amet)");
expect(result.length).toBe(3);
const [lorem, dolor, amet] = result;
expect(lorem).toBe("Lorem McRib");
expect(dolor).toBe("Dolor sit");
expect(amet).toBe("amet");
});
});
describe("Pre-process", () => {
test("Parse Sentences", () => {
const result = preprocessText("Foo Bar. Foo Faz.");
expect(result.length).toBe(2); // Container
expect(result[0].chunks.length).toBe(1); // Sentence
expect(result[0].chunks[0].tokens.length).toBe(2); // Chunk
});
test("Yake example", () => {
const text =
"One of the most iconic 3 software’s on this matter is the Practical Automatic Keyphrase Extraction (KEA), which was ≥ first known on 1999–06–07";
const result = preprocessText(text);
expect(result.length).toBe(1); // Container
expect(result[0].chunks.length).toBe(3); // Sentence
expect(result[0].chunks[0].tokens.length).toBe(17); // Chunk
expect(result[0].chunks[1].tokens.length).toBe(1); // Chunk
expect(result[0].chunks[2].tokens.length).toBe(7); // Chunk
});
});
describe("Tag Terms", () => {
test("Tag Numbers", () => {
const result = tag("123");
expect(result).toBe(TagTypes.Number);
});
test("Tag Spelled Number", () => {
const result = tag("one", false);
expect(result).toBe(TagTypes.Number);
});
test("Tag Money", () => {
const result = tag("$100");
expect(result).toBe(TagTypes.Money);
});
test("Remove Fraction", () => {
const result = tag("1/2");
console.log(result);
expect(result).toBe(TagTypes.Fraction);
});
test("Remove Percent", () => {
const result = tag("50%");
expect(result).toBe(TagTypes.Percent);
});
test("Remove PhoneNumber", () => {
const result = tag("123-456-7890");
expect(result).toBe(TagTypes.Phone);
});
test("Remove Date", () => {
const result = tag("2021-10-10");
expect(result).toBe(TagTypes.Date);
});
});
// Tag parts of speech
// -----------------------------------------------------------------------------
describe("N-gram Stopwords", () => {
test.todo("Tag Determiner", () => {
// Const result = tag("the cat");
});
test.todo("Tag Preposition", () => {
// Const result = tag("bad at golf");
});
test.todo("Tag Conjunction", () => {
// Const result = tag("apple and orange");
});
test.todo("Tag Pronoun", () => {
// Const result = tag("he she");
});
test.todo("Tag Copula", () => {
// Const result = tag("is are");
});
});
// Remove web markup
// -----------------------------------------------------------------------------
describe("N-Gram Web Markup", () => {
test("Remove Url", () => {
const result = tag("https://example.com");
expect(result).toBe(TagTypes.Web);
});
test("Remove unsecured Url", () => {
const result = tag("http://example.com");
expect(result).toBe(TagTypes.Web);
});
test("Remove Url without protocol", () => {
const result = tag("example.com");
expect(result).toBe(TagTypes.Web);
});
test("Remove Url with www", () => {
const result = tag("www.example.com");
expect(result).toBe(TagTypes.Web);
});
test("Remove Url with weird tld", () => {
const result = tag("http://my.blog");
expect(result).toBe(TagTypes.Web);
});
test("Remove Emoticon", () => {
const result = tag(":-)");
expect(result).toBe(TagTypes.Web);
});
test("Remove Email", () => {
const result = tag("email@example.com");
expect(result).toBe(TagTypes.Web);
});
test("Remove HashTag", () => {
const result = tag("#hashtag");
expect(result).toBe(TagTypes.TagHandle);
});
test("Remove AtMention", () => {
const result = tag("@user");
expect(result).toBe(TagTypes.TagHandle);
});
});
describe("Compute Term Statistics", () => {
test("Single token, chunk, and sentence", () => {
const annotatedToken: AnnotatedToken = {
lowercase: "hello",
tag: TagTypes.Uppercase,
isStopword: false,
};
const annotatedText: AnnotatedText = {
sentences: [
{
chunks: [
{
tokens: [annotatedToken],
},
],
},
],
};
const {terms, cooccur} = computeTermStatistics(annotatedText, 2);
expect(Object.keys(terms)).toHaveLength(1);
expect(Object.keys(cooccur)).toHaveLength(0);
});
test("Two tokens, single chunk, single sentence", () => {
const annotatedText: AnnotatedText = {
sentences: [
{
chunks: [
{
tokens: [
{
lowercase: "hello",
tag: TagTypes.Uppercase,
isStopword: false,
},
{
lowercase: "world",
tag: TagTypes.Uppercase,
isStopword: false,
},
],
},
],
},
],
};
const {terms, cooccur} = computeTermStatistics(annotatedText, 1);
// Check the terms
expect(Object.keys(terms).sort()).toEqual(["hello", "world"].sort());
expect(terms.hello.termFrequency).toBe(1);
expect(terms.hello.uppercaseTermFrequency).toBe(1);
expect(terms.world.termFrequency).toBe(1);
expect(terms.world.uppercaseTermFrequency).toBe(1);
// Check cooccurrences
expect(cooccur.hello).toBeDefined();
expect(cooccur.hello.world).toBe(1);
});
test("Acronym and uppercase tokens, single chunk, window=2", () => {
const annotatedToken1: AnnotatedToken = {
lowercase: "nlp",
tag: TagTypes.Acronym,
isStopword: false,
};
const annotatedToken2: AnnotatedToken = {
lowercase: "is",
tag: TagTypes.Uppercase,
isStopword: true,
};
const annotatedToken3: AnnotatedToken = {
lowercase: "cool",
tag: TagTypes.Uppercase,
isStopword: false,
};
const annotatedText: AnnotatedText = {
sentences: [
{
chunks: [
{
tokens: [annotatedToken1, annotatedToken2, annotatedToken3],
},
],
},
],
};
const {terms, cooccur} = computeTermStatistics(annotatedText, 2);
// Check terms
expect(terms.nlp.termFrequency).toBe(1);
expect(terms.nlp.acronymTermFrequency).toBe(1);
expect(terms.is.termFrequency).toBe(1);
expect(terms.is.uppercaseTermFrequency).toBe(1);
expect(terms.cool.termFrequency).toBe(1);
expect(terms.cool.uppercaseTermFrequency).toBe(1);
// Check cooccurrences
// 'nlp' co-occurs with 'is' and 'cool'
expect(cooccur.nlp.is).toBe(1);
expect(cooccur.nlp.cool).toBe(1);
// 'is' co-occurs with 'nlp' and 'cool'
expect(cooccur.is.nlp).toBe(1);
expect(cooccur.is.cool).toBe(1);
// 'cool' co-occurs with 'is' and 'nlp'
expect(cooccur.cool.is).toBe(1);
expect(cooccur.cool.nlp).toBe(1);
});
test("Multiple sentences, checking offsetsSentences", () => {
const annotatedText: AnnotatedText = {
sentences: [
{
chunks: [
{
tokens: [
{
lowercase: "hello",
tag: TagTypes.Uppercase,
isStopword: false,
},
],
},
],
},
{
chunks: [
{
tokens: [
{
lowercase: "world",
tag: TagTypes.Uppercase,
isStopword: false,
},
],
},
],
},
],
};
const {terms, cooccur} = computeTermStatistics(annotatedText, 1);
// Terms should have 'hello' and 'world'.
expect(terms.hello).toBeDefined();
expect(terms.hello.termFrequency).toBe(1);
expect(terms.hello.uppercaseTermFrequency).toBe(1);
expect(terms.world).toBeDefined();
expect(terms.world.termFrequency).toBe(1);
expect(terms.world.uppercaseTermFrequency).toBe(1);
// No cooccurrences, because each is in a separate sentence, no adjacent tokens.
expect(Object.keys(cooccur)).toHaveLength(0);
});
// ADD THESE TESTS
// describe("computeFeatures tests", () => {
// test("Single term, single sentence", () => {
// // 1) Setup
// const terms: Term[] = [
// {
// lowercase: "hello",
// numberOfWords: 1, // TF=1
// numberOfAcronyms: 0,
// numberOfCapitalWords: 1,
// isStopword: false,
// offsetsSentences: [0],
// },
// ];
// const cooccur: Coocurrances = {
// // For one term, there's no real co-occurrence
// hello: {},
// };
// const numberOfSentences = 1;
// // 2) Execute
// const result = computeFeatures(terms, cooccur, numberOfSentences);
// // 3) Assert
// expect(result).toHaveLength(1);
// const [term] = result;
// // Check some basics
// expect(term.termCase).toBeDefined();
// expect(term.termPosition).toBeDefined();
// expect(term.normalizedTermFrequency).toBeDefined();
// expect(term.wordRelatedness).toBeDefined();
// expect(term.wordSentences).toBeDefined();
// // Because numberOfWords=1
// // termCase = max(0,1)/(1 + ln(1)) => 1/1 => 1
// expect(term.termCase).toBeCloseTo(1, 5);
// // TermPosition = ln(3 + median([0])) => ln(3 + 0) => ln(3) ~ 1.0986
// expect(term.termPosition).toBeCloseTo(Math.log(3), 5);
// // NormalizedTermFrequency => 1/(average(1)+std(1)) => 1/(1+0) => 1
// expect(term.normalizedTermFrequency).toBe(1);
// // WordRelatedness => 1 + (0+0)*(1/1) => 1
// // because no left or right neighbors
// expect(term.wordRelatedness).toBe(1);
// // WordSentences => 1/1 => 1
// expect(term.wordSentences).toBe(1);
// });
// test("Two terms, single sentence, simple cooccurrence", () => {
// // 1) Setup
// const terms: Term[] = [
// {
// lowercase: "hello",
// numberOfWords: 2,
// numberOfAcronyms: 0,
// numberOfCapitalWords: 2,
// isStopword: false,
// offsetsSentences: [0],
// },
// {
// lowercase: "world",
// numberOfWords: 1,
// numberOfAcronyms: 0,
// numberOfCapitalWords: 1,
// isStopword: false,
// offsetsSentences: [0],
// },
// ];
// const cooccur: Coocurrances = {
// // 'hello' co-occurs once with 'world'
// hello: {world: 1},
// // 'world' co-occurs once with 'hello'
// world: {hello: 1},
// };
// const numberOfSentences = 1;
// // 2) Execute
// const result = computeFeatures(terms, cooccur, numberOfSentences);
// // 3) Assert
// // Because we have 2 terms: [2, 1] => average=1.5, stdev ~ 0.707..., sum=2.207...
// // highestWordCount=2
// // Check 'hello' stats (TF=2)
// const hello = result.find((t) => t.lowercase === "hello");
// expect(hello).toBeDefined();
// if (hello) {
// // TermCase => max(0,2)/(1+ln(2)) => 2/(1+0.6931) ~ 2/1.6931 => ~1.18
// expect(hello.termCase).toBeGreaterThan(1);
// // TermPosition => ln(3 + median([0])) => ln(3+0) => ln(3) ~1.0986
// expect(hello.termPosition).toBeCloseTo(Math.log(3), 5);
// // NormalizedTermFrequency => 2 / (1.5 + 0.707...) => 2/2.207... ~ 0.905
// expect(hello.normalizedTermFrequency).toBeCloseTo(0.905, 2);
// // For cooccurrence:
// // leftDispersion => from cooccur[*][hello] => 'world' -> cooccur['world']['hello']=1
// // sumOfUniqueLeftNeighbors=1, uniqueLeftNeighbors=1 => 1/1=1
// // rightDispersion => from cooccur[hello][*] => 'world' -> 1
// // sumOfUniqueRightNeighbors=1, uniqueRightNeighbors=1 =>1
// // => wordRelatedness => 1 + (1+1)*(2/2) => 1 + (2*1) => 3
// expect(hello.wordRelatedness).toBeCloseTo(3, 5);
// // WordSentences => 1 / 1 => 1
// expect(hello.wordSentences).toBeCloseTo(1, 5);
// }
// // Check 'world' stats (TF=1)
// const world = result.find((t) => t.lowercase === "world");
// expect(world).toBeDefined();
// if (world) {
// // TermCase => max(0,1)/(1+ln(1)) => 1/1 => 1
// expect(world.termCase).toBe(1);
// // TermPosition => same ln(3+0) => ln(3)
// expect(world.termPosition).toBeCloseTo(Math.log(3), 5);
// // NormalizedTermFrequency => 1 / 2.207 => ~0.452
// expect(world.normalizedTermFrequency).toBeCloseTo(0.45, 2);
// // Cooccur => leftDispersion => cooccur[*][world], i.e. 'hello' -> 1 => sum=1, distinct=1 =>1
// // rightDispersion => cooccur[world][*], i.e. 'hello' ->1 => sum=1, distinct=1 =>1
// // => wordRelatedness => 1 + (1+1)*(1/2) => 1 +2*(0.5) => 2
// expect(world.wordRelatedness).toBeCloseTo(2, 5);
// // WordSentences => 1/1 =>1
// expect(world.wordSentences).toBe(1);
// }
// });
// test("Handle empty validTermFrequencies gracefully", () => {
// // Suppose all terms are stopwords => validTermFrequencies = []
// const terms: Term[] = [
// {
// lowercase: "the",
// numberOfWords: 5,
// numberOfAcronyms: 0,
// numberOfCapitalWords: 0,
// isStopword: true, // All are stopwords
// offsetsSentences: [0],
// },
// {
// lowercase: "and",
// numberOfWords: 3,
// numberOfAcronyms: 0,
// numberOfCapitalWords: 0,
// isStopword: true,
// offsetsSentences: [0],
// },
// ];
// const cooccur: Coocurrances = {the: {}, and: {}};
// // This will lead to average=0, stdev=0 => normalizedWordCount=0 => possible division by zero
// // Let's see how the function handles it
// const result = computeFeatures(terms, cooccur, 1);
// // We expect no crashes
// expect(result).toHaveLength(2);
// // Because averageTermFrequency=0, standardDeviation=0 => normalizedWordCount=0
// // => term.normalizedTermFrequency = numberOfWords / 0 => Infinity or some fallback
// // If your code doesn't guard for that, you might see Infinity or NaN
// // Check if we do see Infinity
// expect(result[0].normalizedTermFrequency).toBe(Infinity);
// expect(result[1].normalizedTermFrequency).toBe(Infinity);
// });
// });
// });
// Describe("N-Gram Basics", () => {
// test("Gets Expected", () => {
// // Original sentence: Education never ends Watson.
// const expected = [
// "education",
// "education does",
// "education does not",
// "does",
// "does not",
// "does not never",
// "not",
// "not never",
// "not never end",
// "never",
// "never end",
// "never end watson",
// "end",
// "end watson",
// "watson",
// ];
// // Const sentences = [["education", "does", "not", "never", "end", "watson"]];
// const terms = preprocessText("Education never ends Watson.");
// const ngrams = getNgrams(terms, 1, 3);
// expect(ngrams.length).toEqual(expected.length);
// });
// test("Sentence Boundaries", () => {
// // Original sentence: Education never ends, Watson.
// const expected = [
// "education",
// "education does",
// "education does not",
// "does",
// "does not",
// "does not never",
// "not",
// "not never",
// "not never end",
// "never",
// "never end",
// "end",
// "watson",
// ];
// // Const sentences = [
// // ["education", "does", "not", "never", "end"],
// // ["watson"],
// // ];
// const terms = preprocessText("Education never ends, Watson.");
// const ngrams = getNgrams(terms, 1, 3);
// // Const terms = explodeTerms(sentences);
// // const result = getNgrams(terms, 1, 3);
// expect(ngrams.length).toEqual(expected.length);
// });
// test("Extraneous POS", () => {
// // Original sentence: Henry is bad at golf #blabla.
// const expected = [
// "henry",
// "henry bad",
// "henry bad golf",
// "bad",
// "bad golf",
// "golf",
// ];
// // Const sentences = [["henry", "bad", "golf"]];
// // const terms = explodeTerms(sentences);
// // const result = getNgrams(terms, 1, 3);
// const terms = preprocessText("Henry is bad at golf #blabla.");
// const ngrams = getNgrams(terms, 1, 3);
// expect(ngrams.length).toEqual(expected.length);
// });
// });
import {isStopword} from "../_words/stopwords.ts";
export function splitAndTrim(text: string, pattern: RegExp) {
const result: string[] = [];
for (const part of text.split(pattern)) {
const value = part.trim();
if (value !== "") result.push(value);
}
return result;
}
export function splitTextIntoSentences(text: string): string[] {
return splitAndTrim(text, /[.?!]\s*/);
}
export function splitSentenceIntoChunks(text: string): string[] {
return splitAndTrim(text, /[:;,$[\]{}()<>]\s*/);
}
export function splitChunkIntoTokens(text: string): string[] {
return splitAndTrim(text, /['’ ]/);
}
// US phone number with optional "+1 " prefix
// Matches either "123-456-7890" or "(123) 456-7890"
const usPhoneNumberPattern =
/^(?:\+1\s)?(?:\d{3}-\d{3}-\d{4}|\(\d{3}\)\s\d{3}-\d{4})$/;
function isUsPhoneNumber(string_: string): boolean {
return usPhoneNumberPattern.test(string_);
}
const percentPattern = /^\d+(\.\d+)?%$/;
function isPercent(string_: string): boolean {
return percentPattern.test(string_);
}
// Matches: $1,234,567.89
const moneyPattern = /^\$\d{1,3}(,\d{3})*(\.\d{2})?$/;
function isMoney(string_: string): boolean {
return moneyPattern.test(string_);
}
// Matches: YYYY-MM-DD with hyphen variants.
const isoDatePattern = new RegExp(
"^" + // 1) Start-of-string anchor
"(\\d{4})" + // 2) Capture a 4-digit year
"[\\-\\u2013\\u2014]" + // 3) Match an ASCII hyphen, en dash, or em dash
"(0[1-9]|1[0-2])" + // 4) Capture the month, allowing 01-12
"[\\-\\u2013\\u2014]" + // 5) Again, match ASCII hyphen, en dash, or em dash
"(0[1-9]|[12]\\d|3[01])" + // 6) Capture the day, allowing 01-31
"$", // 7) End-of-string anchor
);
export function isIsoDate(dateString: string): boolean {
return isoDatePattern.test(dateString);
}
// Matches: Flexible DD/MM/YYYY allowing slash or fullwidth slash
// e.g., "05/01/2025" or "05／01／2025"
const ddMmYyyyPattern = new RegExp(
"^" +
"(0[1-9]|[12]\\d|3[01])" + // Day 01-31
"[/\\uFF0F]" + // Accept / (ASCII) or ／ (fullwidth slash)
"(0[1-9]|1[0-2])" + // Month 01-12
"[/\\uFF0F]" + // Again, slash or fullwidth slash
"(\\d{4})" + // 4-digit year
"$",
);
function isEuDate(dateString: string): boolean {
return ddMmYyyyPattern.test(dateString);
}
// Matches: Flexible DD-MM-YYYY allowing ASCII hyphen, en dash, or em dash
// e.g., "05-01-2025" or "05–01–2025" or "05—01—2025"
const ddMmYyyyDashPattern = new RegExp(
"^" +
"(0[1-9]|[12]\\d|3[01])" + // Day 01-31
"[\\-\\u2013\\u2014]" + // - (ASCII), – (en), — (em)
"(0[1-9]|1[0-2])" + // Month 01-12
"[\\-\\u2013\\u2014]" + // - (ASCII), – (en), — (em)
"(\\d{4})" + // 4-digit year
"$",
);
function isEuDateDash(dateString: string): boolean {
return ddMmYyyyDashPattern.test(dateString);
}
// Matches: -d/-d
const fractionPattern = /^[+-]?\d+\/\d+$/;
function isFraction(string_: string): boolean {
return fractionPattern.test(string_);
}
const numberPattern = /^-?\d{1,3}(,\d{3})*(\.\d+)?|\d+(\.\d+)?$/;
const spelledNumberPattern =
/^(one|two|three|four|five|six|seven|eight|nine|ten)$/i;
export function isNumber(token: string): boolean {
return numberPattern.test(token) || spelledNumberPattern.test(token);
}
// Matches: www.google.com, email@account.com, <head>
// And other items with more than two punctuation symbols.
const webAddressPattern = /^(?:.*[^\w\s]){2,}.*$/;
const tldPattern =
/\.(com|org|net|edu|gov|mil|int|info|biz|name|us|uk|ca|de|au|jp|cn|in|fr|it|xyz|online|site|tech|store|blog|app|dev|io|co|tv|me|ai|pro|health)\b/;
export function isWebAddress(token: string): boolean {
return webAddressPattern.test(token) || tldPattern.test(token);
}
// For "not alpha-numeric", we just need to detect any non-letter-or-digit character.
// The presence of symbols like ‰ or ¥ will trigger this as true.
const nonAlphaNumberPattern = /[^\p{L}\p{N}]/u;
export function isNotAlphaNumeric(token: string): boolean {
// Returns true if there's at least one character that is NOT a letter or digit
return nonAlphaNumberPattern.test(token);
}
// For alpha-numeric, we want at least one letter and at least one digit.
// Everything in the string must be letters or digits (no punctuation, no symbols).
const onlyAlphaNumericPattern = /^(?=.*\p{L})(?=.*\p{N})[\p{L}\p{N}]+$/u;
export function isAlphaNumeric(token: string): boolean {
return onlyAlphaNumericPattern.test(token);
}
const twoPlusUppercasePattern = /^[A-Z.]{2,}$/; // U.S.A. or MIT
export function isAcronym(token: string): boolean {
return twoPlusUppercasePattern.test(token);
}
// Matches an initial uppercase character.
const firstIsUppercasePattern = /^[A-Z]/;
export function isUppercase(
token: string,
isFirstTokenInSentence: boolean,
): boolean {
return isFirstTokenInSentence && firstIsUppercasePattern.test(token);
}
const tagOrHandlePattern = /^[@#]\w[a-z\d]{2,}/;
export function isTagOrHandle(token: string): boolean {
return tagOrHandlePattern.test(token);
}
export enum TagTypes {
Acronym = "Acronym",
AlphaNumeric = "AlphaNumeric",
Date = "Date",
Fraction = "Fraction",
Money = "Money",
NotAlphaNumeric = "NotAlphaNumeric",
Number = "Number",
Parsable = "Parsable",
Percent = "Percent",
Phone = "Phone",
TagHandle = "TagHandle",
Stopword = "Stopword",
Unparsable = "Unparsable",
Uppercase = "Uppercase",
Web = "Web",
}
// Everything except Parsable gets ignored at the moment.
export function tag(token: string, isFirstTokenInSentence = false): TagTypes {
if (isUsPhoneNumber(token)) return TagTypes.Phone;
if (isPercent(token)) return TagTypes.Percent;
if (isFraction(token)) return TagTypes.Fraction;
if (isIsoDate(token)) return TagTypes.Date;
if (isEuDate(token)) return TagTypes.Date;
if (isEuDateDash(token)) return TagTypes.Date;
if (isMoney(token)) return TagTypes.Money;
if (isNumber(token)) return TagTypes.Number;
if (isWebAddress(token)) return TagTypes.Web;
if (isTagOrHandle(token)) return TagTypes.TagHandle;
if (isNotAlphaNumeric(token)) return TagTypes.NotAlphaNumeric;
if (isAlphaNumeric(token)) return TagTypes.AlphaNumeric;
if (isAcronym(token)) return TagTypes.Acronym;
if (isStopword(token)) return TagTypes.Stopword;
if (isUppercase(token, isFirstTokenInSentence)) return TagTypes.Uppercase;
return TagTypes.Parsable;
}
/**
* Helper to compute the median of a numeric array.
* If the array is empty, returns 0 (you can choose another policy).
*/
export function median(values: number[]): number {
if (values.length === 0) return 0;
const sorted = [...values].sort((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
return sorted.length % 2 === 0
? (sorted[mid - 1] + sorted[mid]) / 2
: sorted[mid];
}
/**
* Helper to compute the mean (average) of a numeric array.
* If empty, returns 0.
*/
export function mean(values: number[]): number {
if (values.length === 0) return 0;
const sum = values.reduce((accumulator, value) => accumulator + value, 0);
return sum / values.length;
}
/**
* Helper to compute the sample standard deviation.
* If array length < 2, returns 0.
*/
export function stdev(values: number[]): number {
if (values.length < 2) return 0;
const avg = mean(values);
const variance =
values.reduce((accumulator, value) => accumulator + (value - avg) ** 2, 0) /
(values.length - 1);
return Math.sqrt(variance);
}