|
|
@@ -33,6 +33,9 @@ const FINAL_TOKEN = '' as const; |
|
|
|
|
|
|
|
export const tokenize = (stringValue: string) => ( |
|
|
|
stringValue |
|
|
|
.toLowerCase() |
|
|
|
.trim() |
|
|
|
.replace(/\s+/, ' ') |
|
|
|
.replace(new RegExp(`${TENS_ONES_SEPARATOR}`, 'g'), ' ') |
|
|
|
.split(' ') |
|
|
|
.filter((maybeToken) => maybeToken.length > 0) |
|
|
|