Browse Source

Improve string sanitation for parsing

Ensure strings are properly sanitized before processing.
master
TheoryOfNekomata 1 year ago
parent
commit
bb15edd726
1 changed files with 3 additions and 0 deletions
  1. +3
    -0
      packages/core/src/systems/en-US/parse.ts

+ 3
- 0
packages/core/src/systems/en-US/parse.ts View File

@@ -33,6 +33,9 @@ const FINAL_TOKEN = '' as const;


export const tokenize = (stringValue: string) => ( export const tokenize = (stringValue: string) => (
stringValue stringValue
.toLowerCase()
.trim()
.replace(/\s+/, ' ')
.replace(new RegExp(`${TENS_ONES_SEPARATOR}`, 'g'), ' ') .replace(new RegExp(`${TENS_ONES_SEPARATOR}`, 'g'), ' ')
.split(' ') .split(' ')
.filter((maybeToken) => maybeToken.length > 0) .filter((maybeToken) => maybeToken.length > 0)


Loading…
Cancel
Save