tesseract/packages/text-utils/src/index.ts

import LanguageDetect from 'languagedetect';
import {languages} from 'prismjs/components';

const RESOLVED_ALIASES = Object.fromEntries(
  Object
    .entries(languages)
    .reduce(
      (resolved, [languageId, languageDefinition]) => {
        if (Array.isArray(languageDefinition.alias)) {
          return [
            ...resolved,
            ...(languageDefinition.alias.map((a: string) => [a, { aliasOf: languageId, title: languageDefinition.title, extension: `.${a}`} ])),
            [languageId, { title: languageDefinition.title, extension: `.${languageId}`}],
          ];
        }

        if (typeof languageDefinition.alias === 'string') {
          return [
            ...resolved,
            [languageDefinition.alias, { aliasOf: languageId, title: languageDefinition.title, extension: `.${languageDefinition.alias}`}],
            [languageId, { title: languageDefinition.title, extension: `.${languageId}`}],
          ];
        }

        return [
          ...resolved,
          [languageId, { title: languageDefinition.title, extension: `.${languageId}`}],
        ];
      },
      [] as [string, { aliasOf?: string, title: string, extension: string }][]
    )
);

type Language = string;
type LanguageProbability = number;
type LanguageMatch = [Language, LanguageProbability];

export interface TextMetadata {
  contents?: string;
  scheme?: string;
  schemeTitle?: string;
  languageMatches?: LanguageMatch[];
  lineCount?: number;
  linesOfCode?: number;
}

const countLinesOfCode = (lines: string[], scheme: string): number => {
  // TODO count loc depending on scheme

  //return lines.filter((line) => !line.trim().startsWith('//')).length;

  return lines.filter((line) => line.trim().length > 0).length;
};

export const getTextMetadata = (contents: string, filename?: string): Promise<TextMetadata> => {
  const lineNormalizedContents = contents.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
  const lines = lineNormalizedContents.split('\n');
  const lineCount = lines.length;
  const metadata = Object.entries(RESOLVED_ALIASES).reduce(
    (theMetadata, [key, value]) => {
      if (typeof theMetadata.scheme === 'undefined' && filename?.endsWith(value.extension)) {
        if (value.aliasOf) {
          return {
            ...theMetadata,
            scheme: value.aliasOf,
            schemeTitle: value.title,
            linesOfCode: countLinesOfCode(lines, value.aliasOf),
          };
        }

        return {
          ...theMetadata,
          scheme: key,
          schemeTitle: value.title,
          linesOfCode: countLinesOfCode(lines, key),
        };
      }

      return theMetadata;
    },
    {
      contents,
      lineCount,
    } as TextMetadata,
  );

  if (typeof metadata.scheme !== 'string') {
    const naturalLanguageDetector = new LanguageDetect();
    metadata.languageMatches = naturalLanguageDetector.detect(contents);
  }

  return Promise.resolve(metadata);
};