murasaki-core/src/streams.ts

import { Transform, TransformCallback } from 'stream';
import { xml2json } from 'xml-js';

export interface XmlToJsonLinesTransformStreamOptions {
  entryTagName: string;
}

class XmlToJsonLinesTransformStream extends Transform {
  private charactersToParse?: string;

  private readonly openTag: string;

  private readonly closeTag: string;

  constructor(options: XmlToJsonLinesTransformStreamOptions) {
    super();
    this.openTag = `<${options.entryTagName}>`;
    this.closeTag = `</${options.entryTagName}>`;
  }

  // eslint-disable-next-line no-underscore-dangle
  _transform(chunk: Buffer, _encoding: BufferEncoding, callback: TransformCallback) {
    try {
      const chunkStr = chunk.toString('utf-8');
      if (typeof this.charactersToParse !== 'string') {
        const firstEntryIndex = chunkStr.indexOf(this.openTag);
        this.charactersToParse = chunkStr.slice(firstEntryIndex);
      } else {
        this.charactersToParse += chunkStr;
      }

      let theCharacters = `${this.charactersToParse}`;
      let nextOpenTagIndex = theCharacters.indexOf(this.openTag);
      let nextCloseTagIndex = theCharacters.indexOf(this.closeTag);
      let sliceEnd = nextCloseTagIndex + this.closeTag.length;
      do {
        if (nextOpenTagIndex > -1 && nextCloseTagIndex > -1) {
          const xml = theCharacters
            .slice(nextOpenTagIndex, sliceEnd)
            .replace(/&(.+?);/g, '$1'); // FIXME better handling of XML entities??? This makes the pipe hang for some reason
          const json = xml2json(xml, { compact: true });
          this.push(`${json}\n`);
          theCharacters = theCharacters.slice(sliceEnd);
        }
        nextOpenTagIndex = theCharacters.indexOf(this.openTag);
        nextCloseTagIndex = theCharacters.indexOf(this.closeTag);
        sliceEnd = nextCloseTagIndex + this.closeTag.length;
      } while (nextOpenTagIndex > -1 && nextCloseTagIndex > -1);
      this.charactersToParse = theCharacters;
      callback(null, '');
    } catch (err) {
      callback(err as Error);
    }
  }
}

export const createXmlToJsonLines = (options = {
  entryTagName: 'entry',
} as XmlToJsonLinesTransformStreamOptions) => new XmlToJsonLinesTransformStream(options);