murasaki-core/src/sources/jmdict/downloader.ts

import fetchPonyfill from 'fetch-ponyfill';
import { PassThrough } from 'stream';
import { createGunzip } from 'zlib';
import { SOURCE_ID } from './common';

export interface CreateDownloaderParams {
  type: typeof SOURCE_ID;
  url?: string;
}

const DEFAULT_SOURCE_URL = 'http://ftp.edrdg.org/pub/Nihongo/JMdict.gz' as const;

export const createDownloader = async (params: Omit<CreateDownloaderParams, 'type'>) => {
  const { url = DEFAULT_SOURCE_URL } = params;
  const { fetch } = fetchPonyfill();

  const response = await fetch(url);
  if (!response.ok) {
    throw new Error(`Failed to download: ${url}`);
  }

  const rawStream = response.body as unknown as PassThrough;
  return rawStream
    .pipe(createGunzip());
};