Implement init endpoints

Add endpoints for downloading datasets.

.gitignore

@@ -105,3 +105,4 @@ dist
 .tern-port
 
 .npmrc
+types/

package.json

@@ -1,5 +1,5 @@
 {
-  "name": "murasaki-web-api",
+  "name": "@modal-sh/murasaki-web-api",
   "version": "0.0.0",
   "files": [
     "dist",
@@ -22,7 +22,8 @@
     "vitest": "^0.28.1"
   },
   "dependencies": {
-    "fastify": "^4.12.0"
+    "fastify": "^4.12.0",
+    "@modal-sh/murasaki-core": "link:../core"
   },
   "scripts": {
     "prepublishOnly": "pridepack clean && pridepack build",

src/config.ts

@@ -0,0 +1,4 @@
+export namespace meta {
+  export const port = Number(process.env.PORT ?? 8080);
+  export const host = process.env.HOST ?? '0.0.0.0';
+}

src/index.ts

@@ -1,12 +1,17 @@
-// SERVER
-import SERVER from './server';
+import { createServer } from './server';
+import { addDefaultRoutes, addInitRoutes } from './routes';
+import * as config from './config';
 
-import './routes';
+const server = createServer({
+  logger: process.env.NODE_ENV !== 'test',
+});
+
+addDefaultRoutes(server);
+addInitRoutes(server);
 
-SERVER.listen({ port: 8080, host: '0.0.0.0' }, (err, address) => {
+server.listen({ port: config.meta.port, host: config.meta.host }, (err) => {
   if (err) {
-    SERVER.log.error(err.message);
+    server.log.error(err.message);
     process.exit(1);
   }
-  SERVER.log.info(`server listening on ${address}`);
 });

src/modules/init/InitController.ts

@@ -0,0 +1,18 @@
+import {RouteHandlerMethod} from 'fastify';
+import {CreateDownloaderParams} from '@modal-sh/murasaki-core';
+import {InitService, InitServiceImpl} from './InitService';
+
+export interface InitController {
+  downloadDataset: RouteHandlerMethod;
+}
+
+export class InitControllerImpl implements InitController {
+  constructor(private readonly initService: InitService = new InitServiceImpl()) {
+    // noop
+  }
+
+  readonly downloadDataset: RouteHandlerMethod = async (request, reply) => {
+    const result = await this.initService.downloadDataset(request.body as CreateDownloaderParams);
+    reply.send(result);
+  };
+}

src/modules/init/InitService.ts

@@ -0,0 +1,110 @@
+import {
+  createDownloader,
+  CreateDownloaderParams,
+  Kanjidic,
+  JMdict,
+  JMnedict,
+  KRadFile,
+  RadKFile,
+  createXmlToJsonLines,
+} from '@modal-sh/murasaki-core';
+import { createWriteStream, readFileSync } from 'fs';
+
+export interface InitService {
+  downloadDataset(params: CreateDownloaderParams): Promise<ManifestData>;
+}
+
+interface ManifestDatasetEntry {
+  createdAt: number;
+  lastUpdatedAt: number;
+}
+
+type ManifestData = Record<string, ManifestDatasetEntry>;
+
+export class InitServiceImpl implements InitService {
+  private manifestWriteStream?: NodeJS.WritableStream;
+
+  private readonly manifestFilename = '.murasaki.json' as const;
+
+  private readonly manifestFileEncoding = 'utf-8' as const;
+
+  private readonly data: ManifestData;
+
+  constructor() {
+    try {
+      const dataBuffer = readFileSync(this.manifestFilename);
+      const dataJsonString = dataBuffer.toString(this.manifestFileEncoding);
+      this.data = JSON.parse(dataJsonString) as Record<string, ManifestDatasetEntry>;
+    } catch {
+      this.data = {};
+    }
+  }
+
+  private async commitDatasetMetadata(): Promise<ManifestData> {
+    return new Promise<ManifestData>((resolve, reject) => {
+      this.manifestWriteStream = createWriteStream(this.manifestFilename, {
+        flags: 'w',
+      });
+      this.manifestWriteStream.on('error', reject);
+      this.manifestWriteStream.write(JSON.stringify(this.data));
+      this.manifestWriteStream.end(() => {
+        resolve(this.data);
+      });
+    });
+  }
+
+  async downloadDataset(params: CreateDownloaderParams): Promise<ManifestData> {
+    const downloader = await createDownloader(params);
+
+    return new Promise<ManifestData>((resolve, reject) => {
+      const out = createWriteStream(`${params.type}.jsonl`);
+
+      out.on('finish', () => {
+        const now = Date.now();
+        this.data[params.type] = {
+          ...this.data[params.type],
+          createdAt: this.data[params.type].createdAt ?? now,
+          lastUpdatedAt: now,
+        };
+
+        this.commitDatasetMetadata()
+          .then(resolve)
+          .catch(reject);
+      });
+
+      switch (params.type) {
+        case Kanjidic.SOURCE_ID: {
+          const jsonlParser = createXmlToJsonLines({
+            entryTagName: 'character',
+          });
+
+          downloader
+            .pipe(jsonlParser)
+            .pipe(out);
+        } return;
+        case JMnedict.SOURCE_ID:
+        case JMdict.SOURCE_ID: {
+          const jsonlParser = createXmlToJsonLines({
+            entryTagName: 'entry',
+          });
+
+          downloader
+            .pipe(jsonlParser)
+            .pipe(out);
+        } return;
+        case KRadFile.SOURCE_ID:
+        case RadKFile.SOURCE_ID:
+          downloader.pipe(out);
+          return;
+        default:
+          break;
+      }
+
+      this.commitDatasetMetadata()
+        .then(() => {
+          reject(new Error(`Unknown dataset: ${params.type as unknown as string}`));
+        })
+        .catch(reject);
+    });
+  }
+}

src/routes.ts

@@ -1,5 +1,34 @@
-import SERVER from './server';
+import { FastifyInstance } from 'fastify';
+import { InitController, InitControllerImpl } from './modules/init/InitController';
 
-SERVER.get('/', async (_, reply) => {
-  reply.send({ hello: 'world' })
-});
+export const addDefaultRoutes = (server: FastifyInstance) => {
+  server
+    .route({
+      method: 'GET',
+      url: '/api/health/live',
+      handler: async (_, reply) => {
+        reply.send({
+          status: 'ok',
+        });
+      },
+    })
+    .route({
+      method: 'GET',
+      url: '/api/health/ready',
+      handler: async (_, reply) => {
+        reply.send({
+          status: 'ready',
+        });
+      },
+    });
+};
+
+export const addInitRoutes = (server: FastifyInstance) => {
+  const initController: InitController = new InitControllerImpl();
+  server
+    .route({
+      method: 'POST',
+      url: '/api/download',
+      handler: initController.downloadDataset,
+    });
+};

src/server.ts

@@ -1,7 +1,9 @@
 import fastify from 'fastify';
 
-const SERVER = fastify({
-  logger: true,
-});
+export interface CreateServerOptions {
+  logger?: boolean;
+}
 
-export default SERVER;
+export const createServer = (options: CreateServerOptions) => fastify({
+  logger: options.logger,
+});

yarn.lock

@@ -462,6 +462,10 @@
   resolved "https://registry.yarnpkg.com/@mdn/browser-compat-data/-/browser-compat-data-5.2.54.tgz#253803ffcf8f706155d36c4f3413b9f537f06419"
   integrity sha512-8/W1qTWw/lCf6E01n/Be65LGza/WrDON7ii9F/fKc4EdZad+K1xJWvfYhDSoPpkMCAw0eLIyAB0Z5emQFBVclw==
 
+"@modal-sh/murasaki-core@link:../core":
+  version "0.0.0"
+  uid ""
+
 "@next/eslint-plugin-next@^13.2.4":
   version "13.3.2"
   resolved "https://registry.yarnpkg.com/@next/eslint-plugin-next/-/eslint-plugin-next-13.3.2.tgz#1126508131f85d550da0ad8eb3888ddc5ae4c9c1"
@@ -1828,6 +1832,13 @@ fastq@^1.6.0, fastq@^1.6.1:
   dependencies:
     reusify "^1.0.4"
 
+fetch-ponyfill@^7.1.0:
+  version "7.1.0"
+  resolved "https://js.pack.modal.sh/fetch-ponyfill/-/fetch-ponyfill-7.1.0.tgz#4266ed48b4e64663a50ab7f7fcb8e76f990526d0"
+  integrity sha512-FhbbL55dj/qdVO3YNK7ZEkshvj3eQ7EuIGV2I6ic/2YiocvyWv+7jg2s4AyS0wdRU75s3tA8ZxI/xPigb0v5Aw==
+  dependencies:
+    node-fetch "~2.6.1"
+
 file-entry-cache@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027"
@@ -2657,6 +2668,13 @@ natural-compare@^1.4.0:
   resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
   integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==
 
+node-fetch@~2.6.1:
+  version "2.6.9"
+  resolved "https://js.pack.modal.sh/node-fetch/-/node-fetch-2.6.9.tgz#7c7f744b5cc6eb5fd404e0c7a9fec630a55657e6"
+  integrity sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-releases@^2.0.8:
   version "2.0.10"
   resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.10.tgz#c311ebae3b6a148c89b1813fd7c4d3c024ef537f"
@@ -3202,6 +3220,11 @@ safe-stable-stringify@^2.3.1:
   resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886"
   integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==
 
+sax@^1.2.4:
+  version "1.2.4"
+  resolved "https://js.pack.modal.sh/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
+  integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
+
 secure-json-parse@^2.5.0:
   version "2.7.0"
   resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862"
@@ -3530,6 +3553,11 @@ to-regex-range@^5.0.1:
   dependencies:
     is-number "^7.0.0"
 
+tr46@~0.0.3:
+  version "0.0.3"
+  resolved "https://js.pack.modal.sh/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
+  integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
+
 tsconfig-paths@^3.14.1:
   version "3.14.2"
   resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz#6e32f1f79412decd261f92d633a9dc1cfa99f088"
@@ -3712,6 +3740,19 @@ wcwidth@^1.0.1:
   dependencies:
     defaults "^1.0.3"
 
+webidl-conversions@^3.0.0:
+  version "3.0.1"
+  resolved "https://js.pack.modal.sh/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
+  integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==
+
+whatwg-url@^5.0.0:
+  version "5.0.0"
+  resolved "https://js.pack.modal.sh/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
+  integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==
+  dependencies:
+    tr46 "~0.0.3"
+    webidl-conversions "^3.0.0"
+
 which-boxed-primitive@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"
@@ -3813,6 +3854,13 @@ xdg-basedir@^4.0.0:
   resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13"
   integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==
 
+xml-js@^1.6.11:
+  version "1.6.11"
+  resolved "https://js.pack.modal.sh/xml-js/-/xml-js-1.6.11.tgz#927d2f6947f7f1c19a316dd8eea3614e8b18f8e9"
+  integrity sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==
+  dependencies:
+    sax "^1.2.4"
+
 xml-name-validator@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835"