diff --git a/.gitignore b/.gitignore index 53992de..bea7415 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,4 @@ dist .tern-port .npmrc +types/ diff --git a/package.json b/package.json index 0c45ba1..74877df 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "murasaki-web-api", + "name": "@modal-sh/murasaki-web-api", "version": "0.0.0", "files": [ "dist", @@ -22,7 +22,8 @@ "vitest": "^0.28.1" }, "dependencies": { - "fastify": "^4.12.0" + "fastify": "^4.12.0", + "@modal-sh/murasaki-core": "link:../core" }, "scripts": { "prepublishOnly": "pridepack clean && pridepack build", diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..06cc56f --- /dev/null +++ b/src/config.ts @@ -0,0 +1,4 @@ +export namespace meta { + export const port = Number(process.env.PORT ?? 8080); + export const host = process.env.HOST ?? '0.0.0.0'; +} diff --git a/src/index.ts b/src/index.ts index 6559c5a..2195864 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,12 +1,17 @@ -// SERVER -import SERVER from './server'; +import { createServer } from './server'; +import { addDefaultRoutes, addInitRoutes } from './routes'; +import * as config from './config'; -import './routes'; +const server = createServer({ + logger: process.env.NODE_ENV !== 'test', +}); + +addDefaultRoutes(server); +addInitRoutes(server); -SERVER.listen({ port: 8080, host: '0.0.0.0' }, (err, address) => { +server.listen({ port: config.meta.port, host: config.meta.host }, (err) => { if (err) { - SERVER.log.error(err.message); + server.log.error(err.message); process.exit(1); } - SERVER.log.info(`server listening on ${address}`); }); diff --git a/src/modules/init/InitController.ts b/src/modules/init/InitController.ts new file mode 100644 index 0000000..c9a5337 --- /dev/null +++ b/src/modules/init/InitController.ts @@ -0,0 +1,18 @@ +import {RouteHandlerMethod} from 'fastify'; +import {CreateDownloaderParams} from '@modal-sh/murasaki-core'; +import {InitService, InitServiceImpl} from './InitService'; + +export interface InitController { + downloadDataset: RouteHandlerMethod; +} + +export class InitControllerImpl implements InitController { + constructor(private readonly initService: InitService = new InitServiceImpl()) { + // noop + } + + readonly downloadDataset: RouteHandlerMethod = async (request, reply) => { + const result = await this.initService.downloadDataset(request.body as CreateDownloaderParams); + reply.send(result); + }; +} diff --git a/src/modules/init/InitService.ts b/src/modules/init/InitService.ts new file mode 100644 index 0000000..581f9e3 --- /dev/null +++ b/src/modules/init/InitService.ts @@ -0,0 +1,110 @@ +import { + createDownloader, + CreateDownloaderParams, + Kanjidic, + JMdict, + JMnedict, + KRadFile, + RadKFile, + createXmlToJsonLines, +} from '@modal-sh/murasaki-core'; +import { createWriteStream, readFileSync } from 'fs'; + +export interface InitService { + downloadDataset(params: CreateDownloaderParams): Promise; +} + +interface ManifestDatasetEntry { + createdAt: number; + lastUpdatedAt: number; +} + +type ManifestData = Record; + +export class InitServiceImpl implements InitService { + private manifestWriteStream?: NodeJS.WritableStream; + + private readonly manifestFilename = '.murasaki.json' as const; + + private readonly manifestFileEncoding = 'utf-8' as const; + + private readonly data: ManifestData; + + constructor() { + try { + const dataBuffer = readFileSync(this.manifestFilename); + const dataJsonString = dataBuffer.toString(this.manifestFileEncoding); + this.data = JSON.parse(dataJsonString) as Record; + } catch { + this.data = {}; + } + } + + private async commitDatasetMetadata(): Promise { + return new Promise((resolve, reject) => { + this.manifestWriteStream = createWriteStream(this.manifestFilename, { + flags: 'w', + }); + this.manifestWriteStream.on('error', reject); + this.manifestWriteStream.write(JSON.stringify(this.data)); + this.manifestWriteStream.end(() => { + resolve(this.data); + }); + }); + } + + async downloadDataset(params: CreateDownloaderParams): Promise { + const downloader = await createDownloader(params); + + return new Promise((resolve, reject) => { + const out = createWriteStream(`${params.type}.jsonl`); + + out.on('finish', () => { + const now = Date.now(); + this.data[params.type] = { + ...this.data[params.type], + createdAt: this.data[params.type].createdAt ?? now, + lastUpdatedAt: now, + }; + + this.commitDatasetMetadata() + .then(resolve) + .catch(reject); + }); + + switch (params.type) { + case Kanjidic.SOURCE_ID: { + const jsonlParser = createXmlToJsonLines({ + entryTagName: 'character', + }); + + downloader + .pipe(jsonlParser) + .pipe(out); + } return; + case JMnedict.SOURCE_ID: + case JMdict.SOURCE_ID: { + const jsonlParser = createXmlToJsonLines({ + entryTagName: 'entry', + }); + + downloader + .pipe(jsonlParser) + .pipe(out); + } return; + case KRadFile.SOURCE_ID: + case RadKFile.SOURCE_ID: + downloader.pipe(out); + return; + default: + break; + } + + this.commitDatasetMetadata() + .then(() => { + reject(new Error(`Unknown dataset: ${params.type as unknown as string}`)); + }) + .catch(reject); + }); + } +} diff --git a/src/routes.ts b/src/routes.ts index c9c0f6d..591c550 100644 --- a/src/routes.ts +++ b/src/routes.ts @@ -1,5 +1,34 @@ -import SERVER from './server'; +import { FastifyInstance } from 'fastify'; +import { InitController, InitControllerImpl } from './modules/init/InitController'; -SERVER.get('/', async (_, reply) => { - reply.send({ hello: 'world' }) -}); +export const addDefaultRoutes = (server: FastifyInstance) => { + server + .route({ + method: 'GET', + url: '/api/health/live', + handler: async (_, reply) => { + reply.send({ + status: 'ok', + }); + }, + }) + .route({ + method: 'GET', + url: '/api/health/ready', + handler: async (_, reply) => { + reply.send({ + status: 'ready', + }); + }, + }); +}; + +export const addInitRoutes = (server: FastifyInstance) => { + const initController: InitController = new InitControllerImpl(); + server + .route({ + method: 'POST', + url: '/api/download', + handler: initController.downloadDataset, + }); +}; diff --git a/src/server.ts b/src/server.ts index 8dbaf39..8ac0308 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,7 +1,9 @@ import fastify from 'fastify'; -const SERVER = fastify({ - logger: true, -}); +export interface CreateServerOptions { + logger?: boolean; +} -export default SERVER; +export const createServer = (options: CreateServerOptions) => fastify({ + logger: options.logger, +}); diff --git a/yarn.lock b/yarn.lock index d0ab211..21abb07 100644 --- a/yarn.lock +++ b/yarn.lock @@ -462,6 +462,10 @@ resolved "https://registry.yarnpkg.com/@mdn/browser-compat-data/-/browser-compat-data-5.2.54.tgz#253803ffcf8f706155d36c4f3413b9f537f06419" integrity sha512-8/W1qTWw/lCf6E01n/Be65LGza/WrDON7ii9F/fKc4EdZad+K1xJWvfYhDSoPpkMCAw0eLIyAB0Z5emQFBVclw== +"@modal-sh/murasaki-core@link:../core": + version "0.0.0" + uid "" + "@next/eslint-plugin-next@^13.2.4": version "13.3.2" resolved "https://registry.yarnpkg.com/@next/eslint-plugin-next/-/eslint-plugin-next-13.3.2.tgz#1126508131f85d550da0ad8eb3888ddc5ae4c9c1" @@ -1828,6 +1832,13 @@ fastq@^1.6.0, fastq@^1.6.1: dependencies: reusify "^1.0.4" +fetch-ponyfill@^7.1.0: + version "7.1.0" + resolved "https://js.pack.modal.sh/fetch-ponyfill/-/fetch-ponyfill-7.1.0.tgz#4266ed48b4e64663a50ab7f7fcb8e76f990526d0" + integrity sha512-FhbbL55dj/qdVO3YNK7ZEkshvj3eQ7EuIGV2I6ic/2YiocvyWv+7jg2s4AyS0wdRU75s3tA8ZxI/xPigb0v5Aw== + dependencies: + node-fetch "~2.6.1" + file-entry-cache@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" @@ -2657,6 +2668,13 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== +node-fetch@~2.6.1: + version "2.6.9" + resolved "https://js.pack.modal.sh/node-fetch/-/node-fetch-2.6.9.tgz#7c7f744b5cc6eb5fd404e0c7a9fec630a55657e6" + integrity sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg== + dependencies: + whatwg-url "^5.0.0" + node-releases@^2.0.8: version "2.0.10" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.10.tgz#c311ebae3b6a148c89b1813fd7c4d3c024ef537f" @@ -3202,6 +3220,11 @@ safe-stable-stringify@^2.3.1: resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886" integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g== +sax@^1.2.4: + version "1.2.4" + resolved "https://js.pack.modal.sh/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== + secure-json-parse@^2.5.0: version "2.7.0" resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862" @@ -3530,6 +3553,11 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" +tr46@~0.0.3: + version "0.0.3" + resolved "https://js.pack.modal.sh/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== + tsconfig-paths@^3.14.1: version "3.14.2" resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz#6e32f1f79412decd261f92d633a9dc1cfa99f088" @@ -3712,6 +3740,19 @@ wcwidth@^1.0.1: dependencies: defaults "^1.0.3" +webidl-conversions@^3.0.0: + version "3.0.1" + resolved "https://js.pack.modal.sh/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== + +whatwg-url@^5.0.0: + version "5.0.0" + resolved "https://js.pack.modal.sh/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== + dependencies: + tr46 "~0.0.3" + webidl-conversions "^3.0.0" + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" @@ -3813,6 +3854,13 @@ xdg-basedir@^4.0.0: resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13" integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q== +xml-js@^1.6.11: + version "1.6.11" + resolved "https://js.pack.modal.sh/xml-js/-/xml-js-1.6.11.tgz#927d2f6947f7f1c19a316dd8eea3614e8b18f8e9" + integrity sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g== + dependencies: + sax "^1.2.4" + xml-name-validator@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835"