Bläddra i källkod

Implement init endpoints

Add endpoints for downloading datasets.
master
TheoryOfNekomata 1 år sedan
förälder
incheckning
826f580e65
9 ändrade filer med 234 tillägg och 16 borttagningar
  1. +1
    -0
      .gitignore
  2. +3
    -2
      package.json
  3. +4
    -0
      src/config.ts
  4. +11
    -6
      src/index.ts
  5. +18
    -0
      src/modules/init/InitController.ts
  6. +110
    -0
      src/modules/init/InitService.ts
  7. +33
    -4
      src/routes.ts
  8. +6
    -4
      src/server.ts
  9. +48
    -0
      yarn.lock

+ 1
- 0
.gitignore Visa fil

@@ -105,3 +105,4 @@ dist
.tern-port

.npmrc
types/

+ 3
- 2
package.json Visa fil

@@ -1,5 +1,5 @@
{
"name": "murasaki-web-api",
"name": "@modal-sh/murasaki-web-api",
"version": "0.0.0",
"files": [
"dist",
@@ -22,7 +22,8 @@
"vitest": "^0.28.1"
},
"dependencies": {
"fastify": "^4.12.0"
"fastify": "^4.12.0",
"@modal-sh/murasaki-core": "link:../core"
},
"scripts": {
"prepublishOnly": "pridepack clean && pridepack build",


+ 4
- 0
src/config.ts Visa fil

@@ -0,0 +1,4 @@
export namespace meta {
export const port = Number(process.env.PORT ?? 8080);
export const host = process.env.HOST ?? '0.0.0.0';
}

+ 11
- 6
src/index.ts Visa fil

@@ -1,12 +1,17 @@
// SERVER
import SERVER from './server';
import { createServer } from './server';
import { addDefaultRoutes, addInitRoutes } from './routes';
import * as config from './config';

import './routes';
const server = createServer({
logger: process.env.NODE_ENV !== 'test',
});

addDefaultRoutes(server);
addInitRoutes(server);

SERVER.listen({ port: 8080, host: '0.0.0.0' }, (err, address) => {
server.listen({ port: config.meta.port, host: config.meta.host }, (err) => {
if (err) {
SERVER.log.error(err.message);
server.log.error(err.message);
process.exit(1);
}
SERVER.log.info(`server listening on ${address}`);
});

+ 18
- 0
src/modules/init/InitController.ts Visa fil

@@ -0,0 +1,18 @@
import {RouteHandlerMethod} from 'fastify';
import {CreateDownloaderParams} from '@modal-sh/murasaki-core';
import {InitService, InitServiceImpl} from './InitService';

export interface InitController {
downloadDataset: RouteHandlerMethod;
}

export class InitControllerImpl implements InitController {
constructor(private readonly initService: InitService = new InitServiceImpl()) {
// noop
}

readonly downloadDataset: RouteHandlerMethod = async (request, reply) => {
const result = await this.initService.downloadDataset(request.body as CreateDownloaderParams);
reply.send(result);
};
}

+ 110
- 0
src/modules/init/InitService.ts Visa fil

@@ -0,0 +1,110 @@
import {
createDownloader,
CreateDownloaderParams,
Kanjidic,
JMdict,
JMnedict,
KRadFile,
RadKFile,
createXmlToJsonLines,
} from '@modal-sh/murasaki-core';
import { createWriteStream, readFileSync } from 'fs';

export interface InitService {
downloadDataset(params: CreateDownloaderParams): Promise<ManifestData>;
}

interface ManifestDatasetEntry {
createdAt: number;
lastUpdatedAt: number;
}

type ManifestData = Record<string, ManifestDatasetEntry>;

export class InitServiceImpl implements InitService {
private manifestWriteStream?: NodeJS.WritableStream;

private readonly manifestFilename = '.murasaki.json' as const;

private readonly manifestFileEncoding = 'utf-8' as const;

private readonly data: ManifestData;

constructor() {
try {
const dataBuffer = readFileSync(this.manifestFilename);
const dataJsonString = dataBuffer.toString(this.manifestFileEncoding);
this.data = JSON.parse(dataJsonString) as Record<string, ManifestDatasetEntry>;
} catch {
this.data = {};
}
}

private async commitDatasetMetadata(): Promise<ManifestData> {
return new Promise<ManifestData>((resolve, reject) => {
this.manifestWriteStream = createWriteStream(this.manifestFilename, {
flags: 'w',
});
this.manifestWriteStream.on('error', reject);
this.manifestWriteStream.write(JSON.stringify(this.data));
this.manifestWriteStream.end(() => {
resolve(this.data);
});
});
}

async downloadDataset(params: CreateDownloaderParams): Promise<ManifestData> {
const downloader = await createDownloader(params);

return new Promise<ManifestData>((resolve, reject) => {
const out = createWriteStream(`${params.type}.jsonl`);

out.on('finish', () => {
const now = Date.now();
this.data[params.type] = {
...this.data[params.type],
createdAt: this.data[params.type].createdAt ?? now,
lastUpdatedAt: now,
};

this.commitDatasetMetadata()
.then(resolve)
.catch(reject);
});

switch (params.type) {
case Kanjidic.SOURCE_ID: {
const jsonlParser = createXmlToJsonLines({
entryTagName: 'character',
});

downloader
.pipe(jsonlParser)
.pipe(out);
} return;
case JMnedict.SOURCE_ID:
case JMdict.SOURCE_ID: {
const jsonlParser = createXmlToJsonLines({
entryTagName: 'entry',
});

downloader
.pipe(jsonlParser)
.pipe(out);
} return;
case KRadFile.SOURCE_ID:
case RadKFile.SOURCE_ID:
downloader.pipe(out);
return;
default:
break;
}

this.commitDatasetMetadata()
.then(() => {
reject(new Error(`Unknown dataset: ${params.type as unknown as string}`));
})
.catch(reject);
});
}
}

+ 33
- 4
src/routes.ts Visa fil

@@ -1,5 +1,34 @@
import SERVER from './server';
import { FastifyInstance } from 'fastify';
import { InitController, InitControllerImpl } from './modules/init/InitController';

SERVER.get('/', async (_, reply) => {
reply.send({ hello: 'world' })
});
export const addDefaultRoutes = (server: FastifyInstance) => {
server
.route({
method: 'GET',
url: '/api/health/live',
handler: async (_, reply) => {
reply.send({
status: 'ok',
});
},
})
.route({
method: 'GET',
url: '/api/health/ready',
handler: async (_, reply) => {
reply.send({
status: 'ready',
});
},
});
};

export const addInitRoutes = (server: FastifyInstance) => {
const initController: InitController = new InitControllerImpl();
server
.route({
method: 'POST',
url: '/api/download',
handler: initController.downloadDataset,
});
};

+ 6
- 4
src/server.ts Visa fil

@@ -1,7 +1,9 @@
import fastify from 'fastify';

const SERVER = fastify({
logger: true,
});
export interface CreateServerOptions {
logger?: boolean;
}

export default SERVER;
export const createServer = (options: CreateServerOptions) => fastify({
logger: options.logger,
});

+ 48
- 0
yarn.lock Visa fil

@@ -462,6 +462,10 @@
resolved "https://registry.yarnpkg.com/@mdn/browser-compat-data/-/browser-compat-data-5.2.54.tgz#253803ffcf8f706155d36c4f3413b9f537f06419"
integrity sha512-8/W1qTWw/lCf6E01n/Be65LGza/WrDON7ii9F/fKc4EdZad+K1xJWvfYhDSoPpkMCAw0eLIyAB0Z5emQFBVclw==
"@modal-sh/murasaki-core@link:../core":
version "0.0.0"
uid ""
"@next/eslint-plugin-next@^13.2.4":
version "13.3.2"
resolved "https://registry.yarnpkg.com/@next/eslint-plugin-next/-/eslint-plugin-next-13.3.2.tgz#1126508131f85d550da0ad8eb3888ddc5ae4c9c1"
@@ -1828,6 +1832,13 @@ fastq@^1.6.0, fastq@^1.6.1:
dependencies:
reusify "^1.0.4"
fetch-ponyfill@^7.1.0:
version "7.1.0"
resolved "https://js.pack.modal.sh/fetch-ponyfill/-/fetch-ponyfill-7.1.0.tgz#4266ed48b4e64663a50ab7f7fcb8e76f990526d0"
integrity sha512-FhbbL55dj/qdVO3YNK7ZEkshvj3eQ7EuIGV2I6ic/2YiocvyWv+7jg2s4AyS0wdRU75s3tA8ZxI/xPigb0v5Aw==
dependencies:
node-fetch "~2.6.1"
file-entry-cache@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027"
@@ -2657,6 +2668,13 @@ natural-compare@^1.4.0:
resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==
node-fetch@~2.6.1:
version "2.6.9"
resolved "https://js.pack.modal.sh/node-fetch/-/node-fetch-2.6.9.tgz#7c7f744b5cc6eb5fd404e0c7a9fec630a55657e6"
integrity sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==
dependencies:
whatwg-url "^5.0.0"
node-releases@^2.0.8:
version "2.0.10"
resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.10.tgz#c311ebae3b6a148c89b1813fd7c4d3c024ef537f"
@@ -3202,6 +3220,11 @@ safe-stable-stringify@^2.3.1:
resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886"
integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==
sax@^1.2.4:
version "1.2.4"
resolved "https://js.pack.modal.sh/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
secure-json-parse@^2.5.0:
version "2.7.0"
resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862"
@@ -3530,6 +3553,11 @@ to-regex-range@^5.0.1:
dependencies:
is-number "^7.0.0"
tr46@~0.0.3:
version "0.0.3"
resolved "https://js.pack.modal.sh/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
tsconfig-paths@^3.14.1:
version "3.14.2"
resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz#6e32f1f79412decd261f92d633a9dc1cfa99f088"
@@ -3712,6 +3740,19 @@ wcwidth@^1.0.1:
dependencies:
defaults "^1.0.3"
webidl-conversions@^3.0.0:
version "3.0.1"
resolved "https://js.pack.modal.sh/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==
whatwg-url@^5.0.0:
version "5.0.0"
resolved "https://js.pack.modal.sh/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==
dependencies:
tr46 "~0.0.3"
webidl-conversions "^3.0.0"
which-boxed-primitive@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"
@@ -3813,6 +3854,13 @@ xdg-basedir@^4.0.0:
resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13"
integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==
xml-js@^1.6.11:
version "1.6.11"
resolved "https://js.pack.modal.sh/xml-js/-/xml-js-1.6.11.tgz#927d2f6947f7f1c19a316dd8eea3614e8b18f8e9"
integrity sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==
dependencies:
sax "^1.2.4"
xml-name-validator@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835"


Laddar…
Avbryt
Spara