From 09c65a1dfd942399d623b052fdab3a430ba2a5ea Mon Sep 17 00:00:00 2001 From: TheoryOfNekomata Date: Mon, 3 Jun 2024 19:25:02 +0800 Subject: [PATCH] Update content negotiation logic Ensure both backend and client honors the content negotiation headers. --- packages/core/package.json | 2 + packages/core/src/backend/common.ts | 1 + packages/core/src/common/app.ts | 37 ++++ packages/core/src/common/charset.ts | 17 ++ .../core/src/common/content-negotiation.ts | 48 +++++ packages/core/src/common/index.ts | 2 + packages/core/src/common/language.ts | 16 ++ packages/core/src/common/media-type.ts | 69 +++++- packages/core/src/common/response.ts | 26 ++- .../http-resource-server/test/default.test.ts | 3 +- packages/extenders/http/src/backend/core.ts | 197 +++++++++--------- pnpm-lock.yaml | 10 + 12 files changed, 308 insertions(+), 120 deletions(-) create mode 100644 packages/core/src/common/content-negotiation.ts diff --git a/packages/core/package.json b/packages/core/package.json index d9a7b4c..aa5b8ba 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -13,6 +13,7 @@ "pridepack" ], "devDependencies": { + "@types/negotiator": "^0.6.3", "@types/node": "^20.11.0", "pridepack": "2.6.0", "tslib": "^2.6.2", @@ -44,6 +45,7 @@ "access": "public" }, "dependencies": { + "negotiator": "^0.6.3", "valibot": "^0.30.0" }, "types": "./dist/types/common/index.d.ts", diff --git a/packages/core/src/backend/common.ts b/packages/core/src/backend/common.ts index 27ca780..38b6dec 100644 --- a/packages/core/src/backend/common.ts +++ b/packages/core/src/backend/common.ts @@ -8,6 +8,7 @@ interface BackendParams { export interface ImplementationContext { endpoint: Endpoint; + body?: unknown; params: Record; query?: URLSearchParams; dataSource?: DataSource; diff --git a/packages/core/src/common/app.ts b/packages/core/src/common/app.ts index 265038e..2028e22 100644 --- a/packages/core/src/common/app.ts +++ b/packages/core/src/common/app.ts @@ -1,6 +1,9 @@ import {Endpoint, EndpointOperations} from './endpoint'; import {Operation} from './operation'; import {NamedSet, PredicateMap} from './common'; +import {FALLBACK_LANGUAGE, Language} from './language'; +import {FALLBACK_MEDIA_TYPE, MediaType} from './media-type'; +import {Charset, FALLBACK_CHARSET} from './charset'; export interface BaseAppState { endpoints: unknown; @@ -23,6 +26,13 @@ export interface App; operations: NamedSet; + languages: NamedSet; + mediaTypes: NamedSet; + charsets: NamedSet; + // todo add stateful types with these methods + language(language: Language): this; + mediaType(mediaType: MediaType): this; + charset(charset: Charset): this; operation(newOperation: NewOperation): App< AppName, { @@ -51,6 +61,9 @@ class AppInstance implemen readonly name: Params['name']; readonly endpoints: NamedSet; readonly operations: NamedSet; + readonly languages: NamedSet; + readonly mediaTypes: NamedSet; + readonly charsets: NamedSet; constructor(params: Params) { this.name = params.name; @@ -58,6 +71,30 @@ class AppInstance implemen this.operations = new PredicateMap((newOperation, s) => ( s.method === newOperation.method )); + this.languages = new Map([ + [FALLBACK_LANGUAGE.name, FALLBACK_LANGUAGE], + ]); + this.mediaTypes = new Map([ + [FALLBACK_MEDIA_TYPE.name, FALLBACK_MEDIA_TYPE], + ]); + this.charsets = new Map([ + [FALLBACK_CHARSET.name, FALLBACK_CHARSET], + ]); + } + + language(language: Language): this { + this.languages.set(language.name, language); + return this; + } + + mediaType(mediaType: MediaType): this { + this.mediaTypes.set(mediaType.name, mediaType); + return this; + } + + charset(charset: Charset): this { + this.charsets.set(charset.name, charset); + return this; } operation(newOperation: NewOperation) { diff --git a/packages/core/src/common/charset.ts b/packages/core/src/common/charset.ts index e5086b4..4da6127 100644 --- a/packages/core/src/common/charset.ts +++ b/packages/core/src/common/charset.ts @@ -1,3 +1,6 @@ +import {isTextMediaType} from './media-type'; +import Negotiator from 'negotiator'; + export interface Charset { name: Name; encode: (str: string) => Buffer; @@ -9,3 +12,17 @@ export const FALLBACK_CHARSET = { decode: (buf: Buffer) => buf.toString('utf-8'), name: 'utf-8' as const, } satisfies Charset; + +export const getCharset = (availableCharsets: string[], mediaTypeString: string, charset?: string): Charset['name'] | undefined => { + if (typeof charset === 'undefined') { + return isTextMediaType(mediaTypeString) ? FALLBACK_CHARSET['name'] : undefined; + } + + const negotiator = new Negotiator({ + headers: { + 'accept': `${mediaTypeString};charset=${charset}`, + }, + }); + + return negotiator.charset(availableCharsets); +}; diff --git a/packages/core/src/common/content-negotiation.ts b/packages/core/src/common/content-negotiation.ts new file mode 100644 index 0000000..05e32e4 --- /dev/null +++ b/packages/core/src/common/content-negotiation.ts @@ -0,0 +1,48 @@ +import {App} from './app'; +import {getLanguage} from './language'; +import {getMediaType, parseAcceptString} from './media-type'; +import {getCharset} from './charset'; + +interface ContentNegotiationHeaders { + 'accept-language'?: string; + accept?: string; +} + +export const getContentNegotiationParams = (app: App, headers: ContentNegotiationHeaders) => { + const languageString = getLanguage(Array.from(app.languages.keys()), headers['accept-language']); + const language = ( + typeof languageString !== 'undefined' + ? app.languages.get(languageString) + : undefined + ); + + const parsedAccept = parseAcceptString(headers['accept']); + + const mediaTypeString = ( + typeof parsedAccept !== 'undefined' + ? getMediaType(Array.from(app.mediaTypes.keys()), parsedAccept.mediaType) + : undefined + ); + const mediaType = ( + typeof mediaTypeString !== 'undefined' + ? app.mediaTypes.get(mediaTypeString) + : undefined + ); + + const charsetString = ( + typeof parsedAccept !== 'undefined' && typeof mediaType !== 'undefined' + ? getCharset(Array.from(app.charsets.keys()), mediaType.name, parsedAccept.params.charset) + : undefined + ); + const charset = ( + typeof charsetString !== 'undefined' + ? app.charsets.get(charsetString) + : undefined + ); + + return { + language, + mediaType, + charset, + }; +}; diff --git a/packages/core/src/common/index.ts b/packages/core/src/common/index.ts index 5657c21..575136c 100644 --- a/packages/core/src/common/index.ts +++ b/packages/core/src/common/index.ts @@ -1,5 +1,7 @@ export * from './app'; export * from './charset'; +export * from './common'; +export * from './content-negotiation'; export * from './endpoint'; export * from './language'; export * from './media-type'; diff --git a/packages/core/src/common/language.ts b/packages/core/src/common/language.ts index 92542b7..ad4f73e 100644 --- a/packages/core/src/common/language.ts +++ b/packages/core/src/common/language.ts @@ -1,3 +1,5 @@ +import Negotiator from 'negotiator'; + export type MessageBody = string | string[] | (string | string[])[]; export const LANGUAGE_DEFAULT_ERROR_STATUS_MESSAGE_KEYS = [ @@ -340,3 +342,17 @@ export const FALLBACK_LANGUAGE = { ], }, } satisfies Language; + +export const getLanguage = (availableLanguageNames: string[], languageString?: string): Language['name'] | undefined => { + if (typeof languageString === 'undefined') { + return FALLBACK_LANGUAGE['name']; + } + + const negotiator = new Negotiator({ + headers: { + 'accept-language': languageString, + }, + }); + + return negotiator.language(availableLanguageNames); +}; diff --git a/packages/core/src/common/media-type.ts b/packages/core/src/common/media-type.ts index be412ae..ac4e21f 100644 --- a/packages/core/src/common/media-type.ts +++ b/packages/core/src/common/media-type.ts @@ -1,6 +1,8 @@ +import Negotiator from 'negotiator'; + export interface MediaType< Name extends string = string, - T extends object = object, + T extends object | null = object | null, SerializeOpts extends {} = {}, DeserializeOpts extends {} = {} > { @@ -26,12 +28,59 @@ export const getAcceptPostString = (mediaTypes: Map) => Array .filter((t) => !PATCH_CONTENT_TYPES.includes(t as PatchContentType)) .join(','); -export const isTextMediaType = (mediaType: string) => ( - mediaType.startsWith('text/') - || [ - 'application/json', - 'application/xml', - 'application/x-www-form-urlencoded', - ...PATCH_CONTENT_TYPES, - ].includes(mediaType) -); +export const parseMediaType = (mediaType: string) => { + const [type, subtypeEntirety] = mediaType.split('/').map((s) => s.trim()); + return { + type, + // application/foo+json + subtype: subtypeEntirety.split('+'), + }; +}; + +export const isTextMediaType = (mediaType: string) => { + const { type, subtype } = parseMediaType(mediaType); + if (type === 'text') { + return true; + } + + if (type === 'application' && subtype.length > 0) { + const lastSubtype = subtype.at(-1) as string; + return [ + 'json', + 'xml' + ].includes(lastSubtype); + } + + return false; +}; + +export const getMediaType = (availableMediaTypes: string[], mediaTypeString?: string): MediaType['name'] | undefined => { + if (typeof mediaTypeString === 'undefined') { + return FALLBACK_MEDIA_TYPE['name']; + } + + const negotiator = new Negotiator({ + headers: { + 'accept': mediaTypeString, + }, + }); + return negotiator.mediaType(availableMediaTypes); +}; + +export const parseAcceptString = (acceptString?: string) => { + if (typeof acceptString !== 'string') { + return undefined; + } + const [mediaType, ...acceptParams] = acceptString.split(';'); + const params = Object.fromEntries( + acceptParams.map((s) => { + const [key, ...values] = s.split('='); + return [key.trim(), ...values.map((v) => v.trim()).join('=')]; + }) + ); + + return { + mediaType, + params, + }; +}; diff --git a/packages/core/src/common/response.ts b/packages/core/src/common/response.ts index f09dc70..ba66af9 100644 --- a/packages/core/src/common/response.ts +++ b/packages/core/src/common/response.ts @@ -1,4 +1,5 @@ import {ErrorStatusCode, isErrorStatusCode, StatusCode} from './status-codes'; +import {parseAcceptString, parseMediaType} from './media-type'; type FetchResponse = Awaited>; @@ -68,14 +69,29 @@ export const HttpResponse = < } const contentType = response.headers.get('Content-Type'); - // TODO properly parse media type - if (contentType === 'application/json') { + if (typeof contentType !== 'string') { + return await response.arrayBuffer(); + } + + const parsedAcceptString = parseAcceptString(contentType); + if (typeof parsedAcceptString?.mediaType !== 'string') { + return await response.arrayBuffer(); + } + + const parsedMediaType = parseMediaType(parsedAcceptString.mediaType); + const isJson = ( + ['application', 'text'].includes(parsedMediaType.type) + && parsedMediaType.subtype.at(-1) === 'json' + ); + if (isJson) { return await response.json(); } - const buffer = await response.arrayBuffer(); - return buffer; - // TODO deserialize buffer + if (parsedMediaType.type === 'text') { + return await response.text(); + } + + return await response.arrayBuffer(); }, }; } diff --git a/packages/examples/http-resource-server/test/default.test.ts b/packages/examples/http-resource-server/test/default.test.ts index 730bc59..ed4cae5 100644 --- a/packages/examples/http-resource-server/test/default.test.ts +++ b/packages/examples/http-resource-server/test/default.test.ts @@ -78,10 +78,11 @@ describe('default', () => { ); const response = ResourceItemFetchedResponse.fromFetchResponse(responseRaw); + const body = await response['deserialize'](); expect(response).toHaveProperty('statusCode', statusCodes.HTTP_STATUS_OK); expect(response).toHaveProperty('statusMessage', 'Resource Collection Fetched'); - console.log(responseRaw.headers); + expect(body).toEqual([]); }); it('works for items', async () => { diff --git a/packages/extenders/http/src/backend/core.ts b/packages/extenders/http/src/backend/core.ts index 8389cb3..1e51399 100644 --- a/packages/extenders/http/src/backend/core.ts +++ b/packages/extenders/http/src/backend/core.ts @@ -4,12 +4,11 @@ import { parseToEndpointQueue, ServiceParams, statusCodes, + getContentNegotiationParams, FALLBACK_LANGUAGE, - Language, - FALLBACK_MEDIA_TYPE, - MediaType, FALLBACK_CHARSET, - Charset, + FALLBACK_MEDIA_TYPE, + parseAcceptString, } from '@modal-sh/yasumi'; import { Backend as BaseBackend, @@ -25,84 +24,21 @@ declare module '@modal-sh/yasumi/backend' { interface ServerResponseContext extends http.ServerResponse {} } -const isTextMediaType = (mediaType: string) => { - const { type, subtype } = parseMediaType(mediaType); - if (type === 'text') { - return true; - } - - if (type === 'application' && subtype.length > 0) { - const lastSubtype = subtype.at(-1) as string; - return [ - 'json', - 'xml' - ].includes(lastSubtype); - } - - return false; -}; - -const getLanguage = (languageString?: string): Language => { - if (typeof languageString === 'undefined') { - return FALLBACK_LANGUAGE; - } - // TODO negotiator - return FALLBACK_LANGUAGE; -}; - -const getMediaType = (mediaTypeString?: string): MediaType => { - if (typeof mediaTypeString === 'undefined') { - return FALLBACK_MEDIA_TYPE; - } - return FALLBACK_MEDIA_TYPE; -}; - -const getCharset = (mediaTypeString: string, charset?: string): Charset | undefined => { - if (typeof charset === 'undefined') { - return isTextMediaType(mediaTypeString) ? FALLBACK_CHARSET : undefined; - } - return isTextMediaType(mediaTypeString) ? FALLBACK_CHARSET : undefined; -}; - -const parseMediaType = (mediaType: string) => { - const [type, subtypeEntirety] = mediaType.split('/').map((s) => s.trim()); - return { - type, - // application/foo+json - subtype: subtypeEntirety.split('+'), - }; -}; - -const FALLBACK_BINARY_CHARSET = 'application/octet-stream'; - -const parseAcceptString = (acceptString?: string) => { - if (typeof acceptString !== 'string') { - return undefined; - } - const [mediaType = FALLBACK_BINARY_CHARSET, ...acceptParams] = acceptString.split(';'); - const acceptCharset = acceptParams - .map((s) => s.replace(/\s+/g, '')) - .find((p) => p.startsWith('charset=')); - if (typeof acceptCharset !== 'undefined') { - const [, acceptCharsetValue] = acceptCharset.split('='); - return { - mediaType, - charset: acceptCharsetValue, - }; - } - - if (isTextMediaType(mediaType)) { - return { - // TODO validate if valid mediaType - mediaType, - charset: FALLBACK_CHARSET.name, - }; - } - - return { - mediaType, - }; -}; +const getBody = ( + req: http.IncomingMessage, +) => new Promise((resolve, reject) => { + let body = Buffer.from(''); + req.on('data', (chunk) => { + body = Buffer.concat([body, chunk]); + }); + req.on('end', async () => { + resolve(body); + }); + + req.on('error', (err) => { + reject(err); + }); +}); class ServerInstance implements Server { readonly backend: Backend; @@ -113,31 +49,40 @@ class ServerInstance implements Server { } private readonly requestListener = async (req: ServerRequestContext, res: ServerResponseContext) => { - const { method: methodRaw, url, headers } = req; - const language = getLanguage(headers['accept-language']); - const parsedAccept = parseAcceptString(headers['accept']); - const mediaType = typeof parsedAccept !== 'undefined' ? getMediaType(parsedAccept.mediaType) : undefined; - const charset = typeof parsedAccept !== 'undefined' && typeof mediaType !== 'undefined' ? getCharset(mediaType.name, charsetString) : undefined; + const { method: methodRaw, url, headers: requestHeaders } = req; + const { + language, + mediaType, + charset, + } = getContentNegotiationParams(this.backend.app, requestHeaders); + const fallbackLanguage = language ?? FALLBACK_LANGUAGE; + // TODO use these for errors + const fallbackMediaType = mediaType ?? FALLBACK_MEDIA_TYPE; + const fallbackCharset = charset ?? FALLBACK_CHARSET; + const errorHeaders = { + 'content-language': fallbackLanguage.name, + } as Record; if (typeof methodRaw === 'undefined') { - res.writeHead(statusCodes.HTTP_STATUS_BAD_REQUEST, {}); + res.writeHead(statusCodes.HTTP_STATUS_BAD_REQUEST, errorHeaders); res.end(); return; } if (typeof url === 'undefined') { - res.writeHead(statusCodes.HTTP_STATUS_BAD_REQUEST, {}); + res.writeHead(statusCodes.HTTP_STATUS_BAD_REQUEST, errorHeaders); res.end(); return; } const method = methodRaw.toUpperCase(); + console.log(language?.name, mediaType?.name, charset?.name); console.log(method, url); const endpoints = parseToEndpointQueue(url, this.backend.app.endpoints); const [endpoint, endpointParams] = endpoints.at(-1) ?? []; if (typeof endpoint === 'undefined') { - res.writeHead(statusCodes.HTTP_STATUS_NOT_FOUND); + res.writeHead(statusCodes.HTTP_STATUS_NOT_FOUND, errorHeaders); res.end(); return; } @@ -149,7 +94,7 @@ class ServerInstance implements Server { const doesHeadersMatch = Array.from(op.headers.entries()).reduce( (currentHeadersMatch, [headerKey, opHeaderValue]) => ( // TODO honor content-type matching - currentHeadersMatch && headers[headerKey.toLowerCase()] === opHeaderValue + currentHeadersMatch && requestHeaders[headerKey.toLowerCase()] === opHeaderValue ), true, ); @@ -165,6 +110,7 @@ class ServerInstance implements Server { if (typeof foundAppOperation === 'undefined') { res.writeHead(statusCodes.HTTP_STATUS_METHOD_NOT_ALLOWED, { + ...errorHeaders, 'Allow': appOperations.map((op) => op.method).join(',') }); res.end(); @@ -174,6 +120,7 @@ class ServerInstance implements Server { if (!endpoint.operations.has(foundAppOperation.name)) { const endpointOperations = Array.from(endpoint?.operations ?? []); res.writeHead(statusCodes.HTTP_STATUS_METHOD_NOT_ALLOWED, { + ...errorHeaders, 'Allow': endpointOperations .map((a) => appOperations.find((aa) => aa.name === a)?.method) .join(',') @@ -184,54 +131,96 @@ class ServerInstance implements Server { const implementation = this.backend.implementations.get(foundAppOperation.name); if (typeof implementation === 'undefined') { - res.writeHead(statusCodes.HTTP_STATUS_NOT_IMPLEMENTED); + res.writeHead(statusCodes.HTTP_STATUS_NOT_IMPLEMENTED, errorHeaders); res.end(); return; } const [, search] = url.split('?'); - // TODO get content negotiation params - // TODO add flag on implementation context if CQRS should be enabled try { + // TODO deserialize body, else throw 415 + const requestBodySpecs = parseAcceptString(requestHeaders['content-type']); + let body: unknown = undefined; + // TODO check body + if (typeof requestBodySpecs !== 'undefined') { + const { + mediaType: requestBodyMediaTypeString, + params: requestBodyParams, + } = requestBodySpecs; + const requestBodyCharsetString = requestBodyParams?.charset; + const requestBodyMediaType = this.backend.app.mediaTypes.get(requestBodyMediaTypeString); + if (typeof requestBodyMediaType === 'undefined') { + res.writeHead(statusCodes.HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE, errorHeaders); + res.end(); + return; + } + let decoder = (buf: Buffer, _params?: {}) => buf.toString('binary'); + if (typeof requestBodyCharsetString !== 'undefined') { + const requestBodyCharset = this.backend.app.charsets.get(requestBodyCharsetString); + if (typeof requestBodyCharset !== 'undefined') { + decoder = requestBodyCharset.decode; + } + } + + const bodyBuffer = await getBody(req); + // TODO set params to decoder/deserializer + const bodyDecoded = decoder(bodyBuffer, requestBodyParams); + body = requestBodyMediaType.deserialize(bodyDecoded, requestBodyParams); + } const responseSpec = await implementation({ endpoint, + body, params: endpointParams ?? {}, query: typeof search !== 'undefined' ? new URLSearchParams(search) : undefined, dataSource: this.backend.dataSource, }); if (typeof responseSpec === 'undefined') { - res.writeHead(statusCodes.HTTP_STATUS_UNPROCESSABLE_ENTITY, {}); + res.writeHead(statusCodes.HTTP_STATUS_UNPROCESSABLE_ENTITY, errorHeaders); res.end(); return; } + const responseHeaders = {} as Record; // TODO serialize using content-negotiation params const bodyToSerialize = responseSpec.body; + if (bodyToSerialize instanceof Buffer) { + // TODO make easy and consistent way to set headers without having to mind about casing + responseHeaders['Content-Type'] = responseHeaders['Content-Type'] ?? 'application/octet-stream'; + res.statusMessage = responseSpec.statusMessage; + res.writeHead(responseSpec.statusCode, responseHeaders) + res.end(bodyToSerialize); + return; + } + let encoded = Buffer.from(''); - const headers = {} as Record; - if (typeof bodyToSerialize === 'object' && bodyToSerialize !== null) { + // TODO throw not acceptable when cannot serialize + + if (typeof bodyToSerialize === 'object') { let serialized = ''; - if (typeof mediaType !== 'undefined') { - serialized = mediaType.serialize(bodyToSerialize) as string; - headers['Content-Type'] = mediaType.name; + if (typeof mediaType === 'undefined') { + res.writeHead(statusCodes.HTTP_STATUS_NOT_ACCEPTABLE, errorHeaders); + res.end(); + return; } - if (typeof charset !== 'undefined' && typeof headers['Content-Type'] === 'string') { + serialized = mediaType.serialize(bodyToSerialize) as string; + responseHeaders['Content-Type'] = mediaType.name; + if (typeof charset !== 'undefined' && typeof responseHeaders['Content-Type'] === 'string') { encoded = charset.encode(serialized); - headers['Content-Type'] += `;charset=${charset.name}`; + responseHeaders['Content-Type'] += `;charset=${charset.name}`; } } res.statusMessage = responseSpec.statusMessage; // TODO add default status message per status code - res.writeHead(responseSpec.statusCode, headers); + res.writeHead(responseSpec.statusCode, responseHeaders); res.end(encoded); } catch (errorResponseSpecRaw) { const responseSpec = errorResponseSpecRaw as ErrorResponse; res.statusMessage = responseSpec.statusMessage; // TODO add default status message per status code - res.writeHead(responseSpec.statusCode, {}); + res.writeHead(responseSpec.statusCode, errorHeaders); res.end(); } }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 36a2fe1..9ab9b92 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,10 +8,16 @@ importers: packages/core: dependencies: + negotiator: + specifier: ^0.6.3 + version: 0.6.3 valibot: specifier: ^0.30.0 version: 0.30.0 devDependencies: + '@types/negotiator': + specifier: ^0.6.3 + version: 0.6.3 '@types/node': specifier: ^20.11.0 version: 20.11.0 @@ -759,6 +765,10 @@ packages: resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==} dev: true + /@types/negotiator@0.6.3: + resolution: {integrity: sha512-JkXTOdKs5MF086b/pt8C3+yVp3iDUwG635L7oCH6HvJvvr6lSUU5oe/gLXnPEfYRROHjJIPgCV6cuAg8gGkntQ==} + dev: true + /@types/node@20.11.0: resolution: {integrity: sha512-o9bjXmDNcF7GbM4CNQpmi+TutCgap/K3w1JyKgxAjqx41zp9qlIAVFi0IhCNsJcXolEqLWhbFbEeL0PvYm4pcQ==} dependencies: