Browse Source

Update content negotiation logic

Ensure both backend and client honors the content negotiation headers.
refactor/new-arch
TheoryOfNekomata 6 months ago
parent
commit
09c65a1dfd
12 changed files with 308 additions and 120 deletions
  1. +2
    -0
      packages/core/package.json
  2. +1
    -0
      packages/core/src/backend/common.ts
  3. +37
    -0
      packages/core/src/common/app.ts
  4. +17
    -0
      packages/core/src/common/charset.ts
  5. +48
    -0
      packages/core/src/common/content-negotiation.ts
  6. +2
    -0
      packages/core/src/common/index.ts
  7. +16
    -0
      packages/core/src/common/language.ts
  8. +59
    -10
      packages/core/src/common/media-type.ts
  9. +21
    -5
      packages/core/src/common/response.ts
  10. +2
    -1
      packages/examples/http-resource-server/test/default.test.ts
  11. +93
    -104
      packages/extenders/http/src/backend/core.ts
  12. +10
    -0
      pnpm-lock.yaml

+ 2
- 0
packages/core/package.json View File

@@ -13,6 +13,7 @@
"pridepack"
],
"devDependencies": {
"@types/negotiator": "^0.6.3",
"@types/node": "^20.11.0",
"pridepack": "2.6.0",
"tslib": "^2.6.2",
@@ -44,6 +45,7 @@
"access": "public"
},
"dependencies": {
"negotiator": "^0.6.3",
"valibot": "^0.30.0"
},
"types": "./dist/types/common/index.d.ts",


+ 1
- 0
packages/core/src/backend/common.ts View File

@@ -8,6 +8,7 @@ interface BackendParams<App extends BaseApp> {

export interface ImplementationContext {
endpoint: Endpoint;
body?: unknown;
params: Record<string, unknown>;
query?: URLSearchParams;
dataSource?: DataSource;


+ 37
- 0
packages/core/src/common/app.ts View File

@@ -1,6 +1,9 @@
import {Endpoint, EndpointOperations} from './endpoint';
import {Operation} from './operation';
import {NamedSet, PredicateMap} from './common';
import {FALLBACK_LANGUAGE, Language} from './language';
import {FALLBACK_MEDIA_TYPE, MediaType} from './media-type';
import {Charset, FALLBACK_CHARSET} from './charset';

export interface BaseAppState {
endpoints: unknown;
@@ -23,6 +26,13 @@ export interface App<AppName extends string = string, AppState extends BaseAppSt
name: AppName;
endpoints: NamedSet<Endpoint>;
operations: NamedSet<Operation>;
languages: NamedSet<Language>;
mediaTypes: NamedSet<MediaType>;
charsets: NamedSet<Charset>;
// todo add stateful types with these methods
language(language: Language): this;
mediaType(mediaType: MediaType): this;
charset(charset: Charset): this;
operation<NewOperation extends Operation>(newOperation: NewOperation): App<
AppName,
{
@@ -51,6 +61,9 @@ class AppInstance<Params extends AppParams, State extends BaseAppState> implemen
readonly name: Params['name'];
readonly endpoints: NamedSet<Endpoint>;
readonly operations: NamedSet<Operation>;
readonly languages: NamedSet<Language>;
readonly mediaTypes: NamedSet<MediaType>;
readonly charsets: NamedSet<Charset>;

constructor(params: Params) {
this.name = params.name;
@@ -58,6 +71,30 @@ class AppInstance<Params extends AppParams, State extends BaseAppState> implemen
this.operations = new PredicateMap<Operation['name'], Operation>((newOperation, s) => (
s.method === newOperation.method
));
this.languages = new Map<Language['name'], Language>([
[FALLBACK_LANGUAGE.name, FALLBACK_LANGUAGE],
]);
this.mediaTypes = new Map<MediaType['name'], MediaType>([
[FALLBACK_MEDIA_TYPE.name, FALLBACK_MEDIA_TYPE],
]);
this.charsets = new Map<Charset['name'], Charset>([
[FALLBACK_CHARSET.name, FALLBACK_CHARSET],
]);
}

language(language: Language): this {
this.languages.set(language.name, language);
return this;
}

mediaType(mediaType: MediaType): this {
this.mediaTypes.set(mediaType.name, mediaType);
return this;
}

charset(charset: Charset): this {
this.charsets.set(charset.name, charset);
return this;
}

operation<NewOperation extends Operation>(newOperation: NewOperation) {


+ 17
- 0
packages/core/src/common/charset.ts View File

@@ -1,3 +1,6 @@
import {isTextMediaType} from './media-type';
import Negotiator from 'negotiator';

export interface Charset<Name extends string = string> {
name: Name;
encode: (str: string) => Buffer;
@@ -9,3 +12,17 @@ export const FALLBACK_CHARSET = {
decode: (buf: Buffer) => buf.toString('utf-8'),
name: 'utf-8' as const,
} satisfies Charset;

export const getCharset = (availableCharsets: string[], mediaTypeString: string, charset?: string): Charset['name'] | undefined => {
if (typeof charset === 'undefined') {
return isTextMediaType(mediaTypeString) ? FALLBACK_CHARSET['name'] : undefined;
}

const negotiator = new Negotiator({
headers: {
'accept': `${mediaTypeString};charset=${charset}`,
},
});

return negotiator.charset(availableCharsets);
};

+ 48
- 0
packages/core/src/common/content-negotiation.ts View File

@@ -0,0 +1,48 @@
import {App} from './app';
import {getLanguage} from './language';
import {getMediaType, parseAcceptString} from './media-type';
import {getCharset} from './charset';

interface ContentNegotiationHeaders {
'accept-language'?: string;
accept?: string;
}

export const getContentNegotiationParams = (app: App, headers: ContentNegotiationHeaders) => {
const languageString = getLanguage(Array.from(app.languages.keys()), headers['accept-language']);
const language = (
typeof languageString !== 'undefined'
? app.languages.get(languageString)
: undefined
);

const parsedAccept = parseAcceptString(headers['accept']);

const mediaTypeString = (
typeof parsedAccept !== 'undefined'
? getMediaType(Array.from(app.mediaTypes.keys()), parsedAccept.mediaType)
: undefined
);
const mediaType = (
typeof mediaTypeString !== 'undefined'
? app.mediaTypes.get(mediaTypeString)
: undefined
);

const charsetString = (
typeof parsedAccept !== 'undefined' && typeof mediaType !== 'undefined'
? getCharset(Array.from(app.charsets.keys()), mediaType.name, parsedAccept.params.charset)
: undefined
);
const charset = (
typeof charsetString !== 'undefined'
? app.charsets.get(charsetString)
: undefined
);

return {
language,
mediaType,
charset,
};
};

+ 2
- 0
packages/core/src/common/index.ts View File

@@ -1,5 +1,7 @@
export * from './app';
export * from './charset';
export * from './common';
export * from './content-negotiation';
export * from './endpoint';
export * from './language';
export * from './media-type';


+ 16
- 0
packages/core/src/common/language.ts View File

@@ -1,3 +1,5 @@
import Negotiator from 'negotiator';

export type MessageBody = string | string[] | (string | string[])[];

export const LANGUAGE_DEFAULT_ERROR_STATUS_MESSAGE_KEYS = [
@@ -340,3 +342,17 @@ export const FALLBACK_LANGUAGE = {
],
},
} satisfies Language;

export const getLanguage = (availableLanguageNames: string[], languageString?: string): Language['name'] | undefined => {
if (typeof languageString === 'undefined') {
return FALLBACK_LANGUAGE['name'];
}

const negotiator = new Negotiator({
headers: {
'accept-language': languageString,
},
});

return negotiator.language(availableLanguageNames);
};

+ 59
- 10
packages/core/src/common/media-type.ts View File

@@ -1,6 +1,8 @@
import Negotiator from 'negotiator';

export interface MediaType<
Name extends string = string,
T extends object = object,
T extends object | null = object | null,
SerializeOpts extends {} = {},
DeserializeOpts extends {} = {}
> {
@@ -26,12 +28,59 @@ export const getAcceptPostString = (mediaTypes: Map<string, MediaType>) => Array
.filter((t) => !PATCH_CONTENT_TYPES.includes(t as PatchContentType))
.join(',');

export const isTextMediaType = (mediaType: string) => (
mediaType.startsWith('text/')
|| [
'application/json',
'application/xml',
'application/x-www-form-urlencoded',
...PATCH_CONTENT_TYPES,
].includes(mediaType)
);
export const parseMediaType = (mediaType: string) => {
const [type, subtypeEntirety] = mediaType.split('/').map((s) => s.trim());
return {
type,
// application/foo+json
subtype: subtypeEntirety.split('+'),
};
};

export const isTextMediaType = (mediaType: string) => {
const { type, subtype } = parseMediaType(mediaType);
if (type === 'text') {
return true;
}

if (type === 'application' && subtype.length > 0) {
const lastSubtype = subtype.at(-1) as string;
return [
'json',
'xml'
].includes(lastSubtype);
}

return false;
};

export const getMediaType = (availableMediaTypes: string[], mediaTypeString?: string): MediaType['name'] | undefined => {
if (typeof mediaTypeString === 'undefined') {
return FALLBACK_MEDIA_TYPE['name'];
}

const negotiator = new Negotiator({
headers: {
'accept': mediaTypeString,
},
});
return negotiator.mediaType(availableMediaTypes);
};

export const parseAcceptString = (acceptString?: string) => {
if (typeof acceptString !== 'string') {
return undefined;
}
const [mediaType, ...acceptParams] = acceptString.split(';');
const params = Object.fromEntries(
acceptParams.map((s) => {
const [key, ...values] = s.split('=');
return [key.trim(), ...values.map((v) => v.trim()).join('=')];
})
);

return {
mediaType,
params,
};
};

+ 21
- 5
packages/core/src/common/response.ts View File

@@ -1,4 +1,5 @@
import {ErrorStatusCode, isErrorStatusCode, StatusCode} from './status-codes';
import {parseAcceptString, parseMediaType} from './media-type';

type FetchResponse = Awaited<ReturnType<typeof fetch>>;

@@ -68,14 +69,29 @@ export const HttpResponse = <
}

const contentType = response.headers.get('Content-Type');
// TODO properly parse media type
if (contentType === 'application/json') {
if (typeof contentType !== 'string') {
return await response.arrayBuffer();
}

const parsedAcceptString = parseAcceptString(contentType);
if (typeof parsedAcceptString?.mediaType !== 'string') {
return await response.arrayBuffer();
}

const parsedMediaType = parseMediaType(parsedAcceptString.mediaType);
const isJson = (
['application', 'text'].includes(parsedMediaType.type)
&& parsedMediaType.subtype.at(-1) === 'json'
);
if (isJson) {
return await response.json();
}

const buffer = await response.arrayBuffer();
return buffer;
// TODO deserialize buffer
if (parsedMediaType.type === 'text') {
return await response.text();
}

return await response.arrayBuffer();
},
};
}


+ 2
- 1
packages/examples/http-resource-server/test/default.test.ts View File

@@ -78,10 +78,11 @@ describe('default', () => {
);

const response = ResourceItemFetchedResponse.fromFetchResponse(responseRaw);
const body = await response['deserialize']();

expect(response).toHaveProperty('statusCode', statusCodes.HTTP_STATUS_OK);
expect(response).toHaveProperty('statusMessage', 'Resource Collection Fetched');
console.log(responseRaw.headers);
expect(body).toEqual([]);
});

it('works for items', async () => {


+ 93
- 104
packages/extenders/http/src/backend/core.ts View File

@@ -4,12 +4,11 @@ import {
parseToEndpointQueue,
ServiceParams,
statusCodes,
getContentNegotiationParams,
FALLBACK_LANGUAGE,
Language,
FALLBACK_MEDIA_TYPE,
MediaType,
FALLBACK_CHARSET,
Charset,
FALLBACK_MEDIA_TYPE,
parseAcceptString,
} from '@modal-sh/yasumi';
import {
Backend as BaseBackend,
@@ -25,84 +24,21 @@ declare module '@modal-sh/yasumi/backend' {
interface ServerResponseContext extends http.ServerResponse {}
}

const isTextMediaType = (mediaType: string) => {
const { type, subtype } = parseMediaType(mediaType);
if (type === 'text') {
return true;
}

if (type === 'application' && subtype.length > 0) {
const lastSubtype = subtype.at(-1) as string;
return [
'json',
'xml'
].includes(lastSubtype);
}

return false;
};

const getLanguage = (languageString?: string): Language => {
if (typeof languageString === 'undefined') {
return FALLBACK_LANGUAGE;
}
// TODO negotiator
return FALLBACK_LANGUAGE;
};

const getMediaType = (mediaTypeString?: string): MediaType => {
if (typeof mediaTypeString === 'undefined') {
return FALLBACK_MEDIA_TYPE;
}
return FALLBACK_MEDIA_TYPE;
};

const getCharset = (mediaTypeString: string, charset?: string): Charset | undefined => {
if (typeof charset === 'undefined') {
return isTextMediaType(mediaTypeString) ? FALLBACK_CHARSET : undefined;
}
return isTextMediaType(mediaTypeString) ? FALLBACK_CHARSET : undefined;
};

const parseMediaType = (mediaType: string) => {
const [type, subtypeEntirety] = mediaType.split('/').map((s) => s.trim());
return {
type,
// application/foo+json
subtype: subtypeEntirety.split('+'),
};
};

const FALLBACK_BINARY_CHARSET = 'application/octet-stream';

const parseAcceptString = (acceptString?: string) => {
if (typeof acceptString !== 'string') {
return undefined;
}
const [mediaType = FALLBACK_BINARY_CHARSET, ...acceptParams] = acceptString.split(';');
const acceptCharset = acceptParams
.map((s) => s.replace(/\s+/g, ''))
.find((p) => p.startsWith('charset='));
if (typeof acceptCharset !== 'undefined') {
const [, acceptCharsetValue] = acceptCharset.split('=');
return {
mediaType,
charset: acceptCharsetValue,
};
}

if (isTextMediaType(mediaType)) {
return {
// TODO validate if valid mediaType
mediaType,
charset: FALLBACK_CHARSET.name,
};
}

return {
mediaType,
};
};
const getBody = (
req: http.IncomingMessage,
) => new Promise<Buffer>((resolve, reject) => {
let body = Buffer.from('');
req.on('data', (chunk) => {
body = Buffer.concat([body, chunk]);
});
req.on('end', async () => {
resolve(body);
});

req.on('error', (err) => {
reject(err);
});
});

class ServerInstance<Backend extends BaseBackend> implements Server<Backend> {
readonly backend: Backend;
@@ -113,31 +49,40 @@ class ServerInstance<Backend extends BaseBackend> implements Server<Backend> {
}

private readonly requestListener = async (req: ServerRequestContext, res: ServerResponseContext) => {
const { method: methodRaw, url, headers } = req;
const language = getLanguage(headers['accept-language']);
const parsedAccept = parseAcceptString(headers['accept']);
const mediaType = typeof parsedAccept !== 'undefined' ? getMediaType(parsedAccept.mediaType) : undefined;
const charset = typeof parsedAccept !== 'undefined' && typeof mediaType !== 'undefined' ? getCharset(mediaType.name, charsetString) : undefined;
const { method: methodRaw, url, headers: requestHeaders } = req;
const {
language,
mediaType,
charset,
} = getContentNegotiationParams(this.backend.app, requestHeaders);
const fallbackLanguage = language ?? FALLBACK_LANGUAGE;
// TODO use these for errors
const fallbackMediaType = mediaType ?? FALLBACK_MEDIA_TYPE;
const fallbackCharset = charset ?? FALLBACK_CHARSET;
const errorHeaders = {
'content-language': fallbackLanguage.name,
} as Record<string, string>;

if (typeof methodRaw === 'undefined') {
res.writeHead(statusCodes.HTTP_STATUS_BAD_REQUEST, {});
res.writeHead(statusCodes.HTTP_STATUS_BAD_REQUEST, errorHeaders);
res.end();
return;
}
if (typeof url === 'undefined') {
res.writeHead(statusCodes.HTTP_STATUS_BAD_REQUEST, {});
res.writeHead(statusCodes.HTTP_STATUS_BAD_REQUEST, errorHeaders);
res.end();
return;
}

const method = methodRaw.toUpperCase();
console.log(language?.name, mediaType?.name, charset?.name);
console.log(method, url);

const endpoints = parseToEndpointQueue(url, this.backend.app.endpoints);
const [endpoint, endpointParams] = endpoints.at(-1) ?? [];

if (typeof endpoint === 'undefined') {
res.writeHead(statusCodes.HTTP_STATUS_NOT_FOUND);
res.writeHead(statusCodes.HTTP_STATUS_NOT_FOUND, errorHeaders);
res.end();
return;
}
@@ -149,7 +94,7 @@ class ServerInstance<Backend extends BaseBackend> implements Server<Backend> {
const doesHeadersMatch = Array.from(op.headers.entries()).reduce(
(currentHeadersMatch, [headerKey, opHeaderValue]) => (
// TODO honor content-type matching
currentHeadersMatch && headers[headerKey.toLowerCase()] === opHeaderValue
currentHeadersMatch && requestHeaders[headerKey.toLowerCase()] === opHeaderValue
),
true,
);
@@ -165,6 +110,7 @@ class ServerInstance<Backend extends BaseBackend> implements Server<Backend> {

if (typeof foundAppOperation === 'undefined') {
res.writeHead(statusCodes.HTTP_STATUS_METHOD_NOT_ALLOWED, {
...errorHeaders,
'Allow': appOperations.map((op) => op.method).join(',')
});
res.end();
@@ -174,6 +120,7 @@ class ServerInstance<Backend extends BaseBackend> implements Server<Backend> {
if (!endpoint.operations.has(foundAppOperation.name)) {
const endpointOperations = Array.from(endpoint?.operations ?? []);
res.writeHead(statusCodes.HTTP_STATUS_METHOD_NOT_ALLOWED, {
...errorHeaders,
'Allow': endpointOperations
.map((a) => appOperations.find((aa) => aa.name === a)?.method)
.join(',')
@@ -184,54 +131,96 @@ class ServerInstance<Backend extends BaseBackend> implements Server<Backend> {

const implementation = this.backend.implementations.get(foundAppOperation.name);
if (typeof implementation === 'undefined') {
res.writeHead(statusCodes.HTTP_STATUS_NOT_IMPLEMENTED);
res.writeHead(statusCodes.HTTP_STATUS_NOT_IMPLEMENTED, errorHeaders);
res.end();
return;
}

const [, search] = url.split('?');

// TODO get content negotiation params

// TODO add flag on implementation context if CQRS should be enabled

try {
// TODO deserialize body, else throw 415
const requestBodySpecs = parseAcceptString(requestHeaders['content-type']);
let body: unknown = undefined;
// TODO check body
if (typeof requestBodySpecs !== 'undefined') {
const {
mediaType: requestBodyMediaTypeString,
params: requestBodyParams,
} = requestBodySpecs;
const requestBodyCharsetString = requestBodyParams?.charset;
const requestBodyMediaType = this.backend.app.mediaTypes.get(requestBodyMediaTypeString);
if (typeof requestBodyMediaType === 'undefined') {
res.writeHead(statusCodes.HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE, errorHeaders);
res.end();
return;
}
let decoder = (buf: Buffer, _params?: {}) => buf.toString('binary');
if (typeof requestBodyCharsetString !== 'undefined') {
const requestBodyCharset = this.backend.app.charsets.get(requestBodyCharsetString);
if (typeof requestBodyCharset !== 'undefined') {
decoder = requestBodyCharset.decode;
}
}

const bodyBuffer = await getBody(req);
// TODO set params to decoder/deserializer
const bodyDecoded = decoder(bodyBuffer, requestBodyParams);
body = requestBodyMediaType.deserialize(bodyDecoded, requestBodyParams);
}
const responseSpec = await implementation({
endpoint,
body,
params: endpointParams ?? {},
query: typeof search !== 'undefined' ? new URLSearchParams(search) : undefined,
dataSource: this.backend.dataSource,
});

if (typeof responseSpec === 'undefined') {
res.writeHead(statusCodes.HTTP_STATUS_UNPROCESSABLE_ENTITY, {});
res.writeHead(statusCodes.HTTP_STATUS_UNPROCESSABLE_ENTITY, errorHeaders);
res.end();
return;
}

const responseHeaders = {} as Record<string, string>;
// TODO serialize using content-negotiation params
const bodyToSerialize = responseSpec.body;
if (bodyToSerialize instanceof Buffer) {
// TODO make easy and consistent way to set headers without having to mind about casing
responseHeaders['Content-Type'] = responseHeaders['Content-Type'] ?? 'application/octet-stream';
res.statusMessage = responseSpec.statusMessage;
res.writeHead(responseSpec.statusCode, responseHeaders)
res.end(bodyToSerialize);
return;
}

let encoded = Buffer.from('');
const headers = {} as Record<string, string>;
if (typeof bodyToSerialize === 'object' && bodyToSerialize !== null) {
// TODO throw not acceptable when cannot serialize

if (typeof bodyToSerialize === 'object') {
let serialized = '';
if (typeof mediaType !== 'undefined') {
serialized = mediaType.serialize(bodyToSerialize) as string;
headers['Content-Type'] = mediaType.name;
if (typeof mediaType === 'undefined') {
res.writeHead(statusCodes.HTTP_STATUS_NOT_ACCEPTABLE, errorHeaders);
res.end();
return;
}
if (typeof charset !== 'undefined' && typeof headers['Content-Type'] === 'string') {
serialized = mediaType.serialize(bodyToSerialize) as string;
responseHeaders['Content-Type'] = mediaType.name;
if (typeof charset !== 'undefined' && typeof responseHeaders['Content-Type'] === 'string') {
encoded = charset.encode(serialized);
headers['Content-Type'] += `;charset=${charset.name}`;
responseHeaders['Content-Type'] += `;charset=${charset.name}`;
}
}

res.statusMessage = responseSpec.statusMessage; // TODO add default status message per status code
res.writeHead(responseSpec.statusCode, headers);
res.writeHead(responseSpec.statusCode, responseHeaders);
res.end(encoded);
} catch (errorResponseSpecRaw) {
const responseSpec = errorResponseSpecRaw as ErrorResponse;
res.statusMessage = responseSpec.statusMessage; // TODO add default status message per status code
res.writeHead(responseSpec.statusCode, {});
res.writeHead(responseSpec.statusCode, errorHeaders);
res.end();
}
};


+ 10
- 0
pnpm-lock.yaml View File

@@ -8,10 +8,16 @@ importers:

packages/core:
dependencies:
negotiator:
specifier: ^0.6.3
version: 0.6.3
valibot:
specifier: ^0.30.0
version: 0.30.0
devDependencies:
'@types/negotiator':
specifier: ^0.6.3
version: 0.6.3
'@types/node':
specifier: ^20.11.0
version: 20.11.0
@@ -759,6 +765,10 @@ packages:
resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==}
dev: true

/@types/negotiator@0.6.3:
resolution: {integrity: sha512-JkXTOdKs5MF086b/pt8C3+yVp3iDUwG635L7oCH6HvJvvr6lSUU5oe/gLXnPEfYRROHjJIPgCV6cuAg8gGkntQ==}
dev: true

/@types/node@20.11.0:
resolution: {integrity: sha512-o9bjXmDNcF7GbM4CNQpmi+TutCgap/K3w1JyKgxAjqx41zp9qlIAVFi0IhCNsJcXolEqLWhbFbEeL0PvYm4pcQ==}
dependencies:


Loading…
Cancel
Save