diff --git a/packages/core/src/backend/data-source/common.ts b/packages/core/src/backend/data-source/common.ts index 7346876..e5603b2 100644 --- a/packages/core/src/backend/data-source/common.ts +++ b/packages/core/src/backend/data-source/common.ts @@ -1,6 +1,5 @@ import * as v from 'valibot'; -import {Resource} from '../../common'; -import {DataSourceQuery} from './queries'; +import {Resource, DataSourceQuery} from '../../common'; type IsCreated = boolean; @@ -36,5 +35,3 @@ export interface ResourceIdConfig { export interface GenerationStrategy { (dataSource: DataSource, ...args: unknown[]): Promise; } - -export * from './queries'; diff --git a/packages/core/src/backend/data-source/index.ts b/packages/core/src/backend/data-source/index.ts index 9b1871f..d0b9323 100644 --- a/packages/core/src/backend/data-source/index.ts +++ b/packages/core/src/backend/data-source/index.ts @@ -1,2 +1 @@ export * from './common'; -export * from './queries'; diff --git a/packages/core/src/backend/servers/http/core.ts b/packages/core/src/backend/servers/http/core.ts index 62b7ea1..cd64e95 100644 --- a/packages/core/src/backend/servers/http/core.ts +++ b/packages/core/src/backend/servers/http/core.ts @@ -18,9 +18,10 @@ import { getAcceptPostString, LanguageDefaultErrorStatusMessageKey, PATCH_CONTENT_MAP_TYPE, PATCH_CONTENT_TYPES, - PatchContentType, + PatchContentType, queryMediaTypes, Resource, } from '../../../common'; +import {DataSource} from '../../data-source'; import { handleGetRoot, handleOptions, } from './handlers/default'; @@ -31,6 +32,7 @@ import { handleGetCollection, handleGetItem, handlePatchItem, + handleQueryCollection, } from './handlers/resource'; import {getBody, isTextMediaType} from './utils'; import {decorateRequestWithBackend} from './decorators/backend'; @@ -38,7 +40,6 @@ import {decorateRequestWithMethod} from './decorators/method'; import {decorateRequestWithUrl} from './decorators/url'; import {ErrorPlainResponse, PlainResponse} from './response'; import EventEmitter from 'events'; -import {DataSource} from '../../data-source'; type RequiredResource = Required>['resource']; @@ -112,6 +113,11 @@ const constructPatchSchema = (resource: Resource resource.state.canFetchCollection, + }, { method: 'GET', middleware: handleGetCollection, @@ -211,7 +217,55 @@ export const createServer = (backendState: BackendState, serverParams = {} as Cr return currentHandlerState; } - if (typeof constructBodySchema === 'function') { + if (effectiveMethod === 'QUERY') { + const contentTypeHeader = req.headers['content-type'] ?? 'application/octet-stream'; + const fragments = contentTypeHeader.replace(/\s+/g, ' ').split(';'); + const mediaType = fragments[0]; + const charsetParam = ( + fragments + .map((s) => s.trim()) + .find((f) => f.startsWith('charset=')) + + ?? ( + isTextMediaType(mediaType) + ? 'charset=utf-8' + : 'charset=binary' + ) + ); + const [_charsetKey, charsetRaw] = charsetParam.split('=').map((s) => s.trim()); + const charset = ( + ( + (charsetRaw.startsWith('"') && charsetRaw.endsWith('"')) + || (charsetRaw.startsWith("'") && charsetRaw.endsWith("'")) + ) + ? charsetRaw.slice(1, -1).trim() + : charsetRaw.trim() + ) ?? (isTextMediaType(mediaType) ? 'utf-8' : 'binary'); + + const theBodyBuffer = await getBody(req); + const encodingPair = req.backend.app.charsets.get(charset); + if (typeof encodingPair === 'undefined') { + throw new ErrorPlainResponse('unableToDecodeResource', { + statusCode: constants.HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE, + res: theRes, + }); + } + + const deserializerPair = Object.values(queryMediaTypes) + .find((a) => a.name === mediaType); + if (typeof deserializerPair === 'undefined') { + throw new ErrorPlainResponse( + 'unableToDeserializeRequest', + { + statusCode: constants.HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE, + res: theRes, + }, + ); + } + + const theBodyStr = encodingPair.decode(theBodyBuffer); + req.body = deserializerPair.deserialize(theBodyStr); + } else if (typeof constructBodySchema === 'function') { const bodySchema = constructBodySchema(req.resource, req.resourceId); const contentTypeHeader = req.headers['content-type'] ?? 'application/octet-stream'; const fragments = contentTypeHeader.replace(/\s+/g, ' ').split(';'); diff --git a/packages/core/src/backend/servers/http/handlers/resource.ts b/packages/core/src/backend/servers/http/handlers/resource.ts index a193e57..d8dfb49 100644 --- a/packages/core/src/backend/servers/http/handlers/resource.ts +++ b/packages/core/src/backend/servers/http/handlers/resource.ts @@ -1,27 +1,77 @@ import { constants } from 'http2'; import * as v from 'valibot'; -import {Middleware, convertToDataSourceQuery} from '../../../common'; +import {Middleware} from '../../../common'; import {ErrorPlainResponse, PlainResponse} from '../response'; import assert from 'assert'; import { - applyDelta, + applyDelta, DataSourceQuery, Delta, PATCH_CONTENT_MAP_TYPE, PatchContentType, + queryMediaTypes, } from '../../../../common'; // TODO add handleQueryCollection() +export const handleQueryCollection: Middleware = async (req, res) => { + const { + body, + resource, + backend, + } = req; + + let data: v.Output[]; + let totalItemCount: number | undefined; + try { + // check which attributes have specifics on the queries (e.g. fuzzy search on strings) + const dataSourceQuery = body as DataSourceQuery; + data = await resource.dataSource.getMultiple(dataSourceQuery); // TODO paginated responses per resource + if (backend.showTotalItemCountOnGetCollection && typeof resource.dataSource.getTotalCount === 'function') { + totalItemCount = await resource.dataSource.getTotalCount(dataSourceQuery); + } + } catch (cause) { + throw new ErrorPlainResponse( + 'unableToFetchResourceCollection', + { + cause, + statusCode: constants.HTTP_STATUS_INTERNAL_SERVER_ERROR, + res, + } + ); + } + + const headers: Record = {}; + if (typeof totalItemCount !== 'undefined') { + headers['X-Resource-Total-Item-Count'] = totalItemCount.toString(); + } + + return new PlainResponse({ + headers, + statusCode: constants.HTTP_STATUS_OK, + statusMessage: 'resourceCollectionFetched', + body: data, + res, + }); +}; + export const handleGetCollection: Middleware = async (req, res) => { - const { query, resource, backend } = req; + const { + // TODO don't turn query into URLSearchParams just yet + query, + resource, + backend, + } = req; let data: v.Output[]; let totalItemCount: number | undefined; try { // check which attributes have specifics on the queries (e.g. fuzzy search on strings) - const dataSourceQuery = convertToDataSourceQuery(query); + const dataSourceQuery = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize( + query.toString() + // TODO compute processEntry options based on resource attribute metadata (e.g. fulltext, queryable attributes - firstname, lastname, middlename) + ); data = await resource.dataSource.getMultiple(dataSourceQuery); // TODO paginated responses per resource - if (backend!.showTotalItemCountOnGetCollection && typeof resource.dataSource.getTotalCount === 'function') { + if (backend.showTotalItemCountOnGetCollection && typeof resource.dataSource.getTotalCount === 'function') { totalItemCount = await resource.dataSource.getTotalCount(dataSourceQuery); } } catch (cause) { diff --git a/packages/core/src/common/index.ts b/packages/core/src/common/index.ts index ede5a64..4865f15 100644 --- a/packages/core/src/common/index.ts +++ b/packages/core/src/common/index.ts @@ -8,6 +8,7 @@ export * from './delta'; export * from './media-type'; export * from './resource'; export * from './language'; +export * from './queries'; export * as validation from './validation'; export interface ContentNegotiation { diff --git a/packages/core/src/common/language.ts b/packages/core/src/common/language.ts index 8e67309..92542b7 100644 --- a/packages/core/src/common/language.ts +++ b/packages/core/src/common/language.ts @@ -37,6 +37,7 @@ export const LANGUAGE_DEFAULT_STATUS_MESSAGE_KEYS = [ ...LANGUAGE_DEFAULT_ERROR_STATUS_MESSAGE_KEYS, 'ok', 'resourceCollectionFetched', + 'resourceCollectionQueried', 'resourceFetched', 'resourceDeleted', 'resourcePatched', @@ -80,6 +81,7 @@ export const FALLBACK_LANGUAGE = { ok: 'OK', provideOptions: 'Provide Options', resourceCollectionFetched: '$RESOURCE Collection Fetched', + resourceCollectionQueried: '$RESOURCE Collection Queried', resourceFetched: '$RESOURCE Fetched', resourceNotFound: '$RESOURCE Not Found', deleteNonExistingResource: 'Delete Non-Existing $RESOURCE', diff --git a/packages/core/src/common/queries/common.ts b/packages/core/src/common/queries/common.ts new file mode 100644 index 0000000..5bb878a --- /dev/null +++ b/packages/core/src/common/queries/common.ts @@ -0,0 +1,43 @@ +const DATA_SOURCE_QUERY_OPERATORS = [ + '=', + '!=', + '>=', + '<=', + '>', + '<', + 'LIKE', + 'ILIKE', + 'REGEXP', +] as const; + +type DataSourceQueryOperator = typeof DATA_SOURCE_QUERY_OPERATORS[number]; + +interface DataSourceQueryOperatorExpression { + lhs: string; + operator: DataSourceQueryOperator; + rhs: string | number | boolean | RegExp; +} + +interface DataSourceQueryOrGrouping { + type: 'or'; + expressions: DataSourceQueryOperatorExpression[]; +} + +interface DataSourceQueryAndGrouping { + type: 'and'; + expressions: DataSourceQueryOrGrouping[]; +} + +export type DataSourceQueryExpression = DataSourceQueryOperatorExpression; + +export type DataSourceQuery = DataSourceQueryAndGrouping; + +export interface QueryMediaType< + Name extends string = string, + SerializeOptions extends {} = {}, + DeserializeOptions extends {} = {} +> { + name: Name; + serialize: (object: DataSourceQuery, opts?: SerializeOptions) => string; + deserialize: (s: string, opts?: DeserializeOptions) => DataSourceQuery; +} diff --git a/packages/core/src/common/queries/index.ts b/packages/core/src/common/queries/index.ts new file mode 100644 index 0000000..70ffab8 --- /dev/null +++ b/packages/core/src/common/queries/index.ts @@ -0,0 +1,2 @@ +export * from './common'; +export * as queryMediaTypes from './media-types'; diff --git a/packages/core/src/common/queries/media-types/application/x-www-form-urlencoded.ts b/packages/core/src/common/queries/media-types/application/x-www-form-urlencoded.ts new file mode 100644 index 0000000..6d36f90 --- /dev/null +++ b/packages/core/src/common/queries/media-types/application/x-www-form-urlencoded.ts @@ -0,0 +1,208 @@ +import {QueryMediaType, DataSourceQuery, DataSourceQueryExpression} from '../../common'; + +interface ProcessEntryBase { + type: string; +} + +const AVAILABLE_MATCH_TYPES = [ + 'startsWith', + 'endsWith', + 'includes', + 'regexp', +] as const; + +type MatchType = typeof AVAILABLE_MATCH_TYPES[number]; + +interface ProcessEntryString extends ProcessEntryBase { + type: 'string'; + matchType?: MatchType; + caseInsensitiveMatch?: boolean; +} + +interface ProcessEntryNumber extends ProcessEntryBase { + type: 'number'; + decimal?: boolean; +} + +interface ProcessEntryBoolean extends ProcessEntryBase { + type: 'boolean'; + truthyStrings?: string[]; +} + +export type ProcessEntry = ProcessEntryString | ProcessEntryNumber | ProcessEntryBoolean; + +export const name = 'application/x-www-form-urlencoded' as const; + +const normalizeRhs = (lhs: string, rhs: string, processEntriesMap?: Record) => { + const coerceValues = processEntriesMap?.[lhs] ?? { + type: 'string' + }; + + if (coerceValues?.type === 'number') { + return { + lhs, + operator: '=', + rhs: Number(rhs) + } as DataSourceQueryExpression; + } + + if (coerceValues?.type === 'boolean') { + const truthyStrings = [ + ...(coerceValues.truthyStrings ?? []).map((s) => s.trim().toLowerCase()), + 'true', + ]; + + return { + lhs, + operator: '=', + rhs: truthyStrings.includes(rhs) + } as DataSourceQueryExpression; + } + + if (coerceValues?.type === 'string') { + switch (coerceValues?.matchType) { + case 'startsWith': { + return { + lhs, + operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE', + rhs: `%${rhs}`, + } as DataSourceQueryExpression; + } + case 'endsWith': { + return { + lhs, + operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE', + rhs: `${rhs}%`, + } as DataSourceQueryExpression; + } + case 'includes': { + return { + lhs, + operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE', + rhs: `%${rhs}%`, + } as DataSourceQueryExpression; + } + case 'regexp': { + return { + lhs, + operator: 'REGEXP', + rhs: new RegExp(rhs, coerceValues.caseInsensitiveMatch ? 'i' : ''), + } as DataSourceQueryExpression; + } + default: + break; + } + + return { + lhs, + operator: '=', + rhs, + } as DataSourceQueryExpression; + } + + const unknownCoerceValues = coerceValues as unknown as Record; + throw new Error(`Invalid coercion type: ${unknownCoerceValues.type}`); + // this will be sent to the data source, e.g. the SQL query + // we can also make this function act as a "sanitizer" +} + +interface SerializeOptions {} + +interface DeserializeOptions { + processEntries?: Record; +} + +export const deserialize: QueryMediaType['deserialize'] = ( + s: string, + options = {} as DeserializeOptions +) => { + const q = new URLSearchParams(s); + return Array.from(q.entries()).reduce( + (queries, [key, value]) => { + const existingOr = queries.expressions + .find((ex) => ex.expressions.some((ex2) => ex2.lhs === key)) ?? { + type: 'or', + expressions: [], + }; + const existingLhs = existingOr.expressions.find((ex) => ex.lhs === key); + const newExpression = normalizeRhs(key, value, options.processEntries); + + if (typeof existingLhs === 'undefined') { + return { + ...queries, + expressions: [ + ...queries.expressions, + { + type: 'or', + expressions: [ + newExpression, + ], + }, + ], + }; + } + + return { + ...queries, + expressions: queries.expressions.map((ex) => ( + ex.expressions.some((ex2) => ex2.lhs !== key) + ? ex + : { + ...existingOr, + expressions: [ + ...(existingOr.expressions ?? []), + newExpression, + ], + } + )), + }; + }, + { + type: 'and', + expressions: [], + } as DataSourceQuery + ) +}; + +export const serialize: QueryMediaType['serialize'] = (q: DataSourceQuery) => ( + new URLSearchParams( + q.expressions.map( + (ex) => ex.expressions.map((ex2) => { + if (ex2.rhs instanceof RegExp) { + if (ex2.operator !== 'REGEXP') { + throw new Error(`Invalid rhs given for operator: ${ex2.lhs} ${ex2.operator} `); + } + + return [ex2.lhs, ex2.rhs.toString()]; + } + + switch (typeof ex2.rhs) { + case 'string': { + switch (ex2.operator) { + case 'ILIKE': + case 'LIKE': + return [ex2.lhs, ex2.rhs.replace(/^%+/, '').replace(/%+$/, '')]; + case '=': + return [ex2.lhs, ex2.rhs]; + default: + break; + } + throw new Error(`Invalid operator given for lhs: ${ex2.lhs} ${ex2.rhs}`); + } + case 'number': { + return [ex2.lhs, ex2.rhs.toString()]; + } + case 'boolean': { + return [ex2.lhs, ex2.rhs ? 'true' : 'false']; + } + default: + break; + } + + throw new Error(`Unknown type for rhs: ${ex2.lhs} ${ex2.operator} `); + }) + ) + .flat() + ) + .toString() +); diff --git a/packages/core/src/common/queries/media-types/index.ts b/packages/core/src/common/queries/media-types/index.ts new file mode 100644 index 0000000..96c8ca8 --- /dev/null +++ b/packages/core/src/common/queries/media-types/index.ts @@ -0,0 +1 @@ +export * as applicationXWwwFormUrlencoded from './application/x-www-form-urlencoded'; diff --git a/packages/core/test/features/query.test.ts b/packages/core/test/features/query.test.ts index 7166193..76abd55 100644 --- a/packages/core/test/features/query.test.ts +++ b/packages/core/test/features/query.test.ts @@ -1,10 +1,10 @@ import {describe, expect, it} from 'vitest'; -import { convertToDataSourceQuery } from '../../src/backend'; +import { queryMediaTypes } from '../../src/common'; describe('query', () => { it('returns a data source query collection from search params', () => { const q = new URLSearchParams(); - const collection = convertToDataSourceQuery(q); + const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize(q.toString()); expect(collection.expressions).toBeInstanceOf(Array); }); @@ -13,8 +13,8 @@ describe('query', () => { attr: '2', attr2: '2', }); - const collection = convertToDataSourceQuery( - q, + const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize( + q.toString(), { processEntries: { attr: { @@ -57,8 +57,8 @@ describe('query', () => { attr3: 'true', attr4: 'false', }); - const collection = convertToDataSourceQuery( - q, + const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize( + q.toString(), { processEntries: { attr: { @@ -121,7 +121,9 @@ describe('query', () => { const q = new URLSearchParams({ attr: 'foo' }); - const collection = convertToDataSourceQuery(q); + const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize( + q.toString() + ); expect(collection).toEqual({ type: 'and', expressions: [ @@ -144,7 +146,9 @@ describe('query', () => { ['attr', 'foo'], ['attr2', 'bar'], ]); - const collection = convertToDataSourceQuery(q); + const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize( + q.toString() + ); expect(collection).toEqual({ type: 'and', expressions: [ @@ -177,7 +181,9 @@ describe('query', () => { ['attr', 'foo'], ['attr', 'bar'], ]); - const collection = convertToDataSourceQuery(q); + const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize( + q.toString() + ); expect(collection).toEqual({ type: 'and', expressions: [ @@ -208,7 +214,9 @@ describe('query', () => { ['attr', 'bar'], ['attr2', 'baz'], ]); - const collection = convertToDataSourceQuery(q); + const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize( + q.toString() + ); expect(collection).toEqual({ type: 'and', expressions: [