From 88b4de6a88d244df63fe8492487e0bcdbc1c2ba9 Mon Sep 17 00:00:00 2001 From: TheoryOfNekomata Date: Thu, 16 May 2024 13:14:40 +0800 Subject: [PATCH] Refactor query code Organize queries code to be set apart from the backend. --- packages/core/src/backend/common.ts | 2 - .../{data-source/common.ts => data-source.ts} | 4 +- .../core/src/backend/data-source/index.ts | 1 - .../core/src/backend/data-source/queries.ts | 194 ------------------ packages/core/src/backend/index.ts | 1 + .../servers/http/decorators/method/index.ts | 6 +- packages/core/src/common/media-type.ts | 11 +- packages/core/src/common/queries/common.ts | 44 ++-- .../application/x-www-form-urlencoded.ts | 149 ++++++++------ 9 files changed, 132 insertions(+), 280 deletions(-) rename packages/core/src/backend/{data-source/common.ts => data-source.ts} (91%) delete mode 100644 packages/core/src/backend/data-source/index.ts delete mode 100644 packages/core/src/backend/data-source/queries.ts diff --git a/packages/core/src/backend/common.ts b/packages/core/src/backend/common.ts index bdc79cc..6d95a9c 100644 --- a/packages/core/src/backend/common.ts +++ b/packages/core/src/backend/common.ts @@ -71,5 +71,3 @@ export const getAllowString = (middlewares: AllowedMiddlewareSpecification[]) => const allowedMethods = middlewares.flatMap((m) => m.method === 'GET' ? [m.method, 'HEAD'] : [m.method]); return allowedMethods.join(','); } - -export * from './data-source'; diff --git a/packages/core/src/backend/data-source/common.ts b/packages/core/src/backend/data-source.ts similarity index 91% rename from packages/core/src/backend/data-source/common.ts rename to packages/core/src/backend/data-source.ts index e5603b2..6343a05 100644 --- a/packages/core/src/backend/data-source/common.ts +++ b/packages/core/src/backend/data-source.ts @@ -1,5 +1,5 @@ import * as v from 'valibot'; -import {Resource, DataSourceQuery} from '../../common'; +import {Resource, QueryAndGrouping} from '../common'; type IsCreated = boolean; @@ -7,6 +7,8 @@ type TotalCount = number; type DeleteResult = unknown; +export type DataSourceQuery = QueryAndGrouping; + export interface DataSource< ItemData extends object = object, ID extends unknown = unknown, diff --git a/packages/core/src/backend/data-source/index.ts b/packages/core/src/backend/data-source/index.ts deleted file mode 100644 index d0b9323..0000000 --- a/packages/core/src/backend/data-source/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './common'; diff --git a/packages/core/src/backend/data-source/queries.ts b/packages/core/src/backend/data-source/queries.ts deleted file mode 100644 index f84ea44..0000000 --- a/packages/core/src/backend/data-source/queries.ts +++ /dev/null @@ -1,194 +0,0 @@ -const DATA_SOURCE_QUERY_OPERATORS = [ - '=', - '!=', - '>=', - '<=', - '>', - '<', - 'LIKE', - 'ILIKE', - 'REGEXP' -] as const; - -type DataSourceQueryOperator = typeof DATA_SOURCE_QUERY_OPERATORS[number]; - -interface DataSourceQueryExpression { - lhs: string; - operator: DataSourceQueryOperator; - rhs: string | number | boolean | RegExp; -} - -interface DataSourceQueryOrGrouping { - type: 'or'; - expressions: DataSourceQueryExpression[]; -} - -interface DataSourceQueryAndGrouping { - type: 'and'; - expressions: DataSourceQueryOrGrouping[]; -} - -export type DataSourceQuery = DataSourceQueryAndGrouping; - -interface ProcessEntryBase { - type: string; -} - -const AVAILABLE_MATCH_TYPES = [ - 'startsWith', - 'endsWith', - 'includes', - 'regexp', -] as const; - -type MatchType = typeof AVAILABLE_MATCH_TYPES[number]; - -interface ProcessEntryString extends ProcessEntryBase { - type: 'string'; - matchType?: MatchType; - caseInsensitiveMatch?: boolean; -} - -interface ProcessEntryNumber extends ProcessEntryBase { - type: 'number'; - decimal?: boolean; -} - -interface ProcessEntryBoolean extends ProcessEntryBase { - type: 'boolean'; - truthyStrings?: string[]; -} - -export type ProcessEntry = ProcessEntryString | ProcessEntryNumber | ProcessEntryBoolean; - -interface ConvertToDataSourceQueryCollectionOptions { - processEntries: Record; -} - -const normalizeRhs = (lhs: string, rhs: string, processEntriesMap?: Record) => { - const coerceValues = processEntriesMap?.[lhs] ?? { - type: 'string' - }; - - if (coerceValues?.type === 'number') { - return { - lhs, - operator: '=', - rhs: Number(rhs) - } as DataSourceQueryExpression; - } - - if (coerceValues?.type === 'boolean') { - const truthyStrings = [ - ...(coerceValues.truthyStrings ?? []).map((s) => s.trim().toLowerCase()), - 'true', - ]; - - return { - lhs, - operator: '=', - rhs: truthyStrings.includes(rhs) - } as DataSourceQueryExpression; - } - - if (coerceValues?.type === 'string') { - switch (coerceValues?.matchType) { - case 'startsWith': { - return { - lhs, - operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE', - rhs: `%${rhs}`, - } as DataSourceQueryExpression; - } - case 'endsWith': { - return { - lhs, - operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE', - rhs: `${rhs}%`, - } as DataSourceQueryExpression; - } - case 'includes': { - return { - lhs, - operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE', - rhs: `%${rhs}%`, - } as DataSourceQueryExpression; - } - case 'regexp': { - return { - lhs, - operator: 'REGEXP', - rhs: new RegExp(rhs, coerceValues.caseInsensitiveMatch ? 'i' : ''), - } as DataSourceQueryExpression; - } - default: - break; - } - - return { - lhs, - operator: '=', - rhs, - } as DataSourceQueryExpression; - } - - const unknownCoerceValues = coerceValues as unknown as Record; - throw new Error(`Invalid coercion type: ${unknownCoerceValues.type}`); - // this will be sent to the data source, e.g. the SQL query - // we can also make this function act as a "sanitizer" -} - -export const convertToDataSourceQuery = ( - q: URLSearchParams, - options = {} as ConvertToDataSourceQueryCollectionOptions, -): DataSourceQueryAndGrouping => { - return Array.from(q.entries()).reduce( - (queries, [key, value]) => { - const existingOr = queries.expressions - .find((ex) => ex.expressions.some((ex2) => ex2.lhs === key)) ?? { - type: 'or', - expressions: [], - }; - const existingLhs = existingOr.expressions.find((ex) => ex.lhs === key); - const newExpression = normalizeRhs(key, value, options.processEntries); - - if (typeof existingLhs === 'undefined') { - return { - ...queries, - expressions: [ - ...queries.expressions, - { - type: 'or', - expressions: [ - newExpression, - ], - }, - ], - }; - } - - return { - ...queries, - expressions: queries.expressions.map((ex) => ( - ex.expressions.some((ex2) => ex2.lhs !== key) - ? ex - : { - ...existingOr, - expressions: [ - ...(existingOr.expressions ?? []), - newExpression, - ], - } - )), - }; - }, - { - type: 'and', - expressions: [], - } as DataSourceQueryAndGrouping - ) - - // all the queries are to be treated as "AND", as suggested by the & character to separate the query param entries -}; - -// TODO add more operators diff --git a/packages/core/src/backend/index.ts b/packages/core/src/backend/index.ts index a2abc7e..0fd1885 100644 --- a/packages/core/src/backend/index.ts +++ b/packages/core/src/backend/index.ts @@ -2,4 +2,5 @@ export * from './core'; export * from './common'; export * from './data-source'; +// TODO publish to separate library export * as http from './servers/http'; diff --git a/packages/core/src/backend/servers/http/decorators/method/index.ts b/packages/core/src/backend/servers/http/decorators/method/index.ts index f89fd17..3c61f84 100644 --- a/packages/core/src/backend/servers/http/decorators/method/index.ts +++ b/packages/core/src/backend/servers/http/decorators/method/index.ts @@ -1,13 +1,15 @@ import {RequestDecorator} from '../../../../common'; +const METHOD_SPOOF_HEADER_NAME = 'x-original-method' as const; +const METHOD_SPOOF_ORIGINAL_METHOD = 'POST' as const; const WHITELISTED_METHODS = [ 'QUERY' ] as const; export const decorateRequestWithMethod: RequestDecorator = (req) => { req.method = req.method?.trim().toUpperCase() ?? ''; - if (req.method === 'POST') { - const spoofedMethod = req.headers['x-original-method']; + if (req.method === METHOD_SPOOF_ORIGINAL_METHOD) { + const spoofedMethod = req.headers[METHOD_SPOOF_HEADER_NAME]; if (Array.isArray(spoofedMethod)) { return req; } diff --git a/packages/core/src/common/media-type.ts b/packages/core/src/common/media-type.ts index 3d0c70e..3332340 100644 --- a/packages/core/src/common/media-type.ts +++ b/packages/core/src/common/media-type.ts @@ -1,7 +1,12 @@ -export interface MediaType { +export interface MediaType< + Name extends string = string, + T extends object = object, + SerializeOpts extends unknown[] = [], + DeserializeOpts extends unknown[] = [] +> { name: Name; - serialize: (object: T) => string; - deserialize: (s: string) => T; + serialize: (object: T, ...args: SerializeOpts) => string; + deserialize: (s: string, ...args: DeserializeOpts) => T; } export const FALLBACK_MEDIA_TYPE = { diff --git a/packages/core/src/common/queries/common.ts b/packages/core/src/common/queries/common.ts index 5bb878a..b1500eb 100644 --- a/packages/core/src/common/queries/common.ts +++ b/packages/core/src/common/queries/common.ts @@ -1,4 +1,6 @@ -const DATA_SOURCE_QUERY_OPERATORS = [ +import {MediaType} from '../media-type'; + +const OPERATORS = [ '=', '!=', '>=', @@ -10,34 +12,40 @@ const DATA_SOURCE_QUERY_OPERATORS = [ 'REGEXP', ] as const; -type DataSourceQueryOperator = typeof DATA_SOURCE_QUERY_OPERATORS[number]; +type QueryOperator = typeof OPERATORS[number]; + +type QueryExpressionValue = string | number | boolean; -interface DataSourceQueryOperatorExpression { +interface QueryOperatorExpression { lhs: string; - operator: DataSourceQueryOperator; - rhs: string | number | boolean | RegExp; + operator: QueryOperator; + rhs: QueryExpressionValue | RegExp; +} + +interface QueryFunctionExpression { + name: string; + args: QueryExpressionValue[]; } -interface DataSourceQueryOrGrouping { +export type QueryAnyExpression = QueryOperatorExpression | QueryFunctionExpression; + +export interface QueryOrGrouping { type: 'or'; - expressions: DataSourceQueryOperatorExpression[]; + expressions: QueryAnyExpression[]; } -interface DataSourceQueryAndGrouping { +export interface QueryAndGrouping { type: 'and'; - expressions: DataSourceQueryOrGrouping[]; + expressions: QueryOrGrouping[]; } -export type DataSourceQueryExpression = DataSourceQueryOperatorExpression; - -export type DataSourceQuery = DataSourceQueryAndGrouping; - export interface QueryMediaType< Name extends string = string, SerializeOptions extends {} = {}, DeserializeOptions extends {} = {} -> { - name: Name; - serialize: (object: DataSourceQuery, opts?: SerializeOptions) => string; - deserialize: (s: string, opts?: DeserializeOptions) => DataSourceQuery; -} +> extends MediaType< + Name, + QueryAndGrouping, + [SerializeOptions], + [DeserializeOptions] +> {} diff --git a/packages/core/src/common/queries/media-types/application/x-www-form-urlencoded.ts b/packages/core/src/common/queries/media-types/application/x-www-form-urlencoded.ts index 6d36f90..72b42e5 100644 --- a/packages/core/src/common/queries/media-types/application/x-www-form-urlencoded.ts +++ b/packages/core/src/common/queries/media-types/application/x-www-form-urlencoded.ts @@ -1,4 +1,9 @@ -import {QueryMediaType, DataSourceQuery, DataSourceQueryExpression} from '../../common'; +import { + QueryMediaType, + QueryAndGrouping, + QueryAnyExpression, + QueryOrGrouping, +} from '../../common'; interface ProcessEntryBase { type: string; @@ -34,16 +39,17 @@ export type ProcessEntry = ProcessEntryString | ProcessEntryNumber | ProcessEntr export const name = 'application/x-www-form-urlencoded' as const; const normalizeRhs = (lhs: string, rhs: string, processEntriesMap?: Record) => { - const coerceValues = processEntriesMap?.[lhs] ?? { + const defaultCoerceValues = { type: 'string' - }; + } as ProcessEntry; + const coerceValues = processEntriesMap?.[lhs] ?? defaultCoerceValues; if (coerceValues?.type === 'number') { return { lhs, operator: '=', rhs: Number(rhs) - } as DataSourceQueryExpression; + } as QueryAnyExpression; } if (coerceValues?.type === 'boolean') { @@ -56,7 +62,7 @@ const normalizeRhs = (lhs: string, rhs: string, processEntriesMap?: Record { +const doesGroupHaveExpression = (ex2: QueryAnyExpression, key: string) => { + if ('operator' in ex2) { + return ex2.lhs === key; + } + + if ('name' in ex2) { + return ex2.name === key; + } + + return false; +}; + +export const deserialize: QueryMediaType< + typeof name, + SerializeOptions, + DeserializeOptions +>['deserialize'] = (s: string, options = {} as DeserializeOptions) => { const q = new URLSearchParams(s); + return Array.from(q.entries()).reduce( (queries, [key, value]) => { - const existingOr = queries.expressions - .find((ex) => ex.expressions.some((ex2) => ex2.lhs === key)) ?? { + const defaultOr = { type: 'or', expressions: [], - }; - const existingLhs = existingOr.expressions.find((ex) => ex.lhs === key); + } as QueryOrGrouping; + const existingOr = queries.expressions.find((ex) => ( + ex.expressions.some((ex2) => doesGroupHaveExpression(ex2, key)) + )) ?? defaultOr; + const existingLhs = existingOr.expressions.find((ex) => doesGroupHaveExpression(ex, key)); const newExpression = normalizeRhs(key, value, options.processEntries); if (typeof existingLhs === 'undefined') { @@ -145,7 +167,7 @@ export const deserialize: QueryMediaType ( - ex.expressions.some((ex2) => ex2.lhs !== key) + ex.expressions.some((ex2) => !doesGroupHaveExpression(ex2, key)) ? ex : { ...existingOr, @@ -160,49 +182,58 @@ export const deserialize: QueryMediaType['serialize'] = (q: DataSourceQuery) => ( +const serializeExpression = (ex2: QueryAnyExpression) => { + if ('name' in ex2) { + return [ex2.name, `(${ex2.args.map((s) => s.toString()).join(',')})`]; + } + + if (ex2.rhs instanceof RegExp) { + if (ex2.operator !== 'REGEXP') { + throw new Error(`Invalid rhs given for operator: ${ex2.lhs} ${ex2.operator} `); + } + + return [ex2.lhs, ex2.rhs.toString()]; + } + + switch (typeof ex2.rhs) { + case 'string': { + switch (ex2.operator) { + case 'ILIKE': + case 'LIKE': + return [ex2.lhs, ex2.rhs.replace(/^%+/, '').replace(/%+$/, '')]; + case '=': + return [ex2.lhs, ex2.rhs]; + default: + break; + } + throw new Error(`Invalid operator given for lhs: ${ex2.lhs} ${ex2.rhs}`); + } + case 'number': { + return [ex2.lhs, ex2.rhs.toString()]; + } + case 'boolean': { + return [ex2.lhs, ex2.rhs ? 'true' : 'false']; + } + default: + break; + } + + throw new Error(`Unknown type for rhs: ${ex2.lhs} ${ex2.operator} `); +}; + +export const serialize: QueryMediaType< + typeof name, + SerializeOptions, + DeserializeOptions +>['serialize'] = (q: QueryAndGrouping) => ( new URLSearchParams( - q.expressions.map( - (ex) => ex.expressions.map((ex2) => { - if (ex2.rhs instanceof RegExp) { - if (ex2.operator !== 'REGEXP') { - throw new Error(`Invalid rhs given for operator: ${ex2.lhs} ${ex2.operator} `); - } - - return [ex2.lhs, ex2.rhs.toString()]; - } - - switch (typeof ex2.rhs) { - case 'string': { - switch (ex2.operator) { - case 'ILIKE': - case 'LIKE': - return [ex2.lhs, ex2.rhs.replace(/^%+/, '').replace(/%+$/, '')]; - case '=': - return [ex2.lhs, ex2.rhs]; - default: - break; - } - throw new Error(`Invalid operator given for lhs: ${ex2.lhs} ${ex2.rhs}`); - } - case 'number': { - return [ex2.lhs, ex2.rhs.toString()]; - } - case 'boolean': { - return [ex2.lhs, ex2.rhs ? 'true' : 'false']; - } - default: - break; - } - - throw new Error(`Unknown type for rhs: ${ex2.lhs} ${ex2.operator} `); - }) - ) - .flat() + q.expressions.flatMap((ex) => ( + ex.expressions.map((ex2) => serializeExpression(ex2)) + )) ) .toString() );