Add QUERY method to handlers

Include QUERY method when fetch collection permission is granted for a
resource.

packages/core/src/backend/data-source/common.ts

@@ -1,6 +1,5 @@
 import * as v from 'valibot';
-import {Resource} from '../../common';
-import {DataSourceQuery} from './queries';
+import {Resource, DataSourceQuery} from '../../common';
 
 type IsCreated = boolean;
 
@@ -36,5 +35,3 @@ export interface ResourceIdConfig<IdSchema extends v.BaseSchema> {
 export interface GenerationStrategy {
 	(dataSource: DataSource, ...args: unknown[]): Promise<string | number | unknown>;
 }
-
-export * from './queries';

packages/core/src/backend/data-source/index.ts

@@ -1,2 +1 @@
 export * from './common';
-export * from './queries';

packages/core/src/backend/servers/http/core.ts

@@ -18,9 +18,10 @@ import {
 	getAcceptPostString,
 	LanguageDefaultErrorStatusMessageKey,
 	PATCH_CONTENT_MAP_TYPE, PATCH_CONTENT_TYPES,
-	PatchContentType,
+	PatchContentType, queryMediaTypes,
 	Resource,
 } from '../../../common';
+import {DataSource} from '../../data-source';
 import {
 	handleGetRoot, handleOptions,
 } from './handlers/default';
@@ -31,6 +32,7 @@ import {
 	handleGetCollection,
 	handleGetItem,
 	handlePatchItem,
+	handleQueryCollection,
 } from './handlers/resource';
 import {getBody, isTextMediaType} from './utils';
 import {decorateRequestWithBackend} from './decorators/backend';
@@ -38,7 +40,6 @@ import {decorateRequestWithMethod} from './decorators/method';
 import {decorateRequestWithUrl} from './decorators/url';
 import {ErrorPlainResponse, PlainResponse} from './response';
 import EventEmitter from 'events';
-import {DataSource} from '../../data-source';
 
 type RequiredResource = Required<Pick<RequestContext, 'resource'>>['resource'];
 
@@ -112,6 +113,11 @@ const constructPatchSchema = <T extends v.BaseSchema>(resource: Resource<BaseRes
 };
 // TODO add a way to define custom middlewares
 const defaultCollectionMiddlewares: AllowedMiddlewareSpecification[] = [
+	{
+		method: 'QUERY',
+		middleware: handleQueryCollection,
+		allowed: (resource) => resource.state.canFetchCollection,
+	},
 	{
 		method: 'GET',
 		middleware: handleGetCollection,
@@ -211,7 +217,55 @@ export const createServer = (backendState: BackendState, serverParams = {} as Cr
 			return currentHandlerState;
 		}
 
-		if (typeof constructBodySchema === 'function') {
+		if (effectiveMethod === 'QUERY') {
+			const contentTypeHeader = req.headers['content-type'] ?? 'application/octet-stream';
+			const fragments = contentTypeHeader.replace(/\s+/g, ' ').split(';');
+			const mediaType = fragments[0];
+			const charsetParam = (
+				fragments
+					.map((s) => s.trim())
+					.find((f) => f.startsWith('charset='))
+
+				?? (
+					isTextMediaType(mediaType)
+						? 'charset=utf-8'
+						: 'charset=binary'
+				)
+			);
+			const [_charsetKey, charsetRaw] = charsetParam.split('=').map((s) => s.trim());
+			const charset = (
+				(
+					(charsetRaw.startsWith('"') && charsetRaw.endsWith('"'))
+					|| (charsetRaw.startsWith("'") && charsetRaw.endsWith("'"))
+				)
+					? charsetRaw.slice(1, -1).trim()
+					: charsetRaw.trim()
+			) ?? (isTextMediaType(mediaType) ? 'utf-8' : 'binary');
+
+			const theBodyBuffer = await getBody(req);
+			const encodingPair = req.backend.app.charsets.get(charset);
+			if (typeof encodingPair === 'undefined') {
+				throw new ErrorPlainResponse('unableToDecodeResource', {
+					statusCode: constants.HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE,
+					res: theRes,
+				});
+			}
+
+			const deserializerPair = Object.values(queryMediaTypes)
+				.find((a) => a.name === mediaType);
+			if (typeof deserializerPair === 'undefined') {
+				throw new ErrorPlainResponse(
+					'unableToDeserializeRequest',
+					{
+						statusCode: constants.HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE,
+						res: theRes,
+					},
+				);
+			}
+
+			const theBodyStr = encodingPair.decode(theBodyBuffer);
+			req.body = deserializerPair.deserialize(theBodyStr);
+		} else if (typeof constructBodySchema === 'function') {
 			const bodySchema = constructBodySchema(req.resource, req.resourceId);
 			const contentTypeHeader = req.headers['content-type'] ?? 'application/octet-stream';
 			const fragments = contentTypeHeader.replace(/\s+/g, ' ').split(';');

packages/core/src/backend/servers/http/handlers/resource.ts

@@ -1,27 +1,77 @@
 import { constants } from 'http2';
 import * as v from 'valibot';
-import {Middleware, convertToDataSourceQuery} from '../../../common';
+import {Middleware} from '../../../common';
 import {ErrorPlainResponse, PlainResponse} from '../response';
 import assert from 'assert';
 import {
-	applyDelta,
+	applyDelta, DataSourceQuery,
 	Delta,
 	PATCH_CONTENT_MAP_TYPE,
 	PatchContentType,
+	queryMediaTypes,
 } from '../../../../common';
 
 // TODO add handleQueryCollection()
 
+export const handleQueryCollection: Middleware = async (req, res) => {
+	const {
+		body,
+		resource,
+		backend,
+	} = req;
+
+	let data: v.Output<typeof resource.schema>[];
+	let totalItemCount: number | undefined;
+	try {
+		// check which attributes have specifics on the queries (e.g. fuzzy search on strings)
+		const dataSourceQuery = body as DataSourceQuery;
+		data = await resource.dataSource.getMultiple(dataSourceQuery); // TODO paginated responses per resource
+		if (backend.showTotalItemCountOnGetCollection && typeof resource.dataSource.getTotalCount === 'function') {
+			totalItemCount = await resource.dataSource.getTotalCount(dataSourceQuery);
+		}
+	} catch (cause) {
+		throw new ErrorPlainResponse(
+			'unableToFetchResourceCollection',
+			{
+				cause,
+				statusCode: constants.HTTP_STATUS_INTERNAL_SERVER_ERROR,
+				res,
+			}
+		);
+	}
+
+	const headers: Record<string, string> = {};
+	if (typeof totalItemCount !== 'undefined') {
+		headers['X-Resource-Total-Item-Count'] = totalItemCount.toString();
+	}
+
+	return new PlainResponse({
+		headers,
+		statusCode: constants.HTTP_STATUS_OK,
+		statusMessage: 'resourceCollectionFetched',
+		body: data,
+		res,
+	});
+};
+
 export const handleGetCollection: Middleware = async (req, res) => {
-	const { query, resource, backend } = req;
+	const {
+		// TODO don't turn query into URLSearchParams just yet
+		query,
+		resource,
+		backend,
+	} = req;
 
 	let data: v.Output<typeof resource.schema>[];
 	let totalItemCount: number | undefined;
 	try {
 		// check which attributes have specifics on the queries (e.g. fuzzy search on strings)
-		const dataSourceQuery = convertToDataSourceQuery(query);
+		const dataSourceQuery = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize(
+			query.toString()
+			// TODO compute processEntry options based on resource attribute metadata (e.g. fulltext, queryable attributes - firstname, lastname, middlename)
+		);
 		data = await resource.dataSource.getMultiple(dataSourceQuery); // TODO paginated responses per resource
-		if (backend!.showTotalItemCountOnGetCollection && typeof resource.dataSource.getTotalCount === 'function') {
+		if (backend.showTotalItemCountOnGetCollection && typeof resource.dataSource.getTotalCount === 'function') {
 			totalItemCount = await resource.dataSource.getTotalCount(dataSourceQuery);
 		}
 	} catch (cause) {

packages/core/src/common/index.ts

@@ -8,6 +8,7 @@ export * from './delta';
 export * from './media-type';
 export * from './resource';
 export * from './language';
+export * from './queries';
 export * as validation from './validation';
 
 export interface ContentNegotiation {

packages/core/src/common/language.ts

@@ -37,6 +37,7 @@ export const LANGUAGE_DEFAULT_STATUS_MESSAGE_KEYS = [
 	...LANGUAGE_DEFAULT_ERROR_STATUS_MESSAGE_KEYS,
 	'ok',
 	'resourceCollectionFetched',
+	'resourceCollectionQueried',
 	'resourceFetched',
 	'resourceDeleted',
 	'resourcePatched',
@@ -80,6 +81,7 @@ export const FALLBACK_LANGUAGE = {
 		ok: 'OK',
 		provideOptions: 'Provide Options',
 		resourceCollectionFetched: '$RESOURCE Collection Fetched',
+		resourceCollectionQueried: '$RESOURCE Collection Queried',
 		resourceFetched: '$RESOURCE Fetched',
 		resourceNotFound: '$RESOURCE Not Found',
 		deleteNonExistingResource: 'Delete Non-Existing $RESOURCE',

packages/core/src/common/queries/common.ts

@@ -0,0 +1,43 @@
+const DATA_SOURCE_QUERY_OPERATORS = [
+	'=',
+	'!=',
+	'>=',
+	'<=',
+	'>',
+	'<',
+	'LIKE',
+	'ILIKE',
+	'REGEXP',
+] as const;
+
+type DataSourceQueryOperator = typeof DATA_SOURCE_QUERY_OPERATORS[number];
+
+interface DataSourceQueryOperatorExpression {
+	lhs: string;
+	operator: DataSourceQueryOperator;
+	rhs: string | number | boolean | RegExp;
+}
+
+interface DataSourceQueryOrGrouping {
+	type: 'or';
+	expressions: DataSourceQueryOperatorExpression[];
+}
+
+interface DataSourceQueryAndGrouping {
+	type: 'and';
+	expressions: DataSourceQueryOrGrouping[];
+}
+
+export type DataSourceQueryExpression = DataSourceQueryOperatorExpression;
+
+export type DataSourceQuery = DataSourceQueryAndGrouping;
+
+export interface QueryMediaType<
+	Name extends string = string,
+	SerializeOptions extends {} = {},
+	DeserializeOptions extends {} = {}
+> {
+	name: Name;
+	serialize: (object: DataSourceQuery, opts?: SerializeOptions) => string;
+	deserialize: (s: string, opts?: DeserializeOptions) => DataSourceQuery;
+}

packages/core/src/common/queries/index.ts

@@ -0,0 +1,2 @@
+export * from './common';
+export * as queryMediaTypes from './media-types';

packages/core/src/common/queries/media-types/application/x-www-form-urlencoded.ts

@@ -0,0 +1,208 @@
+import {QueryMediaType, DataSourceQuery, DataSourceQueryExpression} from '../../common';
+
+interface ProcessEntryBase {
+	type: string;
+}
+
+const AVAILABLE_MATCH_TYPES = [
+	'startsWith',
+	'endsWith',
+	'includes',
+	'regexp',
+] as const;
+
+type MatchType = typeof AVAILABLE_MATCH_TYPES[number];
+
+interface ProcessEntryString extends ProcessEntryBase {
+	type: 'string';
+	matchType?: MatchType;
+	caseInsensitiveMatch?: boolean;
+}
+
+interface ProcessEntryNumber extends ProcessEntryBase {
+	type: 'number';
+	decimal?: boolean;
+}
+
+interface ProcessEntryBoolean extends ProcessEntryBase {
+	type: 'boolean';
+	truthyStrings?: string[];
+}
+
+export type ProcessEntry = ProcessEntryString | ProcessEntryNumber | ProcessEntryBoolean;
+
+export const name = 'application/x-www-form-urlencoded' as const;
+
+const normalizeRhs = (lhs: string, rhs: string, processEntriesMap?: Record<string, ProcessEntry>) => {
+	const coerceValues = processEntriesMap?.[lhs] ?? {
+		type: 'string'
+	};
+
+	if (coerceValues?.type === 'number') {
+		return {
+			lhs,
+			operator: '=',
+			rhs: Number(rhs)
+		} as DataSourceQueryExpression;
+	}
+
+	if (coerceValues?.type === 'boolean') {
+		const truthyStrings = [
+			...(coerceValues.truthyStrings ?? []).map((s) => s.trim().toLowerCase()),
+			'true',
+		];
+
+		return {
+			lhs,
+			operator: '=',
+			rhs: truthyStrings.includes(rhs)
+		} as DataSourceQueryExpression;
+	}
+
+	if (coerceValues?.type === 'string') {
+		switch (coerceValues?.matchType) {
+			case 'startsWith': {
+				return {
+					lhs,
+					operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE',
+					rhs: `%${rhs}`,
+				} as DataSourceQueryExpression;
+			}
+			case 'endsWith': {
+				return {
+					lhs,
+					operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE',
+					rhs: `${rhs}%`,
+				} as DataSourceQueryExpression;
+			}
+			case 'includes': {
+				return {
+					lhs,
+					operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE',
+					rhs: `%${rhs}%`,
+				} as DataSourceQueryExpression;
+			}
+			case 'regexp': {
+				return {
+					lhs,
+					operator: 'REGEXP',
+					rhs: new RegExp(rhs, coerceValues.caseInsensitiveMatch ? 'i' : ''),
+				} as DataSourceQueryExpression;
+			}
+			default:
+				break;
+		}
+
+		return {
+			lhs,
+			operator: '=',
+			rhs,
+		} as DataSourceQueryExpression;
+	}
+
+	const unknownCoerceValues = coerceValues as unknown as Record<string, string>;
+	throw new Error(`Invalid coercion type: ${unknownCoerceValues.type}`);
+	// this will be sent to the data source, e.g. the SQL query
+	// we can also make this function act as a "sanitizer"
+}
+
+interface SerializeOptions {}
+
+interface DeserializeOptions {
+	processEntries?: Record<string, ProcessEntry>;
+}
+
+export const deserialize: QueryMediaType<typeof name, SerializeOptions, DeserializeOptions>['deserialize'] = (
+	s: string,
+	options = {} as DeserializeOptions
+) => {
+	const q = new URLSearchParams(s);
+	return Array.from(q.entries()).reduce(
+		(queries, [key, value]) => {
+			const existingOr = queries.expressions
+				.find((ex) => ex.expressions.some((ex2) => ex2.lhs === key)) ?? {
+				type: 'or',
+				expressions: [],
+			};
+			const existingLhs = existingOr.expressions.find((ex) => ex.lhs === key);
+			const newExpression = normalizeRhs(key, value, options.processEntries);
+
+			if (typeof existingLhs === 'undefined') {
+				return {
+					...queries,
+					expressions: [
+						...queries.expressions,
+						{
+							type: 'or',
+							expressions: [
+								newExpression,
+							],
+						},
+					],
+				};
+			}
+
+			return {
+				...queries,
+				expressions: queries.expressions.map((ex) => (
+					ex.expressions.some((ex2) => ex2.lhs !== key)
+						? ex
+						: {
+							...existingOr,
+							expressions: [
+								...(existingOr.expressions ?? []),
+								newExpression,
+							],
+						}
+				)),
+			};
+		},
+		{
+			type: 'and',
+			expressions: [],
+		} as DataSourceQuery
+	)
+};
+
+export const serialize: QueryMediaType<typeof name, SerializeOptions, DeserializeOptions>['serialize'] = (q: DataSourceQuery) => (
+	new URLSearchParams(
+		q.expressions.map(
+			(ex) => ex.expressions.map((ex2) => {
+				if (ex2.rhs instanceof RegExp) {
+					if (ex2.operator !== 'REGEXP') {
+						throw new Error(`Invalid rhs given for operator: ${ex2.lhs} ${ex2.operator} <rhs>`);
+					}
+
+					return [ex2.lhs, ex2.rhs.toString()];
+				}
+
+				switch (typeof ex2.rhs) {
+					case 'string': {
+						switch (ex2.operator) {
+							case 'ILIKE':
+							case 'LIKE':
+								return [ex2.lhs, ex2.rhs.replace(/^%+/, '').replace(/%+$/, '')];
+							case '=':
+								return [ex2.lhs, ex2.rhs];
+							default:
+								break;
+						}
+						throw new Error(`Invalid operator given for lhs: ${ex2.lhs} <op> ${ex2.rhs}`);
+					}
+					case 'number': {
+						return [ex2.lhs, ex2.rhs.toString()];
+					}
+					case 'boolean': {
+						return [ex2.lhs, ex2.rhs ? 'true' : 'false'];
+					}
+					default:
+						break;
+				}
+
+				throw new Error(`Unknown type for rhs: ${ex2.lhs} ${ex2.operator} <rhs>`);
+			})
+		)
+			.flat()
+	)
+		.toString()
+);

packages/core/src/common/queries/media-types/index.ts

@@ -0,0 +1 @@
+export * as applicationXWwwFormUrlencoded from './application/x-www-form-urlencoded';

packages/core/test/features/query.test.ts

@@ -1,10 +1,10 @@
 import {describe, expect, it} from 'vitest';
-import { convertToDataSourceQuery } from '../../src/backend';
+import { queryMediaTypes } from '../../src/common';
 
 describe('query', () => {
 	it('returns a data source query collection from search params', () => {
 		const q = new URLSearchParams();
-		const collection = convertToDataSourceQuery(q);
+		const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize(q.toString());
 		expect(collection.expressions).toBeInstanceOf(Array);
 	});
 
@@ -13,8 +13,8 @@ describe('query', () => {
 			attr: '2',
 			attr2: '2',
 		});
-		const collection = convertToDataSourceQuery(
-			q,
+		const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize(
+			q.toString(),
 			{
 				processEntries: {
 					attr: {
@@ -57,8 +57,8 @@ describe('query', () => {
 			attr3: 'true',
 			attr4: 'false',
 		});
-		const collection = convertToDataSourceQuery(
-			q,
+		const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize(
+			q.toString(),
 			{
 				processEntries: {
 					attr: {
@@ -121,7 +121,9 @@ describe('query', () => {
 		const q = new URLSearchParams({
 			attr: 'foo'
 		});
-		const collection = convertToDataSourceQuery(q);
+		const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize(
+			q.toString()
+		);
 		expect(collection).toEqual({
 			type: 'and',
 			expressions: [
@@ -144,7 +146,9 @@ describe('query', () => {
 			['attr', 'foo'],
 			['attr2', 'bar'],
 		]);
-		const collection = convertToDataSourceQuery(q);
+		const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize(
+			q.toString()
+		);
 		expect(collection).toEqual({
 			type: 'and',
 			expressions: [
@@ -177,7 +181,9 @@ describe('query', () => {
 			['attr', 'foo'],
 			['attr', 'bar'],
 		]);
-		const collection = convertToDataSourceQuery(q);
+		const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize(
+			q.toString()
+		);
 		expect(collection).toEqual({
 			type: 'and',
 			expressions: [
@@ -208,7 +214,9 @@ describe('query', () => {
 			['attr', 'bar'],
 			['attr2', 'baz'],
 		]);
-		const collection = convertToDataSourceQuery(q);
+		const collection = queryMediaTypes.applicationXWwwFormUrlencoded.deserialize(
+			q.toString()
+		);
 		expect(collection).toEqual({
 			type: 'and',
 			expressions: [