Browse Source

Update query

Include string matching operators.
master
TheoryOfNekomata 6 months ago
parent
commit
49d8f3f5d1
7 changed files with 125 additions and 59 deletions
  1. +4
    -0
      README.md
  2. +3
    -1
      packages/core/src/backend/common.ts
  3. +4
    -2
      packages/core/src/backend/data-source/common.ts
  4. +97
    -42
      packages/core/src/backend/data-source/queries.ts
  5. +7
    -4
      packages/core/src/backend/servers/http/handlers/resource.ts
  6. +2
    -2
      packages/core/src/client/index.ts
  7. +8
    -8
      packages/core/test/features/query.test.ts

+ 4
- 0
README.md View File

@@ -77,3 +77,7 @@ See [docs folder](./docs) for more details.
https://url.spec.whatwg.org/#urlencoded-parsing https://url.spec.whatwg.org/#urlencoded-parsing


https://url.spec.whatwg.org/#interface-urlsearchparams https://url.spec.whatwg.org/#interface-urlsearchparams

- 303 See Other

https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/303

+ 3
- 1
packages/core/src/backend/common.ts View File

@@ -47,7 +47,7 @@ export type RequestDecorator = (req: RequestContext) => RequestContext | Promise
export type ParamRequestDecorator<Params extends Array<unknown> = []> = (...args: Params) => RequestDecorator; export type ParamRequestDecorator<Params extends Array<unknown> = []> = (...args: Params) => RequestDecorator;


// TODO put this in HTTP // TODO put this in HTTP
export type Method = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD';
export type Method = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'OPTIONS' | 'HEAD' | 'QUERY';


export interface AllowedMiddlewareSpecification<Schema extends BaseSchema = BaseSchema> { export interface AllowedMiddlewareSpecification<Schema extends BaseSchema = BaseSchema> {
method: Method; method: Method;
@@ -71,3 +71,5 @@ export const getAllowString = (middlewares: AllowedMiddlewareSpecification[]) =>
const allowedMethods = middlewares.flatMap((m) => m.method === 'GET' ? [m.method, 'HEAD'] : [m.method]); const allowedMethods = middlewares.flatMap((m) => m.method === 'GET' ? [m.method, 'HEAD'] : [m.method]);
return allowedMethods.join(','); return allowedMethods.join(',');
} }

export * from './data-source';

+ 4
- 2
packages/core/src/backend/data-source/common.ts View File

@@ -1,6 +1,6 @@
import * as v from 'valibot'; import * as v from 'valibot';
import {Resource} from '../../common'; import {Resource} from '../../common';
import {DataSourceQueryExpression} from './queries';
import {DataSourceQuery} from './queries';


type IsCreated = boolean; type IsCreated = boolean;


@@ -11,7 +11,7 @@ type DeleteResult = unknown;
export interface DataSource< export interface DataSource<
ItemData extends object = object, ItemData extends object = object,
ID extends unknown = unknown, ID extends unknown = unknown,
Query extends DataSourceQueryExpression = DataSourceQueryExpression,
Query extends DataSourceQuery = DataSourceQuery,
> { > {
initialize(): Promise<unknown>; initialize(): Promise<unknown>;
getTotalCount?(query?: Query): Promise<TotalCount>; getTotalCount?(query?: Query): Promise<TotalCount>;
@@ -36,3 +36,5 @@ export interface ResourceIdConfig<IdSchema extends v.BaseSchema> {
export interface GenerationStrategy { export interface GenerationStrategy {
(dataSource: DataSource, ...args: unknown[]): Promise<string | number | unknown>; (dataSource: DataSource, ...args: unknown[]): Promise<string | number | unknown>;
} }

export * from './queries';

+ 97
- 42
packages/core/src/backend/data-source/queries.ts View File

@@ -1,49 +1,60 @@
export const DIRECTIVE_MAP = {
'=': 'equals',
'!=': 'not-equals',
'>=': 'greater-than-equal',
'<=': 'less-than-equal',
'>': 'greater-than',
'<': 'less-than',
};

export const DATA_SOURCE_QUERY_OPERATORS = Object.keys(DIRECTIVE_MAP) as (keyof typeof DIRECTIVE_MAP)[];

export type DataSourceQueryOperator = typeof DATA_SOURCE_QUERY_OPERATORS[number];

export interface DataSourceQueryExpression {
const DATA_SOURCE_QUERY_OPERATORS = [
'=',
'!=',
'>=',
'<=',
'>',
'<',
'LIKE',
'ILIKE',
'REGEXP'
] as const;

type DataSourceQueryOperator = typeof DATA_SOURCE_QUERY_OPERATORS[number];

interface DataSourceQueryExpression {
lhs: string; lhs: string;
operator: DataSourceQueryOperator; operator: DataSourceQueryOperator;
rhs: string | number | boolean;
rhs: string | number | boolean | RegExp;
} }


export interface DataSourceQueryOrGrouping {
interface DataSourceQueryOrGrouping {
type: 'or'; type: 'or';
expressions: DataSourceQueryExpression[]; expressions: DataSourceQueryExpression[];
} }


export interface DataSourceQueryAndGrouping {
interface DataSourceQueryAndGrouping {
type: 'and'; type: 'and';
expressions: DataSourceQueryOrGrouping[]; expressions: DataSourceQueryOrGrouping[];
} }


export type DataSourceQuery = DataSourceQueryAndGrouping;

interface ProcessEntryBase { interface ProcessEntryBase {
type: string; type: string;
} }


export interface ProcessEntryString extends ProcessEntryBase {
const AVAILABLE_MATCH_TYPES = [
'startsWith',
'endsWith',
'includes',
'regexp',
] as const;

type MatchType = typeof AVAILABLE_MATCH_TYPES[number];

interface ProcessEntryString extends ProcessEntryBase {
type: 'string'; type: 'string';
startsWith?: boolean;
endsWith?: boolean;
caseInsensitive?: boolean;
includes?: boolean;
matchType?: MatchType;
caseInsensitiveMatch?: boolean;
} }


export interface ProcessEntryNumber extends ProcessEntryBase {
interface ProcessEntryNumber extends ProcessEntryBase {
type: 'number'; type: 'number';
decimal?: boolean;
} }


export interface ProcessEntryBoolean extends ProcessEntryBase {
interface ProcessEntryBoolean extends ProcessEntryBase {
type: 'boolean'; type: 'boolean';
truthyStrings?: string[]; truthyStrings?: string[];
} }
@@ -52,13 +63,19 @@ export type ProcessEntry = ProcessEntryString | ProcessEntryNumber | ProcessEntr


interface ConvertToDataSourceQueryCollectionOptions { interface ConvertToDataSourceQueryCollectionOptions {
processEntries: Record<string, ProcessEntry>; processEntries: Record<string, ProcessEntry>;
//coerceValues: CoerceValues;
//stringSearch: StringSearch;
} }


const normalizeRhs = (rhs: string, coerceValues?: ProcessEntry) => {
const normalizeRhs = (lhs: string, rhs: string, processEntriesMap?: Record<string, ProcessEntry>) => {
const coerceValues = processEntriesMap?.[lhs] ?? {
type: 'string'
};

if (coerceValues?.type === 'number') { if (coerceValues?.type === 'number') {
return Number(rhs);
return {
lhs,
operator: '=',
rhs: Number(rhs)
} as DataSourceQueryExpression;
} }


if (coerceValues?.type === 'boolean') { if (coerceValues?.type === 'boolean') {
@@ -67,15 +84,61 @@ const normalizeRhs = (rhs: string, coerceValues?: ProcessEntry) => {
'true', 'true',
]; ];


return truthyStrings.includes(rhs);
return {
lhs,
operator: '=',
rhs: truthyStrings.includes(rhs)
} as DataSourceQueryExpression;
}

if (coerceValues?.type === 'string') {
switch (coerceValues?.matchType) {
case 'startsWith': {
return {
lhs,
operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE',
rhs: `%${rhs}`,
} as DataSourceQueryExpression;
}
case 'endsWith': {
return {
lhs,
operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE',
rhs: `${rhs}%`,
} as DataSourceQueryExpression;
}
case 'includes': {
return {
lhs,
operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE',
rhs: `%${rhs}%`,
} as DataSourceQueryExpression;
}
case 'regexp': {
return {
lhs,
operator: 'REGEXP',
rhs: new RegExp(rhs, coerceValues.caseInsensitiveMatch ? 'i' : ''),
} as DataSourceQueryExpression;
}
default:
break;
}

return {
lhs,
operator: '=',
rhs,
} as DataSourceQueryExpression;
} }


return rhs?.toString() ?? '';
const unknownCoerceValues = coerceValues as unknown as Record<string, string>;
throw new Error(`Invalid coercion type: ${unknownCoerceValues.type}`);
// this will be sent to the data source, e.g. the SQL query // this will be sent to the data source, e.g. the SQL query
// we can also make this function act as a "sanitizer" // we can also make this function act as a "sanitizer"
} }


export const convertToDataSourceQueryCollection = (
export const convertToDataSourceQuery = (
q: URLSearchParams, q: URLSearchParams,
options = {} as ConvertToDataSourceQueryCollectionOptions, options = {} as ConvertToDataSourceQueryCollectionOptions,
): DataSourceQueryAndGrouping => { ): DataSourceQueryAndGrouping => {
@@ -87,7 +150,7 @@ export const convertToDataSourceQueryCollection = (
expressions: [], expressions: [],
}; };
const existingLhs = existingOr.expressions.find((ex) => ex.lhs === key); const existingLhs = existingOr.expressions.find((ex) => ex.lhs === key);
const rhs = normalizeRhs(value, options.processEntries?.[key]);
const newExpression = normalizeRhs(key, value, options.processEntries);


if (typeof existingLhs === 'undefined') { if (typeof existingLhs === 'undefined') {
return { return {
@@ -97,11 +160,7 @@ export const convertToDataSourceQueryCollection = (
{ {
type: 'or', type: 'or',
expressions: [ expressions: [
{
lhs: key,
operator: '=',
rhs,
},
newExpression,
], ],
}, },
], ],
@@ -117,11 +176,7 @@ export const convertToDataSourceQueryCollection = (
...existingOr, ...existingOr,
expressions: [ expressions: [
...(existingOr.expressions ?? []), ...(existingOr.expressions ?? []),
{
lhs: key,
operator: '=',
rhs,
},
newExpression,
], ],
} }
)), )),


+ 7
- 4
packages/core/src/backend/servers/http/handlers/resource.ts View File

@@ -1,6 +1,6 @@
import { constants } from 'http2'; import { constants } from 'http2';
import * as v from 'valibot'; import * as v from 'valibot';
import {Middleware} from '../../../common';
import {Middleware, convertToDataSourceQuery} from '../../../common';
import {ErrorPlainResponse, PlainResponse} from '../response'; import {ErrorPlainResponse, PlainResponse} from '../response';
import assert from 'assert'; import assert from 'assert';
import { import {
@@ -10,16 +10,19 @@ import {
PatchContentType, PatchContentType,
} from '../../../../common'; } from '../../../../common';


// TODO add handleQueryCollection()

export const handleGetCollection: Middleware = async (req, res) => { export const handleGetCollection: Middleware = async (req, res) => {
const { query, resource, backend } = req; const { query, resource, backend } = req;


let data: v.Output<typeof resource.schema>[]; let data: v.Output<typeof resource.schema>[];
let totalItemCount: number | undefined; let totalItemCount: number | undefined;
try { try {
// TODO querying mechanism
data = await resource.dataSource.getMultiple(query); // TODO paginated responses per resource
// check which attributes have specifics on the queries (e.g. fuzzy search on strings)
const dataSourceQuery = convertToDataSourceQuery(query);
data = await resource.dataSource.getMultiple(dataSourceQuery); // TODO paginated responses per resource
if (backend!.showTotalItemCountOnGetCollection && typeof resource.dataSource.getTotalCount === 'function') { if (backend!.showTotalItemCountOnGetCollection && typeof resource.dataSource.getTotalCount === 'function') {
totalItemCount = await resource.dataSource.getTotalCount(query);
totalItemCount = await resource.dataSource.getTotalCount(dataSourceQuery);
} }
} catch (cause) { } catch (cause) {
throw new ErrorPlainResponse( throw new ErrorPlainResponse(


+ 2
- 2
packages/core/src/client/index.ts View File

@@ -18,7 +18,7 @@ export interface ClientState {
export interface ClientBuilder { export interface ClientBuilder {
language(languageCode: ClientState['language']['name']): this; language(languageCode: ClientState['language']['name']): this;
charset(charset: ClientState['charset']['name']): this; charset(charset: ClientState['charset']['name']): this;
mediaTyoe(mediaType: ClientState['mediaType']['name']): this;
mediaType(mediaType: ClientState['mediaType']['name']): this;
} }


export interface CreateClientParams { export interface CreateClientParams {
@@ -34,7 +34,7 @@ export const createClient = (params: CreateClientParams) => {
}; };


return { return {
mediaTyoe(mediaTypeName) {
mediaType(mediaTypeName) {
const mediaType = clientState.app.mediaTypes.get(mediaTypeName); const mediaType = clientState.app.mediaTypes.get(mediaTypeName);
clientState.mediaType = mediaType ?? FALLBACK_MEDIA_TYPE; clientState.mediaType = mediaType ?? FALLBACK_MEDIA_TYPE;
return this; return this;


+ 8
- 8
packages/core/test/features/query.test.ts View File

@@ -1,10 +1,10 @@
import {describe, expect, it} from 'vitest'; import {describe, expect, it} from 'vitest';
import { convertToDataSourceQueryCollection } from '../../src/backend';
import { convertToDataSourceQuery } from '../../src/backend';


describe('query', () => { describe('query', () => {
it('returns a data source query collection from search params', () => { it('returns a data source query collection from search params', () => {
const q = new URLSearchParams(); const q = new URLSearchParams();
const collection = convertToDataSourceQueryCollection(q);
const collection = convertToDataSourceQuery(q);
expect(collection.expressions).toBeInstanceOf(Array); expect(collection.expressions).toBeInstanceOf(Array);
}); });


@@ -13,7 +13,7 @@ describe('query', () => {
attr: '2', attr: '2',
attr2: '2', attr2: '2',
}); });
const collection = convertToDataSourceQueryCollection(
const collection = convertToDataSourceQuery(
q, q,
{ {
processEntries: { processEntries: {
@@ -57,7 +57,7 @@ describe('query', () => {
attr3: 'true', attr3: 'true',
attr4: 'false', attr4: 'false',
}); });
const collection = convertToDataSourceQueryCollection(
const collection = convertToDataSourceQuery(
q, q,
{ {
processEntries: { processEntries: {
@@ -121,7 +121,7 @@ describe('query', () => {
const q = new URLSearchParams({ const q = new URLSearchParams({
attr: 'foo' attr: 'foo'
}); });
const collection = convertToDataSourceQueryCollection(q);
const collection = convertToDataSourceQuery(q);
expect(collection).toEqual({ expect(collection).toEqual({
type: 'and', type: 'and',
expressions: [ expressions: [
@@ -144,7 +144,7 @@ describe('query', () => {
['attr', 'foo'], ['attr', 'foo'],
['attr2', 'bar'], ['attr2', 'bar'],
]); ]);
const collection = convertToDataSourceQueryCollection(q);
const collection = convertToDataSourceQuery(q);
expect(collection).toEqual({ expect(collection).toEqual({
type: 'and', type: 'and',
expressions: [ expressions: [
@@ -177,7 +177,7 @@ describe('query', () => {
['attr', 'foo'], ['attr', 'foo'],
['attr', 'bar'], ['attr', 'bar'],
]); ]);
const collection = convertToDataSourceQueryCollection(q);
const collection = convertToDataSourceQuery(q);
expect(collection).toEqual({ expect(collection).toEqual({
type: 'and', type: 'and',
expressions: [ expressions: [
@@ -208,7 +208,7 @@ describe('query', () => {
['attr', 'bar'], ['attr', 'bar'],
['attr2', 'baz'], ['attr2', 'baz'],
]); ]);
const collection = convertToDataSourceQueryCollection(q);
const collection = convertToDataSourceQuery(q);
expect(collection).toEqual({ expect(collection).toEqual({
type: 'and', type: 'and',
expressions: [ expressions: [


Loading…
Cancel
Save