modal-soft
/
yasumi
1
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

Refactor query code 

Organize queries code to be set apart from the backend.
master
TheoryOfNekomata 7 months ago
parent
2250b5b5e9
commit
88b4de6a88
9 changed files with 132 additions and 280 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +0
    -2
      packages/core/src/backend/common.ts
  2. +3
    -1
      packages/core/src/backend/data-source.ts
  3. +0
    -1
      packages/core/src/backend/data-source/index.ts
  4. +0
    -194
      packages/core/src/backend/data-source/queries.ts
  5. +1
    -0
      packages/core/src/backend/index.ts
  6. +4
    -2
      packages/core/src/backend/servers/http/decorators/method/index.ts
  7. +8
    -3
      packages/core/src/common/media-type.ts
  8. +26
    -18
      packages/core/src/common/queries/common.ts
  9. +90
    -59
      packages/core/src/common/queries/media-types/application/x-www-form-urlencoded.ts

+ 0
- 2
packages/core/src/backend/common.ts View File

@@ -71,5 +71,3 @@ export const getAllowString = (middlewares: AllowedMiddlewareSpecification[]) =>
const allowedMethods = middlewares.flatMap((m) => m.method === 'GET' ? [m.method, 'HEAD'] : [m.method]);
return allowedMethods.join(',');
}

export * from './data-source';

packages/core/src/backend/data-source/common.ts → packages/core/src/backend/data-source.ts View File

@@ -1,5 +1,5 @@
import * as v from 'valibot';
import {Resource, DataSourceQuery} from '../../common';
import {Resource, QueryAndGrouping} from '../common';

type IsCreated = boolean;

@@ -7,6 +7,8 @@ type TotalCount = number;

type DeleteResult = unknown;

export type DataSourceQuery = QueryAndGrouping;

export interface DataSource<
ItemData extends object = object,
ID extends unknown = unknown,

+ 0
- 1
packages/core/src/backend/data-source/index.ts View File

@@ -1 +0,0 @@
export * from './common';

+ 0
- 194
packages/core/src/backend/data-source/queries.ts View File

@@ -1,194 +0,0 @@
const DATA_SOURCE_QUERY_OPERATORS = [
'=',
'!=',
'>=',
'<=',
'>',
'<',
'LIKE',
'ILIKE',
'REGEXP'
] as const;

type DataSourceQueryOperator = typeof DATA_SOURCE_QUERY_OPERATORS[number];

interface DataSourceQueryExpression {
lhs: string;
operator: DataSourceQueryOperator;
rhs: string | number | boolean | RegExp;
}

interface DataSourceQueryOrGrouping {
type: 'or';
expressions: DataSourceQueryExpression[];
}

interface DataSourceQueryAndGrouping {
type: 'and';
expressions: DataSourceQueryOrGrouping[];
}

export type DataSourceQuery = DataSourceQueryAndGrouping;

interface ProcessEntryBase {
type: string;
}

const AVAILABLE_MATCH_TYPES = [
'startsWith',
'endsWith',
'includes',
'regexp',
] as const;

type MatchType = typeof AVAILABLE_MATCH_TYPES[number];

interface ProcessEntryString extends ProcessEntryBase {
type: 'string';
matchType?: MatchType;
caseInsensitiveMatch?: boolean;
}

interface ProcessEntryNumber extends ProcessEntryBase {
type: 'number';
decimal?: boolean;
}

interface ProcessEntryBoolean extends ProcessEntryBase {
type: 'boolean';
truthyStrings?: string[];
}

export type ProcessEntry = ProcessEntryString | ProcessEntryNumber | ProcessEntryBoolean;

interface ConvertToDataSourceQueryCollectionOptions {
processEntries: Record<string, ProcessEntry>;
}

const normalizeRhs = (lhs: string, rhs: string, processEntriesMap?: Record<string, ProcessEntry>) => {
const coerceValues = processEntriesMap?.[lhs] ?? {
type: 'string'
};

if (coerceValues?.type === 'number') {
return {
lhs,
operator: '=',
rhs: Number(rhs)
} as DataSourceQueryExpression;
}

if (coerceValues?.type === 'boolean') {
const truthyStrings = [
...(coerceValues.truthyStrings ?? []).map((s) => s.trim().toLowerCase()),
'true',
];

return {
lhs,
operator: '=',
rhs: truthyStrings.includes(rhs)
} as DataSourceQueryExpression;
}

if (coerceValues?.type === 'string') {
switch (coerceValues?.matchType) {
case 'startsWith': {
return {
lhs,
operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE',
rhs: `%${rhs}`,
} as DataSourceQueryExpression;
}
case 'endsWith': {
return {
lhs,
operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE',
rhs: `${rhs}%`,
} as DataSourceQueryExpression;
}
case 'includes': {
return {
lhs,
operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE',
rhs: `%${rhs}%`,
} as DataSourceQueryExpression;
}
case 'regexp': {
return {
lhs,
operator: 'REGEXP',
rhs: new RegExp(rhs, coerceValues.caseInsensitiveMatch ? 'i' : ''),
} as DataSourceQueryExpression;
}
default:
break;
}

return {
lhs,
operator: '=',
rhs,
} as DataSourceQueryExpression;
}

const unknownCoerceValues = coerceValues as unknown as Record<string, string>;
throw new Error(`Invalid coercion type: ${unknownCoerceValues.type}`);
// this will be sent to the data source, e.g. the SQL query
// we can also make this function act as a "sanitizer"
}

export const convertToDataSourceQuery = (
q: URLSearchParams,
options = {} as ConvertToDataSourceQueryCollectionOptions,
): DataSourceQueryAndGrouping => {
return Array.from(q.entries()).reduce(
(queries, [key, value]) => {
const existingOr = queries.expressions
.find((ex) => ex.expressions.some((ex2) => ex2.lhs === key)) ?? {
type: 'or',
expressions: [],
};
const existingLhs = existingOr.expressions.find((ex) => ex.lhs === key);
const newExpression = normalizeRhs(key, value, options.processEntries);

if (typeof existingLhs === 'undefined') {
return {
...queries,
expressions: [
...queries.expressions,
{
type: 'or',
expressions: [
newExpression,
],
},
],
};
}

return {
...queries,
expressions: queries.expressions.map((ex) => (
ex.expressions.some((ex2) => ex2.lhs !== key)
? ex
: {
...existingOr,
expressions: [
...(existingOr.expressions ?? []),
newExpression,
],
}
)),
};
},
{
type: 'and',
expressions: [],
} as DataSourceQueryAndGrouping
)

// all the queries are to be treated as "AND", as suggested by the & character to separate the query param entries
};

// TODO add more operators

+ 1
- 0
packages/core/src/backend/index.ts View File

@@ -2,4 +2,5 @@ export * from './core';
export * from './common';
export * from './data-source';

// TODO publish to separate library
export * as http from './servers/http';

+ 4
- 2
packages/core/src/backend/servers/http/decorators/method/index.ts View File

@@ -1,13 +1,15 @@
import {RequestDecorator} from '../../../../common';

const METHOD_SPOOF_HEADER_NAME = 'x-original-method' as const;
const METHOD_SPOOF_ORIGINAL_METHOD = 'POST' as const;
const WHITELISTED_METHODS = [
'QUERY'
] as const;

export const decorateRequestWithMethod: RequestDecorator = (req) => {
req.method = req.method?.trim().toUpperCase() ?? '';
if (req.method === 'POST') {
const spoofedMethod = req.headers['x-original-method'];
if (req.method === METHOD_SPOOF_ORIGINAL_METHOD) {
const spoofedMethod = req.headers[METHOD_SPOOF_HEADER_NAME];
if (Array.isArray(spoofedMethod)) {
return req;
}


+ 8
- 3
packages/core/src/common/media-type.ts View File

@@ -1,7 +1,12 @@
export interface MediaType<Name extends string = string> {
export interface MediaType<
Name extends string = string,
T extends object = object,
SerializeOpts extends unknown[] = [],
DeserializeOpts extends unknown[] = []
> {
name: Name;
serialize: <T>(object: T) => string;
deserialize: <T>(s: string) => T;
serialize: (object: T, ...args: SerializeOpts) => string;
deserialize: (s: string, ...args: DeserializeOpts) => T;
}

export const FALLBACK_MEDIA_TYPE = {


+ 26
- 18
packages/core/src/common/queries/common.ts View File

@@ -1,4 +1,6 @@
const DATA_SOURCE_QUERY_OPERATORS = [
import {MediaType} from '../media-type';

const OPERATORS = [
'=',
'!=',
'>=',
@@ -10,34 +12,40 @@ const DATA_SOURCE_QUERY_OPERATORS = [
'REGEXP',
] as const;

type DataSourceQueryOperator = typeof DATA_SOURCE_QUERY_OPERATORS[number];
type QueryOperator = typeof OPERATORS[number];

type QueryExpressionValue = string | number | boolean;

interface DataSourceQueryOperatorExpression {
interface QueryOperatorExpression {
lhs: string;
operator: DataSourceQueryOperator;
rhs: string | number | boolean | RegExp;
operator: QueryOperator;
rhs: QueryExpressionValue | RegExp;
}

interface QueryFunctionExpression {
name: string;
args: QueryExpressionValue[];
}

interface DataSourceQueryOrGrouping {
export type QueryAnyExpression = QueryOperatorExpression | QueryFunctionExpression;

export interface QueryOrGrouping {
type: 'or';
expressions: DataSourceQueryOperatorExpression[];
expressions: QueryAnyExpression[];
}

interface DataSourceQueryAndGrouping {
export interface QueryAndGrouping {
type: 'and';
expressions: DataSourceQueryOrGrouping[];
expressions: QueryOrGrouping[];
}

export type DataSourceQueryExpression = DataSourceQueryOperatorExpression;

export type DataSourceQuery = DataSourceQueryAndGrouping;

export interface QueryMediaType<
Name extends string = string,
SerializeOptions extends {} = {},
DeserializeOptions extends {} = {}
> {
name: Name;
serialize: (object: DataSourceQuery, opts?: SerializeOptions) => string;
deserialize: (s: string, opts?: DeserializeOptions) => DataSourceQuery;
}
> extends MediaType<
Name,
QueryAndGrouping,
[SerializeOptions],
[DeserializeOptions]
> {}

+ 90
- 59
packages/core/src/common/queries/media-types/application/x-www-form-urlencoded.ts View File

@@ -1,4 +1,9 @@
import {QueryMediaType, DataSourceQuery, DataSourceQueryExpression} from '../../common';
import {
QueryMediaType,
QueryAndGrouping,
QueryAnyExpression,
QueryOrGrouping,
} from '../../common';

interface ProcessEntryBase {
type: string;
@@ -34,16 +39,17 @@ export type ProcessEntry = ProcessEntryString | ProcessEntryNumber | ProcessEntr
export const name = 'application/x-www-form-urlencoded' as const;

const normalizeRhs = (lhs: string, rhs: string, processEntriesMap?: Record<string, ProcessEntry>) => {
const coerceValues = processEntriesMap?.[lhs] ?? {
const defaultCoerceValues = {
type: 'string'
};
} as ProcessEntry;
const coerceValues = processEntriesMap?.[lhs] ?? defaultCoerceValues;

if (coerceValues?.type === 'number') {
return {
lhs,
operator: '=',
rhs: Number(rhs)
} as DataSourceQueryExpression;
} as QueryAnyExpression;
}

if (coerceValues?.type === 'boolean') {
@@ -56,7 +62,7 @@ const normalizeRhs = (lhs: string, rhs: string, processEntriesMap?: Record<strin
lhs,
operator: '=',
rhs: truthyStrings.includes(rhs)
} as DataSourceQueryExpression;
} as QueryAnyExpression;
}

if (coerceValues?.type === 'string') {
@@ -66,28 +72,28 @@ const normalizeRhs = (lhs: string, rhs: string, processEntriesMap?: Record<strin
lhs,
operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE',
rhs: `%${rhs}`,
} as DataSourceQueryExpression;
} as QueryAnyExpression;
}
case 'endsWith': {
return {
lhs,
operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE',
rhs: `${rhs}%`,
} as DataSourceQueryExpression;
} as QueryAnyExpression;
}
case 'includes': {
return {
lhs,
operator: coerceValues.caseInsensitiveMatch ? 'ILIKE' : 'LIKE',
rhs: `%${rhs}%`,
} as DataSourceQueryExpression;
} as QueryAnyExpression;
}
case 'regexp': {
return {
lhs,
operator: 'REGEXP',
rhs: new RegExp(rhs, coerceValues.caseInsensitiveMatch ? 'i' : ''),
} as DataSourceQueryExpression;
} as QueryAnyExpression;
}
default:
break;
@@ -97,12 +103,12 @@ const normalizeRhs = (lhs: string, rhs: string, processEntriesMap?: Record<strin
lhs,
operator: '=',
rhs,
} as DataSourceQueryExpression;
} as QueryAnyExpression;
}

const unknownCoerceValues = coerceValues as unknown as Record<string, string>;
throw new Error(`Invalid coercion type: ${unknownCoerceValues.type}`);
// this will be sent to the data source, e.g. the SQL query
// this will be sent to the data source, e.g., the SQL query
// we can also make this function act as a "sanitizer"
}

@@ -112,19 +118,35 @@ interface DeserializeOptions {
processEntries?: Record<string, ProcessEntry>;
}

export const deserialize: QueryMediaType<typeof name, SerializeOptions, DeserializeOptions>['deserialize'] = (
s: string,
options = {} as DeserializeOptions
) => {
const doesGroupHaveExpression = (ex2: QueryAnyExpression, key: string) => {
if ('operator' in ex2) {
return ex2.lhs === key;
}

if ('name' in ex2) {
return ex2.name === key;
}

return false;
};

export const deserialize: QueryMediaType<
typeof name,
SerializeOptions,
DeserializeOptions
>['deserialize'] = (s: string, options = {} as DeserializeOptions) => {
const q = new URLSearchParams(s);

return Array.from(q.entries()).reduce(
(queries, [key, value]) => {
const existingOr = queries.expressions
.find((ex) => ex.expressions.some((ex2) => ex2.lhs === key)) ?? {
const defaultOr = {
type: 'or',
expressions: [],
};
const existingLhs = existingOr.expressions.find((ex) => ex.lhs === key);
} as QueryOrGrouping;
const existingOr = queries.expressions.find((ex) => (
ex.expressions.some((ex2) => doesGroupHaveExpression(ex2, key))
)) ?? defaultOr;
const existingLhs = existingOr.expressions.find((ex) => doesGroupHaveExpression(ex, key));
const newExpression = normalizeRhs(key, value, options.processEntries);

if (typeof existingLhs === 'undefined') {
@@ -145,7 +167,7 @@ export const deserialize: QueryMediaType<typeof name, SerializeOptions, Deserial
return {
...queries,
expressions: queries.expressions.map((ex) => (
ex.expressions.some((ex2) => ex2.lhs !== key)
ex.expressions.some((ex2) => !doesGroupHaveExpression(ex2, key))
? ex
: {
...existingOr,
@@ -160,49 +182,58 @@ export const deserialize: QueryMediaType<typeof name, SerializeOptions, Deserial
{
type: 'and',
expressions: [],
} as DataSourceQuery
} as QueryAndGrouping
)
};

export const serialize: QueryMediaType<typeof name, SerializeOptions, DeserializeOptions>['serialize'] = (q: DataSourceQuery) => (
const serializeExpression = (ex2: QueryAnyExpression) => {
if ('name' in ex2) {
return [ex2.name, `(${ex2.args.map((s) => s.toString()).join(',')})`];
}

if (ex2.rhs instanceof RegExp) {
if (ex2.operator !== 'REGEXP') {
throw new Error(`Invalid rhs given for operator: ${ex2.lhs} ${ex2.operator} <rhs>`);
}

return [ex2.lhs, ex2.rhs.toString()];
}

switch (typeof ex2.rhs) {
case 'string': {
switch (ex2.operator) {
case 'ILIKE':
case 'LIKE':
return [ex2.lhs, ex2.rhs.replace(/^%+/, '').replace(/%+$/, '')];
case '=':
return [ex2.lhs, ex2.rhs];
default:
break;
}
throw new Error(`Invalid operator given for lhs: ${ex2.lhs} <op> ${ex2.rhs}`);
}
case 'number': {
return [ex2.lhs, ex2.rhs.toString()];
}
case 'boolean': {
return [ex2.lhs, ex2.rhs ? 'true' : 'false'];
}
default:
break;
}

throw new Error(`Unknown type for rhs: ${ex2.lhs} ${ex2.operator} <rhs>`);
};

export const serialize: QueryMediaType<
typeof name,
SerializeOptions,
DeserializeOptions
>['serialize'] = (q: QueryAndGrouping) => (
new URLSearchParams(
q.expressions.map(
(ex) => ex.expressions.map((ex2) => {
if (ex2.rhs instanceof RegExp) {
if (ex2.operator !== 'REGEXP') {
throw new Error(`Invalid rhs given for operator: ${ex2.lhs} ${ex2.operator} <rhs>`);
}

return [ex2.lhs, ex2.rhs.toString()];
}

switch (typeof ex2.rhs) {
case 'string': {
switch (ex2.operator) {
case 'ILIKE':
case 'LIKE':
return [ex2.lhs, ex2.rhs.replace(/^%+/, '').replace(/%+$/, '')];
case '=':
return [ex2.lhs, ex2.rhs];
default:
break;
}
throw new Error(`Invalid operator given for lhs: ${ex2.lhs} <op> ${ex2.rhs}`);
}
case 'number': {
return [ex2.lhs, ex2.rhs.toString()];
}
case 'boolean': {
return [ex2.lhs, ex2.rhs ? 'true' : 'false'];
}
default:
break;
}

throw new Error(`Unknown type for rhs: ${ex2.lhs} ${ex2.operator} <rhs>`);
})
)
.flat()
q.expressions.flatMap((ex) => (
ex.expressions.map((ex2) => serializeExpression(ex2))
))
)
.toString()
);

Write Preview
Loading…
Cancel
Save