Compare commits

...

2 Commits

Author SHA1 Message Date
Morten Olsen
d02102977a chore: seperate into packages 2025-12-10 10:26:14 +01:00
Morten Olsen
f9494c88e2 update 2025-12-10 09:11:03 +01:00
62 changed files with 1804 additions and 1257 deletions

5
.gitignore vendored
View File

@@ -1,4 +1,5 @@
/node_modules /node_modules/
/packages/*/dist/
.turbo/ .turbo/
/.env /.env
/coverage/ /coverage/

View File

@@ -36,6 +36,26 @@
"packageVersion": "1.0.0", "packageVersion": "1.0.0",
"packageName": "server" "packageName": "server"
} }
},
{
"timestamp": "2025-12-10T07:50:20.652Z",
"template": "pkg",
"values": {
"monoRepo": true,
"packagePrefix": "@morten-olsen/stash-",
"packageVersion": "1.0.0",
"packageName": "query-dsl"
}
},
{
"timestamp": "2025-12-10T08:07:44.756Z",
"template": "pkg",
"values": {
"monoRepo": true,
"packagePrefix": "@morten-olsen/stash-",
"packageVersion": "1.0.0",
"packageName": "runtime"
}
} }
] ]
} }

View File

@@ -10,11 +10,9 @@
"resolveJsonModule": true, "resolveJsonModule": true,
"allowSyntheticDefaultImports": true, "allowSyntheticDefaultImports": true,
"skipLibCheck": true, "skipLibCheck": true,
"noEmit": true,
"jsx": "react-jsx", "jsx": "react-jsx",
"isolatedModules": true, "isolatedModules": true,
"verbatimModuleSyntax": true, "verbatimModuleSyntax": true,
"erasableSyntaxOnly": true, "erasableSyntaxOnly": true
"allowImportingTsExtensions": true
} }
} }

4
packages/query-dsl/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
/node_modules/
/dist/
/coverage/
/.env

View File

@@ -0,0 +1,626 @@
<!-- This is a generated file -->
<!DOCTYPE html>
<meta charset="utf-8">
<style>
body {
background-color: hsl(30, 20%, 95%)
}
</style>
<link rel='stylesheet' href='https://unpkg.com/chevrotain@11.0.3/diagrams/diagrams.css'>
<script src='https://unpkg.com/chevrotain@11.0.3/diagrams/vendor/railroad-diagrams.js'></script>
<script src='https://unpkg.com/chevrotain@11.0.3/diagrams/src/diagrams_builder.js'></script>
<script src='https://unpkg.com/chevrotain@11.0.3/diagrams/src/diagrams_behavior.js'></script>
<script src='https://unpkg.com/chevrotain@11.0.3/diagrams/src/main.js'></script>
<div id="diagrams" align="center"></div>
<script>
window.serializedGrammar = [
{
"type": "Rule",
"name": "orExpression",
"orgText": "",
"definition": [
{
"type": "NonTerminal",
"name": "andExpression",
"idx": 0
},
{
"type": "Repetition",
"idx": 0,
"definition": [
{
"type": "Terminal",
"name": "Or",
"label": "Or",
"idx": 0,
"pattern": "OR"
},
{
"type": "NonTerminal",
"name": "andExpression",
"idx": 2
}
]
}
]
},
{
"type": "Rule",
"name": "andExpression",
"orgText": "",
"definition": [
{
"type": "NonTerminal",
"name": "primaryExpression",
"idx": 0
},
{
"type": "Repetition",
"idx": 0,
"definition": [
{
"type": "Terminal",
"name": "And",
"label": "And",
"idx": 0,
"pattern": "AND"
},
{
"type": "NonTerminal",
"name": "primaryExpression",
"idx": 2
}
]
}
]
},
{
"type": "Rule",
"name": "primaryExpression",
"orgText": "",
"definition": [
{
"type": "Alternation",
"idx": 0,
"definition": [
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "LParen",
"label": "LParen",
"idx": 0,
"pattern": "\\("
},
{
"type": "NonTerminal",
"name": "orExpression",
"idx": 0
},
{
"type": "Terminal",
"name": "RParen",
"label": "RParen",
"idx": 0,
"pattern": "\\)"
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "NonTerminal",
"name": "condition",
"idx": 0
}
]
}
]
}
]
},
{
"type": "Rule",
"name": "condition",
"orgText": "",
"definition": [
{
"type": "NonTerminal",
"name": "fieldReference",
"idx": 0
},
{
"type": "Alternation",
"idx": 0,
"definition": [
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "Is",
"label": "Is",
"idx": 0,
"pattern": "IS"
},
{
"type": "Option",
"idx": 0,
"definition": [
{
"type": "Terminal",
"name": "Not",
"label": "Not",
"idx": 0,
"pattern": "NOT"
}
]
},
{
"type": "Terminal",
"name": "Null",
"label": "Null",
"idx": 0,
"pattern": "NULL"
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "Not",
"label": "Not",
"idx": 2,
"pattern": "NOT"
},
{
"type": "Terminal",
"name": "In",
"label": "In",
"idx": 0,
"pattern": "IN"
},
{
"type": "NonTerminal",
"name": "stringInList",
"idx": 0
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "Not",
"label": "Not",
"idx": 3,
"pattern": "NOT"
},
{
"type": "Terminal",
"name": "In",
"label": "In",
"idx": 2,
"pattern": "IN"
},
{
"type": "NonTerminal",
"name": "numberInList",
"idx": 0
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "Not",
"label": "Not",
"idx": 4,
"pattern": "NOT"
},
{
"type": "Terminal",
"name": "Like",
"label": "Like",
"idx": 0,
"pattern": "LIKE"
},
{
"type": "Terminal",
"name": "StringLiteral",
"label": "StringLiteral",
"idx": 0,
"pattern": "'(?:''|[^'])*'"
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "In",
"label": "In",
"idx": 3,
"pattern": "IN"
},
{
"type": "NonTerminal",
"name": "stringInList",
"idx": 2
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "In",
"label": "In",
"idx": 4,
"pattern": "IN"
},
{
"type": "NonTerminal",
"name": "numberInList",
"idx": 2
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "Like",
"label": "Like",
"idx": 2,
"pattern": "LIKE"
},
{
"type": "Terminal",
"name": "StringLiteral",
"label": "StringLiteral",
"idx": 2,
"pattern": "'(?:''|[^'])*'"
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "Equals",
"label": "Equals",
"idx": 0,
"pattern": "="
},
{
"type": "Terminal",
"name": "StringLiteral",
"label": "StringLiteral",
"idx": 3,
"pattern": "'(?:''|[^'])*'"
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "Equals",
"label": "Equals",
"idx": 2,
"pattern": "="
},
{
"type": "Terminal",
"name": "NumberLiteral",
"label": "NumberLiteral",
"idx": 0,
"pattern": "-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?"
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "Equals",
"label": "Equals",
"idx": 3,
"pattern": "="
},
{
"type": "Terminal",
"name": "Null",
"label": "Null",
"idx": 2,
"pattern": "NULL"
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "NotEquals",
"label": "NotEquals",
"idx": 0,
"pattern": "!="
},
{
"type": "Terminal",
"name": "StringLiteral",
"label": "StringLiteral",
"idx": 4,
"pattern": "'(?:''|[^'])*'"
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "NotEquals",
"label": "NotEquals",
"idx": 2,
"pattern": "!="
},
{
"type": "Terminal",
"name": "NumberLiteral",
"label": "NumberLiteral",
"idx": 2,
"pattern": "-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?"
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "GreaterThan",
"label": "GreaterThan",
"idx": 0,
"pattern": ">"
},
{
"type": "Terminal",
"name": "NumberLiteral",
"label": "NumberLiteral",
"idx": 3,
"pattern": "-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?"
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "GreaterThanOrEqual",
"label": "GreaterThanOrEqual",
"idx": 0,
"pattern": ">="
},
{
"type": "Terminal",
"name": "NumberLiteral",
"label": "NumberLiteral",
"idx": 4,
"pattern": "-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?"
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "LessThan",
"label": "LessThan",
"idx": 0,
"pattern": "<"
},
{
"type": "Terminal",
"name": "NumberLiteral",
"label": "NumberLiteral",
"idx": 5,
"pattern": "-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?"
}
]
},
{
"type": "Alternative",
"definition": [
{
"type": "Terminal",
"name": "LessThanOrEqual",
"label": "LessThanOrEqual",
"idx": 0,
"pattern": "<="
},
{
"type": "Terminal",
"name": "NumberLiteral",
"label": "NumberLiteral",
"idx": 6,
"pattern": "-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?"
}
]
}
]
}
]
},
{
"type": "Rule",
"name": "fieldReference",
"orgText": "",
"definition": [
{
"type": "Terminal",
"name": "Identifier",
"label": "Identifier",
"idx": 0,
"pattern": "[a-zA-Z_][a-zA-Z0-9_]*"
},
{
"type": "Repetition",
"idx": 0,
"definition": [
{
"type": "Terminal",
"name": "Dot",
"label": "Dot",
"idx": 0,
"pattern": "\\."
},
{
"type": "Terminal",
"name": "Identifier",
"label": "Identifier",
"idx": 2,
"pattern": "[a-zA-Z_][a-zA-Z0-9_]*"
}
]
}
]
},
{
"type": "Rule",
"name": "stringInList",
"orgText": "",
"definition": [
{
"type": "Terminal",
"name": "LParen",
"label": "LParen",
"idx": 0,
"pattern": "\\("
},
{
"type": "Terminal",
"name": "StringLiteral",
"label": "StringLiteral",
"idx": 0,
"pattern": "'(?:''|[^'])*'"
},
{
"type": "Repetition",
"idx": 0,
"definition": [
{
"type": "Terminal",
"name": "Comma",
"label": "Comma",
"idx": 0,
"pattern": ","
},
{
"type": "Terminal",
"name": "StringLiteral",
"label": "StringLiteral",
"idx": 2,
"pattern": "'(?:''|[^'])*'"
}
]
},
{
"type": "Terminal",
"name": "RParen",
"label": "RParen",
"idx": 0,
"pattern": "\\)"
}
]
},
{
"type": "Rule",
"name": "numberInList",
"orgText": "",
"definition": [
{
"type": "Terminal",
"name": "LParen",
"label": "LParen",
"idx": 2,
"pattern": "\\("
},
{
"type": "Terminal",
"name": "NumberLiteral",
"label": "NumberLiteral",
"idx": 0,
"pattern": "-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?"
},
{
"type": "Repetition",
"idx": 0,
"definition": [
{
"type": "Terminal",
"name": "Comma",
"label": "Comma",
"idx": 2,
"pattern": ","
},
{
"type": "Terminal",
"name": "NumberLiteral",
"label": "NumberLiteral",
"idx": 2,
"pattern": "-?(?:0|[1-9]\\d*)(?:\\.\\d+)?(?:[eE][+-]?\\d+)?"
}
]
},
{
"type": "Terminal",
"name": "RParen",
"label": "RParen",
"idx": 2,
"pattern": "\\)"
}
]
},
{
"type": "Rule",
"name": "query",
"orgText": "",
"definition": [
{
"type": "NonTerminal",
"name": "orExpression",
"idx": 0
}
]
}
];
</script>
<script>
var diagramsDiv = document.getElementById("diagrams");
main.drawDiagramsFromSerializedGrammar(serializedGrammar, diagramsDiv);
</script>

View File

@@ -0,0 +1,33 @@
{
"type": "module",
"main": "dist/exports.js",
"scripts": {
"build": "tsc --build",
"test:unit": "vitest --run --passWithNoTests",
"test": "pnpm run \"/^test:/\""
},
"packageManager": "pnpm@10.6.0",
"files": [
"dist"
],
"exports": {
".": "./dist/exports.js"
},
"devDependencies": {
"@morten-olsen/stash-configs": "workspace:*",
"@morten-olsen/stash-tests": "workspace:*",
"@types/node": "24.10.2",
"@vitest/coverage-v8": "4.0.15",
"typescript": "5.9.3",
"vitest": "4.0.15"
},
"name": "@morten-olsen/stash-query-dsl",
"version": "1.0.0",
"imports": {
"#root/*": "./src/*"
},
"dependencies": {
"chevrotain": "^11.0.3",
"zod": "4.1.13"
}
}

View File

@@ -0,0 +1,13 @@
import { createSyntaxDiagramsCode } from 'chevrotain';
import { QueryParser } from '../dist/exports.js';
import { mkdir, writeFile } from 'node:fs/promises';
import { resolve } from 'node:path';
const rootPath = resolve('./docs/diagram');
const parser = new QueryParser();
const diagram = createSyntaxDiagramsCode(parser.getSerializedGastProductions());
await mkdir(rootPath, { recursive: true });
await writeFile(resolve(rootPath, 'index.html'), diagram);

View File

@@ -0,0 +1,2 @@
export * from './query-parser.schemas.js';
export { QueryParser } from './query-parser.js';

View File

@@ -0,0 +1,457 @@
import { createToken, Lexer, EmbeddedActionsParser } from 'chevrotain';
import type { QueryFilter, QueryCondition } from './query-parser.schemas.js';
// ----------------- Lexer -----------------
// Whitespace (skipped)
const WhiteSpace = createToken({ name: 'WhiteSpace', pattern: /\s+/, group: Lexer.SKIPPED });
// Keywords (must be defined before Identifier to take precedence)
const And = createToken({ name: 'And', pattern: /AND/i, longer_alt: undefined });
const Or = createToken({ name: 'Or', pattern: /OR/i, longer_alt: undefined });
const Like = createToken({ name: 'Like', pattern: /LIKE/i, longer_alt: undefined });
const Not = createToken({ name: 'Not', pattern: /NOT/i, longer_alt: undefined });
const In = createToken({ name: 'In', pattern: /IN/i, longer_alt: undefined });
const Is = createToken({ name: 'Is', pattern: /IS/i, longer_alt: undefined });
const Null = createToken({ name: 'Null', pattern: /NULL/i, longer_alt: undefined });
// Identifier (must come after keywords)
const Identifier = createToken({ name: 'Identifier', pattern: /[a-zA-Z_][a-zA-Z0-9_]*/ });
// Set longer_alt for keywords to handle cases like "ANDROID" not matching "AND"
And.LONGER_ALT = Identifier;
Or.LONGER_ALT = Identifier;
Like.LONGER_ALT = Identifier;
Not.LONGER_ALT = Identifier;
In.LONGER_ALT = Identifier;
Is.LONGER_ALT = Identifier;
Null.LONGER_ALT = Identifier;
// Literals
const StringLiteral = createToken({
name: 'StringLiteral',
pattern: /'(?:''|[^'])*'/,
});
const NumberLiteral = createToken({
name: 'NumberLiteral',
pattern: /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/,
});
// Operators
const NotEquals = createToken({ name: 'NotEquals', pattern: /!=/ });
const GreaterThanOrEqual = createToken({ name: 'GreaterThanOrEqual', pattern: />=/ });
const LessThanOrEqual = createToken({ name: 'LessThanOrEqual', pattern: /<=/ });
const Equals = createToken({ name: 'Equals', pattern: /=/ });
const GreaterThan = createToken({ name: 'GreaterThan', pattern: />/ });
const LessThan = createToken({ name: 'LessThan', pattern: /</ });
// Punctuation
const LParen = createToken({ name: 'LParen', pattern: /\(/ });
const RParen = createToken({ name: 'RParen', pattern: /\)/ });
const Comma = createToken({ name: 'Comma', pattern: /,/ });
const Dot = createToken({ name: 'Dot', pattern: /\./ });
// Token order matters! More specific patterns first.
const allTokens = [
WhiteSpace,
// Multi-char operators first
NotEquals,
GreaterThanOrEqual,
LessThanOrEqual,
// Single-char operators
Equals,
GreaterThan,
LessThan,
// Punctuation
LParen,
RParen,
Comma,
Dot,
// Keywords (before Identifier)
And,
Or,
Like,
Not,
In,
Is,
Null,
// Literals
StringLiteral,
NumberLiteral,
// Identifier last
Identifier,
];
const QueryLexer = new Lexer(allTokens);
// ----------------- Parser -----------------
class QueryParserParser extends EmbeddedActionsParser {
constructor() {
super(allTokens);
this.performSelfAnalysis();
}
// OR has lowest precedence
#orExpression = this.RULE('orExpression', (): QueryFilter => {
let left = this.SUBRULE(this.#andExpression);
this.MANY(() => {
this.CONSUME(Or);
const right = this.SUBRULE2(this.#andExpression);
left = this.ACTION(() => this.#combineWithOperator(left, right, 'or'));
});
return left;
});
// AND has higher precedence than OR
#andExpression = this.RULE('andExpression', (): QueryFilter => {
let left = this.SUBRULE(this.#primaryExpression);
this.MANY(() => {
this.CONSUME(And);
const right = this.SUBRULE2(this.#primaryExpression);
left = this.ACTION(() => this.#combineWithOperator(left, right, 'and'));
});
return left;
});
// Primary: parenthesized expression or condition
#primaryExpression = this.RULE('primaryExpression', (): QueryFilter => {
return this.OR([
{
ALT: () => {
this.CONSUME(LParen);
const expr = this.SUBRULE(this.#orExpression);
this.CONSUME(RParen);
return expr;
},
},
{ ALT: () => this.SUBRULE(this.#condition) },
]);
});
// Condition: field followed by operator and value(s)
#condition = this.RULE('condition', (): QueryCondition => {
const field = this.SUBRULE(this.#fieldReference);
return this.OR([
// IS NULL / IS NOT NULL
{
ALT: () => {
this.CONSUME(Is);
const isNot = this.OPTION(() => this.CONSUME(Not)) !== undefined;
this.CONSUME(Null);
return this.ACTION(() => ({
type: 'text' as const,
field,
conditions: isNot ? { notEqual: undefined, equal: undefined } : { equal: null },
}));
},
},
// NOT IN (strings) - LA(1)=NOT, LA(2)=IN, LA(3)=(, LA(4)=value
{
GATE: () => this.LA(4).tokenType === StringLiteral,
ALT: () => {
this.CONSUME2(Not);
this.CONSUME(In);
const values = this.SUBRULE(this.#stringInList);
return this.ACTION(() => ({
type: 'text' as const,
field,
conditions: { notIn: values },
}));
},
},
// NOT IN (numbers)
{
GATE: () => this.LA(4).tokenType === NumberLiteral,
ALT: () => {
this.CONSUME3(Not);
this.CONSUME2(In);
const values = this.SUBRULE(this.#numberInList);
return this.ACTION(() => ({
type: 'number' as const,
field,
conditions: { notIn: values },
}));
},
},
// NOT LIKE
{
ALT: () => {
this.CONSUME4(Not);
this.CONSUME(Like);
const pattern = this.CONSUME(StringLiteral);
return this.ACTION(() => ({
type: 'text' as const,
field,
conditions: { notLike: this.#extractStringValue(pattern.image) },
}));
},
},
// IN (strings) - LA(1)=IN, LA(2)=(, LA(3)=value
{
GATE: () => this.LA(3).tokenType === StringLiteral,
ALT: () => {
this.CONSUME3(In);
const values = this.SUBRULE2(this.#stringInList);
return this.ACTION(() => ({
type: 'text' as const,
field,
conditions: { in: values },
}));
},
},
// IN (numbers)
{
GATE: () => this.LA(3).tokenType === NumberLiteral,
ALT: () => {
this.CONSUME4(In);
const values = this.SUBRULE2(this.#numberInList);
return this.ACTION(() => ({
type: 'number' as const,
field,
conditions: { in: values },
}));
},
},
// LIKE
{
ALT: () => {
this.CONSUME2(Like);
const pattern = this.CONSUME2(StringLiteral);
return this.ACTION(() => ({
type: 'text' as const,
field,
conditions: { like: this.#extractStringValue(pattern.image) },
}));
},
},
// = string
{
GATE: () => this.LA(2).tokenType === StringLiteral,
ALT: () => {
this.CONSUME(Equals);
const token = this.CONSUME3(StringLiteral);
return this.ACTION(() => ({
type: 'text' as const,
field,
conditions: { equal: this.#extractStringValue(token.image) },
}));
},
},
// = number
{
GATE: () => this.LA(2).tokenType === NumberLiteral,
ALT: () => {
this.CONSUME2(Equals);
const token = this.CONSUME(NumberLiteral);
return this.ACTION(() => ({
type: 'number' as const,
field,
conditions: { equals: parseFloat(token.image) },
}));
},
},
// = NULL
{
ALT: () => {
this.CONSUME3(Equals);
this.CONSUME2(Null);
return this.ACTION(() => ({
type: 'text' as const,
field,
conditions: { equal: null },
}));
},
},
// != string
{
GATE: () => this.LA(2).tokenType === StringLiteral,
ALT: () => {
this.CONSUME(NotEquals);
const token = this.CONSUME4(StringLiteral);
return this.ACTION(() => ({
type: 'text' as const,
field,
conditions: { notEqual: this.#extractStringValue(token.image) },
}));
},
},
// != number
{
ALT: () => {
this.CONSUME2(NotEquals);
const token = this.CONSUME2(NumberLiteral);
return this.ACTION(() => ({
type: 'number' as const,
field,
conditions: { notEquals: parseFloat(token.image) },
}));
},
},
// > number
{
ALT: () => {
this.CONSUME(GreaterThan);
const token = this.CONSUME3(NumberLiteral);
return this.ACTION(() => ({
type: 'number' as const,
field,
conditions: { greaterThan: parseFloat(token.image) },
}));
},
},
// >= number
{
ALT: () => {
this.CONSUME(GreaterThanOrEqual);
const token = this.CONSUME4(NumberLiteral);
return this.ACTION(() => ({
type: 'number' as const,
field,
conditions: { greaterThanOrEqual: parseFloat(token.image) },
}));
},
},
// < number
{
ALT: () => {
this.CONSUME(LessThan);
const token = this.CONSUME5(NumberLiteral);
return this.ACTION(() => ({
type: 'number' as const,
field,
conditions: { lessThan: parseFloat(token.image) },
}));
},
},
// <= number
{
ALT: () => {
this.CONSUME(LessThanOrEqual);
const token = this.CONSUME6(NumberLiteral);
return this.ACTION(() => ({
type: 'number' as const,
field,
conditions: { lessThanOrEqual: parseFloat(token.image) },
}));
},
},
]);
});
// Field reference: identifier.identifier.identifier...
#fieldReference = this.RULE('fieldReference', (): string[] => {
const parts: string[] = [];
const first = this.CONSUME(Identifier);
this.ACTION(() => parts.push(first.image));
this.MANY(() => {
this.CONSUME(Dot);
const next = this.CONSUME2(Identifier);
this.ACTION(() => parts.push(next.image));
});
return parts;
});
// String IN list: ('val1', 'val2', ...)
#stringInList = this.RULE('stringInList', (): string[] => {
const values: string[] = [];
this.CONSUME(LParen);
const first = this.CONSUME(StringLiteral);
this.ACTION(() => values.push(this.#extractStringValue(first.image)));
this.MANY(() => {
this.CONSUME(Comma);
const next = this.CONSUME2(StringLiteral);
this.ACTION(() => values.push(this.#extractStringValue(next.image)));
});
this.CONSUME(RParen);
return values;
});
// Number IN list: (1, 2, 3, ...)
#numberInList = this.RULE('numberInList', (): number[] => {
const values: number[] = [];
this.CONSUME2(LParen);
const first = this.CONSUME(NumberLiteral);
this.ACTION(() => values.push(parseFloat(first.image)));
this.MANY(() => {
this.CONSUME2(Comma);
const next = this.CONSUME2(NumberLiteral);
this.ACTION(() => values.push(parseFloat(next.image)));
});
this.CONSUME2(RParen);
return values;
});
// Extract string value from quoted literal, handling escaped quotes
#extractStringValue(image: string): string {
// Remove surrounding quotes and unescape doubled quotes
return image.slice(1, -1).replace(/''/g, "'");
}
// Combine two filters with an operator, flattening if possible
#combineWithOperator(left: QueryFilter, right: QueryFilter, operator: 'and' | 'or'): QueryFilter {
if (left.type === 'operator' && left.operator === operator) {
return {
type: 'operator',
operator,
conditions: [...left.conditions, right],
};
}
return {
type: 'operator',
operator,
conditions: [left, right],
};
}
// Entry point
#query = this.RULE('query', (): QueryFilter => {
return this.SUBRULE(this.#orExpression);
});
public parse = (input: string): QueryFilter => {
const lexResult = QueryLexer.tokenize(input);
if (lexResult.errors.length > 0) {
const error = lexResult.errors[0];
// Check if this looks like an unterminated string (starts with ' but lexer failed)
if (error.message.includes("'") || input.slice(error.offset).startsWith("'")) {
// Count unescaped single quotes
const unescapedQuotes = input.replace(/''/g, '').match(/'/g);
if (unescapedQuotes && unescapedQuotes.length % 2 !== 0) {
throw new Error(`Unterminated string starting at position ${error.offset}`);
}
}
throw new Error(`Lexer error at position ${error.offset}: ${error.message}`);
}
this.input = lexResult.tokens;
const result = this.#query();
if (this.errors.length > 0) {
const error = this.errors[0];
throw new Error(`Parse error: ${error.message}`);
}
return result;
};
}
export { QueryParserParser, QueryLexer };

View File

@@ -0,0 +1,85 @@
import { z } from 'zod';
const queryConditionTextSchema = z.object({
type: z.literal('text'),
tableName: z.string().optional(),
field: z.array(z.string()),
conditions: z.object({
equal: z.string().nullish(),
notEqual: z.string().optional(),
like: z.string().optional(),
notLike: z.string().optional(),
in: z.array(z.string()).optional(),
notIn: z.array(z.string()).optional(),
}),
});
type QueryConditionText = z.infer<typeof queryConditionTextSchema>;
const queryConditionNumberSchema = z.object({
type: z.literal('number'),
tableName: z.string().optional(),
field: z.array(z.string()),
conditions: z.object({
equals: z.number().nullish(),
notEquals: z.number().nullish(),
greaterThan: z.number().optional(),
greaterThanOrEqual: z.number().optional(),
lessThan: z.number().optional(),
lessThanOrEqual: z.number().optional(),
in: z.array(z.number()).optional(),
notIn: z.array(z.number()).optional(),
}),
});
type QueryConditionNumber = z.infer<typeof queryConditionNumberSchema>;
const queryConditionSchema = z.discriminatedUnion('type', [queryConditionTextSchema, queryConditionNumberSchema]);
type QueryCondition = z.infer<typeof queryConditionSchema>;
type QueryFilter = QueryCondition | QueryOperator;
type QueryOperator = {
type: 'operator';
operator: 'and' | 'or';
conditions: QueryFilter[];
};
// Create a depth-limited recursive schema for OpenAPI compatibility
// This supports up to 3 levels of nesting, which should be sufficient for most use cases
// OpenAPI cannot handle z.lazy(), so we manually define the nesting
// If you need deeper nesting, you can add more levels (Level3, Level4, etc.)
const queryFilterSchemaLevel0: z.ZodType<QueryFilter> = z.union([
queryConditionSchema,
z.object({
type: z.literal('operator'),
operator: z.enum(['and', 'or']),
conditions: z.array(queryConditionSchema),
}),
]);
const queryFilterSchemaLevel1: z.ZodType<QueryFilter> = z.union([
queryConditionSchema,
z.object({
type: z.literal('operator'),
operator: z.enum(['and', 'or']),
conditions: z.array(queryFilterSchemaLevel0),
}),
]);
const queryFilterSchemaLevel2: z.ZodType<QueryFilter> = z.union([
queryConditionSchema,
z.object({
type: z.literal('operator'),
operator: z.enum(['and', 'or']),
conditions: z.array(queryFilterSchemaLevel1),
}),
]);
// Export the depth-limited schema (supports 3 levels of nesting)
// This works with OpenAPI schema generation
const queryFilterSchema = queryFilterSchemaLevel2;
export type { QueryConditionText, QueryConditionNumber, QueryOperator, QueryCondition, QueryFilter };
export { queryConditionSchema, queryFilterSchema };

View File

@@ -4,7 +4,7 @@ import type {
QueryCondition, QueryCondition,
QueryConditionText, QueryConditionText,
QueryConditionNumber, QueryConditionNumber,
} from '#root/utils/utils.query.ts'; } from './query-parser.schemas.js';
class Stringifier { class Stringifier {
#stringifyFilter = (filter: QueryFilter, needsParens: boolean): string => { #stringifyFilter = (filter: QueryFilter, needsParens: boolean): string => {

View File

@@ -1,8 +1,7 @@
import { describe, it, expect } from 'vitest'; import { describe, it, expect } from 'vitest';
import { QueryParser } from './query-parser.ts'; import { QueryParser } from './query-parser.js';
import type { QueryConditionNumber, QueryConditionText, QueryFilter, QueryOperator } from './query-parser.schemas.js';
import type { QueryConditionNumber, QueryConditionText, QueryFilter, QueryOperator } from '#root/utils/utils.query.ts';
describe('QueryParser', () => { describe('QueryParser', () => {
const parser = new QueryParser(); const parser = new QueryParser();

View File

@@ -0,0 +1,22 @@
import { Stringifier } from './query-parser.stringifier.js';
import { QueryParserParser } from './query-parser.parser.js';
import type { QueryFilter } from './query-parser.schemas.js';
class QueryParser {
#stringifier = new Stringifier();
#parser = new QueryParserParser();
public getSerializedGastProductions() {
return this.#parser.getSerializedGastProductions();
}
public parse = (input: string): QueryFilter => {
return this.#parser.parse(input);
};
public stringify = (filter: QueryFilter): string => {
return this.#stringifier.stringify(filter);
};
}
export { QueryParser };

View File

@@ -0,0 +1,10 @@
{
"compilerOptions": {
"outDir": "./dist",
"rootDir": "./src"
},
"include": [
"src/**/*.ts"
],
"extends": "@morten-olsen/stash-configs/tsconfig.json"
}

View File

@@ -0,0 +1,12 @@
import { defineConfig } from 'vitest/config';
import { getAliases } from '@morten-olsen/stash-tests/vitest';
// eslint-disable-next-line import/no-default-export
export default defineConfig(async () => {
const aliases = await getAliases();
return {
resolve: {
alias: aliases,
},
};
});

4
packages/runtime/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
/node_modules/
/dist/
/coverage/
/.env

View File

@@ -0,0 +1,43 @@
{
"type": "module",
"main": "dist/exports.js",
"scripts": {
"build": "tsc --build",
"test:unit": "vitest --run --passWithNoTests",
"test": "pnpm run \"/^test:/\""
},
"packageManager": "pnpm@10.6.0",
"files": [
"dist"
],
"exports": {
".": "./dist/exports.js"
},
"devDependencies": {
"@morten-olsen/stash-configs": "workspace:*",
"@morten-olsen/stash-tests": "workspace:*",
"@types/deep-equal": "^1.0.4",
"@types/node": "24.10.2",
"@vitest/coverage-v8": "4.0.15",
"typescript": "5.9.3",
"vitest": "4.0.15"
},
"name": "@morten-olsen/stash-runtime",
"version": "1.0.0",
"imports": {
"#root/*": "./src/*"
},
"dependencies": {
"@electric-sql/pglite": "^0.3.14",
"@huggingface/transformers": "^3.8.1",
"@langchain/textsplitters": "^1.0.1",
"@morten-olsen/stash-query-dsl": "workspace:*",
"better-sqlite3": "^12.5.0",
"deep-equal": "^2.2.3",
"knex": "^3.1.0",
"knex-pglite": "^0.13.0",
"pg": "^8.16.3",
"pgvector": "^0.2.1",
"zod": "4.1.13"
}
}

View File

@@ -0,0 +1,4 @@
export { Services } from './utils/utils.services.js';
export { StashRuntime } from './runtime.js';
export * from './services/documents/documents.js';
export * from './services/document-chunks/document-chunks.js';

12
packages/runtime/src/global.d.ts vendored Normal file
View File

@@ -0,0 +1,12 @@
import 'fastify';
import type { Services } from './utils/utils.services.ts';
// eslint-disable-next-line
declare type ExplicitAny = any;
declare module 'fastify' {
// eslint-disable-next-line
export interface FastifyInstance {
services: Services;
}
}

View File

@@ -0,0 +1,32 @@
import { DocumentChunksService } from './exports.js';
import { DatabaseService } from './services/database/database.js';
import { DocumentsService } from './services/documents/documents.js';
import { WarmupService } from './services/warmup/warmup.js';
import { Services } from './utils/utils.services.js';
class StashRuntime {
#services: Services;
constructor(services = new Services()) {
this.#services = services;
services.set(StashRuntime, this);
}
public get database() {
return this.#services.get(DatabaseService);
}
public get documents() {
return this.#services.get(DocumentsService);
}
public get documentChunks() {
return this.#services.get(DocumentChunksService);
}
public get warmup() {
return this.#services.get(WarmupService);
}
}
export { StashRuntime };

View File

@@ -3,9 +3,9 @@ import ClientPgLite from 'knex-pglite';
import { PGlite } from '@electric-sql/pglite'; import { PGlite } from '@electric-sql/pglite';
import { vector } from '@electric-sql/pglite/vector'; import { vector } from '@electric-sql/pglite/vector';
import { migrationSource } from './migrations/migrations.ts'; import { migrationSource } from './migrations/migrations.js';
import { destroy, Services } from '#root/utils/utils.services.ts'; import { destroy, Services } from '#root/utils/utils.services.js';
class DatabaseService { class DatabaseService {
#services: Services; #services: Services;
@@ -50,5 +50,5 @@ class DatabaseService {
}; };
} }
export { type TableRows, tableNames } from './migrations/migrations.ts'; export { type TableRows, tableNames } from './migrations/migrations.js';
export { DatabaseService }; export { DatabaseService };

View File

@@ -1,7 +1,7 @@
import type { Migration } from './migrations.types.ts'; import type { Migration } from './migrations.types.js';
import { EmbeddingsService } from '#root/services/embeddings/embeddings.ts'; import { EmbeddingsService } from '#root/services/embeddings/embeddings.js';
import { EMBEDDING_MODEL } from '#root/utils/utils.consts.ts'; import { EMBEDDING_MODEL } from '#root/utils/utils.consts.js';
const tableNames = { const tableNames = {
documents: 'documents', documents: 'documents',

View File

@@ -1,9 +1,9 @@
import type { Knex } from 'knex'; import type { Knex } from 'knex';
import type { Migration } from './migrations.types.ts'; import type { Migration } from './migrations.types.js';
import { init } from './migrations.001-init.ts'; import { init } from './migrations.001-init.js';
import type { Services } from '#root/utils/utils.services.ts'; import type { Services } from '#root/utils/utils.services.js';
const migrations = [init] satisfies Migration[]; const migrations = [init] satisfies Migration[];
@@ -21,5 +21,5 @@ const migrationSource = (options: MigrationSourceOptions): Knex.MigrationSource<
getMigrations: async () => migrations, getMigrations: async () => migrations,
}); });
export { type TableRows, tableNames } from './migrations.001-init.ts'; export { type TableRows, tableNames } from './migrations.001-init.js';
export { migrationSource }; export { migrationSource };

View File

@@ -1,6 +1,6 @@
import type { Knex } from 'knex'; import type { Knex } from 'knex';
import type { Services } from '#root/utils/utils.services.ts'; import type { Services } from '#root/utils/utils.services.js';
type MigrationOptions = { type MigrationOptions = {
knex: Knex; knex: Knex;

View File

@@ -1,6 +1,6 @@
import type { TableRows } from '../database/database.ts'; import type { TableRows } from '../database/database.js';
import type { DocumentChunk } from './document-chunks.schemas.ts'; import type { DocumentChunk } from './document-chunks.schemas.js';
const mapFromDocumentChunkRow = ( const mapFromDocumentChunkRow = (
row: TableRows['documentChunks'] & { row: TableRows['documentChunks'] & {

View File

@@ -1,7 +1,7 @@
import { z } from 'zod'; import { z } from 'zod';
import { queryFilterSchema } from '@morten-olsen/stash-query-dsl';
import { createListResultSchema } from '#root/utils/utils.schema.ts'; import { createListResultSchema } from '#root/utils/utils.schema.js';
import { queryFilterSchema } from '#root/utils/utils.query.ts';
const documentChunkSchema = z.object({ const documentChunkSchema = z.object({
id: z.string(), id: z.string(),

View File

@@ -1,14 +1,15 @@
import { DatabaseService, tableNames, type TableRows } from '../database/database.ts'; import { QueryParser } from '@morten-olsen/stash-query-dsl';
import { EmbeddingsService } from '../embeddings/embeddings.ts';
import type { DocumentChunkFilter, DocumentChunksFindResult } from './document-chunks.schemas.ts'; import { DatabaseService, tableNames, type TableRows } from '../database/database.js';
import { mapFromDocumentChunkRow } from './document.mappings.ts'; import { EmbeddingsService } from '../embeddings/embeddings.js';
import type { Services } from '#root/utils/utils.services.ts'; import type { DocumentChunkFilter, DocumentChunksFindResult } from './document-chunks.schemas.js';
import { EMBEDDING_MODEL } from '#root/utils/utils.consts.ts'; import { mapFromDocumentChunkRow } from './document-chunks.mappings.js';
import type { Services } from '#root/utils/utils.services.js';
import { EMBEDDING_MODEL } from '#root/utils/utils.consts.js';
import type { ExplicitAny } from '#root/global.js'; import type { ExplicitAny } from '#root/global.js';
import { applyQueryFilter } from '#root/utils/utils.query.ts'; import { applyQueryFilter } from '#root/utils/utils.query.js';
import { QueryParser } from '#root/query-parser/query-parser.ts';
const baseFields = [ const baseFields = [
`${tableNames.documentChunks}.*`, `${tableNames.documentChunks}.*`,
@@ -61,5 +62,5 @@ class DocumentChunksService {
}; };
} }
export * from './document-chunks.schemas.ts'; export * from './document-chunks.schemas.js';
export { DocumentChunksService }; export { DocumentChunksService };

View File

@@ -1,6 +1,6 @@
import type { TableRows } from '../database/database.ts'; import type { TableRows } from '../database/database.js';
import type { Document } from './documents.schemas.ts'; import type { Document } from './documents.schemas.js';
const mapFromDocumentRow = (row: TableRows['documents']): Document => ({ const mapFromDocumentRow = (row: TableRows['documents']): Document => ({
...row, ...row,

View File

@@ -1,7 +1,7 @@
import { z } from 'zod'; import { z } from 'zod';
import { queryFilterSchema } from '@morten-olsen/stash-query-dsl';
import { createListResultSchema } from '#root/utils/utils.schema.ts'; import { createListResultSchema } from '#root/utils/utils.schema.js';
import { queryFilterSchema } from '#root/utils/utils.query.ts';
const documentSchema = z.object({ const documentSchema = z.object({
id: z.string(), id: z.string(),

View File

@@ -1,5 +1,7 @@
import { DatabaseService, tableNames, type TableRows } from '../database/database.ts'; import { QueryParser } from '@morten-olsen/stash-query-dsl';
import { SplittingService } from '../splitter/splitter.ts';
import { DatabaseService, tableNames, type TableRows } from '../database/database.js';
import { SplittingService } from '../splitter/splitter.js';
import type { import type {
Document, Document,
@@ -8,13 +10,12 @@ import type {
DocumentUpsert, DocumentUpsert,
DocumentUpsertResult, DocumentUpsertResult,
} from './documents.schemas.ts'; } from './documents.schemas.ts';
import { mapFromDocumentRow } from './documents.mapping.ts'; import { mapFromDocumentRow } from './documents.mapping.js';
import { EventEmitter } from '#root/utils/utils.event-emitter.ts'; import { EventEmitter } from '#root/utils/utils.event-emitter.js';
import type { Services } from '#root/utils/utils.services.ts'; import type { Services } from '#root/utils/utils.services.js';
import { compareObjectKeys } from '#root/utils/utils.compare.ts'; import { compareObjectKeys } from '#root/utils/utils.compare.js';
import { applyQueryFilter } from '#root/utils/utils.query.ts'; import { applyQueryFilter } from '#root/utils/utils.query.js';
import { QueryParser } from '#root/query-parser/query-parser.ts';
type DocumentsServiceEvents = { type DocumentsServiceEvents = {
upserted: (document: Document) => void; upserted: (document: Document) => void;
@@ -174,5 +175,5 @@ class DocumentsService extends EventEmitter<DocumentsServiceEvents> {
}; };
} }
export * from './documents.schemas.ts'; export * from './documents.schemas.js';
export { DocumentsService }; export { DocumentsService };

View File

@@ -1,6 +1,8 @@
import { pipeline, type FeatureExtractionPipeline } from '@huggingface/transformers'; import { pipeline, type FeatureExtractionPipeline } from '@huggingface/transformers';
import { Vector } from './embeddings.vector.ts'; import { Vector } from './embeddings.vector.js';
import type { ExplicitAny } from '#root/global.js';
type ExtractOptions = { type ExtractOptions = {
input: string[]; input: string[];
@@ -57,4 +59,4 @@ class EmbeddingsService {
}; };
} }
export { EmbeddingsService }; export { EmbeddingsService, Vector };

View File

@@ -1,11 +1,11 @@
import { EmbeddingsService } from '../embeddings/embeddings.ts'; import { EmbeddingsService } from '../embeddings/embeddings.js';
import type { Document } from '../documents/documents.schemas.ts'; import type { Document } from '../documents/documents.schemas.js';
import type { Chunk, Splitter } from './splitter.types.ts'; import type { Chunk, Splitter } from './splitter.types.js';
import { textSplitter } from './splitters/splitters.text.ts'; import { textSplitter } from './splitters/splitters.text.js';
import type { Services } from '#root/utils/utils.services.ts'; import type { Services } from '#root/utils/utils.services.js';
import { EMBEDDING_MODEL } from '#root/utils/utils.consts.ts'; import { EMBEDDING_MODEL } from '#root/utils/utils.consts.js';
class SplittingService { class SplittingService {
#services: Services; #services: Services;
@@ -40,5 +40,5 @@ class SplittingService {
}; };
} }
export * from './splitter.types.ts'; export * from './splitter.types.js';
export { SplittingService }; export { SplittingService };

View File

@@ -1,5 +1,5 @@
import type { Document } from '../documents/documents.schemas.ts'; import type { Document } from '../documents/documents.schemas.js';
import type { Vector } from '../embeddings/embeddings.vector.ts'; import type { Vector } from '../embeddings/embeddings.vector.js';
type Chunk = { type Chunk = {
content: string; content: string;

View File

@@ -1,6 +1,6 @@
import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters'; import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
import type { Splitter } from '../splitter.types.ts'; import type { Splitter } from '../splitter.types.js';
const textSplitter: Splitter = { const textSplitter: Splitter = {
match: (document) => !!document.content, match: (document) => !!document.content,

View File

@@ -1,6 +1,6 @@
import { DatabaseService } from '../database/database.ts'; import { DatabaseService } from '../database/database.js';
import { Services } from '#root/utils/utils.services.ts'; import { Services } from '#root/utils/utils.services.js';
class WarmupService { class WarmupService {
#services: Services; #services: Services;

View File

@@ -1,3 +1,5 @@
import type { ExplicitAny } from '#root/global.js';
type EventListener<T extends unknown[]> = (...args: T) => void | Promise<void>; type EventListener<T extends unknown[]> = (...args: T) => void | Promise<void>;
type OnOptions = { type OnOptions = {

View File

@@ -0,0 +1,161 @@
import type {
QueryCondition,
QueryConditionNumber,
QueryConditionText,
QueryFilter,
} from '@morten-olsen/stash-query-dsl';
import { type Knex } from 'knex';
/**
* Escapes a JSON key for use in PostgreSQL JSON operators.
* Escapes single quotes by doubling them, which is the PostgreSQL standard.
*/
const escapeJsonKey = (key: string): string => {
return key.replace(/'/g, "''");
};
const getFieldSelector = (query: Knex.QueryBuilder, field: string[], tableName?: string) => {
const baseColumn = field[0];
if (field.length === 1) {
return tableName ? `${tableName}.${baseColumn}` : baseColumn;
}
const baseFieldRef = tableName ? query.client.ref(baseColumn).withSchema(tableName) : query.client.ref(baseColumn);
const jsonPath = field.slice(1);
let sqlExpression = baseFieldRef.toString();
for (let i = 0; i < jsonPath.length - 1; i++) {
const escapedKey = escapeJsonKey(jsonPath[i]);
sqlExpression += ` -> '${escapedKey}'`;
}
const finalElement = jsonPath[jsonPath.length - 1];
const escapedFinalKey = escapeJsonKey(finalElement);
sqlExpression += ` ->> '${escapedFinalKey}'`;
return query.client.raw(sqlExpression);
};
const applyQueryConditionText = (query: Knex.QueryBuilder, { field, tableName, conditions }: QueryConditionText) => {
const selector = getFieldSelector(query, field, tableName);
if (conditions.equal) {
query = query.where(selector, '=', conditions.equal);
}
if (conditions.notEqual) {
query = query.where(selector, '<>', conditions.notEqual);
}
if (conditions.like) {
query = query.whereLike(selector, conditions.like);
}
if (conditions.notLike) {
query = query.not.whereLike(selector, conditions.notLike);
}
if (conditions.equal === null) {
query = query.whereNull(selector);
}
if (conditions.notEqual === null) {
query = query.whereNotNull(selector);
}
if (conditions.in) {
query = query.whereIn(selector, conditions.in);
}
if (conditions.notIn) {
query = query.whereNotIn(selector, conditions.notIn);
}
return query;
};
const applyQueryConditionNumber = (
query: Knex.QueryBuilder,
{ field, tableName, conditions }: QueryConditionNumber,
) => {
const selector = getFieldSelector(query, field, tableName);
if (conditions.equals !== undefined && conditions.equals !== null) {
query = query.where(selector, '=', conditions.equals);
}
if (conditions.notEquals !== undefined && conditions.notEquals !== null) {
query = query.where(selector, '<>', conditions.notEquals);
}
if (conditions.equals === null) {
query = query.whereNull(selector);
}
if (conditions.notEquals === null) {
query = query.whereNotNull(selector);
}
if (conditions.greaterThan) {
query = query.where(selector, '>', conditions.greaterThan);
}
if (conditions.greaterThanOrEqual) {
query = query.where(selector, '>=', conditions.greaterThanOrEqual);
}
if (conditions.lessThan) {
query = query.where(selector, '<', conditions.lessThan);
}
if (conditions.lessThanOrEqual) {
query = query.where(selector, '<=', conditions.lessThanOrEqual);
}
if (conditions.in) {
query = query.whereIn(selector, conditions.in);
}
if (conditions.notIn) {
query = query.whereNotIn(selector, conditions.notIn);
}
return query;
};
const applyQueryCondition = (query: Knex.QueryBuilder, options: QueryCondition) => {
switch (options.type) {
case 'text': {
return applyQueryConditionText(query, options);
}
case 'number': {
return applyQueryConditionNumber(query, options);
}
default: {
throw new Error(`Unknown filter type`);
}
}
};
const applyQueryFilter = (query: Knex.QueryBuilder, filter: QueryFilter) => {
if (filter.type === 'operator') {
if (filter.conditions.length === 0) {
return query;
}
switch (filter.operator) {
case 'or': {
return query.where((subquery) => {
let isFirst = true;
for (const condition of filter.conditions) {
if (isFirst) {
applyQueryFilter(subquery, condition);
isFirst = false;
} else {
subquery.orWhere((subSubquery) => {
applyQueryFilter(subSubquery, condition);
});
}
}
});
}
case 'and': {
return query.where((subquery) => {
let isFirst = true;
for (const condition of filter.conditions) {
if (isFirst) {
applyQueryFilter(subquery, condition);
isFirst = false;
} else {
subquery.andWhere((subSubquery) => {
applyQueryFilter(subSubquery, condition);
});
}
}
});
}
}
} else {
return applyQueryCondition(query, filter);
}
};
export { applyQueryCondition, applyQueryFilter };

View File

@@ -0,0 +1,10 @@
{
"compilerOptions": {
"outDir": "./dist",
"rootDir": "./src"
},
"include": [
"src/**/*.ts"
],
"extends": "@morten-olsen/stash-configs/tsconfig.json"
}

View File

@@ -0,0 +1,12 @@
import { defineConfig } from 'vitest/config';
import { getAliases } from '@morten-olsen/stash-tests/vitest';
// eslint-disable-next-line import/no-default-export
export default defineConfig(async () => {
const aliases = await getAliases();
return {
resolve: {
alias: aliases,
},
};
});

View File

@@ -31,21 +31,14 @@
"#root/*": "./src/*" "#root/*": "./src/*"
}, },
"dependencies": { "dependencies": {
"@electric-sql/pglite": "^0.3.14",
"@fastify/cors": "11.1.0", "@fastify/cors": "11.1.0",
"@fastify/swagger": "9.6.1", "@fastify/swagger": "9.6.1",
"@fastify/websocket": "11.2.0", "@fastify/websocket": "11.2.0",
"@huggingface/transformers": "^3.8.1", "@morten-olsen/stash-query-dsl": "workspace:*",
"@langchain/textsplitters": "^1.0.1", "@morten-olsen/stash-runtime": "workspace:*",
"@scalar/fastify-api-reference": "1.40.2", "@scalar/fastify-api-reference": "1.40.2",
"better-sqlite3": "^12.5.0",
"deep-equal": "^2.2.3",
"fastify": "5.6.2", "fastify": "5.6.2",
"fastify-type-provider-zod": "6.1.0", "fastify-type-provider-zod": "6.1.0",
"knex": "^3.1.0",
"knex-pglite": "^0.13.0",
"pg": "^8.16.3",
"pgvector": "^0.2.1",
"zod": "4.1.13", "zod": "4.1.13",
"zod-to-json-schema": "3.25.0" "zod-to-json-schema": "3.25.0"
} }

View File

@@ -9,13 +9,12 @@ import {
validatorCompiler, validatorCompiler,
type ZodTypeProvider, type ZodTypeProvider,
} from 'fastify-type-provider-zod'; } from 'fastify-type-provider-zod';
import { StashRuntime } from '@morten-olsen/stash-runtime';
import { Services } from './utils/utils.services.ts'; import { systemEndpoints } from './endpoints/system/system.js';
import { systemEndpoints } from './endpoints/system/system.ts'; import { documentEndpoints } from './endpoints/documents/documents.js';
import { WarmupService } from './services/warmup/warmup.ts'; import { documentFilterEndpoints } from './endpoints/document-filters/document-filters.js';
import { documentEndpoints } from './endpoints/documents/documents.ts'; import { documentChunkFilterEndpoints } from './endpoints/document-chunk-filters/document-chunk-filters.js';
import { documentFilterEndpoints } from './endpoints/document-filters/document-filters.ts';
import { documentChunkFilterEndpoints } from './endpoints/document-chunk-filters/document-chunk-filters.ts';
class BaseError extends Error { class BaseError extends Error {
public statusCode: number; public statusCode: number;
@@ -26,12 +25,12 @@ class BaseError extends Error {
} }
} }
const createApi = async (services: Services = new Services()) => { const createApi = async (runtime: StashRuntime = new StashRuntime()) => {
const app = fastify().withTypeProvider<ZodTypeProvider>(); const app = fastify().withTypeProvider<ZodTypeProvider>();
app.setValidatorCompiler(validatorCompiler); app.setValidatorCompiler(validatorCompiler);
app.setSerializerCompiler(serializerCompiler); app.setSerializerCompiler(serializerCompiler);
app.decorate('services', services); app.decorate('runtime', runtime);
app.register(fastifyCors); app.register(fastifyCors);
app.register(fastifySwagger, { app.register(fastifySwagger, {
@@ -92,8 +91,7 @@ const createApi = async (services: Services = new Services()) => {
}); });
app.addHook('onReady', async () => { app.addHook('onReady', async () => {
const warmupService = app.services.get(WarmupService); app.runtime.warmup.ensure();
await warmupService.ensure();
}); });
await app.register(systemEndpoints, { prefix: '/system' }); await app.register(systemEndpoints, { prefix: '/system' });

View File

@@ -1,11 +1,10 @@
import { StashRuntime, type DocumentUpsert } from '@morten-olsen/stash-runtime';
import { createApi } from './api.js'; import { createApi } from './api.js';
import { DocumentsService, type DocumentUpsert } from './services/documents/documents.ts';
import { Services } from './utils/utils.services.ts';
const services = new Services(); const runtime = new StashRuntime();
const server = await createApi(services); const server = await createApi(runtime);
const documentsService = services.get(DocumentsService);
const documents: DocumentUpsert[] = [ const documents: DocumentUpsert[] = [
{ {
metadata: { metadata: {
@@ -31,7 +30,7 @@ const documents: DocumentUpsert[] = [
}, },
]; ];
await Promise.all(documents.map((document) => documentsService.upsert(document))); await Promise.all(documents.map((document) => runtime.documents.upsert(document)));
await server.listen({ await server.listen({
port: 3400, port: 3400,

View File

@@ -1,11 +1,6 @@
import { documentChunkFilterSchema, documentChunksFindResultSchema } from '@morten-olsen/stash-runtime';
import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
import {
documentChunkFilterSchema,
documentChunksFindResultSchema,
DocumentChunksService,
} from '#root/services/document-chunks/document-chunks.ts';
const documentChunkFilterEndpoints: FastifyPluginAsyncZod = async (instance) => { const documentChunkFilterEndpoints: FastifyPluginAsyncZod = async (instance) => {
instance.route({ instance.route({
method: 'POST', method: 'POST',
@@ -20,9 +15,8 @@ const documentChunkFilterEndpoints: FastifyPluginAsyncZod = async (instance) =>
}, },
}, },
handler: async (req, reply) => { handler: async (req, reply) => {
const { services } = instance; const { runtime } = instance;
const documentChunksService = services.get(DocumentChunksService); const response = await runtime.documentChunks.find(req.body);
const response = await documentChunksService.find(req.body);
await reply.send(response); await reply.send(response);
}, },
}); });

View File

@@ -1,11 +1,6 @@
import { documentFilterSchema, documentFindResultSchema } from '@morten-olsen/stash-runtime';
import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
import {
documentFilterSchema,
documentFindResultSchema,
DocumentsService,
} from '#root/services/documents/documents.ts';
const documentFilterEndpoints: FastifyPluginAsyncZod = async (instance) => { const documentFilterEndpoints: FastifyPluginAsyncZod = async (instance) => {
instance.route({ instance.route({
method: 'POST', method: 'POST',
@@ -20,9 +15,8 @@ const documentFilterEndpoints: FastifyPluginAsyncZod = async (instance) => {
}, },
}, },
handler: async (req, reply) => { handler: async (req, reply) => {
const { services } = instance; const { runtime } = instance;
const documentsService = services.get(DocumentsService); const response = await runtime.documents.find(req.body);
const response = await documentsService.find(req.body);
await reply.send(response); await reply.send(response);
}, },
}); });

View File

@@ -1,11 +1,6 @@
import { documentUpsertResultSchema, documentUpsertSchema } from '@morten-olsen/stash-runtime';
import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
import {
DocumentsService,
documentUpsertResultSchema,
documentUpsertSchema,
} from '#root/services/documents/documents.ts';
const documentEndpoints: FastifyPluginAsyncZod = async (instance) => { const documentEndpoints: FastifyPluginAsyncZod = async (instance) => {
instance.route({ instance.route({
method: 'POST', method: 'POST',
@@ -20,9 +15,8 @@ const documentEndpoints: FastifyPluginAsyncZod = async (instance) => {
}, },
}, },
handler: async (req, reply) => { handler: async (req, reply) => {
const { services } = instance; const { runtime } = instance;
const documentsService = services.get(DocumentsService); const response = await runtime.documents.upsert(req.body);
const response = await documentsService.upsert(req.body);
await reply.send(response); await reply.send(response);
}, },
}); });

View File

@@ -1,8 +1,6 @@
import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'; import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
import { z } from 'zod'; import { z } from 'zod';
import { DatabaseService } from '#root/services/database/database.ts';
const systemEndpoints: FastifyPluginAsyncZod = async (instance) => { const systemEndpoints: FastifyPluginAsyncZod = async (instance) => {
instance.route({ instance.route({
method: 'GET', method: 'GET',
@@ -18,9 +16,8 @@ const systemEndpoints: FastifyPluginAsyncZod = async (instance) => {
}, },
}, },
handler: async (_, reply) => { handler: async (_, reply) => {
const { services } = instance; const { runtime } = instance;
const databaseService = services.get(DatabaseService); const db = await runtime.database.getInstance();
const db = await databaseService.getInstance();
await db.raw('SELECT 1=1'); await db.raw('SELECT 1=1');
await reply.send({ await reply.send({
status: 'ok', status: 'ok',

View File

@@ -1,5 +1,5 @@
import 'fastify'; import 'fastify';
import type { Services } from './utils/utils.services.ts'; import type { StashRuntime } from '@morten-olsen/stash-runtime';
// eslint-disable-next-line // eslint-disable-next-line
declare type ExplicitAny = any; declare type ExplicitAny = any;
@@ -7,6 +7,6 @@ declare type ExplicitAny = any;
declare module 'fastify' { declare module 'fastify' {
// eslint-disable-next-line // eslint-disable-next-line
export interface FastifyInstance { export interface FastifyInstance {
services: Services; runtime: StashRuntime;
} }
} }

View File

@@ -1,202 +0,0 @@
import type { Token } from './query-parser.types.ts';
class Lexer {
#input: string;
#position = 0;
#tokens: Token[] = [];
constructor(input: string) {
this.#input = input;
}
#skipWhitespace = (): void => {
while (this.#position < this.#input.length && /\s/.test(this.#input[this.#position])) {
this.#position++;
}
};
#nextToken = (): Token | null => {
const char = this.#input[this.#position];
const startPosition = this.#position;
// Single character tokens
if (char === '(') {
this.#position++;
return { type: 'LPAREN', value: '(', position: startPosition };
}
if (char === ')') {
this.#position++;
return { type: 'RPAREN', value: ')', position: startPosition };
}
if (char === ',') {
this.#position++;
return { type: 'COMMA', value: ',', position: startPosition };
}
if (char === '.') {
this.#position++;
return { type: 'DOT', value: '.', position: startPosition };
}
// Two-character operators
if (char === '!' && this.#input[this.#position + 1] === '=') {
this.#position += 2;
return { type: 'NOT_EQUALS', value: '!=', position: startPosition };
}
if (char === '>' && this.#input[this.#position + 1] === '=') {
this.#position += 2;
return { type: 'GREATER_THAN_OR_EQUAL', value: '>=', position: startPosition };
}
if (char === '<' && this.#input[this.#position + 1] === '=') {
this.#position += 2;
return { type: 'LESS_THAN_OR_EQUAL', value: '<=', position: startPosition };
}
// Single character operators
if (char === '=') {
this.#position++;
return { type: 'EQUALS', value: '=', position: startPosition };
}
if (char === '>') {
this.#position++;
return { type: 'GREATER_THAN', value: '>', position: startPosition };
}
if (char === '<') {
this.#position++;
return { type: 'LESS_THAN', value: '<', position: startPosition };
}
// String literal
if (char === "'") {
return this.#readString();
}
// Number
if (/[0-9]/.test(char) || (char === '-' && /[0-9]/.test(this.#input[this.#position + 1]))) {
return this.#readNumber();
}
// Identifier or keyword
if (/[a-zA-Z_]/.test(char)) {
return this.#readIdentifierOrKeyword();
}
throw new Error(`Unexpected character '${char}' at position ${this.#position}`);
};
#readString = (): Token => {
const startPosition = this.#position;
this.#position++; // Skip opening quote
let value = '';
while (this.#position < this.#input.length) {
const char = this.#input[this.#position];
if (char === "'") {
// Check for escaped quote
if (this.#input[this.#position + 1] === "'") {
value += "'";
this.#position += 2;
} else {
this.#position++; // Skip closing quote
return { type: 'STRING', value, position: startPosition };
}
} else {
value += char;
this.#position++;
}
}
throw new Error(`Unterminated string starting at position ${startPosition}`);
};
#readNumber = (): Token => {
const startPosition = this.#position;
let value = '';
// Optional minus sign
if (this.#input[this.#position] === '-') {
value += '-';
this.#position++;
}
// Integer part
while (this.#position < this.#input.length && /[0-9]/.test(this.#input[this.#position])) {
value += this.#input[this.#position];
this.#position++;
}
// Decimal part
if (this.#input[this.#position] === '.' && /[0-9]/.test(this.#input[this.#position + 1])) {
value += '.';
this.#position++;
while (this.#position < this.#input.length && /[0-9]/.test(this.#input[this.#position])) {
value += this.#input[this.#position];
this.#position++;
}
}
// Scientific notation
if (this.#input[this.#position] === 'e' || this.#input[this.#position] === 'E') {
value += this.#input[this.#position];
this.#position++;
if (this.#input[this.#position] === '+' || this.#input[this.#position] === '-') {
value += this.#input[this.#position];
this.#position++;
}
while (this.#position < this.#input.length && /[0-9]/.test(this.#input[this.#position])) {
value += this.#input[this.#position];
this.#position++;
}
}
return { type: 'NUMBER', value, position: startPosition };
};
#readIdentifierOrKeyword = (): Token => {
const startPosition = this.#position;
let value = '';
while (this.#position < this.#input.length && /[a-zA-Z0-9_]/.test(this.#input[this.#position])) {
value += this.#input[this.#position];
this.#position++;
}
const upperValue = value.toUpperCase();
// Keywords
switch (upperValue) {
case 'AND':
return { type: 'AND', value, position: startPosition };
case 'OR':
return { type: 'OR', value, position: startPosition };
case 'LIKE':
return { type: 'LIKE', value, position: startPosition };
case 'NOT':
return { type: 'NOT', value, position: startPosition };
case 'IN':
return { type: 'IN', value, position: startPosition };
case 'IS':
return { type: 'IS', value, position: startPosition };
case 'NULL':
return { type: 'NULL', value, position: startPosition };
default:
return { type: 'IDENTIFIER', value, position: startPosition };
}
};
public tokenize = (): Token[] => {
while (this.#position < this.#input.length) {
this.#skipWhitespace();
if (this.#position >= this.#input.length) break;
const token = this.#nextToken();
if (token) {
this.#tokens.push(token);
}
}
this.#tokens.push({ type: 'EOF', value: '', position: this.#position });
return this.#tokens;
};
}
export { Lexer };

View File

@@ -1,317 +0,0 @@
import { Lexer } from './query-parser.lexer.ts';
import type { Token, TokenType } from './query-parser.types.ts';
import type { QueryConditionText, QueryConditionNumber, QueryFilter, QueryCondition } from '#root/utils/utils.query.ts';
class Parser {
#tokens: Token[] = [];
#position = 0;
#current = (): Token => {
return this.#tokens[this.#position];
};
#advance = (): Token => {
const token = this.#current();
this.#position++;
return token;
};
#expect = (type: TokenType): Token => {
const token = this.#current();
if (token.type !== type) {
throw new Error(`Expected ${type} but got ${token.type} at position ${token.position}`);
}
return this.#advance();
};
#parseExpression = (): QueryFilter => {
return this.#parseOr();
};
#parseOr = (): QueryFilter => {
let left = this.#parseAnd();
while (this.#current().type === 'OR') {
this.#advance();
const right = this.#parseAnd();
left = this.#combineWithOperator(left, right, 'or');
}
return left;
};
#parseAnd = (): QueryFilter => {
let left = this.#parsePrimary();
while (this.#current().type === 'AND') {
this.#advance();
const right = this.#parsePrimary();
left = this.#combineWithOperator(left, right, 'and');
}
return left;
};
#combineWithOperator = (left: QueryFilter, right: QueryFilter, operator: 'and' | 'or'): QueryFilter => {
// If left is already an operator of the same type, add to its conditions
if (left.type === 'operator' && left.operator === operator) {
return {
type: 'operator',
operator,
conditions: [...left.conditions, right],
};
}
return {
type: 'operator',
operator,
conditions: [left, right],
};
};
#parsePrimary = (): QueryFilter => {
// Handle parenthesized expressions
if (this.#current().type === 'LPAREN') {
this.#advance();
const expr = this.#parseExpression();
this.#expect('RPAREN');
return expr;
}
// Must be a condition
return this.#parseCondition();
};
#parseCondition = (): QueryCondition => {
const field = this.#parseField();
const token = this.#current();
// IS NULL / IS NOT NULL
if (token.type === 'IS') {
this.#advance();
const isNot = this.#current().type === 'NOT';
if (isNot) {
this.#advance();
}
this.#expect('NULL');
// IS NULL / IS NOT NULL could be either text or number - default to text
return {
type: 'text',
field,
conditions: isNot ? { notEqual: undefined, equal: undefined } : { equal: null },
} satisfies QueryConditionText;
}
// NOT IN / NOT LIKE
if (token.type === 'NOT') {
this.#advance();
const nextToken = this.#current();
if (nextToken.type === 'IN') {
this.#advance();
return this.#parseInCondition(field, true);
}
if (nextToken.type === 'LIKE') {
this.#advance();
const pattern = this.#expect('STRING').value;
return {
type: 'text',
field,
conditions: { notLike: pattern },
};
}
throw new Error(`Expected IN or LIKE after NOT at position ${nextToken.position}`);
}
// IN
if (token.type === 'IN') {
this.#advance();
return this.#parseInCondition(field, false);
}
// LIKE
if (token.type === 'LIKE') {
this.#advance();
const pattern = this.#expect('STRING').value;
return {
type: 'text',
field,
conditions: { like: pattern },
};
}
// Comparison operators
if (token.type === 'EQUALS') {
this.#advance();
return this.#parseValueCondition(field, 'equals');
}
if (token.type === 'NOT_EQUALS') {
this.#advance();
return this.#parseValueCondition(field, 'notEquals');
}
if (token.type === 'GREATER_THAN') {
this.#advance();
const value = this.#parseNumber();
return {
type: 'number',
field,
conditions: { greaterThan: value },
};
}
if (token.type === 'GREATER_THAN_OR_EQUAL') {
this.#advance();
const value = this.#parseNumber();
return {
type: 'number',
field,
conditions: { greaterThanOrEqual: value },
};
}
if (token.type === 'LESS_THAN') {
this.#advance();
const value = this.#parseNumber();
return {
type: 'number',
field,
conditions: { lessThan: value },
};
}
if (token.type === 'LESS_THAN_OR_EQUAL') {
this.#advance();
const value = this.#parseNumber();
return {
type: 'number',
field,
conditions: { lessThanOrEqual: value },
};
}
throw new Error(`Unexpected token '${token.value}' at position ${token.position}`);
};
#parseField = (): string[] => {
const parts: string[] = [];
parts.push(this.#expect('IDENTIFIER').value);
while (this.#current().type === 'DOT') {
this.#advance();
parts.push(this.#expect('IDENTIFIER').value);
}
return parts;
};
#parseValueCondition = (field: string[], operator: 'equals' | 'notEquals'): QueryCondition => {
const token = this.#current();
if (token.type === 'STRING') {
this.#advance();
const textCondition: QueryConditionText = {
type: 'text',
field,
conditions: operator === 'equals' ? { equal: token.value } : { notEqual: token.value },
};
return textCondition;
}
if (token.type === 'NUMBER') {
this.#advance();
const value = parseFloat(token.value);
const numCondition: QueryConditionNumber = {
type: 'number',
field,
conditions: operator === 'equals' ? { equals: value } : { notEquals: value },
};
return numCondition;
}
if (token.type === 'NULL') {
this.#advance();
// NULL equality - default to text type
return {
type: 'text',
field,
conditions: operator === 'equals' ? { equal: null } : {},
} as QueryConditionText;
}
throw new Error(`Expected value but got ${token.type} at position ${token.position}`);
};
#parseNumber = (): number => {
const token = this.#expect('NUMBER');
return parseFloat(token.value);
};
#parseInCondition = (field: string[], isNot: boolean): QueryCondition => {
this.#expect('LPAREN');
const firstToken = this.#current();
if (firstToken.type === 'STRING') {
// Text IN
const values: string[] = [];
values.push(this.#advance().value);
while (this.#current().type === 'COMMA') {
this.#advance();
values.push(this.#expect('STRING').value);
}
this.#expect('RPAREN');
return {
type: 'text',
field,
conditions: isNot ? { notIn: values } : { in: values },
};
}
if (firstToken.type === 'NUMBER') {
// Numeric IN
const values: number[] = [];
values.push(parseFloat(this.#advance().value));
while (this.#current().type === 'COMMA') {
this.#advance();
values.push(parseFloat(this.#expect('NUMBER').value));
}
this.#expect('RPAREN');
return {
type: 'number',
field,
conditions: isNot ? { notIn: values } : { in: values },
};
}
throw new Error(`Expected STRING or NUMBER in IN list at position ${firstToken.position}`);
};
public parse(input: string): QueryFilter {
const lexer = new Lexer(input);
this.#tokens = lexer.tokenize();
this.#position = 0;
const result = this.#parseExpression();
if (this.#current().type !== 'EOF') {
throw new Error(`Unexpected token '${this.#current().value}' at position ${this.#current().position}`);
}
return result;
}
}
export { Parser };

View File

@@ -1,19 +0,0 @@
import { Stringifier } from './query-parser.stringifier.ts';
import { Parser } from './query-parser.parser.ts';
import type { QueryFilter } from '#root/utils/utils.query.ts';
class QueryParser {
private parser = new Parser();
private stringifier = new Stringifier();
public parse = (input: string): QueryFilter => {
return this.parser.parse(input);
};
public stringify = (filter: QueryFilter): string => {
return this.stringifier.stringify(filter);
};
}
export { QueryParser };

View File

@@ -1,30 +0,0 @@
type TokenType =
| 'IDENTIFIER'
| 'STRING'
| 'NUMBER'
| 'AND'
| 'OR'
| 'LIKE'
| 'NOT'
| 'IN'
| 'IS'
| 'NULL'
| 'EQUALS'
| 'NOT_EQUALS'
| 'GREATER_THAN'
| 'GREATER_THAN_OR_EQUAL'
| 'LESS_THAN'
| 'LESS_THAN_OR_EQUAL'
| 'LPAREN'
| 'RPAREN'
| 'COMMA'
| 'DOT'
| 'EOF';
type Token = {
type: TokenType;
value: string;
position: number;
};
export type { TokenType, Token };

View File

@@ -1,549 +0,0 @@
import { type Knex } from 'knex';
import { z } from 'zod';
/**
* Escapes a JSON key for use in PostgreSQL JSON operators.
* Escapes single quotes by doubling them, which is the PostgreSQL standard.
*/
const escapeJsonKey = (key: string): string => {
return key.replace(/'/g, "''");
};
const getFieldSelector = (query: Knex.QueryBuilder, field: string[], tableName?: string) => {
const baseColumn = field[0];
if (field.length === 1) {
return tableName ? `${tableName}.${baseColumn}` : baseColumn;
}
const baseFieldRef = tableName ? query.client.ref(baseColumn).withSchema(tableName) : query.client.ref(baseColumn);
const jsonPath = field.slice(1);
let sqlExpression = baseFieldRef.toString();
for (let i = 0; i < jsonPath.length - 1; i++) {
const escapedKey = escapeJsonKey(jsonPath[i]);
sqlExpression += ` -> '${escapedKey}'`;
}
const finalElement = jsonPath[jsonPath.length - 1];
const escapedFinalKey = escapeJsonKey(finalElement);
sqlExpression += ` ->> '${escapedFinalKey}'`;
return query.client.raw(sqlExpression);
};
const queryConditionTextSchema = z
.object({
type: z.literal('text'),
tableName: z.string().optional(),
field: z.array(z.string()),
conditions: z.object({
equal: z.string().nullish(),
notEqual: z.string().optional(),
like: z.string().optional(),
notLike: z.string().optional(),
in: z.array(z.string()).optional(),
notIn: z.array(z.string()).optional(),
}),
})
.meta({
example: {
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'bar',
},
},
examples: [
{
summary: 'Equal condition',
value: {
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'bar',
},
},
},
{
summary: 'Like condition',
value: {
type: 'text',
field: ['content'],
conditions: {
like: '%cat%',
},
},
},
{
summary: 'In condition',
value: {
type: 'text',
field: ['type'],
conditions: {
in: ['demo', 'article', 'post'],
},
},
},
{
summary: 'Null check',
value: {
type: 'text',
field: ['source'],
conditions: {
equal: null,
},
},
},
],
});
type QueryConditionText = z.infer<typeof queryConditionTextSchema>;
const applyQueryConditionText = (query: Knex.QueryBuilder, { field, tableName, conditions }: QueryConditionText) => {
const selector = getFieldSelector(query, field, tableName);
if (conditions.equal) {
query = query.where(selector, '=', conditions.equal);
}
if (conditions.notEqual) {
query = query.where(selector, '<>', conditions.notEqual);
}
if (conditions.like) {
query = query.whereLike(selector, conditions.like);
}
if (conditions.notLike) {
query = query.not.whereLike(selector, conditions.notLike);
}
if (conditions.equal === null) {
query = query.whereNull(selector);
}
if (conditions.notEqual === null) {
query = query.whereNotNull(selector);
}
if (conditions.in) {
query = query.whereIn(selector, conditions.in);
}
if (conditions.notIn) {
query = query.whereNotIn(selector, conditions.notIn);
}
return query;
};
const queryConditionNumberSchema = z
.object({
type: z.literal('number'),
tableName: z.string().optional(),
field: z.array(z.string()),
conditions: z.object({
equals: z.number().nullish(),
notEquals: z.number().nullish(),
greaterThan: z.number().optional(),
greaterThanOrEqual: z.number().optional(),
lessThan: z.number().optional(),
lessThanOrEqual: z.number().optional(),
in: z.array(z.number()).optional(),
notIn: z.array(z.number()).optional(),
}),
})
.meta({
example: {
type: 'number',
field: ['typeVersion'],
conditions: {
equals: 1,
},
},
examples: [
{
summary: 'Equals condition',
value: {
type: 'number',
field: ['typeVersion'],
conditions: {
equals: 1,
},
},
},
{
summary: 'Greater than condition',
value: {
type: 'number',
field: ['typeVersion'],
conditions: {
greaterThan: 0,
},
},
},
{
summary: 'Range condition',
value: {
type: 'number',
field: ['typeVersion'],
conditions: {
greaterThanOrEqual: 1,
lessThanOrEqual: 10,
},
},
},
{
summary: 'In condition',
value: {
type: 'number',
field: ['typeVersion'],
conditions: {
in: [1, 2, 3],
},
},
},
],
});
type QueryConditionNumber = z.infer<typeof queryConditionNumberSchema>;
const applyQueryConditionNumber = (
query: Knex.QueryBuilder,
{ field, tableName, conditions }: QueryConditionNumber,
) => {
const selector = getFieldSelector(query, field, tableName);
if (conditions.equals !== undefined && conditions.equals !== null) {
query = query.where(selector, '=', conditions.equals);
}
if (conditions.notEquals !== undefined && conditions.notEquals !== null) {
query = query.where(selector, '<>', conditions.notEquals);
}
if (conditions.equals === null) {
query = query.whereNull(selector);
}
if (conditions.notEquals === null) {
query = query.whereNotNull(selector);
}
if (conditions.greaterThan) {
query = query.where(selector, '>', conditions.greaterThan);
}
if (conditions.greaterThanOrEqual) {
query = query.where(selector, '>=', conditions.greaterThanOrEqual);
}
if (conditions.lessThan) {
query = query.where(selector, '<', conditions.lessThan);
}
if (conditions.lessThanOrEqual) {
query = query.where(selector, '<=', conditions.lessThanOrEqual);
}
if (conditions.in) {
query = query.whereIn(selector, conditions.in);
}
if (conditions.notIn) {
query = query.whereNotIn(selector, conditions.notIn);
}
return query;
};
const queryConditionSchema = z.discriminatedUnion('type', [queryConditionTextSchema, queryConditionNumberSchema]);
type QueryCondition = z.infer<typeof queryConditionSchema>;
const applyQueryCondition = (query: Knex.QueryBuilder, options: QueryCondition) => {
switch (options.type) {
case 'text': {
return applyQueryConditionText(query, options);
}
case 'number': {
return applyQueryConditionNumber(query, options);
}
default: {
throw new Error(`Unknown filter type`);
}
}
};
type QueryFilter = QueryCondition | QueryOperator;
type QueryOperator = {
type: 'operator';
operator: 'and' | 'or';
conditions: QueryFilter[];
};
// Create a depth-limited recursive schema for OpenAPI compatibility
// This supports up to 3 levels of nesting, which should be sufficient for most use cases
// OpenAPI cannot handle z.lazy(), so we manually define the nesting
// If you need deeper nesting, you can add more levels (Level3, Level4, etc.)
const queryFilterSchemaLevel0: z.ZodType<QueryFilter> = z.union([
queryConditionSchema,
z
.object({
type: z.literal('operator'),
operator: z.enum(['and', 'or']),
conditions: z.array(queryConditionSchema),
})
.meta({
example: {
type: 'operator',
operator: 'and',
conditions: [
{
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'bar',
},
},
],
},
examples: [
{
summary: 'AND operator',
value: {
type: 'operator',
operator: 'and',
conditions: [
{
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'bar',
},
},
{
type: 'text',
field: ['type'],
conditions: {
equal: 'demo',
},
},
],
},
},
{
summary: 'OR operator',
value: {
type: 'operator',
operator: 'or',
conditions: [
{
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'bar',
},
},
{
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'baz',
},
},
],
},
},
],
}),
]);
const queryFilterSchemaLevel1: z.ZodType<QueryFilter> = z.union([
queryConditionSchema,
z
.object({
type: z.literal('operator'),
operator: z.enum(['and', 'or']),
conditions: z.array(queryFilterSchemaLevel0),
})
.meta({
example: {
type: 'operator',
operator: 'or',
conditions: [
{
type: 'operator',
operator: 'and',
conditions: [
{
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'bar',
},
},
],
},
{
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'baz',
},
},
],
},
examples: [
{
summary: 'Nested AND within OR',
value: {
type: 'operator',
operator: 'or',
conditions: [
{
type: 'operator',
operator: 'and',
conditions: [
{
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'bar',
},
},
{
type: 'text',
field: ['type'],
conditions: {
equal: 'demo',
},
},
],
},
{
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'baz',
},
},
],
},
},
],
}),
]);
const queryFilterSchemaLevel2: z.ZodType<QueryFilter> = z.union([
queryConditionSchema,
z
.object({
type: z.literal('operator'),
operator: z.enum(['and', 'or']),
conditions: z.array(queryFilterSchemaLevel1),
})
.meta({
example: {
type: 'operator',
operator: 'and',
conditions: [
{
type: 'operator',
operator: 'or',
conditions: [
{
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'bar',
},
},
{
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'baz',
},
},
],
},
{
type: 'text',
field: ['type'],
conditions: {
equal: 'demo',
},
},
],
},
examples: [
{
summary: 'Complex nested query',
value: {
type: 'operator',
operator: 'and',
conditions: [
{
type: 'operator',
operator: 'or',
conditions: [
{
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'bar',
},
},
{
type: 'text',
field: ['metadata', 'foo'],
conditions: {
equal: 'baz',
},
},
],
},
{
type: 'text',
field: ['type'],
conditions: {
equal: 'demo',
},
},
],
},
},
],
}),
]);
// Export the depth-limited schema (supports 3 levels of nesting)
// This works with OpenAPI schema generation
const queryFilterSchema = queryFilterSchemaLevel2;
const applyQueryFilter = (query: Knex.QueryBuilder, filter: QueryFilter) => {
if (filter.type === 'operator') {
if (filter.conditions.length === 0) {
return query;
}
switch (filter.operator) {
case 'or': {
return query.where((subquery) => {
let isFirst = true;
for (const condition of filter.conditions) {
if (isFirst) {
applyQueryFilter(subquery, condition);
isFirst = false;
} else {
subquery.orWhere((subSubquery) => {
applyQueryFilter(subSubquery, condition);
});
}
}
});
}
case 'and': {
return query.where((subquery) => {
let isFirst = true;
for (const condition of filter.conditions) {
if (isFirst) {
applyQueryFilter(subquery, condition);
isFirst = false;
} else {
subquery.andWhere((subSubquery) => {
applyQueryFilter(subSubquery, condition);
});
}
}
});
}
}
} else {
return applyQueryCondition(query, filter);
}
};
export type { QueryConditionText, QueryConditionNumber, QueryOperator, QueryCondition, QueryFilter };
export { applyQueryCondition, queryConditionSchema, queryFilterSchema, applyQueryFilter };

View File

@@ -1,6 +1,7 @@
{ {
"compilerOptions": { "compilerOptions": {
"outDir": "./dist", "outDir": "./dist",
"rootDir": "./src",
"paths": { "paths": {
"#root/*": [ "#root/*": [
"./src/*" "./src/*"

152
pnpm-lock.yaml generated
View File

@@ -50,41 +50,54 @@ importers:
packages/configs: {} packages/configs: {}
packages/server: packages/query-dsl:
dependencies:
chevrotain:
specifier: ^11.0.3
version: 11.0.3
zod:
specifier: 4.1.13
version: 4.1.13
devDependencies:
'@morten-olsen/stash-configs':
specifier: workspace:*
version: link:../configs
'@morten-olsen/stash-tests':
specifier: workspace:*
version: link:../tests
'@types/node':
specifier: 24.10.2
version: 24.10.2
'@vitest/coverage-v8':
specifier: 4.0.15
version: 4.0.15(vitest@4.0.15(@types/node@24.10.2)(tsx@4.21.0)(yaml@2.8.2))
typescript:
specifier: 5.9.3
version: 5.9.3
vitest:
specifier: 4.0.15
version: 4.0.15(@types/node@24.10.2)(tsx@4.21.0)(yaml@2.8.2)
packages/runtime:
dependencies: dependencies:
'@electric-sql/pglite': '@electric-sql/pglite':
specifier: ^0.3.14 specifier: ^0.3.14
version: 0.3.14 version: 0.3.14
'@fastify/cors':
specifier: 11.1.0
version: 11.1.0
'@fastify/swagger':
specifier: 9.6.1
version: 9.6.1
'@fastify/websocket':
specifier: 11.2.0
version: 11.2.0
'@huggingface/transformers': '@huggingface/transformers':
specifier: ^3.8.1 specifier: ^3.8.1
version: 3.8.1 version: 3.8.1
'@langchain/textsplitters': '@langchain/textsplitters':
specifier: ^1.0.1 specifier: ^1.0.1
version: 1.0.1(@langchain/core@1.1.4) version: 1.0.1(@langchain/core@1.1.4)
'@scalar/fastify-api-reference': '@morten-olsen/stash-query-dsl':
specifier: 1.40.2 specifier: workspace:*
version: 1.40.2 version: link:../query-dsl
better-sqlite3: better-sqlite3:
specifier: ^12.5.0 specifier: ^12.5.0
version: 12.5.0 version: 12.5.0
deep-equal: deep-equal:
specifier: ^2.2.3 specifier: ^2.2.3
version: 2.2.3 version: 2.2.3
fastify:
specifier: 5.6.2
version: 5.6.2
fastify-type-provider-zod:
specifier: 6.1.0
version: 6.1.0(@fastify/swagger@9.6.1)(fastify@5.6.2)(openapi-types@12.1.3)(zod@4.1.13)
knex: knex:
specifier: ^3.1.0 specifier: ^3.1.0
version: 3.1.0(better-sqlite3@12.5.0)(pg@8.16.3) version: 3.1.0(better-sqlite3@12.5.0)(pg@8.16.3)
@@ -100,6 +113,58 @@ importers:
zod: zod:
specifier: 4.1.13 specifier: 4.1.13
version: 4.1.13 version: 4.1.13
devDependencies:
'@morten-olsen/stash-configs':
specifier: workspace:*
version: link:../configs
'@morten-olsen/stash-tests':
specifier: workspace:*
version: link:../tests
'@types/deep-equal':
specifier: ^1.0.4
version: 1.0.4
'@types/node':
specifier: 24.10.2
version: 24.10.2
'@vitest/coverage-v8':
specifier: 4.0.15
version: 4.0.15(vitest@4.0.15(@types/node@24.10.2)(tsx@4.21.0)(yaml@2.8.2))
typescript:
specifier: 5.9.3
version: 5.9.3
vitest:
specifier: 4.0.15
version: 4.0.15(@types/node@24.10.2)(tsx@4.21.0)(yaml@2.8.2)
packages/server:
dependencies:
'@fastify/cors':
specifier: 11.1.0
version: 11.1.0
'@fastify/swagger':
specifier: 9.6.1
version: 9.6.1
'@fastify/websocket':
specifier: 11.2.0
version: 11.2.0
'@morten-olsen/stash-query-dsl':
specifier: workspace:*
version: link:../query-dsl
'@morten-olsen/stash-runtime':
specifier: workspace:*
version: link:../runtime
'@scalar/fastify-api-reference':
specifier: 1.40.2
version: 1.40.2
fastify:
specifier: 5.6.2
version: 5.6.2
fastify-type-provider-zod:
specifier: 6.1.0
version: 6.1.0(@fastify/swagger@9.6.1)(fastify@5.6.2)(openapi-types@12.1.3)(zod@4.1.13)
zod:
specifier: 4.1.13
version: 4.1.13
zod-to-json-schema: zod-to-json-schema:
specifier: 3.25.0 specifier: 3.25.0
version: 3.25.0(zod@4.1.13) version: 3.25.0(zod@4.1.13)
@@ -181,6 +246,21 @@ packages:
'@cfworker/json-schema@4.1.1': '@cfworker/json-schema@4.1.1':
resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==}
'@chevrotain/cst-dts-gen@11.0.3':
resolution: {integrity: sha512-BvIKpRLeS/8UbfxXxgC33xOumsacaeCKAjAeLyOn7Pcp95HiRbrpl14S+9vaZLolnbssPIUuiUd8IvgkRyt6NQ==}
'@chevrotain/gast@11.0.3':
resolution: {integrity: sha512-+qNfcoNk70PyS/uxmj3li5NiECO+2YKZZQMbmjTqRI3Qchu8Hig/Q9vgkHpI3alNjr7M+a2St5pw5w5F6NL5/Q==}
'@chevrotain/regexp-to-ast@11.0.3':
resolution: {integrity: sha512-1fMHaBZxLFvWI067AVbGJav1eRY7N8DDvYCTwGBiE/ytKBgP8azTdgyrKyWZ9Mfh09eHWb5PgTSO8wi7U824RA==}
'@chevrotain/types@11.0.3':
resolution: {integrity: sha512-gsiM3G8b58kZC2HaWR50gu6Y1440cHiJ+i3JUvcp/35JchYejb2+5MVeJK0iKThYpAa/P2PYFV4hoi44HD+aHQ==}
'@chevrotain/utils@11.0.3':
resolution: {integrity: sha512-YslZMgtJUyuMbZ+aKvfF3x1f5liK4mWNxghFRv7jqRR9C3R3fAOGTTKvxXDa2Y1s9zSbcpuO0cAxDYsc9SrXoQ==}
'@electric-sql/pglite@0.3.14': '@electric-sql/pglite@0.3.14':
resolution: {integrity: sha512-3DB258dhqdsArOI1fIt7cb9RpUOgcDg5hXWVgVHAeqVQ/qxtFy605QKs4gx6mFq3jWsSPqDN8TgSEsqC3OfV9Q==} resolution: {integrity: sha512-3DB258dhqdsArOI1fIt7cb9RpUOgcDg5hXWVgVHAeqVQ/qxtFy605QKs4gx6mFq3jWsSPqDN8TgSEsqC3OfV9Q==}
@@ -1422,6 +1502,9 @@ packages:
resolution: {integrity: sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==} resolution: {integrity: sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==}
engines: {node: '>=10'} engines: {node: '>=10'}
chevrotain@11.0.3:
resolution: {integrity: sha512-ci2iJH6LeIkvP9eJW6gpueU8cnZhv85ELY8w8WiFtNjMHA5ad6pQLaJo9mEly/9qUyCpvqX8/POVUTf18/HFdw==}
chownr@1.1.4: chownr@1.1.4:
resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
@@ -2288,6 +2371,9 @@ packages:
resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==}
engines: {node: '>=10'} engines: {node: '>=10'}
lodash-es@4.17.21:
resolution: {integrity: sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==}
lodash.merge@4.6.2: lodash.merge@4.6.2:
resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==}
@@ -3373,6 +3459,23 @@ snapshots:
'@cfworker/json-schema@4.1.1': {} '@cfworker/json-schema@4.1.1': {}
'@chevrotain/cst-dts-gen@11.0.3':
dependencies:
'@chevrotain/gast': 11.0.3
'@chevrotain/types': 11.0.3
lodash-es: 4.17.21
'@chevrotain/gast@11.0.3':
dependencies:
'@chevrotain/types': 11.0.3
lodash-es: 4.17.21
'@chevrotain/regexp-to-ast@11.0.3': {}
'@chevrotain/types@11.0.3': {}
'@chevrotain/utils@11.0.3': {}
'@electric-sql/pglite@0.3.14': {} '@electric-sql/pglite@0.3.14': {}
'@emnapi/runtime@1.7.1': '@emnapi/runtime@1.7.1':
@@ -4580,6 +4683,15 @@ snapshots:
char-regex@1.0.2: {} char-regex@1.0.2: {}
chevrotain@11.0.3:
dependencies:
'@chevrotain/cst-dts-gen': 11.0.3
'@chevrotain/gast': 11.0.3
'@chevrotain/regexp-to-ast': 11.0.3
'@chevrotain/types': 11.0.3
'@chevrotain/utils': 11.0.3
lodash-es: 4.17.21
chownr@1.1.4: {} chownr@1.1.4: {}
chownr@3.0.0: {} chownr@3.0.0: {}
@@ -5567,6 +5679,8 @@ snapshots:
dependencies: dependencies:
p-locate: 5.0.0 p-locate: 5.0.0
lodash-es@4.17.21: {}
lodash.merge@4.6.2: {} lodash.merge@4.6.2: {}
lodash@4.17.21: {} lodash@4.17.21: {}

14
tsconfig.json Normal file
View File

@@ -0,0 +1,14 @@
{
"include": [],
"references": [
{
"path": "./packages/query-dsl/tsconfig.json"
},
{
"path": "./packages/runtime/tsconfig.json"
},
{
"path": "./packages/server/tsconfig.json"
}
]
}

View File

@@ -1,8 +1,8 @@
import { defineConfig, type UserConfigExport } from 'vitest/config'; import { defineConfig, type ViteUserConfigExport } from 'vitest/config';
// eslint-disable-next-line import/no-default-export // eslint-disable-next-line import/no-default-export
export default defineConfig(async () => { export default defineConfig(async () => {
const config: UserConfigExport = { const config: ViteUserConfigExport = {
test: { test: {
coverage: { coverage: {
provider: 'v8', provider: 'v8',