feat: improved zod handling

packages/query-dsl/src/exports.ts

@@ -1,3 +1,4 @@
 export * from './query-parser.schemas.js';
-export { QueryParser } from './query-parser.js';
+export { QueryParser, queryParser } from './query-parser.js';
 export * from './utils.filter.js';
+export * from './query-parser.codec.js';

packages/query-dsl/src/query-parser.codec.ts

@@ -0,0 +1,20 @@
+import { z } from 'zod';
+
+import { queryFilterSchema } from './query-parser.schemas.js';
+import { queryParser } from './query-parser.js';
+
+const queryStringSchema: typeof queryFilterSchema = z
+  .codec(z.string(), queryFilterSchema, {
+    encode: (filter) => {
+      return queryParser.stringify(filter);
+    },
+    decode: (input) => {
+      return queryParser.parse(input);
+    },
+    // eslint-disable-next-line
+  }).meta({ id: 'QueryString', examples: ["metadata.foo = 'bar'"] }) as any;
+
+// eslint-disable-next-line
+const querySchema: typeof queryFilterSchema = z.union([queryStringSchema, queryFilterSchema]) as any
+
+export { querySchema };

packages/query-dsl/src/query-parser.ts

@@ -19,4 +19,6 @@ class QueryParser {
   };
 }
 
-export { QueryParser };
+const queryParser = new QueryParser();
+
+export { QueryParser, queryParser };

packages/runtime/src/services/document-chunks/document-chunks.mappings.ts

@@ -1,13 +1,11 @@
 import type { TableRows } from '../database/database.js';
 
-import type { DocumentChunk } from './document-chunks.schemas.js';
+import { documentChunkWithDistanceSchema } from './document-chunks.schemas.js';
 
 const mapFromDocumentChunkRow = (
   row: TableRows['documentChunks'] & {
-    metadata: unknown;
+    metadata?: unknown;
   },
-): DocumentChunk => ({
+) => documentChunkWithDistanceSchema.parse(row);
-  ...row,
-});
 
 export { mapFromDocumentChunkRow };

packages/runtime/src/services/document-chunks/document-chunks.schemas.ts

@@ -1,7 +1,7 @@
 import { z } from 'zod';
-import { queryFilterSchema } from '@morten-olsen/stash-query-dsl';
+import { querySchema } from '@morten-olsen/stash-query-dsl';
 
-import { createListResultSchema, queryDSLSchema } from '../../utils/utils.schema.js';
+import { createListResultSchema } from '../../utils/utils.schema.js';
 
 const documentChunkSchema = z
   .object({
@@ -19,21 +19,28 @@ const documentChunkFilterSchema = z
     limit: z.number().default(20),
     offset: z.number().default(0),
     semanticText: z.string().optional(),
-    conditions: z.union([queryDSLSchema, queryFilterSchema]).optional(),
+    conditions: querySchema.optional(),
   })
   .meta({ id: 'DocumentChunkFilter' });
 
 type DocumentChunkFilter = z.infer<typeof documentChunkFilterSchema>;
 
-const documentChunksFindResultSchema = createListResultSchema(
+const documentChunkWithDistanceSchema = documentChunkSchema
-  documentChunkSchema
   .extend({
     distance: z.number().optional(),
   })
-    .meta({ id: 'DocumentChunkWithDistance' }),
+  .meta({ id: 'DocumentChunkWithDistance' });
-).meta({ id: 'DocumentChunkFindResult' });
+
+const documentChunksFindResultSchema = createListResultSchema(documentChunkWithDistanceSchema).meta({
+  id: 'DocumentChunkFindResult',
+});
 
 type DocumentChunksFindResult = z.infer<typeof documentChunksFindResultSchema>;
 
 export type { DocumentChunk, DocumentChunkFilter, DocumentChunksFindResult };
-export { documentChunkSchema, documentChunkFilterSchema, documentChunksFindResultSchema };
+export {
+  documentChunkSchema,
+  documentChunkFilterSchema,
+  documentChunksFindResultSchema,
+  documentChunkWithDistanceSchema,
+};

packages/runtime/src/services/document-chunks/document-chunks.ts

@@ -1,5 +1,3 @@
-import { QueryParser } from '@morten-olsen/stash-query-dsl';
-
 import { DatabaseService, tableNames, type TableRows } from '../database/database.js';
 import { EmbeddingsService } from '../embeddings/embeddings.js';
 import type { Services } from '../../utils/utils.services.js';
@@ -44,11 +42,7 @@ class DocumentChunksService {
       query = query.orderBy('createdAt', 'desc');
     }
     if (filter.conditions) {
-      const parser = this.#services.get(QueryParser);
+      query = applyQueryFilter(query, filter.conditions);
-      query = applyQueryFilter(
-        query,
-        typeof filter.conditions === 'string' ? parser.parse(filter.conditions) : filter.conditions,
-      );
     }
 
     query = query.limit(filter.limit).offset(filter.offset);
@@ -56,7 +50,7 @@ class DocumentChunksService {
     const items = await query;
 
     return {
-      items: items.map(mapFromDocumentChunkRow as ExplicitAny),
+      items: items.map(mapFromDocumentChunkRow),
     };
   };
 }

packages/runtime/src/services/documents/documents.mapping.ts

@@ -1,12 +1,7 @@
 import type { TableRows } from '../database/database.js';
 
-import type { Document } from './documents.schemas.js';
+import { documentSchema, type Document } from './documents.schemas.js';
 
-const mapFromDocumentRow = (row: TableRows['documents']): Document => ({
+const mapFromDocumentRow = (row: TableRows['documents']): Document => documentSchema.parse(row);
-  ...row,
-  createdAt: row.createdAt.toISOString?.(),
-  updatedAt: row.updatedAt.toISOString?.(),
-  deletedAt: row.deletedAt?.toISOString?.() || null,
-});
 
 export { mapFromDocumentRow };

packages/runtime/src/services/documents/documents.schemas.ts

@@ -6,10 +6,10 @@ import { createListResultSchema, queryDSLSchema } from '../../utils/utils.schema
 const documentSchema = z
   .object({
     id: z.guid(),
-    owner: z.string().nullable(),
+    owner: z.guid().nullable(),
-    createdAt: z.iso.datetime(),
+    createdAt: z.coerce.date(),
-    updatedAt: z.iso.datetime(),
+    updatedAt: z.coerce.date(),
-    deletedAt: z.iso.datetime().nullable(),
+    deletedAt: z.coerce.date().nullable(),
     contentType: z.string().nullable(),
     text: z.string().nullable(),
     source: z.string().nullable(),

packages/runtime/src/utils/utils.schema.ts

@@ -1,5 +1,8 @@
+import { QueryParser } from '@morten-olsen/stash-query-dsl';
 import { z, type ZodType } from 'zod';
 
+const parser = new QueryParser();
+
 const createListResultSchema = <T extends ZodType>(schema: T) =>
   z.object({
     items: z.array(schema),
@@ -8,8 +11,15 @@ const createListResultSchema = <T extends ZodType>(schema: T) =>
 const queryDSLSchema = z
   .string()
   .describe('Query DSL based filter')
+  .superRefine((value, context) => {
+    try {
+      parser.parse(value);
+    } catch (err) {
+      context.addIssue(String(err));
+    }
+  })
   .meta({
-    id: 'QueryDSLString',
+    id: 'QueryDQLString',
     examples: ["metadata.foo = 'bar'"],
   });
 

packages/server/src/api.ts

@@ -31,7 +31,7 @@ const createApi = async (runtime: StashRuntime = new StashRuntime()) => {
   runtime.documents.subscribe({
     filter: "metadata.foo = 'bar'",
     fn: (document) => {
-      console.log(document);
+      // console.log(document);
     },
   });
   const app = fastify().withTypeProvider<ZodTypeProvider>();