feat: support views as data catalogs

packages/server/package.json

@@ -2,6 +2,7 @@
   "type": "module",
   "main": "dist/exports.js",
   "scripts": {
+    "dev": "node --no-warnings --watch src/dev.ts",
     "build": "tsc --build",
     "test:unit": "vitest --run --passWithNoTests",
     "test": "pnpm run \"/^test:/\""
@@ -18,6 +19,7 @@
     "@morten-olsen/reservoir-tests": "workspace:*",
     "@types/node": "24.10.0",
     "@vitest/coverage-v8": "4.0.6",
+    "dotenv": "^17.2.3",
     "typescript": "5.9.3",
     "vitest": "4.0.6"
   },
@@ -27,6 +29,7 @@
     "#root/*": "./src/*"
   },
   "dependencies": {
+    "@fastify/sensible": "^6.0.3",
     "@fastify/swagger": "^9.5.2",
     "@scalar/fastify-api-reference": "^1.38.1",
     "better-sqlite3": "^12.4.1",
@@ -38,6 +41,7 @@
     "pg": "^8.16.3",
     "pino": "^10.1.0",
     "pino-pretty": "^13.1.2",
+    "yaml": "^2.8.1",
     "zod": "^4.1.12"
   }
 }

packages/server/src/api/api.documents.ts

@@ -11,7 +11,7 @@ const documentsPlugin: FastifyPluginAsyncZod = async (app) => {
     method: 'POST',
     url: '',
     schema: {
-      operationId: 'v1.documents.post',
+      operationId: 'v1.documents.upsert',
       tags: ['documents'],
       summary: 'Upsert documents',
       body: upsertDocumentRequestSchema,

packages/server/src/api/api.ts

@@ -1,9 +1,12 @@
 import fastify from 'fastify';
+import fastifySensible from '@fastify/sensible';
 import fastifySwagger from '@fastify/swagger';
 import fastifyScalar from '@scalar/fastify-api-reference';
+import YAML from 'yaml';
 import { jsonSchemaTransform, serializerCompiler, validatorCompiler } from 'fastify-type-provider-zod';
 
 import { documentsPlugin } from './api.documents.ts';
+import { viewsPlugin } from './api.views.ts';
 
 import { Services } from '#root/utils/utils.services.ts';
 import { DatabaseService } from '#root/database/database.ts';
@@ -22,6 +25,16 @@ const createApi = async (services: Services = new Services()) => {
   app.setValidatorCompiler(validatorCompiler);
   app.setSerializerCompiler(serializerCompiler);
   app.decorate('services', services);
+  app.addContentTypeParser('application/yaml', { parseAs: 'buffer' }, (_, body, done) => {
+    try {
+      const parsed = YAML.parse(body.toString('utf8')); // Parse the buffer as YAML
+      done(null, parsed); // Call done with null for error and the parsed object
+    } catch {
+      done(app.httpErrors.badRequest('Invalid YAML format'));
+    }
+  });
+
+  await app.register(fastifySensible);
 
   await app.register(fastifySwagger, {
     openapi: {
@@ -42,6 +55,10 @@ const createApi = async (services: Services = new Services()) => {
     prefix: '/api/v1/documents',
   });
 
+  await app.register(viewsPlugin, {
+    prefix: '/api/v1/views',
+  });
+
   app.addHook('onReady', async () => {
     app.swagger();
   });

packages/server/src/api/api.views.ts

@@ -0,0 +1,86 @@
+import type { FastifyPluginAsyncZod } from 'fastify-type-provider-zod';
+import { z } from 'zod';
+
+import { viewInfoSchema, viewUpsertRequestSchema } from '#root/services/views/views.schemas.ts';
+import { ViewsService } from '#root/services/views/views.ts';
+
+const viewsPlugin: FastifyPluginAsyncZod = async (app) => {
+  app.route({
+    method: 'GET',
+    url: '',
+    schema: {
+      operationId: 'v1.views.list',
+      tags: ['views'],
+      summary: 'List views',
+      response: {
+        200: z.object({
+          items: z.array(viewInfoSchema),
+        }),
+      },
+    },
+    handler: async (req, reply) => {
+      const viewsService = app.services.get(ViewsService);
+      const items = await viewsService.list();
+      return reply.send({ items });
+    },
+  });
+
+  app.route({
+    method: 'PUT',
+    url: '/:name',
+    schema: {
+      operationId: 'v1.views.list',
+      tags: ['views'],
+      summary: 'Upsert view',
+      params: z.object({
+        name: z.string(),
+      }),
+      body: viewUpsertRequestSchema.omit({
+        name: true,
+      }),
+      response: {
+        200: z.object({
+          name: z.string(),
+        }),
+      },
+    },
+    handler: async (req, reply) => {
+      const input = {
+        ...req.body,
+        name: req.params.name,
+      };
+      const viewsService = app.services.get(ViewsService);
+      await viewsService.upsert(input);
+      reply.send({
+        name: req.params.name,
+      });
+    },
+  });
+
+  app.route({
+    method: 'DELETE',
+    url: '/:name',
+    schema: {
+      operationId: 'v1.views.delete',
+      tags: ['views'],
+      summary: 'Delete view',
+      params: z.object({
+        name: z.string(),
+      }),
+      response: {
+        200: z.object({
+          name: z.string(),
+        }),
+      },
+    },
+    handler: async (req, reply) => {
+      const viewsService = app.services.get(ViewsService);
+      await viewsService.remove(req.params.name);
+      reply.send({
+        name: req.params.name,
+      });
+    },
+  });
+};
+
+export { viewsPlugin };

packages/server/src/dev.ts

@@ -0,0 +1,8 @@
+import 'dotenv/config';
+import { createApi } from './api/api.ts';
+
+const app = await createApi();
+await app.listen({
+  port: 9111,
+  host: process.env.SERVER_HOST,
+});

packages/server/src/services/views/views.schemas.ts

@@ -0,0 +1,35 @@
+import { z } from 'zod';
+
+const columnInfoSchema = z.object({
+  name: z.string(),
+  dataType: z.string(),
+  isNullable: z.boolean(),
+});
+
+type ColumnInfo = z.infer<typeof columnInfoSchema>;
+
+const viewInfoSchema = z.object({
+  name: z.string(),
+  columns: z.array(columnInfoSchema),
+});
+
+type ViewInfo = z.infer<typeof viewInfoSchema>;
+
+const viewUpsertRequestSchema = z.object({
+  name: z.string(),
+  description: z.string().nullable(),
+  columns: z
+    .record(
+      z.string(),
+      z.object({
+        description: z.string().nullish(),
+      }),
+    )
+    .optional(),
+  query: z.string(),
+});
+
+type ViewUpsertRequest = z.infer<typeof viewUpsertRequestSchema>;
+
+export type { ColumnInfo, ViewInfo, ViewUpsertRequest };
+export { columnInfoSchema, viewInfoSchema, viewUpsertRequestSchema };

packages/server/src/services/views/views.ts

@@ -0,0 +1,100 @@
+import type { ViewInfo, ViewUpsertRequest } from './views.schemas.ts';
+
+import { DatabaseService } from '#root/database/database.ts';
+import type { Services } from '#root/utils/utils.services.ts';
+
+class ViewsService {
+  #services: Services;
+
+  constructor(services: Services) {
+    this.#services = services;
+  }
+
+  #listFromPostgres = async () => {
+    const dbService = this.#services.get(DatabaseService);
+    const db = await dbService.getInstance();
+    const pgViewsResult = await db.raw<{ rows: { schema_name: string; view_name: string }[] }>(`
+          SELECT
+              schemaname AS schema_name,
+              viewname AS view_name
+          FROM
+              pg_views
+          WHERE
+              schemaname NOT IN ('pg_catalog', 'information_schema', 'information_schema')
+              AND viewname NOT LIKE 'pg_%'
+              AND viewname NOT LIKE 'sql_%';
+        `);
+    const views: ViewInfo[] = await Promise.all(
+      pgViewsResult.rows.map(async (row) => {
+        const pgColumnsResult = await db.raw<{
+          rows: { column_name: string; data_type: string; is_nullable: 'YES' | 'NO' }[];
+        }>(
+          `
+            SELECT
+                column_name,
+                data_type,
+                is_nullable
+            FROM
+                information_schema.columns
+            WHERE
+                table_schema = ?
+                AND table_name = ?
+            ORDER BY
+                ordinal_position;
+          `,
+          [row.schema_name, row.view_name],
+        );
+        const result: ViewInfo = {
+          name: row.view_name,
+          columns: pgColumnsResult.rows.map((column) => ({
+            name: column.column_name,
+            dataType: column.data_type,
+            isNullable: column.is_nullable === 'YES',
+          })),
+        };
+        return result;
+      }),
+    );
+    return views;
+  };
+
+  public list = async () => {
+    const dbService = this.#services.get(DatabaseService);
+    const db = await dbService.getInstance();
+
+    switch (db.client.config.client) {
+      case 'pg':
+        return this.#listFromPostgres();
+      default:
+        throw new Error(`Client ${db.client} is not supported`);
+    }
+  };
+
+  public upsert = async (options: ViewUpsertRequest) => {
+    const { name, columns = {}, query, description } = options;
+    const dbService = this.#services.get(DatabaseService);
+    const db = await dbService.getInstance();
+    const subquery = db.raw(query);
+    await db.schema.createViewOrReplace(name, (view) => {
+      // view.columns(columns);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      view.as(subquery as any);
+    });
+    if (description) {
+      const sql = db.raw(`COMMENT ON VIEW ?? IS ?;`, [name, description]);
+      await db.raw(sql.toQuery());
+    }
+    for (const [columnName, info] of Object.entries(columns)) {
+      const sql = db.raw(`COMMENT ON COLUMN ??.?? IS ?;`, [name, columnName, info.description || null]);
+      await db.raw(sql.toQuery());
+    }
+  };
+
+  public remove = async (name: string) => {
+    const dbService = this.#services.get(DatabaseService);
+    const db = await dbService.getInstance();
+    await db.schema.dropViewIfExists(name);
+  };
+}
+
+export { ViewsService };