From 59d6faaafca40b050e8d2279e73f6f4cd518eb83 Mon Sep 17 00:00:00 2001 From: Morten Olsen Date: Fri, 12 Jan 2024 14:14:40 +0100 Subject: [PATCH] feat: switched from worker API to fs based --- docs/first-workload.md | 15 +++-- docs/interacting-with-server.md | 4 +- packages/cli/package.json | 1 + packages/cli/src/client/client.ts | 11 ++- .../src/commands/artifacts/artifacts.list.ts | 4 +- .../src/commands/artifacts/artifacts.pull.ts | 32 +++++++++ .../commands/artifacts/artifacts.remove.ts | 59 ++++++++++++++++ .../cli/src/commands/artifacts/artifacts.ts | 4 ++ packages/cli/src/commands/auth/auth.login.ts | 25 ++++++- packages/cli/src/commands/loads/loads.list.ts | 8 ++- packages/cli/src/commands/loads/loads.push.ts | 4 +- packages/cli/src/commands/logs/logs.list.ts | 6 +- packages/cli/src/commands/logs/logs.remove.ts | 63 +++++++++++++++++ packages/cli/src/commands/logs/logs.ts | 2 + packages/cli/src/commands/runs/runs.create.ts | 4 +- packages/cli/src/commands/runs/runs.list.ts | 6 +- .../cli/src/commands/secrets/secrets.list.ts | 4 +- .../src/commands/secrets/secrets.remove.ts | 4 +- .../cli/src/commands/secrets/secrets.set.ts | 4 +- packages/cli/src/context/context.ts | 50 ++++++++++++++ packages/cli/src/utils/step.ts | 4 +- packages/examples/src/simple.ts | 7 +- .../mini-loader/src/artifacts/artifacts.ts | 4 +- packages/mini-loader/src/input/input.ts | 13 +++- packages/mini-loader/src/logger/logger.ts | 16 ++--- packages/mini-loader/src/secrets/secrets.ts | 5 +- packages/mini-loader/src/utils.ts | 30 +++++++-- packages/runner/package.json | 5 +- packages/runner/src/index.ts | 67 ++++++++++++++++--- packages/server/src/index.ts | 4 ++ .../server/src/repos/artifacts/artifacts.ts | 12 +++- packages/server/src/repos/logs/logs.ts | 5 +- .../server/src/router/router.artifacts.ts | 12 +++- packages/server/src/router/router.logs.ts | 2 +- packages/server/src/router/router.utils.ts | 3 +- packages/server/src/runner/runner.instance.ts | 2 +- packages/server/src/server/server.ts | 13 ++++ pnpm-lock.yaml | 11 +++ 38 files changed, 458 insertions(+), 67 deletions(-) create mode 100644 packages/cli/src/commands/artifacts/artifacts.pull.ts create mode 100644 packages/cli/src/commands/artifacts/artifacts.remove.ts create mode 100644 packages/cli/src/commands/logs/logs.remove.ts create mode 100644 packages/cli/src/context/context.ts diff --git a/docs/first-workload.md b/docs/first-workload.md index 54c6f89..add5ba4 100644 --- a/docs/first-workload.md +++ b/docs/first-workload.md @@ -15,14 +15,19 @@ npm install -g @morten-olsen/mini-loader-cli Now, let's write a basic script that outputs a single artifact named “hello”. Create a new file with the following JavaScript code: ```javascript -import { artifacts } from "@morten-olsen/mini-loader"; +import { artifacts } from '@morten-olsen/mini-loader'; -artifacts.create('hello', 'world'); +const run = async () => { + artifacts.create('hello', 'world'); +}; + +run(); ``` -Save this file as `script.mjs`. +Save this file as `script.js`. #### A Note on Dependencies + In this script, we're using the `@morten-olsen/mini-loader` package, which might not be installed in your local environment. No worries though, as mini loader can automatically download necessary packages when preparing the script. Alternatively, for a more structured approach (especially if you're using TypeScript), you can initialize a Node.js project and install the dependencies for complete access to typings. ### Step 3: Run the Script Locally @@ -30,7 +35,7 @@ In this script, we're using the `@morten-olsen/mini-loader` package, which might To validate that your script is functioning correctly, execute it locally using the following command: ```bash -mini-loader local run script.mjs -ai +mini-loader local run script.js -ai ``` The `-ai` flag instructs the CLI to automatically download any referenced packages when bundling the script. @@ -41,4 +46,4 @@ After running the command, you should see an output confirming that a new artifa Congratulations on setting up and running your first script with mini loader! You're now ready to take the next step. -[Next: Setting Up the Server](./setup-server.md) \ No newline at end of file +[Next: Setting Up the Server](./setup-server.md) diff --git a/docs/interacting-with-server.md b/docs/interacting-with-server.md index 28caa65..9ec55fc 100644 --- a/docs/interacting-with-server.md +++ b/docs/interacting-with-server.md @@ -58,7 +58,7 @@ mini-loader artifacts ls To download a specific artifact: ```bash -mini-loader artifacts pull > myfile.txt +mini-loader artifacts pull myfile.txt ``` Replace `` with the identifier of the artifact you wish to download. @@ -67,4 +67,4 @@ Replace `` with the identifier of the artifact you wish to download. You're now equipped to manage loads, runs, logs, and artifacts using the mini loader CLI. For advanced usage, such as managing secrets, proceed to the next section. -[Next: Managing Secrets](./managing-secrets.md) \ No newline at end of file +[Next: Managing Secrets](./managing-secrets.md) diff --git a/packages/cli/package.json b/packages/cli/package.json index c910443..48c3d97 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -27,6 +27,7 @@ "@rollup/plugin-sucrase": "^5.0.2", "@trpc/client": "^10.45.0", "commander": "^11.1.0", + "env-paths": "^3.0.0", "inquirer": "^9.2.12", "ora": "^8.0.1", "rollup": "^4.9.4", diff --git a/packages/cli/src/client/client.ts b/packages/cli/src/client/client.ts index dc0e281..09f7aea 100644 --- a/packages/cli/src/client/client.ts +++ b/packages/cli/src/client/client.ts @@ -2,13 +2,20 @@ import { createTRPCProxyClient, httpBatchLink } from '@trpc/client'; import superjson from 'superjson'; import type { Runtime } from '@morten-olsen/mini-loader-server'; import type { RootRouter } from '@morten-olsen/mini-loader-server'; +import { Context } from '../context/context.js'; -const createClient = () => { +const createClient = (context: Context) => { + if (!context.host || !context.token) { + throw new Error('Not signed in'); + } const client = createTRPCProxyClient({ transformer: superjson, links: [ httpBatchLink({ - url: 'http://localhost:4500/trpc', + url: `${context.host}/trpc`, + headers: { + authorization: `Bearer ${context.token}`, + }, }), ], }); diff --git a/packages/cli/src/commands/artifacts/artifacts.list.ts b/packages/cli/src/commands/artifacts/artifacts.list.ts index 809fa4a..6c9f1f3 100644 --- a/packages/cli/src/commands/artifacts/artifacts.list.ts +++ b/packages/cli/src/commands/artifacts/artifacts.list.ts @@ -1,6 +1,7 @@ import { Command } from 'commander'; import { createClient } from '../../client/client.js'; import { step } from '../../utils/step.js'; +import { Context } from '../../context/context.js'; const list = new Command('list'); @@ -20,8 +21,9 @@ list .option('-a, --limit ', 'Limit', '1000') .action(async () => { const { runId, loadId, offset, limit } = list.opts(); + const context = new Context(); const client = await step('Connecting to server', async () => { - return createClient(); + return createClient(context); }); const artifacts = await step('Getting artifacts', async () => { return await client.artifacts.find.query({ diff --git a/packages/cli/src/commands/artifacts/artifacts.pull.ts b/packages/cli/src/commands/artifacts/artifacts.pull.ts new file mode 100644 index 0000000..829d25a --- /dev/null +++ b/packages/cli/src/commands/artifacts/artifacts.pull.ts @@ -0,0 +1,32 @@ +import { Command } from 'commander'; +import { createClient } from '../../client/client.js'; +import { step } from '../../utils/step.js'; +import { Context } from '../../context/context.js'; +import { dirname, resolve } from 'path'; +import { mkdir, writeFile } from 'fs/promises'; + +const pull = new Command('pull'); + +pull + .description('Download artifact') + .argument('', 'Artifact ID') + .argument('', 'File to save') + .action(async (id, file) => { + const context = new Context(); + const target = resolve(file); + const client = await step('Connecting to server', async () => { + return createClient(context); + }); + const artifact = await step('Getting artifact', async () => { + const result = await client.artifacts.get.query(id); + if (!result) { + throw new Error('Artifact not found'); + } + return result; + }); + await mkdir(dirname(target), { recursive: true }); + const data = Buffer.from(artifact.data, 'base64').toString('utf-8'); + await writeFile(target, data, 'utf-8'); + }); + +export { pull }; diff --git a/packages/cli/src/commands/artifacts/artifacts.remove.ts b/packages/cli/src/commands/artifacts/artifacts.remove.ts new file mode 100644 index 0000000..95a4f2c --- /dev/null +++ b/packages/cli/src/commands/artifacts/artifacts.remove.ts @@ -0,0 +1,59 @@ +import { Command } from 'commander'; +import { createClient } from '../../client/client.js'; +import { step } from '../../utils/step.js'; +import { Context } from '../../context/context.js'; +import inquirer from 'inquirer'; + +const remove = new Command('remove'); + +const toInt = (value?: string) => { + if (!value) { + return undefined; + } + return parseInt(value, 10); +}; + +remove + .alias('ls') + .description('List logs') + .option('-r, --run-id ', 'Run ID') + .option('-l, --load-id ', 'Load ID') + .option('-o, --offset ', 'Offset') + .option('-a, --limit ', 'Limit', '1000') + .action(async () => { + const { runId, loadId, offset, limit } = remove.opts(); + const context = new Context(); + const client = await step('Connecting to server', async () => { + return createClient(context); + }); + const response = await step('Preparing to delete', async () => { + return await client.artifacts.prepareRemove.query({ + runId, + loadId, + offset: toInt(offset), + limit: toInt(limit), + }); + }); + + if (!response.ids.length) { + console.log('No logs to delete'); + return; + } + const { confirm } = await inquirer.prompt([ + { + type: 'confirm', + name: 'confirm', + message: `Are you sure you want to delete ${response.ids.length} logs?`, + }, + ]); + + if (!confirm) { + return; + } + + await step('Deleting artifacts', async () => { + await client.artifacts.remove.mutate(response); + }); + }); + +export { remove }; diff --git a/packages/cli/src/commands/artifacts/artifacts.ts b/packages/cli/src/commands/artifacts/artifacts.ts index f441ae9..9a041e8 100644 --- a/packages/cli/src/commands/artifacts/artifacts.ts +++ b/packages/cli/src/commands/artifacts/artifacts.ts @@ -1,7 +1,11 @@ import { Command } from 'commander'; import { list } from './artifacts.list.js'; +import { remove } from './artifacts.remove.js'; +import { pull } from './artifacts.pull.js'; const artifacts = new Command('artifacts'); artifacts.addCommand(list); +artifacts.addCommand(remove); +artifacts.addCommand(pull); export { artifacts }; diff --git a/packages/cli/src/commands/auth/auth.login.ts b/packages/cli/src/commands/auth/auth.login.ts index 920e72e..401a7c5 100644 --- a/packages/cli/src/commands/auth/auth.login.ts +++ b/packages/cli/src/commands/auth/auth.login.ts @@ -1,16 +1,19 @@ import { Command } from 'commander'; import inquerer from 'inquirer'; +import { Context } from '../../context/context.js'; +import { step } from '../../utils/step.js'; const login = new Command('login'); login.description('Login to your account'); login.action(async () => { + const context = new Context(); const { host, token } = await inquerer.prompt([ { type: 'input', name: 'host', message: 'Enter the host of your server', - default: 'http://localhost:4500', + default: context.host ?? 'http://localhost:4500', }, { type: 'password', @@ -19,7 +22,25 @@ login.action(async () => { }, ]); - console.log(host, token); + const healthResponse = await step('Getting auth status', async () => { + return await fetch(`${host}/health`, { + headers: { + authorization: `Bearer ${token}`, + }, + }); + }); + + if (!healthResponse.ok) { + throw new Error('Invalid token'); + } + const health = await healthResponse.json(); + if (!health.authorized) { + throw new Error('Invalid token'); + } + + await step('Saving login', async () => { + await context.saveLogin(host, token); + }); }); export { login }; diff --git a/packages/cli/src/commands/loads/loads.list.ts b/packages/cli/src/commands/loads/loads.list.ts index 25d441e..ba26373 100644 --- a/packages/cli/src/commands/loads/loads.list.ts +++ b/packages/cli/src/commands/loads/loads.list.ts @@ -1,6 +1,7 @@ import { Command } from 'commander'; import { createClient } from '../../client/client.js'; import { step } from '../../utils/step.js'; +import { Context } from '../../context/context.js'; const list = new Command('list'); @@ -8,11 +9,12 @@ list .alias('ls') .description('List loads') .action(async () => { + const context = new Context(); const client = await step('Connecting to server', async () => { - return createClient(); + return createClient(context); }); - const loads = step('Getting data', async () => { - await client.loads.find.query({}); + const loads = await step('Getting data', async () => { + return await client.loads.find.query({}); }); console.table(loads); }); diff --git a/packages/cli/src/commands/loads/loads.push.ts b/packages/cli/src/commands/loads/loads.push.ts index 630349a..dc21b2a 100644 --- a/packages/cli/src/commands/loads/loads.push.ts +++ b/packages/cli/src/commands/loads/loads.push.ts @@ -3,6 +3,7 @@ import { resolve } from 'path'; import { createClient } from '../../client/client.js'; import { bundle } from '../../bundler/bundler.js'; import { step } from '../../utils/step.js'; +import { Context } from '../../context/context.js'; const push = new Command('push'); @@ -14,9 +15,10 @@ push .option('-ai, --auto-install', 'Auto install dependencies', false) .action(async (script) => { const opts = push.opts(); + const context = new Context(); const location = resolve(script); const client = await step('Connecting to server', async () => { - return createClient(); + return createClient(context); }); const code = await step('Bundling', async () => { return await bundle({ entry: location, autoInstall: opts.autoInstall }); diff --git a/packages/cli/src/commands/logs/logs.list.ts b/packages/cli/src/commands/logs/logs.list.ts index 07eb93f..e1bfba2 100644 --- a/packages/cli/src/commands/logs/logs.list.ts +++ b/packages/cli/src/commands/logs/logs.list.ts @@ -1,6 +1,7 @@ import { Command } from 'commander'; import { createClient } from '../../client/client.js'; import { step } from '../../utils/step.js'; +import { Context } from '../../context/context.js'; const list = new Command('list'); @@ -22,8 +23,9 @@ list .option('-s, --sort ', 'Sort', 'desc') .action(async () => { const { runId, loadId, severities, offset, limit, order } = list.opts(); + const context = new Context(); const client = await step('Connecting to server', async () => { - return createClient(); + return createClient(context); }); const logs = await step('Getting logs', async () => { return await client.logs.find.query({ @@ -35,7 +37,7 @@ list order, }); }); - console.table(logs.reverse()); + console.table(logs); }); export { list }; diff --git a/packages/cli/src/commands/logs/logs.remove.ts b/packages/cli/src/commands/logs/logs.remove.ts new file mode 100644 index 0000000..1adbc48 --- /dev/null +++ b/packages/cli/src/commands/logs/logs.remove.ts @@ -0,0 +1,63 @@ +import { Command } from 'commander'; +import { createClient } from '../../client/client.js'; +import { step } from '../../utils/step.js'; +import { Context } from '../../context/context.js'; +import inquirer from 'inquirer'; + +const remove = new Command('remove'); + +const toInt = (value?: string) => { + if (!value) { + return undefined; + } + return parseInt(value, 10); +}; + +remove + .alias('ls') + .description('List logs') + .option('-r, --run-id ', 'Run ID') + .option('-l, --load-id ', 'Load ID') + .option('--severities ', 'Severities') + .option('-o, --offset ', 'Offset') + .option('-a, --limit ', 'Limit', '1000') + .option('-s, --sort ', 'Sort', 'desc') + .action(async () => { + const { runId, loadId, severities, offset, limit, order } = remove.opts(); + const context = new Context(); + const client = await step('Connecting to server', async () => { + return createClient(context); + }); + const response = await step('Preparing to delete', async () => { + return await client.logs.prepareRemove.query({ + runId, + loadId, + severities, + offset: toInt(offset), + limit: toInt(limit), + order, + }); + }); + + if (!response.ids.length) { + console.log('No logs to delete'); + return; + } + const { confirm } = await inquirer.prompt([ + { + type: 'confirm', + name: 'confirm', + message: `Are you sure you want to delete ${response.ids.length} logs?`, + }, + ]); + + if (!confirm) { + return; + } + + await step('Deleting logs', async () => { + await client.logs.remove.mutate(response); + }); + }); + +export { remove }; diff --git a/packages/cli/src/commands/logs/logs.ts b/packages/cli/src/commands/logs/logs.ts index 19431c6..ef6b69a 100644 --- a/packages/cli/src/commands/logs/logs.ts +++ b/packages/cli/src/commands/logs/logs.ts @@ -1,7 +1,9 @@ import { Command } from 'commander'; import { list } from './logs.list.js'; +import { remove } from './logs.remove.js'; const logs = new Command('logs'); logs.addCommand(list); +logs.addCommand(remove); export { logs }; diff --git a/packages/cli/src/commands/runs/runs.create.ts b/packages/cli/src/commands/runs/runs.create.ts index 6e8296c..2cc2279 100644 --- a/packages/cli/src/commands/runs/runs.create.ts +++ b/packages/cli/src/commands/runs/runs.create.ts @@ -1,6 +1,7 @@ import { Command } from 'commander'; import { createClient } from '../../client/client.js'; import { step } from '../../utils/step.js'; +import { Context } from '../../context/context.js'; const create = new Command('create'); @@ -8,8 +9,9 @@ create .description('Create a new run') .argument('load-id', 'Load ID') .action(async (loadId) => { + const context = new Context(); const client = await step('Connecting to server', async () => { - return createClient(); + return createClient(context); }); await step('Creating run', async () => { await client.runs.create.mutate({ loadId }); diff --git a/packages/cli/src/commands/runs/runs.list.ts b/packages/cli/src/commands/runs/runs.list.ts index 3ac822a..dca5c20 100644 --- a/packages/cli/src/commands/runs/runs.list.ts +++ b/packages/cli/src/commands/runs/runs.list.ts @@ -1,16 +1,18 @@ import { Command } from 'commander'; import { createClient } from '../../client/client.js'; import { step } from '../../utils/step.js'; +import { Context } from '../../context/context.js'; -const list = new Command('create'); +const list = new Command('list'); list .alias('ls') .description('Find a run') .argument('[load-id]', 'Load ID') .action(async (loadId) => { + const context = new Context(); const client = await step('Connecting to server', async () => { - return createClient(); + return createClient(context); }); const runs = await step('Getting runs', async () => { return await client.runs.find.query({ loadId }); diff --git a/packages/cli/src/commands/secrets/secrets.list.ts b/packages/cli/src/commands/secrets/secrets.list.ts index 9093d06..7cca018 100644 --- a/packages/cli/src/commands/secrets/secrets.list.ts +++ b/packages/cli/src/commands/secrets/secrets.list.ts @@ -1,6 +1,7 @@ import { Command } from 'commander'; import { createClient } from '../../client/client.js'; import { step } from '../../utils/step.js'; +import { Context } from '../../context/context.js'; const list = new Command('list'); @@ -18,8 +19,9 @@ list .option('-a, --limit ', 'Limit', '1000') .action(async () => { const { offset, limit } = list.opts(); + const context = new Context(); const client = await step('Connecting to server', async () => { - return createClient(); + return createClient(context); }); const secrets = await step('Getting secrets', async () => { return await client.secrets.find.query({ diff --git a/packages/cli/src/commands/secrets/secrets.remove.ts b/packages/cli/src/commands/secrets/secrets.remove.ts index 8dab67b..e3ba3ea 100644 --- a/packages/cli/src/commands/secrets/secrets.remove.ts +++ b/packages/cli/src/commands/secrets/secrets.remove.ts @@ -1,6 +1,7 @@ import { Command } from 'commander'; import { createClient } from '../../client/client.js'; import { step } from '../../utils/step.js'; +import { Context } from '../../context/context.js'; const remove = new Command('remove'); @@ -8,8 +9,9 @@ remove .alias('rm') .argument('') .action(async (id) => { + const context = new Context(); const client = await step('Connecting to server', async () => { - return createClient(); + return createClient(context); }); await step('Removing', async () => { await client.secrets.remove.mutate({ diff --git a/packages/cli/src/commands/secrets/secrets.set.ts b/packages/cli/src/commands/secrets/secrets.set.ts index c0cab84..f4f2df3 100644 --- a/packages/cli/src/commands/secrets/secrets.set.ts +++ b/packages/cli/src/commands/secrets/secrets.set.ts @@ -1,6 +1,7 @@ import { Command } from 'commander'; import { createClient } from '../../client/client.js'; import { step } from '../../utils/step.js'; +import { Context } from '../../context/context.js'; const set = new Command('set'); @@ -8,8 +9,9 @@ set .argument('') .argument('[value]') .action(async (id, value) => { + const context = new Context(); const client = await step('Connecting to server', async () => { - return createClient(); + return createClient(context); }); await step('Setting secret', async () => { await client.secrets.set.mutate({ diff --git a/packages/cli/src/context/context.ts b/packages/cli/src/context/context.ts new file mode 100644 index 0000000..4b056bf --- /dev/null +++ b/packages/cli/src/context/context.ts @@ -0,0 +1,50 @@ +import envPaths from 'env-paths'; +import { existsSync, readFileSync, writeFileSync } from 'fs'; +import { mkdir } from 'fs/promises'; +import { dirname } from 'path'; + +type ContextValues = { + host: string; + token: string; +}; + +class Context { + #location: string; + #config?: ContextValues; + + constructor() { + const paths = envPaths('dws'); + this.#location = paths.config; + if (existsSync(this.#location)) { + this.#config = JSON.parse(readFileSync(this.#location, 'utf-8')); + } + } + + public get host() { + return this.#config?.host; + } + + public get token() { + return this.#config?.token; + } + + public saveLogin = (host: string, token: string) => { + this.#config = { + ...(this.#config || {}), + host, + token, + }; + this.save(); + }; + + public save = async () => { + if (!this.#config) { + return; + } + const json = JSON.stringify(this.#config); + mkdir(dirname(this.#location), { recursive: true }); + writeFileSync(this.#location, json); + }; +} + +export { Context }; diff --git a/packages/cli/src/utils/step.ts b/packages/cli/src/utils/step.ts index d8f15d2..2f63ff5 100644 --- a/packages/cli/src/utils/step.ts +++ b/packages/cli/src/utils/step.ts @@ -4,10 +4,10 @@ const step = async (message: string, fn: () => Promise): Promise => { const spinner = ora(message).start(); try { const result = await fn(); - spinner.succeed(); + await spinner.succeed(); return result; } catch (err) { - spinner.fail(); + await spinner.fail(); throw err; } }; diff --git a/packages/examples/src/simple.ts b/packages/examples/src/simple.ts index 3bd613d..4515951 100644 --- a/packages/examples/src/simple.ts +++ b/packages/examples/src/simple.ts @@ -1,5 +1,8 @@ import { artifacts, logger } from '@morten-olsen/mini-loader'; -logger.info('Hello world'); +const run = async () => { + await logger.info('Hello world'); + await artifacts.create('foo', 'bar'); +}; -artifacts.create('foo', 'bar'); +run(); diff --git a/packages/mini-loader/src/artifacts/artifacts.ts b/packages/mini-loader/src/artifacts/artifacts.ts index 7776fd0..ed0770b 100644 --- a/packages/mini-loader/src/artifacts/artifacts.ts +++ b/packages/mini-loader/src/artifacts/artifacts.ts @@ -8,8 +8,8 @@ type ArtifactCreateEvent = { }; }; -const create = (name: string, data: Buffer | string) => { - send({ +const create = async (name: string, data: Buffer | string) => { + await send({ type: 'artifact:create', payload: { name, diff --git a/packages/mini-loader/src/input/input.ts b/packages/mini-loader/src/input/input.ts index 5b54871..a451a98 100644 --- a/packages/mini-loader/src/input/input.ts +++ b/packages/mini-loader/src/input/input.ts @@ -1,7 +1,14 @@ -import { workerData } from 'worker_threads'; +import { existsSync } from 'fs'; +import { readFile } from 'fs/promises'; -const get = () => { - return workerData as T; +const path = process.env.INPUT_PATH; +const hasInput = path ? existsSync(path) : false; + +const get = () => { + if (!hasInput || !path) { + return undefined; + } + return readFile(path, 'utf-8'); }; const input = { diff --git a/packages/mini-loader/src/logger/logger.ts b/packages/mini-loader/src/logger/logger.ts index 9c231f8..fd1da50 100644 --- a/packages/mini-loader/src/logger/logger.ts +++ b/packages/mini-loader/src/logger/logger.ts @@ -9,31 +9,31 @@ type LoggerEvent = { }; }; -const sendLog = (event: LoggerEvent['payload']) => { - send({ +const sendLog = async (event: LoggerEvent['payload']) => { + await send({ type: 'log', payload: event, }); }; -const info = (message: string, data?: unknown) => { - sendLog({ +const info = async (message: string, data?: unknown) => { + await sendLog({ severity: 'info', message, data, }); }; -const warn = (message: string, data?: unknown) => { - sendLog({ +const warn = async (message: string, data?: unknown) => { + await sendLog({ severity: 'warning', message, data, }); }; -const error = (message: string, data?: unknown) => { - sendLog({ +const error = async (message: string, data?: unknown) => { + await sendLog({ severity: 'error', message, data, diff --git a/packages/mini-loader/src/secrets/secrets.ts b/packages/mini-loader/src/secrets/secrets.ts index 1541605..ae1156d 100644 --- a/packages/mini-loader/src/secrets/secrets.ts +++ b/packages/mini-loader/src/secrets/secrets.ts @@ -1,8 +1,7 @@ -import { workerData } from 'worker_threads'; +const secretData = JSON.parse(process.env.SECRETS || '{}'); const get = (id: string) => { - const items = workerData?.secrets ?? {}; - return items[id]; + return secretData[id]; }; const secrets = { diff --git a/packages/mini-loader/src/utils.ts b/packages/mini-loader/src/utils.ts index 9f5d260..8f911ab 100644 --- a/packages/mini-loader/src/utils.ts +++ b/packages/mini-loader/src/utils.ts @@ -1,8 +1,28 @@ -import { parentPort } from 'worker_threads'; +import { Socket, createConnection } from 'net'; -const send = (data: any) => { - const cleaned = JSON.parse(JSON.stringify(data)); - parentPort?.postMessage(cleaned); -}; +const connect = () => + new Promise((resolve, reject) => { + const current = createConnection(process.env.HOST_SOCKET!); + + current.on('connect', () => { + resolve(current); + }); + current.on('error', (error) => { + reject(error); + }); + }); + +const send = async (data: any) => + new Promise(async (resolve, reject) => { + const connection = await connect(); + const cleaned = JSON.parse(JSON.stringify(data)); + connection.write(JSON.stringify(cleaned), 'utf-8', (err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); export { send }; diff --git a/packages/runner/package.json b/packages/runner/package.json index 3f95156..e45aa9d 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -16,12 +16,13 @@ } }, "devDependencies": { - "@morten-olsen/mini-loader-configs": "workspace:^", "@morten-olsen/mini-loader": "workspace:^", + "@morten-olsen/mini-loader-configs": "workspace:^", "@types/node": "^20.10.8", "typescript": "^5.3.3" }, "dependencies": { - "eventemitter3": "^5.0.1" + "eventemitter3": "^5.0.1", + "nanoid": "^5.0.4" } } \ No newline at end of file diff --git a/packages/runner/src/index.ts b/packages/runner/src/index.ts index c469a41..6d83869 100644 --- a/packages/runner/src/index.ts +++ b/packages/runner/src/index.ts @@ -1,6 +1,11 @@ import { Worker } from 'worker_threads'; +import os from 'os'; import { EventEmitter } from 'eventemitter3'; import { Event } from '@morten-olsen/mini-loader'; +import { join } from 'path'; +import { createServer } from 'http'; +import { nanoid } from 'nanoid'; +import { chmod, mkdir, rm, writeFile } from 'fs/promises'; type RunEvents = { message: (event: Event) => void; @@ -10,29 +15,75 @@ type RunEvents = { type RunOptions = { script: string; - input?: unknown; + input?: Buffer | string; secrets?: Record; }; const run = async ({ script, input, secrets }: RunOptions) => { + const dataDir = join(os.tmpdir(), 'mini-loader', nanoid()); + await mkdir(dataDir, { recursive: true }); + await chmod(dataDir, 0o700); + const hostSocket = join(dataDir, 'host'); + const server = createServer(); + const inputLocation = join(dataDir, 'input'); + + if (input) { + await writeFile(inputLocation, input); + } + const emitter = new EventEmitter(); + + server.on('connection', (socket) => { + socket.on('data', (data) => { + const message = JSON.parse(data.toString()); + emitter.emit('message', message); + }); + }); + server.listen(hostSocket); + const worker = new Worker(script, { eval: true, - env: secrets, + stdin: false, + stdout: false, + stderr: false, + env: { + HOST_SOCKET: hostSocket, + SECRETS: JSON.stringify(secrets), + INPUT_PATH: inputLocation, + }, workerData: { input, - secrets, }, }); - const promise = new Promise((resolve, reject) => { - worker.on('message', (message: Event) => { - emitter.emit('message', message); + worker.stdout?.on('data', (data) => { + emitter.emit('message', { + type: 'log', + payload: { + severity: 'info', + message: data.toString(), + }, }); - worker.on('exit', () => { + }); + + worker.stderr?.on('data', (data) => { + emitter.emit('message', { + type: 'log', + payload: { + severity: 'error', + message: data.toString(), + }, + }); + }); + + const promise = new Promise((resolve, reject) => { + worker.on('exit', async () => { + server.close(); + await rm(dataDir, { recursive: true, force: true }); resolve(); }); - worker.on('error', (error) => { + worker.on('error', async (error) => { + server.close(); reject(error); }); }); diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 870c6d2..bfefcf9 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -27,5 +27,9 @@ program.addCommand(createToken); await program.parseAsync(process.argv); +process.on('unhandledRejection', (reason, p) => { + console.log('Unhandled Rejection at: Promise', p, 'reason:', reason); +}); + export type { Runtime } from './runtime/runtime.js'; export type { RootRouter } from './router/router.js'; diff --git a/packages/server/src/repos/artifacts/artifacts.ts b/packages/server/src/repos/artifacts/artifacts.ts index c40729a..13544e4 100644 --- a/packages/server/src/repos/artifacts/artifacts.ts +++ b/packages/server/src/repos/artifacts/artifacts.ts @@ -18,6 +18,13 @@ class ArtifactRepo extends EventEmitter { this.#options = options; } + public get = async (id: string) => { + const { database } = this.#options; + const db = await database.instance; + const result = await db('artifacts').where({ id }).first(); + return result || null; + }; + public add = async (options: AddArtifactOptions) => { const { database } = this.#options; const db = await database.instance; @@ -59,8 +66,9 @@ class ArtifactRepo extends EventEmitter { query.limit(options.limit); } - const ids = await query; - const token = ids.map((id) => Buffer.from(id.id).toString('base64')).join('|'); + const result = await query; + const ids = result.map((row) => row.id); + const token = ids.map((id) => Buffer.from(id).toString('base64')).join('|'); const hash = createHash('sha256').update(token).digest('hex'); return { ids, diff --git a/packages/server/src/repos/logs/logs.ts b/packages/server/src/repos/logs/logs.ts index a38d080..ed090dd 100644 --- a/packages/server/src/repos/logs/logs.ts +++ b/packages/server/src/repos/logs/logs.ts @@ -56,8 +56,9 @@ class LogRepo extends EventEmitter { query.whereIn('severity', options.severities); } - const ids = await query; - const token = ids.map((id) => Buffer.from(id.id).toString('base64')).join('|'); + const result = await query; + const ids = result.map((row) => row.id); + const token = ids.map((id) => Buffer.from(id).toString('base64')).join('|'); const hash = createHash('sha256').update(token).digest('hex'); return { ids, diff --git a/packages/server/src/router/router.artifacts.ts b/packages/server/src/router/router.artifacts.ts index f2dcce1..8f92eb1 100644 --- a/packages/server/src/router/router.artifacts.ts +++ b/packages/server/src/router/router.artifacts.ts @@ -11,12 +11,21 @@ const find = publicProcedure.input(findArtifactsSchema).query(async ({ input, ct return result; }); +const get = publicProcedure.input(z.string()).query(async ({ input, ctx }) => { + const { runtime } = ctx; + const { repos } = runtime; + const { artifacts } = repos; + + const result = await artifacts.get(input); + return result; +}); + const prepareRemove = publicProcedure.input(findArtifactsSchema).query(async ({ input, ctx }) => { const { runtime } = ctx; const { repos } = runtime; const { artifacts } = repos; - await artifacts.prepareRemove(input); + return await artifacts.prepareRemove(input); }); const remove = publicProcedure @@ -35,6 +44,7 @@ const remove = publicProcedure }); const artifactsRouter = router({ + get, find, remove, prepareRemove, diff --git a/packages/server/src/router/router.logs.ts b/packages/server/src/router/router.logs.ts index d4af353..7e73e38 100644 --- a/packages/server/src/router/router.logs.ts +++ b/packages/server/src/router/router.logs.ts @@ -16,7 +16,7 @@ const prepareRemove = publicProcedure.input(findLogsSchema).query(async ({ input const { repos } = runtime; const { logs } = repos; - await logs.prepareRemove(input); + return await logs.prepareRemove(input); }); const remove = publicProcedure diff --git a/packages/server/src/router/router.utils.ts b/packages/server/src/router/router.utils.ts index 04da662..dbc7fd0 100644 --- a/packages/server/src/router/router.utils.ts +++ b/packages/server/src/router/router.utils.ts @@ -14,7 +14,8 @@ const createContext = async ({ runtime }: ContextOptions) => { if (!authorization) { throw new Error('No authorization header'); } - await auth.validateToken(authorization); + const [, token] = authorization.split(' '); + await auth.validateToken(token); return { runtime, }; diff --git a/packages/server/src/runner/runner.instance.ts b/packages/server/src/runner/runner.instance.ts index 9486bc4..ebb0da9 100644 --- a/packages/server/src/runner/runner.instance.ts +++ b/packages/server/src/runner/runner.instance.ts @@ -54,7 +54,7 @@ class RunnerInstance extends EventEmitter { const { runs, secrets } = repos; try { const { script: scriptHash, input } = await runs.getById(id); - const scriptLocation = resolve(config.files.location, 'script', `${scriptHash}.js`); + const scriptLocation = resolve(config.files.location, 'scripts', `${scriptHash}.js`); const script = await readFile(scriptLocation, 'utf-8'); const allSecrets = await secrets.getAll(); await runs.started(id); diff --git a/packages/server/src/server/server.ts b/packages/server/src/server/server.ts index 310c17b..a044531 100644 --- a/packages/server/src/server/server.ts +++ b/packages/server/src/server/server.ts @@ -10,6 +10,19 @@ const createServer = async (runtime: Runtime) => { return { hello: 'world' }; }); + server.get('/health', async (req) => { + let authorized = false; + try { + const { authorization } = req.headers; + if (authorization) { + const [, token] = authorization.split(' '); + await runtime.auth.validateToken(token); + authorized = true; + } + } catch (error) {} + return { authorized, status: 'ok' }; + }); + server.register(fastifyTRPCPlugin, { prefix: '/trpc', trpcOptions: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7a21f65..71d1cb6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -60,6 +60,9 @@ importers: commander: specifier: ^11.1.0 version: 11.1.0 + env-paths: + specifier: ^3.0.0 + version: 3.0.0 inquirer: specifier: ^9.2.12 version: 9.2.12 @@ -132,6 +135,9 @@ importers: eventemitter3: specifier: ^5.0.1 version: 5.0.1 + nanoid: + specifier: ^5.0.4 + version: 5.0.4 devDependencies: '@morten-olsen/mini-loader': specifier: workspace:^ @@ -2206,6 +2212,11 @@ packages: dev: false optional: true + /env-paths@3.0.0: + resolution: {integrity: sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + /err-code@2.0.3: resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} requiresBuild: true