5 Commits
0.1.8 ... 0.2.0

Author SHA1 Message Date
Morten Olsen
1115ce2fb3 feat: add http gateway (#3) 2024-01-12 21:10:48 +01:00
Morten Olsen
9c5249956e chore: create devcontainer.json (#1) 2024-01-12 17:40:44 +01:00
Morten Olsen
b5d8cf3a51 feat: support multiple contexts 2024-01-12 15:31:44 +01:00
Morten Olsen
5154fbb4a5 ci: add NPM publish 2024-01-12 15:07:26 +01:00
Morten Olsen
59d6faaafc feat: switched from worker API to fs based 2024-01-12 14:35:16 +01:00
56 changed files with 674 additions and 132 deletions

View File

@@ -0,0 +1,5 @@
{
"image": "mcr.microsoft.com/devcontainers/universal:2",
"features": {
}
}

View File

@@ -91,33 +91,33 @@ jobs:
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
# release-npm:
# if: github.ref == 'refs/heads/main'
# runs-on: ubuntu-latest
# needs: [build, update-release-draft]
# permissions:
# contents: read
# packages: write
# steps:
# - uses: actions/checkout@v3
# with:
# fetch-depth: 0
# - run: corepack enable
# - uses: actions/setup-node@v3
# with:
# cache: '${{ env.NODE_CACHE }}'
# node-version: '${{ env.NODE_VERSION }}'
# scope: '${{ env.NODE_SCOPE }}'
# - uses: actions/download-artifact@v3
# with:
# name: lib
# path: ./
# - run: |
# pnpm config set //registry.npmjs.org/:_authToken ${NPM_TOKEN}
# pnpm install
# git config user.name "Github Actions Bot"
# git config user.email "<>"
# node scripts/set-version.ts $(git describe --tag --abbrev=0)
# pnpm publish -r --publish-branch main --access public --no-git-checks
# env:
# NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
release-npm:
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
needs: [build, update-release-draft]
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- run: corepack enable
- uses: actions/setup-node@v3
with:
cache: '${{ env.NODE_CACHE }}'
node-version: '${{ env.NODE_VERSION }}'
scope: '${{ env.NODE_SCOPE }}'
- uses: actions/download-artifact@v3
with:
name: lib
path: ./
- run: |
pnpm config set //registry.npmjs.org/:_authToken ${NPM_TOKEN}
pnpm install
git config user.name "Github Actions Bot"
git config user.email "<>"
node scripts/set-version.mjs $(git describe --tag --abbrev=0)
pnpm publish -r --publish-branch main --access public --no-git-checks
env:
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

View File

@@ -15,14 +15,19 @@ npm install -g @morten-olsen/mini-loader-cli
Now, let's write a basic script that outputs a single artifact named “hello”. Create a new file with the following JavaScript code:
```javascript
import { artifacts } from "@morten-olsen/mini-loader";
import { artifacts } from '@morten-olsen/mini-loader';
artifacts.create('hello', 'world');
const run = async () => {
artifacts.create('hello', 'world');
};
run();
```
Save this file as `script.mjs`.
Save this file as `script.js`.
#### A Note on Dependencies
In this script, we're using the `@morten-olsen/mini-loader` package, which might not be installed in your local environment. No worries though, as mini loader can automatically download necessary packages when preparing the script. Alternatively, for a more structured approach (especially if you're using TypeScript), you can initialize a Node.js project and install the dependencies for complete access to typings.
### Step 3: Run the Script Locally
@@ -30,7 +35,7 @@ In this script, we're using the `@morten-olsen/mini-loader` package, which might
To validate that your script is functioning correctly, execute it locally using the following command:
```bash
mini-loader local run script.mjs -ai
mini-loader local run script.js -ai
```
The `-ai` flag instructs the CLI to automatically download any referenced packages when bundling the script.
@@ -41,4 +46,4 @@ After running the command, you should see an output confirming that a new artifa
Congratulations on setting up and running your first script with mini loader! You're now ready to take the next step.
[Next: Setting Up the Server](./setup-server.md)
[Next: Setting Up the Server](./setup-server.md)

View File

@@ -1,6 +1,7 @@
{
"name": "@morten-olsen/mini-loader-repo",
"private": "true",
"license": "GPL-3.0",
"packageManager": "pnpm@8.10.4",
"version": "1.0.0",
"scripts": {
@@ -11,7 +12,6 @@
},
"keywords": [],
"author": "",
"license": "ISC",
"devDependencies": {
"@react-native-community/eslint-config": "^3.2.0",
"eslint": "^8.53.0",
@@ -23,5 +23,10 @@
"@pnpm/find-workspace-packages": "^6.0.9",
"@types/node": "^20.10.8",
"ts-node": "^10.9.2"
},
"homepage": "https://github.com/morten-olsen/mini-loader",
"repository": {
"type": "git",
"url": "https://github.com/morten-olsen/mini-loader"
}
}

1
packages/cli/README.md Normal file
View File

@@ -0,0 +1 @@
[Go to documentation](https://github.com/morten-olsen/mini-loader)

View File

@@ -3,6 +3,7 @@
"version": "1.0.0",
"main": "./dist/esm/index.js",
"types": "./dist/esm/index.d.ts",
"license": "GPL-3.0",
"bin": {
"mini-loader": "./bin/index.mjs"
},
@@ -41,5 +42,10 @@
"@morten-olsen/mini-loader-server": "workspace:^",
"@types/inquirer": "^9.0.7",
"typescript": "^5.3.3"
},
"homepage": "https://github.com/morten-olsen/mini-loader",
"repository": {
"type": "git",
"url": "https://github.com/morten-olsen/mini-loader"
}
}

View File

@@ -17,12 +17,12 @@ const bundle = async ({ entry, autoInstall }: BundleOptions) => {
const entryFile = resolve(entry);
const codeBundler = await rollup({
plugins: [
fix(json)(),
fix(sucrase)({
transforms: ['typescript', 'jsx'],
}),
...[autoInstall ? fix(auto) : []],
nodeResolve({ extensions: ['.js', '.jsx', '.ts', '.tsx'] }),
fix(json)(),
nodeResolve({ preferBuiltins: true, extensions: ['.js', '.jsx', '.ts', '.tsx'] }),
fix(commonjs)({ include: /node_modules/ }),
],
input: entryFile,

View File

@@ -2,6 +2,7 @@ import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
const list = new Command('list');
@@ -21,7 +22,8 @@ list
.option('-a, --limit <limit>', 'Limit', '1000')
.action(async () => {
const { runId, loadId, offset, limit } = list.opts();
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});

View File

@@ -4,6 +4,7 @@ import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { dirname, resolve } from 'path';
import { mkdir, writeFile } from 'fs/promises';
import { Config } from '../../config/config.js';
const pull = new Command('pull');
@@ -12,7 +13,8 @@ pull
.argument('<artifact-id>', 'Artifact ID')
.argument('<file>', 'File to save')
.action(async (id, file) => {
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const target = resolve(file);
const client = await step('Connecting to server', async () => {
return createClient(context);

View File

@@ -3,6 +3,7 @@ import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import inquirer from 'inquirer';
import { Config } from '../../config/config.js';
const remove = new Command('remove');
@@ -22,7 +23,8 @@ remove
.option('-a, --limit <limit>', 'Limit', '1000')
.action(async () => {
const { runId, loadId, offset, limit } = remove.opts();
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});

View File

@@ -2,12 +2,14 @@ import { Command } from 'commander';
import inquerer from 'inquirer';
import { Context } from '../../context/context.js';
import { step } from '../../utils/step.js';
import { Config } from '../../config/config.js';
const login = new Command('login');
login.description('Login to your account');
login.action(async () => {
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const { host, token } = await inquerer.prompt([
{
type: 'input',

View File

@@ -0,0 +1,10 @@
import { Command } from 'commander';
import { Config } from '../../config/config.js';
const current = new Command('current');
current.action(async () => {
const config = new Config();
console.log(config.context);
});
export { current };

View File

@@ -0,0 +1,11 @@
import { Command } from 'commander';
import { Context } from '../../context/context.js';
const list = new Command('list');
list.alias('ls').description('List contexts');
list.action(async () => {
const contexts = await Context.list();
console.table(contexts);
});
export { list };

View File

@@ -0,0 +1,12 @@
import { Command } from 'commander';
import { list } from './contexts.list.js';
import { use } from './contexts.use.js';
import { current } from './contexts.current.js';
const contexts = new Command('contexts');
contexts.description('Manage contexts');
contexts.addCommand(list);
contexts.addCommand(use);
contexts.addCommand(current);
export { contexts };

View File

@@ -0,0 +1,11 @@
import { Command } from 'commander';
import { Config } from '../../config/config.js';
const use = new Command('use');
use.argument('<name>').action(async (name) => {
const config = new Config();
await config.setContext(name);
});
export { use };

View File

@@ -2,6 +2,7 @@ import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
const list = new Command('list');
@@ -9,7 +10,8 @@ list
.alias('ls')
.description('List loads')
.action(async () => {
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});

View File

@@ -4,6 +4,7 @@ import { createClient } from '../../client/client.js';
import { bundle } from '../../bundler/bundler.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
const push = new Command('push');
@@ -15,7 +16,8 @@ push
.option('-ai, --auto-install', 'Auto install dependencies', false)
.action(async (script) => {
const opts = push.opts();
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const location = resolve(script);
const client = await step('Connecting to server', async () => {
return createClient(context);
@@ -23,7 +25,7 @@ push
const code = await step('Bundling', async () => {
return await bundle({ entry: location, autoInstall: opts.autoInstall });
});
const id = await step('Creating load', async () => {
const id = await step(`Creating load ${(code.length / 1024).toFixed(0)}`, async () => {
return await client.loads.set.mutate({
id: opts.id,
name: opts.name,
@@ -32,9 +34,10 @@ push
});
console.log('created load with id', id);
if (opts.run) {
await step('Creating run', async () => {
await client.runs.create.mutate({ loadId: id });
const runId = await step('Creating run', async () => {
return await client.runs.create.mutate({ loadId: id });
});
console.log('created run with id', runId);
}
});

View File

@@ -2,6 +2,7 @@ import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
const list = new Command('list');
@@ -23,7 +24,8 @@ list
.option('-s, --sort <order>', 'Sort', 'desc')
.action(async () => {
const { runId, loadId, severities, offset, limit, order } = list.opts();
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
@@ -37,7 +39,7 @@ list
order,
});
});
console.table(logs.reverse());
console.table(logs);
});
export { list };

View File

@@ -3,6 +3,7 @@ import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import inquirer from 'inquirer';
import { Config } from '../../config/config.js';
const remove = new Command('remove');
@@ -24,7 +25,8 @@ remove
.option('-s, --sort <order>', 'Sort', 'desc')
.action(async () => {
const { runId, loadId, severities, offset, limit, order } = remove.opts();
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});

View File

@@ -2,6 +2,7 @@ import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
const create = new Command('create');
@@ -9,7 +10,8 @@ create
.description('Create a new run')
.argument('load-id', 'Load ID')
.action(async (loadId) => {
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});

View File

@@ -2,6 +2,7 @@ import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
const list = new Command('list');
@@ -10,7 +11,8 @@ list
.description('Find a run')
.argument('[load-id]', 'Load ID')
.action(async (loadId) => {
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});

View File

@@ -0,0 +1,59 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import inquirer from 'inquirer';
import { Config } from '../../config/config.js';
const remove = new Command('remove');
const toInt = (value?: string) => {
if (!value) {
return undefined;
}
return parseInt(value, 10);
};
remove
.alias('ls')
.description('List logs')
.option('-l, --load-id <loadId>', 'Load ID')
.option('-o, --offset <offset>', 'Offset')
.option('-a, --limit <limit>', 'Limit', '1000')
.action(async () => {
const { loadId, offset, limit } = remove.opts();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const response = await step('Preparing to delete', async () => {
return await client.runs.prepareRemove.query({
loadId,
offset: toInt(offset),
limit: toInt(limit),
});
});
if (!response.ids.length) {
console.log('No logs to delete');
return;
}
const { confirm } = await inquirer.prompt([
{
type: 'confirm',
name: 'confirm',
message: `Are you sure you want to delete ${response.ids.length} logs?`,
},
]);
if (!confirm) {
return;
}
await step('Deleting artifacts', async () => {
await client.runs.remove.mutate(response);
});
});
export { remove };

View File

@@ -0,0 +1,23 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
const terminate = new Command('terminate');
terminate
.description('Terminate an in progress run')
.argument('run-id', 'Run ID')
.action(async (runId) => {
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
await step('Terminating run', async () => {
await client.runs.terminate.mutate(runId);
});
});
export { terminate };

View File

@@ -1,8 +1,14 @@
import { Command } from 'commander';
import { create } from './runs.create.js';
import { list } from './runs.list.js';
import { remove } from './runs.remove.js';
import { terminate } from './runs.terminate.js';
const runs = new Command('runs');
runs.description('Manage runs').addCommand(create).addCommand(list);
runs.description('Manage runs');
runs.addCommand(create);
runs.addCommand(list);
runs.addCommand(remove);
runs.addCommand(terminate);
export { runs };

View File

@@ -2,6 +2,7 @@ import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
const list = new Command('list');
@@ -19,7 +20,8 @@ list
.option('-a, --limit <limit>', 'Limit', '1000')
.action(async () => {
const { offset, limit } = list.opts();
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});

View File

@@ -2,6 +2,7 @@ import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
const remove = new Command('remove');
@@ -9,7 +10,8 @@ remove
.alias('rm')
.argument('<id>')
.action(async (id) => {
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});

View File

@@ -2,6 +2,7 @@ import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
const set = new Command('set');
@@ -9,7 +10,8 @@ set
.argument('<id>')
.argument('[value]')
.action(async (id, value) => {
const context = new Context();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});

View File

@@ -0,0 +1,44 @@
import envPaths from 'env-paths';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import { mkdir } from 'fs/promises';
import { join, dirname } from 'path';
type ConfigValues = {
context?: string;
};
class Config {
#location: string;
#config?: ConfigValues;
constructor() {
const paths = envPaths('mini-loader');
this.#location = join(paths.config, 'config.json');
if (existsSync(this.#location)) {
this.#config = JSON.parse(readFileSync(this.#location, 'utf-8'));
}
}
public get context() {
return this.#config?.context || 'default';
}
public setContext = (context: string) => {
this.#config = {
...(this.#config || {}),
context,
};
this.save();
};
public save = async () => {
if (!this.#config) {
return;
}
const json = JSON.stringify(this.#config);
mkdir(dirname(this.#location), { recursive: true });
writeFileSync(this.#location, json);
};
}
export { Config };

View File

@@ -1,7 +1,7 @@
import envPaths from 'env-paths';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import { mkdir } from 'fs/promises';
import { dirname } from 'path';
import { mkdir, readdir } from 'fs/promises';
import { dirname, join } from 'path';
type ContextValues = {
host: string;
@@ -12,9 +12,9 @@ class Context {
#location: string;
#config?: ContextValues;
constructor() {
const paths = envPaths('dws');
this.#location = paths.config;
constructor(name: string) {
const paths = envPaths('mini-loader');
this.#location = join(paths.config, 'contexts', name);
if (existsSync(this.#location)) {
this.#config = JSON.parse(readFileSync(this.#location, 'utf-8'));
}
@@ -45,6 +45,15 @@ class Context {
mkdir(dirname(this.#location), { recursive: true });
writeFileSync(this.#location, json);
};
public static list = async () => {
const paths = envPaths('mini-loader');
const location = join(paths.config, 'contexts');
if (!existsSync(location)) {
return [];
}
return await readdir(location);
};
}
export { Context };

View File

@@ -6,6 +6,7 @@ import { artifacts } from './commands/artifacts/artifacts.js';
import { secrets } from './commands/secrets/secrets.js';
import { local } from './commands/local/local.js';
import { auth } from './commands/auth/auth.js';
import { contexts } from './commands/contexts/contexts.js';
program.addCommand(loads);
program.addCommand(runs);
@@ -14,5 +15,6 @@ program.addCommand(artifacts);
program.addCommand(secrets);
program.addCommand(local);
program.addCommand(auth);
program.addCommand(contexts);
await program.parseAsync();

View File

@@ -1,6 +1,7 @@
{
"name": "@morten-olsen/mini-loader-configs",
"version": "1.0.0",
"private": true,
"description": "",
"main": "index.js",
"scripts": {
@@ -8,5 +9,10 @@
},
"keywords": [],
"author": "",
"license": "ISC"
"license": "GPL-3.0",
"homepage": "https://github.com/morten-olsen/mini-loader",
"repository": {
"type": "git",
"url": "https://github.com/morten-olsen/mini-loader"
}
}

View File

@@ -1,6 +1,8 @@
{
"name": "@morten-olsen/mini-loader-examples",
"version": "1.0.0",
"license": "GPL-3.0",
"private": true,
"main": "./dist/esm/index.js",
"types": "./dist/esm/index.d.ts",
"scripts": {
@@ -16,10 +18,18 @@
}
},
"devDependencies": {
"@morten-olsen/mini-loader-configs": "workspace:^",
"@morten-olsen/mini-loader-cli": "workspace:^",
"@morten-olsen/mini-loader": "workspace:^",
"@morten-olsen/mini-loader-cli": "workspace:^",
"@morten-olsen/mini-loader-configs": "workspace:^",
"@types/node": "^20.10.8",
"typescript": "^5.3.3"
},
"homepage": "https://github.com/morten-olsen/mini-loader",
"repository": {
"type": "git",
"url": "https://github.com/morten-olsen/mini-loader"
},
"dependencies": {
"fastify": "^4.25.2"
}
}

View File

@@ -0,0 +1,12 @@
import { http } from '@morten-olsen/mini-loader';
import fastify from 'fastify';
const server = fastify();
server.all('*', async (req) => {
return req.url;
});
server.listen({
path: http.getPath(),
});

View File

@@ -1,5 +1,9 @@
import { artifacts, logger } from '@morten-olsen/mini-loader';
await logger.info('Hello world');
const run = async () => {
await logger.info('Hello world');
await artifacts.create('foo', 'bar');
process.exit(0);
};
await artifacts.create('foo', 'bar');
run();

View File

@@ -0,0 +1 @@
[Go to documentation](https://github.com/morten-olsen/mini-loader)

View File

@@ -1,6 +1,7 @@
{
"name": "@morten-olsen/mini-loader",
"version": "1.0.0",
"license": "GPL-3.0",
"main": "./dist/esm/index.js",
"types": "./dist/esm/index.d.ts",
"scripts": {
@@ -19,5 +20,10 @@
"@morten-olsen/mini-loader-configs": "workspace:^",
"@types/node": "^20.10.8",
"typescript": "^5.3.3"
},
"homepage": "https://github.com/morten-olsen/mini-loader",
"repository": {
"type": "git",
"url": "https://github.com/morten-olsen/mini-loader"
}
}

View File

@@ -0,0 +1,7 @@
const getPath = () => process.env.HTTP_GATEWAY_PATH!;
const http = {
getPath,
};
export { http };

View File

@@ -8,3 +8,4 @@ export { logger } from './logger/logger.js';
export { artifacts } from './artifacts/artifacts.js';
export { input } from './input/input.js';
export { secrets } from './secrets/secrets.js';
export { http } from './http/http.js';

View File

@@ -12,11 +12,9 @@ const connect = () =>
});
});
const connectionRequest = connect();
const send = async (data: any) =>
new Promise<void>(async (resolve, reject) => {
const connection = await connectionRequest;
const connection = await connect();
const cleaned = JSON.parse(JSON.stringify(data));
connection.write(JSON.stringify(cleaned), 'utf-8', (err) => {
if (err) {

View File

@@ -0,0 +1 @@
[Go to documentation](https://github.com/morten-olsen/mini-loader)

View File

@@ -1,6 +1,7 @@
{
"name": "@morten-olsen/mini-loader-runner",
"version": "1.0.0",
"license": "GPL-3.0",
"main": "./dist/esm/index.js",
"types": "./dist/esm/index.d.ts",
"scripts": {
@@ -16,13 +17,18 @@
}
},
"devDependencies": {
"@morten-olsen/mini-loader": "workspace:^",
"@morten-olsen/mini-loader-configs": "workspace:^",
"@types/node": "^20.10.8",
"typescript": "^5.3.3"
},
"dependencies": {
"@morten-olsen/mini-loader": "workspace:^",
"eventemitter3": "^5.0.1",
"nanoid": "^5.0.4"
},
"homepage": "https://github.com/morten-olsen/mini-loader",
"repository": {
"type": "git",
"url": "https://github.com/morten-olsen/mini-loader"
}
}
}

View File

@@ -1,17 +1,5 @@
import { Worker } from 'worker_threads';
import os from 'os';
import { EventEmitter } from 'eventemitter3';
import { Event } from '@morten-olsen/mini-loader';
import { join } from 'path';
import { createServer } from 'http';
import { nanoid } from 'nanoid';
import { chmod, mkdir, unlink, writeFile } from 'fs/promises';
type RunEvents = {
message: (event: Event) => void;
error: (error: Error) => void;
exit: () => void;
};
import { setup } from './setup/setup.js';
type RunOptions = {
script: string;
@@ -20,58 +8,55 @@ type RunOptions = {
};
const run = async ({ script, input, secrets }: RunOptions) => {
const dataDir = join(os.tmpdir(), 'mini-loader', nanoid());
await mkdir(dataDir, { recursive: true });
await chmod(dataDir, 0o700);
const hostSocket = join(dataDir, 'host');
const server = createServer();
const inputLocation = join(dataDir, 'input');
const info = await setup({ script, input, secrets });
if (input) {
await writeFile(inputLocation, input);
}
const worker = new Worker(info.scriptLocation, {
stdin: false,
stdout: false,
stderr: false,
env: info.env,
});
const emitter = new EventEmitter<RunEvents>();
server.on('connection', (socket) => {
socket.on('data', (data) => {
const message = JSON.parse(data.toString());
emitter.emit('message', message);
worker.stdout?.on('data', (data) => {
info.emitter.emit('message', {
type: 'log',
payload: {
severity: 'info',
message: data.toString(),
},
});
});
const worker = new Worker(script, {
eval: true,
env: {
HOST_SOCKET: hostSocket,
SECRETS: JSON.stringify(secrets),
INPUT_PATH: inputLocation,
},
workerData: {
input,
},
worker.stderr?.on('data', (data) => {
info.emitter.emit('message', {
type: 'log',
payload: {
severity: 'error',
message: data.toString(),
},
});
});
const promise = new Promise<void>((resolve, reject) => {
worker.on('message', (message: Event) => {
emitter.emit('message', message);
});
worker.on('exit', async () => {
server.close();
await unlink(hostSocket);
await info.teardown();
resolve();
});
worker.on('error', async (error) => {
server.close();
await unlink(hostSocket);
reject(error);
});
});
return {
emitter,
...info,
teardown: async () => {
worker.terminate();
},
promise,
};
};
type RunInfo = Awaited<ReturnType<typeof run>>;
export type { RunInfo };
export { run };

View File

@@ -0,0 +1,71 @@
import { join } from 'path';
import os from 'os';
import { nanoid } from 'nanoid';
import { chmod, mkdir, rm, writeFile } from 'fs/promises';
import { createServer } from 'net';
import { EventEmitter } from 'eventemitter3';
type SetupOptions = {
input?: Buffer | string;
script: string;
secrets?: Record<string, string>;
};
type RunEvents = {
message: (event: any) => void;
error: (error: Error) => void;
exit: () => void;
};
const setup = async (options: SetupOptions) => {
const { input, script, secrets } = options;
const emitter = new EventEmitter<RunEvents>();
const dataDir = join(os.tmpdir(), 'mini-loader', nanoid());
await mkdir(dataDir, { recursive: true });
await chmod(dataDir, 0o700);
const hostSocket = join(dataDir, 'host');
const httpGatewaySocket = join(dataDir, 'socket');
const server = createServer();
const inputLocation = join(dataDir, 'input');
const scriptLocation = join(dataDir, 'script.js');
if (input) {
await writeFile(inputLocation, input);
}
await writeFile(scriptLocation, script);
const env = {
HOST_SOCKET: hostSocket,
SECRETS: JSON.stringify(secrets || {}),
INPUT_PATH: inputLocation,
HTTP_GATEWAY_PATH: httpGatewaySocket,
};
const teardown = async () => {
server.close();
await rm(dataDir, { recursive: true, force: true });
};
server.on('connection', (socket) => {
socket.on('data', (data) => {
const message = JSON.parse(data.toString());
emitter.emit('message', message);
});
});
server.listen(hostSocket);
return {
env,
emitter,
teardown,
httpGatewaySocket,
scriptLocation,
hostSocket,
};
};
type Setup = Awaited<ReturnType<typeof setup>>;
export type { Setup };
export { setup };

View File

@@ -0,0 +1 @@
[Go to documentation](https://github.com/morten-olsen/mini-loader)

View File

@@ -1,6 +1,7 @@
{
"name": "@morten-olsen/mini-loader-server",
"version": "1.0.0",
"license": "GPL-3.0",
"main": "./dist/esm/index.js",
"types": "./dist/esm/index.d.ts",
"bin": {
@@ -26,6 +27,7 @@
"typescript": "^5.3.3"
},
"dependencies": {
"@fastify/reply-from": "^9.7.0",
"@trpc/client": "^10.45.0",
"@trpc/server": "^10.45.0",
"commander": "^11.1.0",
@@ -38,5 +40,10 @@
"sqlite3": "^5.1.7",
"superjson": "^2.2.1",
"zod": "^3.22.4"
},
"homepage": "https://github.com/morten-olsen/mini-loader",
"repository": {
"type": "git",
"url": "https://github.com/morten-olsen/mini-loader"
}
}

View File

@@ -0,0 +1,34 @@
import { FastifyPluginAsync } from 'fastify';
import FastifyReplyFrom from '@fastify/reply-from';
import { escape } from 'querystring';
import { Runtime } from '../runtime/runtime.js';
type Options = {
runtime: Runtime;
};
const gateway: FastifyPluginAsync<Options> = async (fastify, { runtime }) => {
await fastify.register(FastifyReplyFrom, {
http: {},
});
fastify.all('/gateway/*', (req, res) => {
const [runId, ...pathSegments] = (req.params as any)['*'].split('/').filter(Boolean);
const run = runtime.runner.getInstance(runId);
if (!run) {
res.statusCode = 404;
res.send({ error: 'Run not found' });
return;
}
const socketPath = run.run?.httpGatewaySocket;
if (!socketPath) {
res.statusCode = 404;
res.send({ error: 'No socket path to run' });
return;
}
const path = pathSegments.join('/');
res.from(`unix+http://${escape(socketPath)}/${path}`);
});
};
export { gateway };

View File

@@ -27,5 +27,9 @@ program.addCommand(createToken);
await program.parseAsync(process.argv);
process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
});
export type { Runtime } from './runtime/runtime.js';
export type { RootRouter } from './router/router.js';

View File

@@ -3,6 +3,7 @@ import { EventEmitter } from 'eventemitter3';
import { Database } from '../../database/database.js';
import { CreateRunOptions, FindRunsOptions, UpdateRunOptions } from './runs.schemas.js';
import { LoadRepo } from '../loads/loads.js';
import { createHash } from 'crypto';
type RunRepoEvents = {
created: (args: { id: string; loadId: string }) => void;
@@ -18,13 +19,22 @@ type RunRepoOptions = {
class RunRepo extends EventEmitter<RunRepoEvents> {
#options: RunRepoOptions;
#isReady: Promise<void>;
constructor(options: RunRepoOptions) {
super();
this.#options = options;
this.#isReady = this.#setup();
}
#setup = async () => {
const { database } = this.#options;
const db = await database.instance;
await db('runs').update({ status: 'failed', error: 'server was shut down' }).where({ status: 'running' });
};
public getById = async (id: string) => {
await this.#isReady;
const { database } = this.#options;
const db = await database.instance;
@@ -36,6 +46,7 @@ class RunRepo extends EventEmitter<RunRepoEvents> {
};
public getByLoadId = async (loadId: string) => {
await this.#isReady;
const { database } = this.#options;
const db = await database.instance;
@@ -44,6 +55,7 @@ class RunRepo extends EventEmitter<RunRepoEvents> {
};
public find = async (options: FindRunsOptions) => {
await this.#isReady;
const { database } = this.#options;
const db = await database.instance;
const query = db('runs').select(['id', 'status', 'startedAt', 'status', 'error', 'endedAt']);
@@ -62,19 +74,41 @@ class RunRepo extends EventEmitter<RunRepoEvents> {
return runs;
};
public remove = async (options: FindRunsOptions) => {
public prepareRemove = async (options: FindRunsOptions) => {
await this.#isReady;
const { database } = this.#options;
const db = await database.instance;
const query = db('runs');
const query = db('runs').select('id');
if (options.loadId) {
query.where({ loadId: options.loadId });
}
await query.del();
const result = await query;
const ids = result.map((row) => row.id);
const token = ids.map((id) => Buffer.from(id).toString('base64')).join('|');
const hash = createHash('sha256').update(token).digest('hex');
return {
ids,
hash,
};
};
public remove = async (hash: string, ids: string[]) => {
const { database } = this.#options;
const db = await database.instance;
const token = ids.map((id) => Buffer.from(id).toString('base64')).join('|');
const actualHash = createHash('sha256').update(token).digest('hex');
if (hash !== actualHash) {
throw new Error('Invalid hash');
}
await db('runs').whereIn('id', ids).delete();
};
public started = async (id: string) => {
await this.#isReady;
const { database } = this.#options;
const db = await database.instance;
const current = await this.getById(id);
@@ -92,6 +126,7 @@ class RunRepo extends EventEmitter<RunRepoEvents> {
};
public finished = async (id: string, options: UpdateRunOptions) => {
await this.#isReady;
const { database } = this.#options;
const db = await database.instance;
const { loadId } = await this.getById(id);
@@ -114,6 +149,7 @@ class RunRepo extends EventEmitter<RunRepoEvents> {
};
public create = async (options: CreateRunOptions) => {
await this.#isReady;
const { database, loads } = this.#options;
const id = nanoid();
const db = await database.instance;

View File

@@ -1,3 +1,4 @@
import { z } from 'zod';
import { createRunSchema, findRunsSchema } from '../repos/repos.js';
import { publicProcedure, router } from './router.utils.js';
@@ -17,17 +18,50 @@ const find = publicProcedure.input(findRunsSchema).query(async ({ input, ctx })
return results;
});
const remove = publicProcedure.input(findRunsSchema).mutation(async ({ input, ctx }) => {
const prepareRemove = publicProcedure.input(findRunsSchema).query(async ({ input, ctx }) => {
const { runtime } = ctx;
const { repos } = runtime;
const { runs } = repos;
await runs.remove(input);
return await runs.prepareRemove(input);
});
const remove = publicProcedure
.input(
z.object({
hash: z.string(),
ids: z.array(z.string()),
}),
)
.mutation(async ({ input, ctx }) => {
const { runtime } = ctx;
const { repos } = runtime;
const { runs } = repos;
for (const id of input.ids) {
const instance = runtime.runner.getInstance(id);
if (instance) {
await instance.run?.teardown();
}
}
await runs.remove(input.hash, input.ids);
});
const terminate = publicProcedure.input(z.string()).mutation(async ({ input, ctx }) => {
const { runtime } = ctx;
const { runner } = runtime;
const instance = runner.getInstance(input);
if (!instance || !instance.run) {
return;
}
await instance.run.teardown();
});
const runsRouter = router({
create,
find,
remove,
prepareRemove,
terminate,
});
export { runsRouter };

View File

@@ -1,5 +1,5 @@
import { EventEmitter } from 'eventemitter3';
import { run } from '@morten-olsen/mini-loader-runner';
import { RunInfo, run } from '@morten-olsen/mini-loader-runner';
import { Repos } from '../repos/repos.js';
import { LoggerEvent } from '../../../mini-loader/dist/esm/logger/logger.js';
import { ArtifactCreateEvent } from '../../../mini-loader/dist/esm/artifacts/artifacts.js';
@@ -20,12 +20,17 @@ type RunnerInstanceOptions = {
class RunnerInstance extends EventEmitter<RunnerInstanceEvents> {
#options: RunnerInstanceOptions;
#run?: RunInfo;
constructor(options: RunnerInstanceOptions) {
super();
this.#options = options;
}
public get run() {
return this.#run;
}
#addLog = async (event: LoggerEvent['payload']) => {
const { repos, id, loadId } = this.#options;
const { logs } = repos;
@@ -58,11 +63,13 @@ class RunnerInstance extends EventEmitter<RunnerInstanceEvents> {
const script = await readFile(scriptLocation, 'utf-8');
const allSecrets = await secrets.getAll();
await runs.started(id);
const { promise, emitter } = await run({
const current = await run({
script,
secrets: allSecrets,
input,
});
this.#run = current;
const { promise, emitter } = current;
emitter.on('message', (message) => {
switch (message.type) {
case 'log': {
@@ -84,9 +91,11 @@ class RunnerInstance extends EventEmitter<RunnerInstanceEvents> {
}
await runs.finished(id, { status: 'failed', error: errorMessage });
} finally {
this.#run = undefined;
this.emit('completed', { id });
}
};
}
export type { RunInfo };
export { RunnerInstance };

View File

@@ -36,6 +36,10 @@ class Runner {
this.#instances.set(args.id, instance);
await instance.start();
};
public getInstance = (id: string) => {
return this.#instances.get(id);
};
}
export { Runner };

View File

@@ -3,9 +3,16 @@ import fastify from 'fastify';
import { RootRouter, rootRouter } from '../router/router.js';
import { createContext } from '../router/router.utils.js';
import { Runtime } from '../runtime/runtime.js';
import { gateway } from '../gateway/gateway.js';
const createServer = async (runtime: Runtime) => {
const server = fastify({});
const server = fastify({
maxParamLength: 10000,
bodyLimit: 30 * 1024 * 1024,
logger: {
level: 'warn',
},
});
server.get('/', async () => {
return { hello: 'world' };
});
@@ -33,6 +40,14 @@ const createServer = async (runtime: Runtime) => {
},
} satisfies FastifyTRPCPluginOptions<RootRouter>['trpcOptions'],
});
server.register(gateway, {
runtime,
});
server.addHook('onError', async (request, reply, error) => {
console.error(error);
});
await server.ready();
return server;

47
pnpm-lock.yaml generated
View File

@@ -101,6 +101,10 @@ importers:
packages/configs: {}
packages/examples:
dependencies:
fastify:
specifier: ^4.25.2
version: 4.25.2
devDependencies:
'@morten-olsen/mini-loader':
specifier: workspace:^
@@ -132,6 +136,9 @@ importers:
packages/runner:
dependencies:
'@morten-olsen/mini-loader':
specifier: workspace:^
version: link:../mini-loader
eventemitter3:
specifier: ^5.0.1
version: 5.0.1
@@ -139,9 +146,6 @@ importers:
specifier: ^5.0.4
version: 5.0.4
devDependencies:
'@morten-olsen/mini-loader':
specifier: workspace:^
version: link:../mini-loader
'@morten-olsen/mini-loader-configs':
specifier: workspace:^
version: link:../configs
@@ -154,6 +158,9 @@ importers:
packages/server:
dependencies:
'@fastify/reply-from':
specifier: ^9.7.0
version: 9.7.0
'@trpc/client':
specifier: ^10.45.0
version: 10.45.0(@trpc/server@10.45.0)
@@ -476,6 +483,11 @@ packages:
fast-uri: 2.3.0
dev: false
/@fastify/busboy@2.1.0:
resolution: {integrity: sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==}
engines: {node: '>=14'}
dev: false
/@fastify/deepmerge@1.3.0:
resolution: {integrity: sha512-J8TOSBq3SoZbDhM9+R/u77hP93gz/rajSA+K2kGyijPpORPWUXHUpTaleoj+92As0S9uPRP7Oi8IqMf0u+ro6A==}
dev: false
@@ -490,6 +502,19 @@ packages:
fast-json-stringify: 5.10.0
dev: false
/@fastify/reply-from@9.7.0:
resolution: {integrity: sha512-/F1QBl3FGlTqStjmiuoLRDchVxP967TZh6FZPwQteWhdLsDec8mqSACE+cRzw6qHUj3v9hfdd7JNgmb++fyFhQ==}
dependencies:
'@fastify/error': 3.4.1
end-of-stream: 1.4.4
fast-content-type-parse: 1.1.0
fast-querystring: 1.1.2
fastify-plugin: 4.5.1
pump: 3.0.0
tiny-lru: 11.2.5
undici: 5.28.2
dev: false
/@gar/promisify@1.1.3:
resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==}
requiresBuild: true
@@ -2683,6 +2708,10 @@ packages:
resolution: {integrity: sha512-eel5UKGn369gGEWOqBShmFJWfq/xSJvsgDzgLYC845GneayWvXBf0lJCBn5qTABfewy1ZDPoaR5OZCP+kssfuw==}
dev: false
/fastify-plugin@4.5.1:
resolution: {integrity: sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ==}
dev: false
/fastify@4.25.2:
resolution: {integrity: sha512-SywRouGleDHvRh054onj+lEZnbC1sBCLkR0UY3oyJwjD4BdZJUrxBqfkfCaqn74pVCwBaRHGuL3nEWeHbHzAfw==}
dependencies:
@@ -5151,6 +5180,11 @@ packages:
engines: {node: '>=8'}
dev: false
/tiny-lru@11.2.5:
resolution: {integrity: sha512-JpqM0K33lG6iQGKiigcwuURAKZlq6rHXfrgeL4/I8/REoyJTGU+tEMszvT/oTRVHG2OiylhGDjqPp1jWMlr3bw==}
engines: {node: '>=12'}
dev: false
/tmp@0.0.33:
resolution: {integrity: sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==}
engines: {node: '>=0.6.0'}
@@ -5368,6 +5402,13 @@ packages:
/undici-types@5.26.5:
resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
/undici@5.28.2:
resolution: {integrity: sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==}
engines: {node: '>=14.0'}
dependencies:
'@fastify/busboy': 2.1.0
dev: false
/unique-filename@1.1.1:
resolution: {integrity: sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==}
requiresBuild: true

View File

@@ -1,7 +1,9 @@
import { findWorkspacePackages } from '@pnpm/find-workspace-packages';
import { writeFile } from 'fs/promises';
import { readFile, writeFile } from 'fs/promises';
import { join } from 'path';
const sharedData = JSON.parse(await readFile(join(process.cwd(), 'scripts/shared-data.json')));
const version = process.argv[2];
if (!version) {
throw new Error('Version is required');
@@ -11,6 +13,9 @@ const packages = await findWorkspacePackages(process.cwd());
for (const { manifest, dir } of packages) {
console.log(dir, version);
for (let [key, value] of Object.entries(sharedData || {})) {
manifest[key] = value;
}
manifest.version = version;
await writeFile(join(dir, 'package.json'), JSON.stringify(manifest, null, 2));
}

8
scripts/shared-data.json Normal file
View File

@@ -0,0 +1,8 @@
{
"license": "GPL-3.0",
"homepage": "https://github.com/morten-olsen/mini-loader",
"repository": {
"type": "git",
"url": "https://github.com/morten-olsen/mini-loader"
}
}

View File

@@ -1,15 +1,15 @@
{
"include": [],
"references": [
{
"path": "./packages/mini-loader/tsconfig.json"
},
{
"path": "./packages/runner/tsconfig.json"
},
{
"path": "./packages/server/tsconfig.json"
},
{
"path": "./packages/mini-loader/tsconfig.json"
},
{
"path": "./packages/cli/tsconfig.json"
},