5 Commits

Author SHA1 Message Date
Morten Olsen
d3ad83ddf7 qa: add e2e tests 2024-01-17 08:14:19 +01:00
Morten Olsen
e5064ca905 fix: include runner in dependencies and await context create (#43)
Co-authored-by: Simple Systemer <simplesystemer@users.noreply.github.com>
Co-authored-by: denissb <spawnenius@gmail.com>
2024-01-15 21:54:33 +01:00
Morten Olsen
1c3b993ab2 docs: completed v1 docs (#36) 2024-01-15 15:10:19 +01:00
Morten Olsen
161a098c9f ci: docker fix (#37) 2024-01-15 15:09:51 +01:00
Morten Olsen
a08f9e1c91 chore: docker multi arch (#35) 2024-01-15 13:50:06 +01:00
87 changed files with 1791 additions and 612 deletions

View File

@@ -3,7 +3,7 @@ name: Node.js Package
on:
push:
pull_request:
types: [opened]
types: [opened, synchronize, reopened]
# release:
# types: [created]
@@ -18,6 +18,7 @@ env:
permissions:
contents: read
packages: read
pull-requests: write
jobs:
build:
@@ -34,6 +35,11 @@ jobs:
- run: pnpm install
- run: pnpm run test
- name: 'Report Coverage'
# Set if: always() to also generate the report if tests are failing
# Only works if you set `reportOnFailure: true` in your vite config as specified above
if: always()
uses: davelosert/vitest-coverage-report-action@v2
- run: pnpm run build
- uses: actions/upload-artifact@v3
with:
@@ -71,12 +77,24 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Retrieve version
run: |
echo "TAG_NAME=$(git describe --tag --abbrev=0)" >> $GITHUB_OUTPUT
id: version
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
@@ -84,11 +102,16 @@ jobs:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
latest
${{ steps.version.outputs.TAG_NAME }}
- name: Build and push Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: .
file: ./docker/Dockerfile
platforms: linux/amd64,linux/arm64
cache-from: type=gha
cache-to: type=gha,mode=max
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

2
.gitignore vendored
View File

@@ -2,3 +2,5 @@
.turbo/
/.pnpm-store/
/out/
/coverage/
/e2e-tmp/

View File

@@ -1,10 +1,8 @@
![banner](./assets/banner.png)
# Welcome to Mini Loader! 🌐
Welcome to mini loader, a lightweight, Docker-based server solution for managing and executing workloads with ease. Designed for developers, small teams, and anyone in need of a simple yet powerful tool for running tasks, hosting API servers, or scheduling routine jobs.
Welcome to mini loader, a lightweight server solution for managing and executing workloads with ease. Designed for developers, small teams, and anyone in need of a simple yet powerful tool for running tasks, hosting API servers, or scheduling routine jobs.
## Features
@@ -15,20 +13,37 @@ Welcome to mini loader, a lightweight, Docker-based server solution for managing
- **Task Scheduling**: Built-in support for cron-like job scheduling.
- **HTTP Gateway**: Expose a HTTP server from your workloads
Also see [anti-features and limitations](./docs/anti-features.md)
Also see [anti-features and limitations](./docs/02-anti-features.md)
:construction: This project is under active development and has not reached v1.0 yet. Expect some bugs and potential breaking changes in APIs. We appreciate your patience and welcome your feedback as we work towards a stable release!
For an overview of what's coming next, check out our roadmap at [GitHub Milestones](https://github.com/morten-olsen/mini-loader/milestones).
## Quick Start
Get up and running with mini loader in just a few steps:
1. **Install the CLI**: `npm install -g @morten-olsen/mini-loader-cli`
2. **Deploy the Server**: `docker run -p 4500:4500 —-name mini-loader ghcr.io/morten-olsen/mini-loader`.
3. **Get your access token**: `docker exec mini-loader mini-loader-server create-token`
4. **Login**: `mini-loader auth login http://localhost:4500`
5. **Push Your First Load**: `mini-loader loads push script.mjs -r -i first`
6. **See the logs**: `mini-loader logs ls -l first`
```bash
# Install the CLI and the server
npm i -g @morten-olsen/mini-loader-cli @morten-olsen/mini-loader-server
For a detailed guide on getting started, please refer to the [Getting Started Tutorial](./docs/getting-started.md).
# Start the server
mini-loader-server start &
# Get your access token
mini-loader-server create-token
# Authenticate the CLI
mini-loader auth login
# Push your first workload
mini-loader loads push -r -ai my-script.js -i first
# See the output logs
mini-loader logs ls -l first
```
For a detailed guide on getting started, please refer to the [Getting Started Tutorial](./docs/01-getting-started.md).
## Support and Contributions
@@ -43,4 +58,4 @@ mini loader is open-source software licensed under the [GPL-3 License](./LICENSE
## Let's Get Started!
Dive into the world of simplified workload management with mini loader. Start with our [Getting Started Tutorial](./docs/getting-started.md) and unleash the full potential of your tasks and applications!
Dive into the world of simplified workload management with mini loader. Start with our [Getting Started Tutorial](./docs/01-getting-started.md) and unleash the full potential of your tasks and applications!

View File

@@ -15,20 +15,18 @@ Before diving into mini loader, ensure you have the following:
## Contents
- [Creating you first workload](./installation.md): Learn how to write workloads and execute them locally with the mini loader CLI
- [Running the server](./pushing-managing-loads.md): Instructions on how to run the server locally using docker.
- [Interacting with the server](./interacting-with-server.md): Learn the basic commands used to manage workloads.
- [Managing secrets](./managing-secrets.md): Upload secrets to the server that can be used inside your scripts.
- [Authorization](./setting-up-oidc.md): Extend the authorization using OIDC
- [Create an API](./creating-an-api.md): Create a workload which exposes a HTTP api
- [Creating you first workload](./03-tutorial/01-first-workload.md): Learn how to write workloads and execute them locally with the mini loader CLI
- [Running the server](./03-tutorial/02-setup-server.md): Instructions on how to run the server locally using docker.
- [Interacting with the server](./03-tutorial/03-interacting-with-server.md): Learn the basic commands used to manage workloads.
- [Managing secrets](./03-tutorial/04-managing-secrets.md): Upload secrets to the server that can be used inside your scripts.
- [Create an API](./03-tutorial/05-creating-an-api.md): Create a workload which exposes a HTTP api
## Getting Help
If you encounter any issues or have questions, please refer to the [FAQs](./faqs.md)
If you encounter any issues or have questions, please refer to the [FAQs](./04-faqs.md)
## Let's Get Started!
Ready to streamline your workload management? Let's jump right into [creating your first workload](./first-workload.md) and set up the mini loader CLI!
Ready to streamline your workload management? Let's jump right into [creating your first workload](./03-tutorial/01-first-workload.md) and set up the mini loader CLI!
[Next: create a workload](./first-workload.md)
[Next: create a workload](./03-tutorial/01-first-workload.md)

View File

@@ -46,4 +46,4 @@ After running the command, you should see an output confirming that a new artifa
Congratulations on setting up and running your first script with mini loader! You're now ready to take the next step.
[Next: Setting Up the Server](./setup-server.md)
[Next: Setting Up the Server](./02-setup-server.md)

View File

@@ -1,7 +1,8 @@
Certainly! Here's a revised version of your documentation page to make it
## Quick Start with mini loader using Docker
This guide will help you quickly set up and run a mini loader server using Docker. Follow these simple steps to deploy your server and start interacting with it using the [mini-loader CLI](./first-workload.md).
This guide will help you quickly set up and run a mini loader server using Docker. Follow these simple steps to deploy your server and start interacting with it using the [mini-loader CLI](./01-first-workload.md).
### Step 1: Deploy the mini loader Container
@@ -57,4 +58,4 @@ This command lists all the loads currently on your server, confirming that the C
You've successfully deployed and configured your mini loader server using Docker! You're now ready to start interacting with the server.
[Next: Interacting with the Server](./interacting-with-server.md)
[Next: Interacting with the Server](./03-interacting-with-server.md)

View File

@@ -67,4 +67,4 @@ Replace `<id>` with the identifier of the artifact you wish to download.
You're now equipped to manage loads, runs, logs, and artifacts using the mini loader CLI. For advanced usage, such as managing secrets, proceed to the next section.
[Next: Managing Secrets](./managing-secrets.md)
[Next: Managing Secrets](./04-managing-secrets.md)

View File

@@ -0,0 +1,81 @@
## Managing Secrets
### Introduction
In many workflows, accessing sensitive data such as API tokens or credentials is essential. To handle this securely, you can use secrets management. This tutorial demonstrates how to manage secrets using the CLI and implement them in a simple Node.js workload.
### Creating Secrets with the CLI
To create a new secret, use the `mini-loader` CLI as follows:
```bash
mini-loader secrets set <id>
```
For example, to store a GitHub personal access token, you would use:
```bash
mini-loader secrets set githubtoken
```
Upon execution, you'll be prompted to enter your access token.
### Implementing Secrets in Your Workload
Next, let's create a Node.js script (`github.js`) that uses this token to fetch your GitHub username and saves it as an artifact.
1. **Create `github.js` File:**
```javascript
import { secrets, artifacts } from '@morten-olsen/mini-loader';
import { Octokit } from '@octokit/rest';
// Retrieve the secret
const accessToken = secrets.get('githubtoken');
// Main async function to fetch and save GitHub username
async function run() {
const octokit = new Octokit({ auth: accessToken });
const user = await octokit.users.getAuthenticated();
await artifacts.create('user', JSON.stringify(user.data.login));
}
// Execute the function
run().catch(console.error);
```
This script initializes the Octokit client with the access token, fetches the authenticated user's data, and then saves the username as an artifact.
2. **Run the Script:**
Execute your script with `mini-loader`:
```bash
mini-loader loads push github.js -r -ai
```
### Managing Local Secrets
If you're running the script locally, you can manage secrets either by using a `.secrets` file or setting an environment variable.
1. **Using a `.secrets` File:**
Create a file named `.secrets` and add your token:
```
githubtoken=<your-token>
```
2. **Using Environment Variables:**
Prefix your environment variable with `ML_S_` and run the script:
```bash
ML_S_githubtoken=<your-token> mini-loader local run github.js -ai
```
### Conclusion
By following these steps, you can securely manage and use secrets within your workloads, enhancing the security and integrity of your applications.
[Next: Creating an API](./05-creating-an-api.md)

View File

@@ -0,0 +1,52 @@
## Creating an API Inside Your Workload
Workloads in mini loader can set up simple HTTP servers by connecting to a socket file, a feature supported by many JavaScript server libraries.
### Binding Your Workload to an HTTP Endpoint
To expose your workload as an HTTP server, specify the path parameter using the `getPath()` method provided by the `@morten-olsen/mini-loader` package. This method dynamically assigns a path for your API.
### Important Note
Please be aware that the gateway provided by mini loader isn't fully featured. As such, certain functionalities like streaming and WebSockets may not be supported.
### Example: Setting Up a Server with Fastify
Here's how you can create a simple API server using Fastify in TypeScript:
```typescript
import { http } from '@morten-olsen/mini-loader';
import fastify from 'fastify';
const server = fastify();
// Handling all requests and returning the requested URL
server.all('*', async (req) => {
return req.url;
});
// Listening on the path provided by mini loader
server.listen({
path: http.getPath(),
});
```
With this setup, your server will respond to all incoming requests by returning the requested URL.
### Deploying Your Workload
Now, you can push and run your workload just like any other script:
```bash
mini-loader loads push -r my-script.ts
```
### Accessing Your Server
After pushing your workload, mini loader will display the run ID. You can use this ID to access your server. For example, to make a request to your server, you can use `curl`:
```bash
curl http://localhost:4500/gateway/{your-run-id}
```
Replace `{your-run-id}` with the actual run ID provided by mini loader.

View File

@@ -8,21 +8,25 @@
"build": "turbo build",
"build:dev": "tsc --build --watch",
"test:lint": "eslint ./packages/*/src",
"test": "pnpm run test:lint"
"test:code": "vitest --coverage",
"test": "pnpm run test:lint && pnpm run test:code"
},
"keywords": [],
"author": "",
"devDependencies": {
"@react-native-community/eslint-config": "^3.2.0",
"@vitest/coverage-v8": "^1.2.0",
"eslint": "^8.53.0",
"prettier": "^2.8.8",
"turbo": "^1.10.16",
"typescript": "^5.3.3"
"typescript": "^5.3.3",
"vite": "^5.0.11"
},
"dependencies": {
"@pnpm/find-workspace-packages": "^6.0.9",
"@types/node": "^20.10.8",
"ts-node": "^10.9.2"
"ts-node": "^10.9.2",
"vitest": "^1.2.0"
},
"homepage": "https://github.com/morten-olsen/mini-loader",
"repository": {

View File

@@ -1,4 +1,4 @@
#!/usr/bin/env node
import 'source-map-support/register.js';
import '../dist/esm/src/index.js';
import '../dist/esm/bin.js';

View File

@@ -1,8 +1,8 @@
{
"name": "@morten-olsen/mini-loader-cli",
"version": "1.0.0",
"main": "./dist/esm/src/index.js",
"types": "./dist/esm/src/index.d.ts",
"main": "./dist/esm/index.js",
"types": "./dist/esm/index.d.ts",
"license": "GPL-3.0",
"bin": {
"mini-loader": "./bin/index.mjs"
@@ -14,9 +14,12 @@
"files": [
"./dist"
],
"imports": {
"#pkg": "./package.json"
},
"exports": {
".": {
"import": "./dist/esm/src/index.js"
"import": "./dist/esm/index.js"
}
},
"dependencies": {
@@ -37,7 +40,8 @@
"rollup": "^4.9.4",
"rollup-plugin-node-polyfills": "^0.2.1",
"source-map-support": "^0.5.21",
"superjson": "^2.2.1"
"superjson": "^2.2.1",
"typedi": "^0.10.0"
},
"devDependencies": {
"@morten-olsen/mini-loader-configs": "workspace:^",

View File

@@ -0,0 +1,46 @@
import { ContainerInstance, Service } from 'typedi';
import { Context } from '../context/context.js';
import { Config } from '../config/config.js';
import { Client, createClient } from '../client/client.js';
import type { Runtime } from '@morten-olsen/mini-loader-server';
import { Terminal } from './output.js';
@Service()
class CliApi {
#container: ContainerInstance;
#client?: Client;
constructor(container: ContainerInstance) {
this.#container = container;
}
public get log() {
return this.#container.get(Terminal).log;
}
public get step() {
return this.#container.get(Terminal).step;
}
public get output() {
return this.#container.get(Terminal).output;
}
public get config() {
return this.#container.get(Config);
}
public get client() {
if (!this.#client) {
this.#client = createClient(this.context);
}
return this.#client;
}
public get context() {
return new Context(this.config);
}
}
export type { Runtime };
export { CliApi };

View File

@@ -0,0 +1,27 @@
import ora from 'ora';
import { Service } from 'typedi';
@Service()
class Terminal {
public log = (message: string) => {
console.log(message);
};
public output = (data: unknown) => {
console.table(data);
};
public step = async <T>(message: string, action: () => Promise<T>) => {
const spinner = ora(message).start();
try {
const result = await action();
await spinner.succeed();
return result;
} catch (err) {
await spinner.fail();
throw err;
}
};
}
export { Terminal };

7
packages/cli/src/bin.ts Normal file
View File

@@ -0,0 +1,7 @@
import { ContainerInstance } from 'typedi';
import { createClientCli } from './index.js';
const container = new ContainerInstance('client');
const program = createClientCli(container);
await program.parseAsync();

View File

@@ -2,7 +2,6 @@ import { createTRPCProxyClient, httpBatchLink } from '@trpc/client';
import superjson from 'superjson';
import type { Runtime } from '@morten-olsen/mini-loader-server';
import type { RootRouter } from '@morten-olsen/mini-loader-server';
import pkg from '../../package.json';
import { Context } from '../context/context.js';
const createClient = (context: Context) => {
@@ -15,7 +14,6 @@ const createClient = (context: Context) => {
httpBatchLink({
url: `${context.host}/trpc`,
headers: {
'x-version': pkg.version,
authorization: `Bearer ${context.token}`,
},
}),

View File

@@ -1,8 +1,5 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const list = new Command('list');
@@ -22,11 +19,7 @@ list
.option('-a, --limit <limit>', 'Limit', '1000')
.action(async () => {
const { runId, loadId, offset, limit } = list.opts();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const { step, client } = getApi(list);
const artifacts = await step('Getting artifacts', async () => {
return await client.artifacts.find.query({
runId,

View File

@@ -1,10 +1,7 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { dirname, resolve } from 'path';
import { mkdir, writeFile } from 'fs/promises';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const pull = new Command('pull');
@@ -13,12 +10,8 @@ pull
.argument('<artifact-id>', 'Artifact ID')
.argument('<file>', 'File to save')
.action(async (id, file) => {
const config = new Config();
const context = new Context(config.context);
const { step, client } = getApi(pull);
const target = resolve(file);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const artifact = await step('Getting artifact', async () => {
const result = await client.artifacts.get.query(id);
if (!result) {

View File

@@ -1,9 +1,6 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import inquirer from 'inquirer';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const remove = new Command('remove');
@@ -22,12 +19,8 @@ remove
.option('-o, --offset <offset>', 'Offset')
.option('-a, --limit <limit>', 'Limit', '1000')
.action(async () => {
const { step, client } = getApi(remove);
const { runId, loadId, offset, limit } = remove.opts();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const response = await step('Preparing to delete', async () => {
return await client.artifacts.prepareRemove.query({
runId,

View File

@@ -1,28 +1,37 @@
import { Command } from 'commander';
import inquerer from 'inquirer';
import { Context } from '../../context/context.js';
import { step } from '../../utils/step.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const login = new Command('login');
login.description('Login to your account');
login.option('--host <host>', 'The host of your server');
login.option('--token <token>', 'The token of your account');
login.action(async () => {
const config = new Config();
const context = new Context(config.context);
const { host, token } = await inquerer.prompt([
let { host, token } = login.opts();
const { step, context } = getApi(login);
if (!host) {
const answers = await inquerer.prompt([
{
type: 'input',
name: 'host',
message: 'Enter the host of your server',
default: context.host ?? 'http://localhost:4500',
default: context.host || 'http://localhost:4500',
},
]);
host = answers.host;
}
if (!token) {
const answers = await inquerer.prompt([
{
type: 'password',
name: 'token',
message: 'Enter your token',
},
]);
token = answers.token;
}
const healthResponse = await step('Getting auth status', async () => {
return await fetch(`${host}/health`, {

View File

@@ -1,9 +1,9 @@
import { Command } from 'commander';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const current = new Command('current');
current.action(async () => {
const config = new Config();
const { config } = getApi(current);
console.log(config.context);
});

View File

@@ -1,10 +1,10 @@
import { Command } from 'commander';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const use = new Command('use');
use.argument('<name>').action(async (name) => {
const config = new Config();
const { config } = getApi(use);
await config.setContext(name);
});

View File

@@ -1,8 +1,5 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const list = new Command('list');
@@ -10,15 +7,11 @@ list
.alias('ls')
.description('List loads')
.action(async () => {
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const { output, step, client } = getApi(list);
const loads = await step('Getting data', async () => {
return await client.loads.find.query({});
});
console.table(loads);
await output(loads);
});
export { list };

View File

@@ -1,10 +1,7 @@
import { Command } from 'commander';
import { resolve } from 'path';
import { createClient } from '../../client/client.js';
import { bundle } from '../../bundler/bundler.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const push = new Command('push');
@@ -16,12 +13,8 @@ push
.option('-ai, --auto-install', 'Auto install dependencies', false)
.action(async (script) => {
const opts = push.opts();
const config = new Config();
const context = new Context(config.context);
const { step, log, client } = getApi(push);
const location = resolve(script);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const code = await step('Bundling', async () => {
return await bundle({ entry: location, autoInstall: opts.autoInstall });
});
@@ -32,12 +25,12 @@ push
script: code,
});
});
console.log('created load with id', id);
await log(`created load with id ${id}`);
if (opts.run) {
const runId = await step('Creating run', async () => {
return await client.runs.create.mutate({ loadId: id });
});
console.log('created run with id', runId);
await log(`created run with id ${runId}`);
}
});

View File

@@ -2,9 +2,8 @@ import { Command } from 'commander';
import { resolve } from 'path';
import { run as runLoad } from '@morten-olsen/mini-loader-runner';
import { bundle } from '../../bundler/bundler.js';
import { step } from '../../utils/step.js';
import { readSecrets } from './local.utils.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const run = new Command('run');
@@ -12,8 +11,8 @@ run
.option('-ai, --auto-install', 'Auto install dependencies', false)
.argument('script')
.action(async (script) => {
const { step, config } = getApi(run);
const location = resolve(script);
const config = new Config();
const { autoInstall } = run.opts();
const secrets = await readSecrets();

View File

@@ -6,7 +6,7 @@ import { join } from 'path';
const ENV_PREFIX = 'ML_S_';
const readSecrets = async () => {
let secretLocation = join(process.cwd(), '.secret');
let secretLocation = join(process.cwd(), '.secrets');
let secrets: Record<string, string> = {};

View File

@@ -1,8 +1,5 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const list = new Command('list');
@@ -24,11 +21,7 @@ list
.option('-s, --sort <order>', 'Sort', 'desc')
.action(async () => {
const { runId, loadId, severities, offset, limit, order } = list.opts();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const { step, output, client } = getApi(list);
const logs = await step('Getting logs', async () => {
return await client.logs.find.query({
runId,
@@ -39,7 +32,7 @@ list
order,
});
});
console.table(logs);
output(logs);
});
export { list };

View File

@@ -1,9 +1,6 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import inquirer from 'inquirer';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const remove = new Command('remove');
@@ -24,12 +21,8 @@ remove
.option('-a, --limit <limit>', 'Limit', '1000')
.option('-s, --sort <order>', 'Sort', 'desc')
.action(async () => {
const { step, client } = getApi(remove);
const { runId, loadId, severities, offset, limit, order } = remove.opts();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const response = await step('Preparing to delete', async () => {
return await client.logs.prepareRemove.query({
runId,

View File

@@ -1,8 +1,5 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const create = new Command('create');
@@ -10,11 +7,7 @@ create
.description('Create a new run')
.argument('load-id', 'Load ID')
.action(async (loadId) => {
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const { step, client } = getApi(create);
await step('Creating run', async () => {
await client.runs.create.mutate({ loadId });
});

View File

@@ -1,8 +1,5 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const list = new Command('list');
@@ -11,15 +8,11 @@ list
.description('Find a run')
.argument('[load-id]', 'Load ID')
.action(async (loadId) => {
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const { step, output, client } = getApi(list);
const runs = await step('Getting runs', async () => {
return await client.runs.find.query({ loadId });
});
console.table(runs);
await output(runs);
});
export { list };

View File

@@ -1,9 +1,6 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import inquirer from 'inquirer';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const remove = new Command('remove');
@@ -21,12 +18,8 @@ remove
.option('-o, --offset <offset>', 'Offset')
.option('-a, --limit <limit>', 'Limit', '1000')
.action(async () => {
const { step, client } = getApi(remove);
const { loadId, offset, limit } = remove.opts();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const response = await step('Preparing to delete', async () => {
return await client.runs.prepareRemove.query({
loadId,

View File

@@ -1,8 +1,5 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const terminate = new Command('terminate');
@@ -10,11 +7,7 @@ terminate
.description('Terminate an in progress run')
.argument('run-id', 'Run ID')
.action(async (runId) => {
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const { step, client } = getApi(terminate);
await step('Terminating run', async () => {
await client.runs.terminate.mutate(runId);
});

View File

@@ -1,8 +1,5 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const add = new Command('add');
@@ -12,12 +9,8 @@ add
.argument('<cron>', 'Cron')
.option('-n, --name <name>', 'Name')
.action(async (loadId, cron) => {
const config = new Config();
const context = new Context(config.context);
const { step, client } = getApi(add);
const { name } = add.opts();
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const id = await step('Adding schedule', async () => {
return await client.schedules.add.mutate({
name,

View File

@@ -1,8 +1,5 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const list = new Command('list');
@@ -20,12 +17,8 @@ list
.option('-o, --offset <offset>', 'Offset')
.option('-a, --limit <limit>', 'Limit', '1000')
.action(async () => {
const { step, output, client } = getApi(list);
const { loadIds, offset, limit } = list.opts();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const schedules = await step('Getting schedules', async () => {
return await client.schedules.find.query({
loadIds,
@@ -33,7 +26,7 @@ list
limit: toInt(limit),
});
});
console.table(schedules);
output(schedules);
});
export { list };

View File

@@ -1,9 +1,6 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import inquirer from 'inquirer';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const remove = new Command('remove');
@@ -22,12 +19,8 @@ remove
.option('-o, --offset <offset>', 'Offset')
.option('-a, --limit <limit>', 'Limit', '1000')
.action(async () => {
const { step, client } = getApi(remove);
const { ids, loadIds, offset, limit } = remove.opts();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const response = await step('Preparing to delete', async () => {
return await client.schedules.prepareRemove.query({
ids,

View File

@@ -1,8 +1,5 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const list = new Command('list');
@@ -15,16 +12,12 @@ const toInt = (value?: string) => {
list
.alias('ls')
.description('List logs')
.description('List secrets')
.option('-o, --offset <offset>', 'Offset')
.option('-a, --limit <limit>', 'Limit', '1000')
.action(async () => {
const { step, client } = getApi(list);
const { offset, limit } = list.opts();
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const secrets = await step('Getting secrets', async () => {
return await client.secrets.find.query({
offset: toInt(offset),

View File

@@ -1,8 +1,5 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const remove = new Command('remove');
@@ -10,11 +7,7 @@ remove
.alias('rm')
.argument('<id>')
.action(async (id) => {
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const { step, client } = getApi(remove);
await step('Removing', async () => {
await client.secrets.remove.mutate({
id,

View File

@@ -1,8 +1,5 @@
import { Command } from 'commander';
import { createClient } from '../../client/client.js';
import { step } from '../../utils/step.js';
import { Context } from '../../context/context.js';
import { Config } from '../../config/config.js';
import { getApi } from '../../utils/command.js';
const set = new Command('set');
@@ -10,11 +7,7 @@ set
.argument('<id>')
.argument('[value]')
.action(async (id, value) => {
const config = new Config();
const context = new Context(config.context);
const client = await step('Connecting to server', async () => {
return createClient(context);
});
const { step, client } = getApi(set);
await step('Setting secret', async () => {
await client.secrets.set.mutate({
id,

View File

@@ -1,20 +1,22 @@
import envPaths from 'env-paths';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import { mkdir } from 'fs/promises';
import { existsSync, readFileSync } from 'fs';
import { mkdir, writeFile } from 'fs/promises';
import { join, dirname } from 'path';
import { ContainerInstance, Service } from 'typedi';
import { Paths } from '../paths/paths.js';
type ConfigValues = {
context?: string;
};
const paths = envPaths('mini-loader');
@Service()
class Config {
#paths: Paths;
#location: string;
#config?: ConfigValues;
constructor() {
this.#location = join(paths.config, 'config.json');
constructor(contianer: ContainerInstance) {
this.#paths = contianer.get(Paths);
this.#location = join(this.#paths.config, 'config.json');
if (existsSync(this.#location)) {
this.#config = JSON.parse(readFileSync(this.#location, 'utf-8'));
}
@@ -24,8 +26,12 @@ class Config {
return this.#config?.context || 'default';
}
public get location() {
return this.#paths.config;
}
public get cacheLocation() {
return join(paths.cache, this.context);
return join(this.#paths.cache, this.context);
}
public setContext = (context: string) => {
@@ -42,7 +48,7 @@ class Config {
}
const json = JSON.stringify(this.#config);
mkdir(dirname(this.#location), { recursive: true });
writeFileSync(this.#location, json);
writeFile(this.#location, json);
};
}

View File

@@ -1,7 +1,8 @@
import envPaths from 'env-paths';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import { mkdir, readdir } from 'fs/promises';
import { existsSync, readFileSync } from 'fs';
import { mkdir, readdir, writeFile } from 'fs/promises';
import { dirname, join } from 'path';
import { Config } from '../config/config.js';
type ContextValues = {
host: string;
@@ -12,9 +13,8 @@ class Context {
#location: string;
#config?: ContextValues;
constructor(name: string) {
const paths = envPaths('mini-loader');
this.#location = join(paths.config, 'contexts', name);
constructor(config: Config) {
this.#location = join(config.location, 'contexts', config.context);
if (existsSync(this.#location)) {
this.#config = JSON.parse(readFileSync(this.#location, 'utf-8'));
}
@@ -28,13 +28,13 @@ class Context {
return this.#config?.token;
}
public saveLogin = (host: string, token: string) => {
public saveLogin = async (host: string, token: string) => {
this.#config = {
...(this.#config || {}),
host,
token,
};
this.save();
await this.save();
};
public save = async () => {
@@ -42,8 +42,8 @@ class Context {
return;
}
const json = JSON.stringify(this.#config);
mkdir(dirname(this.#location), { recursive: true });
writeFileSync(this.#location, json);
await mkdir(dirname(this.#location), { recursive: true });
await writeFile(this.#location, json);
};
public static list = async () => {

View File

@@ -1,5 +1,4 @@
import { Command, program } from 'commander';
import pkg from '../package.json';
import { Command } from 'commander';
import { loads } from './commands/loads/loads.js';
import { runs } from './commands/runs/runs.js';
import { logs } from './commands/logs/logs.js';
@@ -9,7 +8,12 @@ import { local } from './commands/local/local.js';
import { auth } from './commands/auth/auth.js';
import { contexts } from './commands/contexts/contexts.js';
import { schedules } from './commands/schedules/schedules.js';
import { ContainerInstance } from 'typedi';
const createClientCli = (container: ContainerInstance) => {
const program = new Command();
program.exitOverride();
program.setOptionValue('_container', container);
program.addCommand(loads);
program.addCommand(runs);
program.addCommand(logs);
@@ -20,12 +24,11 @@ program.addCommand(auth);
program.addCommand(contexts);
program.addCommand(schedules);
program.version(pkg.version);
return program;
};
const version = new Command('version');
version.action(() => {
console.log(pkg.version);
});
program.addCommand(version);
await program.parseAsync();
export { CliApi } from './api/api.js';
export { Context } from './context/context.js';
export { Terminal } from './api/output.js';
export { Paths } from './paths/paths.js';
export { createClientCli };

View File

@@ -0,0 +1,17 @@
import envPaths from 'env-paths';
import { Service } from 'typedi';
const paths = envPaths('mini-loader');
@Service()
class Paths {
public get config() {
return paths.config;
}
public get cache() {
return paths.cache;
}
}
export { Paths };

View File

@@ -0,0 +1,11 @@
import { Command } from 'commander';
import { ContainerInstance } from 'typedi';
import { CliApi } from '../api/api.js';
const getApi = (command: Command) => {
const { _container } = command.optsWithGlobals() as { _container: ContainerInstance };
const api = _container.get(CliApi);
return api;
};
export { getApi };

View File

@@ -1,15 +0,0 @@
import ora from 'ora';
const step = async <T>(message: string, fn: () => Promise<T>): Promise<T> => {
const spinner = ora(message).start();
try {
const result = await fn();
await spinner.succeed();
return result;
} catch (err) {
await spinner.fail();
throw err;
}
};
export { step };

View File

@@ -2,6 +2,7 @@
"extends": "@morten-olsen/mini-loader-configs/tsconfig.esm.json",
"compilerOptions": {
"outDir": "dist/esm",
"rootDir": "src"
},
"include": [
"./src/**/*.ts"

View File

@@ -7,6 +7,7 @@
"declarationMap": true,
"sourceMap": true,
"esModuleInterop": true,
"experimentalDecorators": true,
"strict": true,
"resolveJsonModule": true,
"allowSyntheticDefaultImports": true,

View File

@@ -0,0 +1 @@
demo=foobar

View File

@@ -0,0 +1,3 @@
import { secrets } from '@morten-olsen/mini-loader';
console.log(secrets.get('demo'));

View File

@@ -3,7 +3,6 @@ import { artifacts, logger } from '@morten-olsen/mini-loader';
const run = async () => {
await logger.info('Hello world');
await artifacts.create('foo', 'bar');
process.exit(0);
};
run();

View File

@@ -1,4 +1,5 @@
import { Socket, createConnection } from 'net';
import { Event } from './index.js';
const connect = () =>
new Promise<Socket>((resolve, reject) => {
@@ -12,7 +13,7 @@ const connect = () =>
});
});
const send = async (data: any) =>
const send = async (data: Event) =>
new Promise<void>(async (resolve, reject) => {
const connection = await connect();
const cleaned = JSON.parse(JSON.stringify(data));

View File

@@ -1,4 +1,4 @@
#!/usr/bin/env node
import 'source-map-support/register.js';
import '../dist/esm/src/index.js';
import '../dist/esm/bin.js';

View File

@@ -2,8 +2,8 @@
"name": "@morten-olsen/mini-loader-server",
"version": "1.0.0",
"license": "GPL-3.0",
"main": "./dist/esm/src/index.js",
"types": "./dist/esm/src/index.d.ts",
"main": "./dist/esm/index.js",
"types": "./dist/esm/index.d.ts",
"bin": {
"mini-loader-server": "./bin/index.mjs"
},
@@ -14,20 +14,23 @@
"files": [
"./dist"
],
"imports": {
"#pkg": "./package.json"
},
"exports": {
".": {
"import": "./dist/esm/src/index.js"
"import": "./dist/esm/index.js"
}
},
"devDependencies": {
"@morten-olsen/mini-loader-configs": "workspace:^",
"@morten-olsen/mini-loader-runner": "workspace:^",
"@types/jsonwebtoken": "^9.0.5",
"@types/node": "^20.10.8",
"typescript": "^5.3.3"
},
"dependencies": {
"@fastify/reply-from": "^9.7.0",
"@morten-olsen/mini-loader-runner": "workspace:^",
"@trpc/client": "^10.45.0",
"@trpc/server": "^10.45.0",
"commander": "^11.1.0",
@@ -38,9 +41,11 @@
"jsonwebtoken": "^9.0.2",
"knex": "^3.1.0",
"nanoid": "^5.0.4",
"reflect-metadata": "^0.2.1",
"source-map-support": "^0.5.21",
"sqlite3": "^5.1.7",
"superjson": "^2.2.1",
"typedi": "^0.10.0",
"zod": "^3.22.4"
},
"homepage": "https://github.com/morten-olsen/mini-loader",

View File

@@ -4,17 +4,21 @@ import { existsSync } from 'fs';
import { mkdir, readFile, writeFile } from 'fs/promises';
import jwt from 'jsonwebtoken';
import { Config } from '../config/config.js';
import { ContainerInstance, Service } from 'typedi';
type AuthOptions = {
config: Config;
};
@Service()
class Auth {
#options: AuthOptions;
#data: Promise<{ secret: string }>;
constructor(options: AuthOptions) {
this.#options = options;
constructor(container: ContainerInstance) {
this.#options = {
config: container.get(Config),
};
this.#data = this.#setup();
}

View File

@@ -0,0 +1,14 @@
import { ContainerInstance } from 'typedi';
import { createServerCli } from './index.js';
const program = createServerCli(new ContainerInstance('server'));
program.setOptionValue('output', (data: unknown) => {
console.log('got data', data);
});
await program.parseAsync(process.argv);
process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
});

View File

@@ -1,10 +1,22 @@
import { Knex } from 'knex';
import { Service } from 'typedi';
import envPaths from 'env-paths';
import { join } from 'path';
type Config = {
database: Omit<Knex.Config, 'migrations'>;
files: {
data: string;
cache: string;
const paths = envPaths('mini-loader-server');
@Service()
class Config {
database: Omit<Knex.Config, 'migrations'> = {
client: 'sqlite3',
connection: {
filename: join(paths.data, 'db.sqlite'),
},
useNullAsDefault: true,
};
files = {
data: paths.data,
cache: paths.cache,
};
auth?: {
oidc?: {
@@ -15,6 +27,6 @@ type Config = {
};
};
};
};
}
export type { Config };
export { Config };

View File

@@ -1,18 +1,24 @@
import knex, { Knex } from 'knex';
import { Service, ContainerInstance } from 'typedi';
import { Config } from '../config/config.js';
import { source } from './migrations/migrations.source.js';
import { mkdir } from 'fs/promises';
import { dirname } from 'path';
const tableNames = {
loads: 'loads',
};
@Service()
class Database {
#instance?: Promise<Knex>;
#config: Knex.Config;
constructor(config: Knex.Config) {
constructor(container: ContainerInstance) {
const config = container.get(Config);
this.#config = {
...config,
...config.database,
migrations: {
migrationSource: source,
},
@@ -20,6 +26,15 @@ class Database {
}
#setup = async (config: Knex.Config) => {
if (
config.connection &&
typeof config.connection !== 'string' &&
'filename' in config.connection &&
typeof config.connection.filename === 'string' &&
config.connection.filename !== ':memory:'
) {
await mkdir(dirname(config.connection.filename), { recursive: true });
}
const db = knex(config);
await db.migrate.latest();
return db;

View File

@@ -0,0 +1,9 @@
import { nanoid } from 'nanoid';
import { Service } from 'typedi';
@Service()
class IdGenerator {
public generate = () => nanoid();
}
export { IdGenerator };

View File

@@ -1,13 +1,18 @@
import { program, Command } from 'commander';
import 'reflect-metadata';
import { Command } from 'commander';
import { Runtime } from './runtime/runtime.js';
import { createServer } from './server/server.js';
import { ContainerInstance } from 'typedi';
const createServerCli = (container: ContainerInstance) => {
const program = new Command();
const start = new Command('start');
start.action(async () => {
const port = 4500;
const runtime = await Runtime.create();
const runtime = container.get(Runtime);
await runtime.scheduler.start();
const server = await createServer(runtime);
const server = await createServer(container);
await server.listen({
port,
host: '0.0.0.0',
@@ -18,7 +23,7 @@ start.action(async () => {
const createToken = new Command('create-token');
createToken.action(async () => {
const runtime = await Runtime.create();
const runtime = container.get(Runtime);
const token = await runtime.auth.createToken({
policy: {
'*:*': ['*'],
@@ -29,12 +34,9 @@ createToken.action(async () => {
program.addCommand(start);
program.addCommand(createToken);
return program;
};
await program.parseAsync(process.argv);
process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
});
export type { Runtime } from './runtime/runtime.js';
export { createServerCli, createServer, Runtime };
export { Config } from './config/config.js';
export type { RootRouter } from './router/router.js';

View File

@@ -1,21 +1,27 @@
import { EventEmitter } from 'eventemitter3';
import { Database } from '../../database/database.js';
import { nanoid } from 'nanoid';
import { AddArtifactOptions, FindArtifactsOptions } from './artifacts.schemas.js';
import { createHash } from 'crypto';
import { ContainerInstance, Service } from 'typedi';
import { IdGenerator } from '../../id/id.js';
type ArtifactRepoEvents = {};
type ArtifactRepoOptions = {
database: Database;
idGenerator: IdGenerator;
};
@Service()
class ArtifactRepo extends EventEmitter<ArtifactRepoEvents> {
#options: ArtifactRepoOptions;
constructor(options: ArtifactRepoOptions) {
constructor(container: ContainerInstance) {
super();
this.#options = options;
this.#options = {
database: container.get(Database),
idGenerator: container.get(IdGenerator),
};
}
public get = async (id: string) => {
@@ -26,9 +32,9 @@ class ArtifactRepo extends EventEmitter<ArtifactRepoEvents> {
};
public add = async (options: AddArtifactOptions) => {
const { database } = this.#options;
const { database, idGenerator } = this.#options;
const db = await database.instance;
const id = nanoid();
const id = idGenerator.generate();
await db('artifacts').insert({
id,

View File

@@ -1,11 +1,12 @@
import { EventEmitter } from 'eventemitter3';
import { Database } from '../../database/database.js';
import { FindLoadsOptions, SetLoadOptions } from './loads.schemas.js';
import { nanoid } from 'nanoid';
import { createHash } from 'crypto';
import { Config } from '../../config/config.js';
import { mkdir, writeFile } from 'fs/promises';
import { resolve } from 'path';
import { ContainerInstance, Service } from 'typedi';
import { IdGenerator } from '../../id/id.js';
type LoadRepoEvents = {
created: (id: string) => void;
@@ -16,14 +17,20 @@ type LoadRepoEvents = {
type LoadRepoOptions = {
database: Database;
config: Config;
idGenerator: IdGenerator;
};
@Service()
class LoadRepo extends EventEmitter<LoadRepoEvents> {
#options: LoadRepoOptions;
constructor(options: LoadRepoOptions) {
constructor(container: ContainerInstance) {
super();
this.#options = options;
this.#options = {
database: container.get(Database),
config: container.get(Config),
idGenerator: container.get(IdGenerator),
};
}
public getById = async (id: string) => {
@@ -58,9 +65,9 @@ class LoadRepo extends EventEmitter<LoadRepoEvents> {
};
public set = async (options: SetLoadOptions) => {
const { database } = this.#options;
const { database, idGenerator } = this.#options;
const db = await database.instance;
const id = options.id || nanoid();
const id = options.id || idGenerator.generate();
const script = createHash('sha256').update(options.script).digest('hex');
const scriptDir = resolve(this.#options.config.files.data, 'scripts');
await mkdir(scriptDir, { recursive: true });

View File

@@ -1,27 +1,33 @@
import { EventEmitter } from 'eventemitter3';
import { Database } from '../../database/database.js';
import { AddLogOptions, FindLogsOptions } from './logs.schemas.js';
import { nanoid } from 'nanoid';
import { createHash } from 'crypto';
import { ContainerInstance, Service } from 'typedi';
import { IdGenerator } from '../../id/id.js';
type LogRepoEvents = {};
type LogRepoOptions = {
database: Database;
idGenerator: IdGenerator;
};
@Service()
class LogRepo extends EventEmitter<LogRepoEvents> {
#options: LogRepoOptions;
constructor(options: LogRepoOptions) {
constructor(container: ContainerInstance) {
super();
this.#options = options;
this.#options = {
database: container.get(Database),
idGenerator: container.get(IdGenerator),
};
}
public add = async (options: AddLogOptions) => {
const { database } = this.#options;
const { database, idGenerator } = this.#options;
const db = await database.instance;
const id = nanoid();
const id = idGenerator.generate();
await db('logs').insert({
id,

View File

@@ -1,5 +1,4 @@
import { Config } from '../config/config.js';
import { Database } from '../database/database.js';
import { ContainerInstance, Service } from 'typedi';
import { ArtifactRepo } from './artifacts/artifacts.js';
import { LoadRepo } from './loads/loads.js';
import { LogRepo } from './logs/logs.js';
@@ -7,64 +6,36 @@ import { RunRepo } from './runs/runs.js';
import { ScheduleRepo } from './schedules/schedules.js';
import { SecretRepo } from './secrets/secrets.js';
type ReposOptions = {
database: Database;
config: Config;
};
@Service()
class Repos {
#loads: LoadRepo;
#runs: RunRepo;
#logs: LogRepo;
#artifacts: ArtifactRepo;
#secrets: SecretRepo;
#schedule: ScheduleRepo;
#container: ContainerInstance;
constructor({ database, config }: ReposOptions) {
this.#loads = new LoadRepo({
database,
config,
});
this.#runs = new RunRepo({
database,
loads: this.#loads,
});
this.#logs = new LogRepo({
database,
});
this.#artifacts = new ArtifactRepo({
database,
});
this.#secrets = new SecretRepo({
database,
});
this.#schedule = new ScheduleRepo({
database,
});
constructor(container: ContainerInstance) {
this.#container = container;
}
public get loads() {
return this.#loads;
return this.#container.get(LoadRepo);
}
public get runs() {
return this.#runs;
return this.#container.get(RunRepo);
}
public get logs() {
return this.#logs;
return this.#container.get(LogRepo);
}
public get artifacts() {
return this.#artifacts;
return this.#container.get(ArtifactRepo);
}
public get secrets() {
return this.#secrets;
return this.#container.get(SecretRepo);
}
public get schedules() {
return this.#schedule;
return this.#container.get(ScheduleRepo);
}
}

View File

@@ -1,9 +1,10 @@
import { nanoid } from 'nanoid';
import { EventEmitter } from 'eventemitter3';
import { Database } from '../../database/database.js';
import { CreateRunOptions, FindRunsOptions, UpdateRunOptions } from './runs.schemas.js';
import { LoadRepo } from '../loads/loads.js';
import { createHash } from 'crypto';
import { ContainerInstance, Service } from 'typedi';
import { IdGenerator } from '../../id/id.js';
type RunRepoEvents = {
created: (args: { id: string; loadId: string }) => void;
@@ -15,16 +16,21 @@ type RunRepoEvents = {
type RunRepoOptions = {
database: Database;
loads: LoadRepo;
idGenerator: IdGenerator;
};
@Service()
class RunRepo extends EventEmitter<RunRepoEvents> {
#options: RunRepoOptions;
#isReady: Promise<void>;
#isSetup?: Promise<void>;
constructor(options: RunRepoOptions) {
constructor(container: ContainerInstance) {
super();
this.#options = options;
this.#isReady = this.#setup();
this.#options = {
database: container.get(Database),
loads: container.get(LoadRepo),
idGenerator: container.get(IdGenerator),
};
}
#setup = async () => {
@@ -33,6 +39,13 @@ class RunRepo extends EventEmitter<RunRepoEvents> {
await db('runs').update({ status: 'failed', error: 'server was shut down' }).where({ status: 'running' });
};
get #isReady() {
if (!this.#isSetup) {
this.#isSetup = this.#setup();
}
return this.#isSetup;
}
public getById = async (id: string) => {
await this.#isReady;
const { database } = this.#options;
@@ -150,8 +163,8 @@ class RunRepo extends EventEmitter<RunRepoEvents> {
public create = async (options: CreateRunOptions) => {
await this.#isReady;
const { database, loads } = this.#options;
const id = nanoid();
const { database, loads, idGenerator } = this.#options;
const id = idGenerator.generate();
const db = await database.instance;
const script = await loads.getScript(options.loadId);

View File

@@ -1,8 +1,9 @@
import { EventEmitter } from 'eventemitter3';
import { Database } from '../../database/database.js';
import { nanoid } from 'nanoid';
import { AddScheduleOptions, FindSchedulesOptions } from './schedules.schemas.js';
import { createHash } from 'crypto';
import { ContainerInstance, Service } from 'typedi';
import { IdGenerator } from '../../id/id.js';
type ScheduleRepoEvents = {
added: (id: string) => void;
@@ -11,14 +12,19 @@ type ScheduleRepoEvents = {
type ScheduleRepoOptions = {
database: Database;
idGenerator: IdGenerator;
};
@Service()
class ScheduleRepo extends EventEmitter<ScheduleRepoEvents> {
#options: ScheduleRepoOptions;
constructor(options: ScheduleRepoOptions) {
constructor(container: ContainerInstance) {
super();
this.#options = options;
this.#options = {
database: container.get(Database),
idGenerator: container.get(IdGenerator),
};
}
public get = async (id: string) => {
@@ -29,9 +35,9 @@ class ScheduleRepo extends EventEmitter<ScheduleRepoEvents> {
};
public add = async (options: AddScheduleOptions) => {
const { database } = this.#options;
const { database, idGenerator } = this.#options;
const db = await database.instance;
const id = nanoid();
const id = idGenerator.generate();
await db('schedules').insert({
id,

View File

@@ -1,6 +1,7 @@
import { EventEmitter } from 'eventemitter3';
import { Database } from '../../database/database.js';
import { FindSecretOptions, SetSecretOptions } from './secrets.schemas.js';
import { ContainerInstance, Service } from 'typedi';
type LogRepoEvents = {};
@@ -8,12 +9,15 @@ type LogRepoOptions = {
database: Database;
};
@Service()
class SecretRepo extends EventEmitter<LogRepoEvents> {
#options: LogRepoOptions;
constructor(options: LogRepoOptions) {
constructor(container: ContainerInstance) {
super();
this.#options = options;
this.#options = {
database: container.get(Database),
};
}
public set = async (options: SetSecretOptions) => {

View File

@@ -2,13 +2,15 @@ import { initTRPC } from '@trpc/server';
import { CreateFastifyContextOptions } from '@trpc/server/adapters/fastify';
import superjson from 'superjson';
import { Runtime } from '../runtime/runtime.js';
import { ContainerInstance } from 'typedi';
type ContextOptions = {
runtime: Runtime;
container: ContainerInstance;
};
const createContext = async ({ runtime }: ContextOptions) => {
const createContext = async ({ container }: ContextOptions) => {
return async ({ req }: CreateFastifyContextOptions) => {
const runtime = container.get(Runtime);
const { authorization } = req.headers;
const { auth } = runtime;
if (!authorization) {
@@ -18,6 +20,7 @@ const createContext = async ({ runtime }: ContextOptions) => {
await auth.validateToken(token);
return {
runtime,
container,
};
};
};

View File

@@ -1,3 +1,4 @@
import { ContainerInstance, Service } from 'typedi';
import { Config } from '../config/config.js';
import { Repos } from '../repos/repos.js';
import { RunnerInstance } from './runner.instance.js';
@@ -7,13 +8,17 @@ type RunnerOptions = {
config: Config;
};
@Service()
class Runner {
#options: RunnerOptions;
#instances: Map<string, RunnerInstance> = new Map();
constructor(options: RunnerOptions) {
this.#options = options;
const { repos } = options;
constructor(container: ContainerInstance) {
this.#options = {
repos: container.get(Repos),
config: container.get(Config),
};
const { repos } = this.#options;
repos.runs.on('created', this.#start);
}

View File

@@ -1,26 +1,21 @@
import { resolve } from 'path';
import envPaths from 'env-paths';
import { Database } from '../database/database.js';
import { ContainerInstance, Service } from 'typedi';
import { Repos } from '../repos/repos.js';
import { Runner } from '../runner/runner.js';
import { Config } from '../config/config.js';
import { Auth } from '../auth/auth.js';
import { Scheduler } from '../scheduler/scheduler.js';
const paths = envPaths('mini-loader-server');
@Service()
class Runtime {
#repos: Repos;
#runner: Runner;
#auth: Auth;
#scheduler: Scheduler;
constructor(options: Config) {
const database = new Database(options.database);
this.#repos = new Repos({ database, config: options });
this.#runner = new Runner({ repos: this.#repos, config: options });
this.#auth = new Auth({ config: options });
this.#scheduler = new Scheduler({ runs: this.#repos.runs, schedules: this.#repos.schedules });
constructor(container: ContainerInstance) {
this.#repos = container.get(Repos);
this.#runner = container.get(Runner);
this.#auth = container.get(Auth);
this.#scheduler = container.get(Scheduler);
}
public get repos() {
@@ -38,24 +33,6 @@ class Runtime {
public get scheduler() {
return this.#scheduler;
}
public static create = async () => {
const runtime = new Runtime({
database: {
client: 'sqlite3',
connection: {
filename: resolve(paths.data, 'database.sqlite'),
},
useNullAsDefault: true,
},
files: {
data: process.env.DATA_DIR || resolve(paths.data, 'data', 'files'),
cache: process.env.CACHE_DIR || resolve(paths.cache, 'data', 'cache'),
},
});
return runtime;
};
}
export { Runtime };

View File

@@ -1,6 +1,7 @@
import { CronJob } from 'cron';
import { ScheduleRepo } from '../repos/schedules/schedules.js';
import { RunRepo } from '../repos/runs/runs.js';
import { ContainerInstance, Service } from 'typedi';
type SchedulerOptions = {
runs: RunRepo;
@@ -13,12 +14,16 @@ type RunningSchedule = {
stop: () => Promise<void>;
};
@Service()
class Scheduler {
#running: RunningSchedule[] = [];
#options: SchedulerOptions;
constructor(options: SchedulerOptions) {
this.#options = options;
constructor(container: ContainerInstance) {
this.#options = {
runs: container.get(RunRepo),
schedules: container.get(ScheduleRepo),
};
const { schedules } = this.#options;
schedules.on('added', this.#add);
schedules.on('removed', this.#remove);

View File

@@ -1,12 +1,13 @@
import pkg from '../../package.json';
import { fastifyTRPCPlugin, FastifyTRPCPluginOptions } from '@trpc/server/adapters/fastify';
import fastify from 'fastify';
import { RootRouter, rootRouter } from '../router/router.js';
import { createContext } from '../router/router.utils.js';
import { Runtime } from '../runtime/runtime.js';
import { gateway } from '../gateway/gateway.js';
import { ContainerInstance } from 'typedi';
import { Runtime } from '../runtime/runtime.js';
const createServer = async (runtime: Runtime) => {
const createServer = async (container: ContainerInstance) => {
const runtime = container.get(Runtime);
const server = fastify({
maxParamLength: 10000,
bodyLimit: 30 * 1024 * 1024,
@@ -25,14 +26,14 @@ const createServer = async (runtime: Runtime) => {
authorized = true;
}
} catch (error) {}
return { authorized, status: 'ok', version: pkg.version };
return { authorized, status: 'ok' };
});
server.register(fastifyTRPCPlugin, {
prefix: '/trpc',
trpcOptions: {
router: rootRouter,
createContext: await createContext({ runtime }),
createContext: await createContext({ container }),
onError({ error }) {
console.error(error);
},

View File

@@ -2,6 +2,7 @@
"extends": "@morten-olsen/mini-loader-configs/tsconfig.esm.json",
"compilerOptions": {
"outDir": "dist/esm",
"rootDir": "src"
},
"include": [
"src"

4
packages/tests/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
/dist/
/node_modules/
/coverage/
/e2e-tmp/

1
packages/tests/README.md Normal file
View File

@@ -0,0 +1 @@
[Go to documentation](https://github.com/morten-olsen/mini-loader)

View File

@@ -0,0 +1 @@
console.log('simple');

View File

@@ -0,0 +1,40 @@
{
"name": "@morten-olsen/mini-loader-tests",
"private": "true",
"version": "1.0.0",
"license": "GPL-3.0",
"main": "./dist/esm/index.js",
"types": "./dist/esm/index.d.ts",
"type": "module",
"files": [
"./dist"
],
"exports": {
".": {
"import": "./dist/esm/index.js"
}
},
"devDependencies": {
"@morten-olsen/mini-loader-configs": "workspace:^",
"@types/node": "^20.10.8",
"fastify": "^4.25.2",
"sqlite3": "^5.1.7",
"typescript": "^5.3.3",
"vite": "^5.0.11"
},
"dependencies": {
"@morten-olsen/mini-loader": "workspace:^",
"@morten-olsen/mini-loader-cli": "workspace:^",
"@morten-olsen/mini-loader-server": "workspace:^",
"get-port": "^7.0.0",
"memfs": "^4.6.0",
"nanoid": "^5.0.4",
"typedi": "^0.10.0",
"vitest": "^1.2.0"
},
"homepage": "https://github.com/morten-olsen/mini-loader",
"repository": {
"type": "git",
"url": "https://github.com/morten-olsen/mini-loader"
}
}

View File

@@ -0,0 +1,56 @@
import { describe, it, beforeAll, afterAll, vi } from 'vitest';
import { TestEnv, createEnv } from './utils/env.js';
import { writeFile } from 'fs/promises';
import { resolve } from 'path';
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
describe('Quick start', () => {
let env: TestEnv;
beforeAll(async () => {
vi.mock('nanoid', () => {
let currentId = 0;
return {
nanoid: () => `id-${currentId++}`,
};
});
env = await createEnv();
});
afterAll(async () => {
vi.clearAllMocks();
await env.cleanup();
});
it('should be able to start the server', async () => {
const { server, port } = env;
await server.listen({ port });
});
it('should be able to login', async () => {
const { clientRun, port, token } = env;
await clientRun(['auth', 'login', '--host', `http://localhost:${port}`, '--token', token]);
});
it('should be able to push a load', async () => {
const { clientRun, location } = env;
const scriptPath = resolve(location, 'simple.js');
await writeFile(scriptPath, 'console.log("hello world")', 'utf-8');
await clientRun(['loads', 'push', scriptPath, '-i', 'demo', '-r']);
await sleep(100);
});
it('should be able to get the run', async () => {
const { clientRun } = env;
await clientRun(['runs', 'ls', 'demo']);
});
it('should be able to get logs', async () => {
const { clientRun, clientOutput } = env;
await clientRun(['logs', 'ls', '-l', 'demo']);
console.log(JSON.stringify(clientOutput, null, 2));
});
});

View File

@@ -0,0 +1,136 @@
import { Paths, Terminal, createClientCli } from '@morten-olsen/mini-loader-cli';
import { Runtime, createServer, Config } from '@morten-olsen/mini-loader-server';
import { FastifyInstance } from 'fastify';
import { ContainerInstance } from 'typedi';
import { join, resolve } from 'path';
import getPort from 'get-port';
import { rm } from 'fs/promises';
const cacheLocation = './e2e-tmp';
const getUniqueIdentifier = () => {
return Math.random().toString(36).substr(2, 9);
};
type CliActionOutput =
| {
type: 'stdout';
data: string;
}
| {
type: 'stderr';
data: string;
}
| {
type: 'step';
message: string;
state: 'pending' | 'success' | 'error';
}
| {
type: 'output';
data: unknown;
};
const createEnv = async () => {
const location = resolve(cacheLocation, getUniqueIdentifier());
const port = await getPort();
let clientOutput: CliActionOutput[] = [];
const serverContainer = new ContainerInstance(getUniqueIdentifier());
const clientRun = async (params: string[]) => {
const clientContainer = new ContainerInstance(getUniqueIdentifier());
clientContainer.set(Paths, {
config: join(location, 'client', 'config'),
cache: join(location, 'client', 'cache'),
});
clientContainer.set(Terminal, {
log: async (message: any) => {
clientOutput.push({
type: 'stdout',
data: message,
});
},
output: async (data: any) => {
clientOutput.push({
type: 'output',
data,
});
},
step: async (message: any, action: any) => {
clientOutput.push({
type: 'step',
message,
state: 'pending',
});
try {
const result = await action();
clientOutput.push({
type: 'step',
message,
state: 'success',
});
return result;
} catch (err) {
clientOutput.push({
type: 'step',
message,
state: 'error',
});
throw err;
}
},
});
const client = createClientCli(clientContainer);
client.configureOutput({
writeOut: (data) => {
clientOutput.push({
type: 'stdout',
data,
});
},
writeErr: (data) => {
clientOutput.push({
type: 'stderr',
data,
});
throw new Error(`Error in client ${data}`);
},
});
await client.parseAsync(['', '', ...params]);
};
serverContainer.set(Config, {
database: {
client: 'sqlite3',
connection: {
filename: join(location, 'server', 'data', 'db.sqlite'),
},
useNullAsDefault: true,
},
files: {
data: join(location, 'server', 'data'),
cache: join(location, 'server', 'cache'),
},
});
const server = await createServer(serverContainer);
const runtime = serverContainer.get(Runtime);
const token = await runtime.auth.createToken({
policy: {
'*:*': ['*'],
},
});
const cleanup = async () => {
await server.close();
await rm(location, { recursive: true, force: true });
};
return { clientRun, clientOutput, runtime, server, token, location, port, cleanup };
};
type TestEnv = Awaited<ReturnType<typeof createEnv>>;
export type { TestEnv, FastifyInstance };
export { createEnv };

View File

@@ -0,0 +1,9 @@
{
"extends": "@morten-olsen/mini-loader-configs/tsconfig.esm.json",
"compilerOptions": {
"outDir": "dist/esm",
},
"include": [
"src"
],
}

View File

@@ -0,0 +1,22 @@
/// <reference types="vitest" />
import { createRequire } from 'module';
import { defineConfig } from 'vite';
const require = createRequire(import.meta.url);
const server = require.resolve('../server/src/index.ts');
const cli = require.resolve('../cli/src/index.ts');
const runner = require.resolve('../runner/src/index.ts');
const sdk = require.resolve('../mini-loader/src/index.ts');
export default defineConfig({
test: {
alias: {
'@morten-olsen/mini-loader-server': server,
'@morten-olsen/mini-loader-cli': cli,
'@morten-olsen/mini-loader-runner': runner,
'@morten-olsen/mini-loader': sdk,
},
},
});

1004
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

14
vitest.config.ts Normal file
View File

@@ -0,0 +1,14 @@
/// <reference types="vitest" />
import { defineConfig } from 'vite';
export default defineConfig({
test: {
coverage: {
reporter: ['text', 'json-summary', 'json', 'html'],
reportOnFailure: true,
include: ['packages/*/src/**/*.ts'],
exclude: ['packages/examples/**/*', 'packages/tests/**/*'],
},
},
});

1
vitest.workspace.ts Normal file
View File

@@ -0,0 +1 @@
export default ['packages/*'];