From 1c3b993ab229423b624a052d9353410a32ea9f18 Mon Sep 17 00:00:00 2001 From: Morten Olsen Date: Mon, 15 Jan 2024 15:10:19 +0100 Subject: [PATCH] docs: completed v1 docs (#36) --- README.md | 39 ++++++--- ...tting-started.md => 01-getting-started.md} | 18 ++--- .../{anti-features.md => 02-anti-features.md} | 0 .../01-first-workload.md} | 2 +- .../02-setup-server.md} | 7 +- .../03-interacting-with-server.md} | 2 +- docs/03-tutorial/04-managing-secrets.md | 81 +++++++++++++++++++ docs/03-tutorial/05-creating-an-api.md | 52 ++++++++++++ docs/{faq.md => 04-faq.md} | 0 docs/creating-an-api.md | 0 docs/managing-secrets.md | 0 docs/setting-up-oidc.md | 0 packages/cli/bin/index.mjs | 2 +- packages/cli/package.json | 9 ++- packages/cli/src/client/client.ts | 6 +- .../cli/src/commands/local/local.utils.ts | 2 +- packages/cli/src/index.ts | 7 +- packages/cli/tsconfig.json | 1 + packages/examples/.secrets | 1 + packages/examples/src/secrets.ts | 3 + packages/examples/src/simple.ts | 1 - packages/mini-loader/src/utils.ts | 3 +- packages/server/bin/index.mjs | 2 +- packages/server/package.json | 9 ++- packages/server/src/database/database.ts | 11 +++ packages/server/src/server/server.ts | 8 +- packages/server/tsconfig.json | 1 + 27 files changed, 226 insertions(+), 41 deletions(-) rename docs/{getting-started.md => 01-getting-started.md} (52%) rename docs/{anti-features.md => 02-anti-features.md} (100%) rename docs/{first-workload.md => 03-tutorial/01-first-workload.md} (97%) rename docs/{setup-server.md => 03-tutorial/02-setup-server.md} (91%) rename docs/{interacting-with-server.md => 03-tutorial/03-interacting-with-server.md} (96%) create mode 100644 docs/03-tutorial/04-managing-secrets.md create mode 100644 docs/03-tutorial/05-creating-an-api.md rename docs/{faq.md => 04-faq.md} (100%) delete mode 100644 docs/creating-an-api.md delete mode 100644 docs/managing-secrets.md delete mode 100644 docs/setting-up-oidc.md create mode 100644 packages/examples/.secrets create mode 100644 packages/examples/src/secrets.ts diff --git a/README.md b/README.md index 94b41ee..2962be4 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,8 @@ - - ![banner](./assets/banner.png) # Welcome to Mini Loader! 🌐 -Welcome to mini loader, a lightweight, Docker-based server solution for managing and executing workloads with ease. Designed for developers, small teams, and anyone in need of a simple yet powerful tool for running tasks, hosting API servers, or scheduling routine jobs. +Welcome to mini loader, a lightweight server solution for managing and executing workloads with ease. Designed for developers, small teams, and anyone in need of a simple yet powerful tool for running tasks, hosting API servers, or scheduling routine jobs. ## Features @@ -15,20 +13,37 @@ Welcome to mini loader, a lightweight, Docker-based server solution for managing - **Task Scheduling**: Built-in support for cron-like job scheduling. - **HTTP Gateway**: Expose a HTTP server from your workloads -Also see [anti-features and limitations](./docs/anti-features.md) +Also see [anti-features and limitations](./docs/02-anti-features.md) + +:construction: This project is under active development and has not reached v1.0 yet. Expect some bugs and potential breaking changes in APIs. We appreciate your patience and welcome your feedback as we work towards a stable release! + +For an overview of what's coming next, check out our roadmap at [GitHub Milestones](https://github.com/morten-olsen/mini-loader/milestones). ## Quick Start Get up and running with mini loader in just a few steps: -1. **Install the CLI**: `npm install -g @morten-olsen/mini-loader-cli` -2. **Deploy the Server**: `docker run -p 4500:4500 —-name mini-loader ghcr.io/morten-olsen/mini-loader`. -3. **Get your access token**: `docker exec mini-loader mini-loader-server create-token` -4. **Login**: `mini-loader auth login http://localhost:4500` -5. **Push Your First Load**: `mini-loader loads push script.mjs -r -i first` -6. **See the logs**: `mini-loader logs ls -l first` +```bash +# Install the CLI and the server +npm i -g @morten-olsen/mini-loader-cli @morten-olsen/mini-loader-server -For a detailed guide on getting started, please refer to the [Getting Started Tutorial](./docs/getting-started.md). +# Start the server +mini-loader-server start & + +# Get your access token +mini-loader-server create-token + +# Authenticate the CLI +mini-loader auth login + +# Push your first workload +mini-loader loads push -r -ai my-script.js -i first + +# See the output logs +mini-loader logs ls -l first +``` + +For a detailed guide on getting started, please refer to the [Getting Started Tutorial](./docs/01-getting-started.md). ## Support and Contributions @@ -43,4 +58,4 @@ mini loader is open-source software licensed under the [GPL-3 License](./LICENSE ## Let's Get Started! -Dive into the world of simplified workload management with mini loader. Start with our [Getting Started Tutorial](./docs/getting-started.md) and unleash the full potential of your tasks and applications! +Dive into the world of simplified workload management with mini loader. Start with our [Getting Started Tutorial](./docs/01-getting-started.md) and unleash the full potential of your tasks and applications! diff --git a/docs/getting-started.md b/docs/01-getting-started.md similarity index 52% rename from docs/getting-started.md rename to docs/01-getting-started.md index 4c8a4a8..58ca15f 100644 --- a/docs/getting-started.md +++ b/docs/01-getting-started.md @@ -15,20 +15,18 @@ Before diving into mini loader, ensure you have the following: ## Contents -- [Creating you first workload](./installation.md): Learn how to write workloads and execute them locally with the mini loader CLI -- [Running the server](./pushing-managing-loads.md): Instructions on how to run the server locally using docker. -- [Interacting with the server](./interacting-with-server.md): Learn the basic commands used to manage workloads. -- [Managing secrets](./managing-secrets.md): Upload secrets to the server that can be used inside your scripts. -- [Authorization](./setting-up-oidc.md): Extend the authorization using OIDC -- [Create an API](./creating-an-api.md): Create a workload which exposes a HTTP api +- [Creating you first workload](./03-tutorial/01-first-workload.md): Learn how to write workloads and execute them locally with the mini loader CLI +- [Running the server](./03-tutorial/02-setup-server.md): Instructions on how to run the server locally using docker. +- [Interacting with the server](./03-tutorial/03-interacting-with-server.md): Learn the basic commands used to manage workloads. +- [Managing secrets](./03-tutorial/04-managing-secrets.md): Upload secrets to the server that can be used inside your scripts. +- [Create an API](./03-tutorial/05-creating-an-api.md): Create a workload which exposes a HTTP api ## Getting Help -If you encounter any issues or have questions, please refer to the [FAQs](./faqs.md) +If you encounter any issues or have questions, please refer to the [FAQs](./04-faqs.md) ## Let's Get Started! -Ready to streamline your workload management? Let's jump right into [creating your first workload](./first-workload.md) and set up the mini loader CLI! +Ready to streamline your workload management? Let's jump right into [creating your first workload](./03-tutorial/01-first-workload.md) and set up the mini loader CLI! - -[Next: create a workload](./first-workload.md) \ No newline at end of file +[Next: create a workload](./03-tutorial/01-first-workload.md) diff --git a/docs/anti-features.md b/docs/02-anti-features.md similarity index 100% rename from docs/anti-features.md rename to docs/02-anti-features.md diff --git a/docs/first-workload.md b/docs/03-tutorial/01-first-workload.md similarity index 97% rename from docs/first-workload.md rename to docs/03-tutorial/01-first-workload.md index add5ba4..a3e880a 100644 --- a/docs/first-workload.md +++ b/docs/03-tutorial/01-first-workload.md @@ -46,4 +46,4 @@ After running the command, you should see an output confirming that a new artifa Congratulations on setting up and running your first script with mini loader! You're now ready to take the next step. -[Next: Setting Up the Server](./setup-server.md) +[Next: Setting Up the Server](./02-setup-server.md) diff --git a/docs/setup-server.md b/docs/03-tutorial/02-setup-server.md similarity index 91% rename from docs/setup-server.md rename to docs/03-tutorial/02-setup-server.md index d8f062b..cc7498e 100644 --- a/docs/setup-server.md +++ b/docs/03-tutorial/02-setup-server.md @@ -1,7 +1,8 @@ -Certainly! Here's a revised version of your documentation page to make it +Certainly! Here's a revised version of your documentation page to make it + ## Quick Start with mini loader using Docker -This guide will help you quickly set up and run a mini loader server using Docker. Follow these simple steps to deploy your server and start interacting with it using the [mini-loader CLI](./first-workload.md). +This guide will help you quickly set up and run a mini loader server using Docker. Follow these simple steps to deploy your server and start interacting with it using the [mini-loader CLI](./01-first-workload.md). ### Step 1: Deploy the mini loader Container @@ -57,4 +58,4 @@ This command lists all the loads currently on your server, confirming that the C You've successfully deployed and configured your mini loader server using Docker! You're now ready to start interacting with the server. -[Next: Interacting with the Server](./interacting-with-server.md) \ No newline at end of file +[Next: Interacting with the Server](./03-interacting-with-server.md) diff --git a/docs/interacting-with-server.md b/docs/03-tutorial/03-interacting-with-server.md similarity index 96% rename from docs/interacting-with-server.md rename to docs/03-tutorial/03-interacting-with-server.md index 9ec55fc..4286fb8 100644 --- a/docs/interacting-with-server.md +++ b/docs/03-tutorial/03-interacting-with-server.md @@ -67,4 +67,4 @@ Replace `` with the identifier of the artifact you wish to download. You're now equipped to manage loads, runs, logs, and artifacts using the mini loader CLI. For advanced usage, such as managing secrets, proceed to the next section. -[Next: Managing Secrets](./managing-secrets.md) +[Next: Managing Secrets](./04-managing-secrets.md) diff --git a/docs/03-tutorial/04-managing-secrets.md b/docs/03-tutorial/04-managing-secrets.md new file mode 100644 index 0000000..a2c7c2e --- /dev/null +++ b/docs/03-tutorial/04-managing-secrets.md @@ -0,0 +1,81 @@ +## Managing Secrets + +### Introduction + +In many workflows, accessing sensitive data such as API tokens or credentials is essential. To handle this securely, you can use secrets management. This tutorial demonstrates how to manage secrets using the CLI and implement them in a simple Node.js workload. + +### Creating Secrets with the CLI + +To create a new secret, use the `mini-loader` CLI as follows: + +```bash +mini-loader secrets set +``` + +For example, to store a GitHub personal access token, you would use: + +```bash +mini-loader secrets set githubtoken +``` + +Upon execution, you'll be prompted to enter your access token. + +### Implementing Secrets in Your Workload + +Next, let's create a Node.js script (`github.js`) that uses this token to fetch your GitHub username and saves it as an artifact. + +1. **Create `github.js` File:** + + ```javascript + import { secrets, artifacts } from '@morten-olsen/mini-loader'; + import { Octokit } from '@octokit/rest'; + + // Retrieve the secret + const accessToken = secrets.get('githubtoken'); + + // Main async function to fetch and save GitHub username + async function run() { + const octokit = new Octokit({ auth: accessToken }); + const user = await octokit.users.getAuthenticated(); + await artifacts.create('user', JSON.stringify(user.data.login)); + } + + // Execute the function + run().catch(console.error); + ``` + + This script initializes the Octokit client with the access token, fetches the authenticated user's data, and then saves the username as an artifact. + +2. **Run the Script:** + + Execute your script with `mini-loader`: + + ```bash + mini-loader loads push github.js -r -ai + ``` + +### Managing Local Secrets + +If you're running the script locally, you can manage secrets either by using a `.secrets` file or setting an environment variable. + +1. **Using a `.secrets` File:** + + Create a file named `.secrets` and add your token: + + ``` + githubtoken= + ``` + +2. **Using Environment Variables:** + + Prefix your environment variable with `ML_S_` and run the script: + + ```bash + ML_S_githubtoken= mini-loader local run github.js -ai + ``` + +### Conclusion + +By following these steps, you can securely manage and use secrets within your workloads, enhancing the security and integrity of your applications. + +[Next: Creating an API](./05-creating-an-api.md) diff --git a/docs/03-tutorial/05-creating-an-api.md b/docs/03-tutorial/05-creating-an-api.md new file mode 100644 index 0000000..cdb7aa9 --- /dev/null +++ b/docs/03-tutorial/05-creating-an-api.md @@ -0,0 +1,52 @@ +## Creating an API Inside Your Workload + +Workloads in mini loader can set up simple HTTP servers by connecting to a socket file, a feature supported by many JavaScript server libraries. + +### Binding Your Workload to an HTTP Endpoint + +To expose your workload as an HTTP server, specify the path parameter using the `getPath()` method provided by the `@morten-olsen/mini-loader` package. This method dynamically assigns a path for your API. + +### Important Note + +Please be aware that the gateway provided by mini loader isn't fully featured. As such, certain functionalities like streaming and WebSockets may not be supported. + +### Example: Setting Up a Server with Fastify + +Here's how you can create a simple API server using Fastify in TypeScript: + +```typescript +import { http } from '@morten-olsen/mini-loader'; +import fastify from 'fastify'; + +const server = fastify(); + +// Handling all requests and returning the requested URL +server.all('*', async (req) => { + return req.url; +}); + +// Listening on the path provided by mini loader +server.listen({ + path: http.getPath(), +}); +``` + +With this setup, your server will respond to all incoming requests by returning the requested URL. + +### Deploying Your Workload + +Now, you can push and run your workload just like any other script: + +```bash +mini-loader loads push -r my-script.ts +``` + +### Accessing Your Server + +After pushing your workload, mini loader will display the run ID. You can use this ID to access your server. For example, to make a request to your server, you can use `curl`: + +```bash +curl http://localhost:4500/gateway/{your-run-id} +``` + +Replace `{your-run-id}` with the actual run ID provided by mini loader. diff --git a/docs/faq.md b/docs/04-faq.md similarity index 100% rename from docs/faq.md rename to docs/04-faq.md diff --git a/docs/creating-an-api.md b/docs/creating-an-api.md deleted file mode 100644 index e69de29..0000000 diff --git a/docs/managing-secrets.md b/docs/managing-secrets.md deleted file mode 100644 index e69de29..0000000 diff --git a/docs/setting-up-oidc.md b/docs/setting-up-oidc.md deleted file mode 100644 index e69de29..0000000 diff --git a/packages/cli/bin/index.mjs b/packages/cli/bin/index.mjs index 4ec2560..efb0ee3 100755 --- a/packages/cli/bin/index.mjs +++ b/packages/cli/bin/index.mjs @@ -1,4 +1,4 @@ #!/usr/bin/env node import 'source-map-support/register.js'; -import '../dist/esm/src/index.js'; +import '../dist/esm/index.js'; diff --git a/packages/cli/package.json b/packages/cli/package.json index 633cd4e..cda0c1e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,8 +1,8 @@ { "name": "@morten-olsen/mini-loader-cli", "version": "1.0.0", - "main": "./dist/esm/src/index.js", - "types": "./dist/esm/src/index.d.ts", + "main": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", "license": "GPL-3.0", "bin": { "mini-loader": "./bin/index.mjs" @@ -14,9 +14,12 @@ "files": [ "./dist" ], + "imports": { + "#pkg": "./package.json" + }, "exports": { ".": { - "import": "./dist/esm/src/index.js" + "import": "./dist/esm/index.js" } }, "dependencies": { diff --git a/packages/cli/src/client/client.ts b/packages/cli/src/client/client.ts index 32a7891..91a3af1 100644 --- a/packages/cli/src/client/client.ts +++ b/packages/cli/src/client/client.ts @@ -1,10 +1,14 @@ import { createTRPCProxyClient, httpBatchLink } from '@trpc/client'; import superjson from 'superjson'; +import { createRequire } from 'module'; import type { Runtime } from '@morten-olsen/mini-loader-server'; import type { RootRouter } from '@morten-olsen/mini-loader-server'; -import pkg from '../../package.json'; import { Context } from '../context/context.js'; +import { readFile } from 'fs/promises'; +const require = createRequire(import.meta.url); + +const pkg = JSON.parse(await readFile(require.resolve('#pkg'), 'utf-8')); const createClient = (context: Context) => { if (!context.host || !context.token) { throw new Error('Not signed in'); diff --git a/packages/cli/src/commands/local/local.utils.ts b/packages/cli/src/commands/local/local.utils.ts index 8a8da08..937ef4c 100644 --- a/packages/cli/src/commands/local/local.utils.ts +++ b/packages/cli/src/commands/local/local.utils.ts @@ -6,7 +6,7 @@ import { join } from 'path'; const ENV_PREFIX = 'ML_S_'; const readSecrets = async () => { - let secretLocation = join(process.cwd(), '.secret'); + let secretLocation = join(process.cwd(), '.secrets'); let secrets: Record = {}; diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 996b4ad..364e892 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,5 +1,5 @@ import { Command, program } from 'commander'; -import pkg from '../package.json'; +import { createRequire } from 'module'; import { loads } from './commands/loads/loads.js'; import { runs } from './commands/runs/runs.js'; import { logs } from './commands/logs/logs.js'; @@ -9,6 +9,11 @@ import { local } from './commands/local/local.js'; import { auth } from './commands/auth/auth.js'; import { contexts } from './commands/contexts/contexts.js'; import { schedules } from './commands/schedules/schedules.js'; +import { readFile } from 'fs/promises'; + +const require = createRequire(import.meta.url); + +const pkg = JSON.parse(await readFile(require.resolve('#pkg'), 'utf-8')); program.addCommand(loads); program.addCommand(runs); diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index bfca80f..2941828 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -2,6 +2,7 @@ "extends": "@morten-olsen/mini-loader-configs/tsconfig.esm.json", "compilerOptions": { "outDir": "dist/esm", + "rootDir": "src" }, "include": [ "./src/**/*.ts" diff --git a/packages/examples/.secrets b/packages/examples/.secrets new file mode 100644 index 0000000..f23501f --- /dev/null +++ b/packages/examples/.secrets @@ -0,0 +1 @@ +demo=foobar \ No newline at end of file diff --git a/packages/examples/src/secrets.ts b/packages/examples/src/secrets.ts new file mode 100644 index 0000000..490d330 --- /dev/null +++ b/packages/examples/src/secrets.ts @@ -0,0 +1,3 @@ +import { secrets } from '@morten-olsen/mini-loader'; + +console.log(secrets.get('demo')); diff --git a/packages/examples/src/simple.ts b/packages/examples/src/simple.ts index 5e148fe..4515951 100644 --- a/packages/examples/src/simple.ts +++ b/packages/examples/src/simple.ts @@ -3,7 +3,6 @@ import { artifacts, logger } from '@morten-olsen/mini-loader'; const run = async () => { await logger.info('Hello world'); await artifacts.create('foo', 'bar'); - process.exit(0); }; run(); diff --git a/packages/mini-loader/src/utils.ts b/packages/mini-loader/src/utils.ts index 8f911ab..8ad64c7 100644 --- a/packages/mini-loader/src/utils.ts +++ b/packages/mini-loader/src/utils.ts @@ -1,4 +1,5 @@ import { Socket, createConnection } from 'net'; +import { Event } from './index.js'; const connect = () => new Promise((resolve, reject) => { @@ -12,7 +13,7 @@ const connect = () => }); }); -const send = async (data: any) => +const send = async (data: Event) => new Promise(async (resolve, reject) => { const connection = await connect(); const cleaned = JSON.parse(JSON.stringify(data)); diff --git a/packages/server/bin/index.mjs b/packages/server/bin/index.mjs index 4ec2560..efb0ee3 100755 --- a/packages/server/bin/index.mjs +++ b/packages/server/bin/index.mjs @@ -1,4 +1,4 @@ #!/usr/bin/env node import 'source-map-support/register.js'; -import '../dist/esm/src/index.js'; +import '../dist/esm/index.js'; diff --git a/packages/server/package.json b/packages/server/package.json index 2b4c93d..cf65858 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -2,8 +2,8 @@ "name": "@morten-olsen/mini-loader-server", "version": "1.0.0", "license": "GPL-3.0", - "main": "./dist/esm/src/index.js", - "types": "./dist/esm/src/index.d.ts", + "main": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", "bin": { "mini-loader-server": "./bin/index.mjs" }, @@ -14,9 +14,12 @@ "files": [ "./dist" ], + "imports": { + "#pkg": "./package.json" + }, "exports": { ".": { - "import": "./dist/esm/src/index.js" + "import": "./dist/esm/index.js" } }, "devDependencies": { diff --git a/packages/server/src/database/database.ts b/packages/server/src/database/database.ts index e96c8a4..acfe5eb 100644 --- a/packages/server/src/database/database.ts +++ b/packages/server/src/database/database.ts @@ -1,6 +1,8 @@ import knex, { Knex } from 'knex'; import { source } from './migrations/migrations.source.js'; +import { mkdir } from 'fs/promises'; +import { dirname } from 'path'; const tableNames = { loads: 'loads', @@ -20,6 +22,15 @@ class Database { } #setup = async (config: Knex.Config) => { + if ( + config.connection && + typeof config.connection !== 'string' && + 'filename' in config.connection && + typeof config.connection.filename === 'string' && + config.connection.filename !== ':memory:' + ) { + await mkdir(dirname(config.connection.filename), { recursive: true }); + } const db = knex(config); await db.migrate.latest(); return db; diff --git a/packages/server/src/server/server.ts b/packages/server/src/server/server.ts index d3808f7..f4e7ab3 100644 --- a/packages/server/src/server/server.ts +++ b/packages/server/src/server/server.ts @@ -1,12 +1,18 @@ -import pkg from '../../package.json'; import { fastifyTRPCPlugin, FastifyTRPCPluginOptions } from '@trpc/server/adapters/fastify'; import fastify from 'fastify'; import { RootRouter, rootRouter } from '../router/router.js'; import { createContext } from '../router/router.utils.js'; import { Runtime } from '../runtime/runtime.js'; import { gateway } from '../gateway/gateway.js'; +import { createRequire } from 'module'; +import { readFile } from 'fs/promises'; + +const require = createRequire(import.meta.url); const createServer = async (runtime: Runtime) => { + const pkgLocation = require.resolve('#pkg'); + const pkg = JSON.parse(await readFile(pkgLocation, 'utf-8')); + const server = fastify({ maxParamLength: 10000, bodyLimit: 30 * 1024 * 1024, diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index 067be8e..90d2562 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -2,6 +2,7 @@ "extends": "@morten-olsen/mini-loader-configs/tsconfig.esm.json", "compilerOptions": { "outDir": "dist/esm", + "rootDir": "src" }, "include": [ "src"