mirror of
https://github.com/morten-olsen/homelab-apps.git
synced 2026-02-08 01:36:28 +01:00
add database
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -3,4 +3,7 @@
|
||||
**/charts/*.tgz
|
||||
charts/*.tgz
|
||||
|
||||
**/__pycache__/
|
||||
__pycache__/
|
||||
|
||||
**/Chart.lock
|
||||
|
||||
25
AGENTS.md
25
AGENTS.md
@@ -203,20 +203,27 @@ env:
|
||||
|
||||
The `PostgresDatabase` resource automatically provisions PostgreSQL databases.
|
||||
|
||||
**New Version (Recommended):**
|
||||
|
||||
Create `templates/database.yaml`:
|
||||
```yaml
|
||||
apiVersion: homelab.mortenolsen.pro/v1
|
||||
kind: PostgresDatabase
|
||||
metadata:
|
||||
name: '{{ .Release.Name }}'
|
||||
spec:
|
||||
environment: '{{ .Values.globals.environment }}'
|
||||
{{ include "common.database" . }}
|
||||
```
|
||||
|
||||
Add to `values.yaml`:
|
||||
```yaml
|
||||
database:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
**Legacy Version (Deprecated):**
|
||||
|
||||
The legacy `homelab.mortenolsen.pro/v1` API version is deprecated. For new charts, use the common library template which uses the new `postgres.homelab.mortenolsen.pro/v1` API version.
|
||||
|
||||
**What it does:**
|
||||
- Creates a PostgreSQL database with the same name as your release
|
||||
- Creates a user with appropriate permissions
|
||||
- Generates a Kubernetes secret named `{{ .Release.Name }}-database` containing:
|
||||
- Generates a Kubernetes secret named `{{ .Release.Name }}-connection` containing:
|
||||
- `url`: Complete PostgreSQL connection URL
|
||||
- `host`: Database hostname
|
||||
- `port`: Database port
|
||||
@@ -230,10 +237,12 @@ env:
|
||||
- name: DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: "{{ .Release.Name }}-database"
|
||||
name: "{{ .Release.Name }}-connection"
|
||||
key: url
|
||||
```
|
||||
|
||||
**Note:** The secret name changed from `{release}-pg-connection` (legacy) to `{release}-connection` (new version). The common library template handles this automatically.
|
||||
|
||||
### 3. Secret Generation
|
||||
|
||||
The `GenerateSecret` resource creates secure random secrets.
|
||||
|
||||
@@ -1,6 +1,2 @@
|
||||
apiVersion: homelab.mortenolsen.pro/v1
|
||||
kind: PostgresDatabase
|
||||
metadata:
|
||||
name: '{{ .Release.Name }}'
|
||||
spec:
|
||||
environment: '{{ .Values.globals.environment }}'
|
||||
{{ include "common.database" . }}
|
||||
|
||||
|
||||
1
apps/charts/blinko/templates/oidc.yaml
Normal file
1
apps/charts/blinko/templates/oidc.yaml
Normal file
@@ -0,0 +1 @@
|
||||
{{ include "common.oidc" . }}
|
||||
@@ -0,0 +1 @@
|
||||
{{ include "common.externalSecrets.externalSecrets" . }}
|
||||
@@ -0,0 +1 @@
|
||||
{{ include "common.externalSecrets.passwordGenerators" . }}
|
||||
@@ -1,9 +0,0 @@
|
||||
apiVersion: homelab.mortenolsen.pro/v1
|
||||
kind: GenerateSecret
|
||||
metadata:
|
||||
name: '{{ .Release.Name }}-secrets'
|
||||
spec:
|
||||
fields:
|
||||
- name: betterauth
|
||||
encoding: base64
|
||||
length: 64
|
||||
@@ -34,6 +34,28 @@ persistentVolumeClaims:
|
||||
- name: data
|
||||
size: 1Gi
|
||||
|
||||
# OIDC client
|
||||
oidc:
|
||||
enabled: true
|
||||
redirectUris:
|
||||
- "/api/auth/callback/authentik"
|
||||
|
||||
# Database configuration
|
||||
database:
|
||||
enabled: true
|
||||
|
||||
# External Secrets configuration
|
||||
externalSecrets:
|
||||
- name: "{release}-secrets"
|
||||
passwords:
|
||||
- name: betterauth
|
||||
length: 64
|
||||
allowRepeat: true # Required for longer passwords
|
||||
noUpper: false
|
||||
encoding: hex # hex encoding for the secret
|
||||
secretKeys:
|
||||
- betterauth # Use this key name in the secret instead of default "password"
|
||||
|
||||
# VirtualService configuration
|
||||
virtualService:
|
||||
enabled: true
|
||||
@@ -56,5 +78,5 @@ env:
|
||||
DATABASE_URL:
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: "{release}-pg-connection"
|
||||
name: "{release}-connection"
|
||||
key: url
|
||||
|
||||
@@ -90,6 +90,16 @@ virtualService:
|
||||
public: true
|
||||
private: true
|
||||
|
||||
# OIDC client configuration (if applicable)
|
||||
oidc:
|
||||
enabled: true
|
||||
redirectUris:
|
||||
- "/api/auth/callback/authentik"
|
||||
|
||||
# Database configuration (if applicable)
|
||||
database:
|
||||
enabled: true
|
||||
|
||||
# Environment variables
|
||||
env:
|
||||
MY_VAR: "value"
|
||||
@@ -148,6 +158,36 @@ Replace your template files with simple includes:
|
||||
{{ include "common.oidc" . }}
|
||||
```
|
||||
|
||||
#### database.yaml (if applicable)
|
||||
**Before:** ~10 lines
|
||||
**After:**
|
||||
```yaml
|
||||
{{ include "common.database" . }}
|
||||
```
|
||||
|
||||
#### secret.yaml (if using External Secrets)
|
||||
**Before:** ~10 lines (GenerateSecret)
|
||||
**After (recommended - split files for correct ordering):**
|
||||
|
||||
Create two files:
|
||||
|
||||
`templates/secret-password-generators.yaml`:
|
||||
```yaml
|
||||
{{ include "common.externalSecrets.passwordGenerators" . }}
|
||||
```
|
||||
|
||||
`templates/secret-external-secrets.yaml`:
|
||||
```yaml
|
||||
{{ include "common.externalSecrets.externalSecrets" . }}
|
||||
```
|
||||
|
||||
**Alternative (single file):**
|
||||
```yaml
|
||||
{{ include "common.externalSecrets" . }}
|
||||
```
|
||||
|
||||
**Note:** Splitting into separate files ensures Password generators are created before ExternalSecrets, which prevents sync errors.
|
||||
|
||||
### Step 4: Update Dependencies
|
||||
|
||||
Build the chart dependencies:
|
||||
@@ -274,6 +314,44 @@ volumes:
|
||||
persistentVolumeClaim: books # Uses PVC name as-is (not prefixed)
|
||||
```
|
||||
|
||||
### External Secrets (Password Generation)
|
||||
|
||||
```yaml
|
||||
# External Secrets configuration
|
||||
externalSecrets:
|
||||
- name: "{release}-secrets" # Secret name (supports placeholders)
|
||||
passwords:
|
||||
- name: betterauth # Generator name (used in generator resource name)
|
||||
length: 64 # Password length (default: 32)
|
||||
allowRepeat: true # Allow repeated characters (default: false)
|
||||
# Required for passwords longer than ~50 characters
|
||||
noUpper: false # Disable uppercase (default: false)
|
||||
encoding: hex # Encoding format: raw (default), hex, base64, base64url, base32
|
||||
secretKeys: # Required: sets the key name in the secret
|
||||
- betterauth # Without this, the key defaults to "password"
|
||||
- name: apitoken # Generator name
|
||||
length: 32
|
||||
allowRepeat: false # Can be false for shorter passwords
|
||||
secretKeys: # Required: sets the key name in the secret
|
||||
- apitoken # Without this, the key defaults to "password"
|
||||
```
|
||||
|
||||
**Important:** For passwords longer than approximately 50 characters, you must set `allowRepeat: true`. The default character set (uppercase, lowercase, digits) doesn't have enough unique characters to generate very long passwords without repeats.
|
||||
|
||||
**Multiple secrets:**
|
||||
|
||||
```yaml
|
||||
externalSecrets:
|
||||
- name: "{release}-secrets"
|
||||
passwords:
|
||||
- name: password
|
||||
length: 32
|
||||
- name: "{release}-api-keys"
|
||||
passwords:
|
||||
- name: apikey
|
||||
length: 64
|
||||
```
|
||||
|
||||
## Available Placeholders
|
||||
|
||||
See [TEMPLATING.md](./TEMPLATING.md) for complete placeholder documentation.
|
||||
@@ -325,15 +403,246 @@ See [TEMPLATING.md](./TEMPLATING.md) for complete placeholder documentation.
|
||||
- Secret references use `{release}` placeholder
|
||||
- Cleaner, more maintainable values.yaml
|
||||
|
||||
## Database Configuration
|
||||
|
||||
The common library supports the new PostgreSQL database resource (API version `postgres.homelab.mortenolsen.pro/v1`).
|
||||
|
||||
### Enabling Database Support
|
||||
|
||||
Add to your `values.yaml`:
|
||||
|
||||
```yaml
|
||||
# Database configuration
|
||||
database:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
### Database Template
|
||||
|
||||
Create `templates/database.yaml`:
|
||||
|
||||
```yaml
|
||||
{{ include "common.database" . }}
|
||||
```
|
||||
|
||||
### Generated Secret
|
||||
|
||||
The PostgresDatabase resource creates a secret named `{release}-connection` containing:
|
||||
- `url` - Complete PostgreSQL connection URL
|
||||
- `host` - Database hostname
|
||||
- `port` - Database port
|
||||
- `database` - Database name
|
||||
- `username` - Database username
|
||||
- `password` - Database password
|
||||
|
||||
### Using Database Secrets
|
||||
|
||||
Reference the database secret in your environment variables:
|
||||
|
||||
```yaml
|
||||
env:
|
||||
DATABASE_URL:
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: "{release}-connection"
|
||||
key: url
|
||||
DB_HOST:
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: "{release}-connection"
|
||||
key: host
|
||||
```
|
||||
|
||||
### Global Configuration
|
||||
|
||||
The database resource requires global configuration in `apps/root/values.yaml`:
|
||||
|
||||
```yaml
|
||||
globals:
|
||||
database:
|
||||
ref:
|
||||
name: postgres
|
||||
namespace: shared
|
||||
```
|
||||
|
||||
### Migration from Legacy PostgresDatabase
|
||||
|
||||
If migrating from the legacy `homelab.mortenolsen.pro/v1` PostgresDatabase:
|
||||
|
||||
1. **Update API version**: Changed from `homelab.mortenolsen.pro/v1` to `postgres.homelab.mortenolsen.pro/v1`
|
||||
2. **Update spec**: Changed from `environment` to `clusterRef` with `name` and `namespace`
|
||||
3. **Update secret name**: Changed from `{release}-pg-connection` to `{release}-connection`
|
||||
4. **Add namespace**: Metadata now includes `namespace: {{ .Release.Namespace }}`
|
||||
|
||||
The common library template handles all of this automatically.
|
||||
|
||||
### Migrating Database from Old Server to New Server
|
||||
|
||||
When migrating databases from the old PostgreSQL server (`prod-postgres-cluster-0` in `homelab` namespace) to the new server (`postgres-statefulset-0` in `shared` namespace), use the migration script.
|
||||
|
||||
#### Database Naming Convention
|
||||
|
||||
Database names follow the pattern `{namespace}_{name}` where:
|
||||
- `{namespace}` is the Kubernetes namespace (default: `prod`)
|
||||
- `{name}` is the application name (release name)
|
||||
|
||||
**Examples:**
|
||||
- `prod_blinko` - blinko app in prod namespace
|
||||
- `prod_gitea` - gitea app in prod namespace
|
||||
- `shared_authentik-db` - authentik app in shared namespace
|
||||
|
||||
#### Using the Migration Script
|
||||
|
||||
The migration script is located at `scripts/migrate_database.py` and handles:
|
||||
- Dumping the database from the old server
|
||||
- Restoring to the new server
|
||||
- Fixing permissions and ownership automatically
|
||||
|
||||
**Basic Usage:**
|
||||
```bash
|
||||
./scripts/migrate_database.py <source_db_name> <dest_db_name>
|
||||
```
|
||||
|
||||
**Example:**
|
||||
```bash
|
||||
# Migrate prod_blinko database (same name on both servers)
|
||||
./scripts/migrate_database.py prod_blinko prod_blinko
|
||||
```
|
||||
|
||||
**With Different Database Names:**
|
||||
```bash
|
||||
# Migrate from old_name to new_name
|
||||
./scripts/migrate_database.py old_name new_name
|
||||
```
|
||||
|
||||
**With Custom PostgreSQL Users:**
|
||||
```bash
|
||||
# If the PostgreSQL users differ from defaults
|
||||
./scripts/migrate_database.py prod_blinko prod_blinko \
|
||||
--source-user homelab \
|
||||
--dest-user postgres
|
||||
```
|
||||
|
||||
**Overwriting Existing Data:**
|
||||
```bash
|
||||
# Use --clean flag to drop existing objects before restoring
|
||||
# WARNING: This will DELETE all existing data in the destination database!
|
||||
./scripts/migrate_database.py prod_blinko prod_blinko --clean
|
||||
```
|
||||
|
||||
#### Behavior with Existing Databases
|
||||
|
||||
**Without `--clean` flag:**
|
||||
- The script will attempt to restore objects to the destination database
|
||||
- If tables/objects already exist, `pg_restore` may:
|
||||
- Fail with errors (e.g., "relation already exists")
|
||||
- Cause data conflicts (duplicate key violations)
|
||||
- Partially restore data
|
||||
- **This will NOT automatically overwrite existing data**
|
||||
|
||||
**With `--clean` flag:**
|
||||
- Drops all existing objects (tables, sequences, functions, etc.) before restoring
|
||||
- **WARNING: This will DELETE all existing data in the destination database**
|
||||
- Use this when you want to completely replace the destination database with source data
|
||||
- Recommended for initial migrations or when you're sure you want to overwrite
|
||||
|
||||
**Best Practice:**
|
||||
- For initial migrations: Use `--clean` to ensure a clean restore
|
||||
- For updates/re-syncs: Use `--clean` only if you're certain you want to replace all data
|
||||
- For incremental updates: Consider using application-specific sync mechanisms instead
|
||||
|
||||
#### Prerequisites
|
||||
|
||||
1. **Destination database must exist** - The script will verify but not create the database
|
||||
2. **Both pods must be running** - The script checks this automatically
|
||||
3. **Source database must exist** - The script verifies this before starting
|
||||
|
||||
#### What the Script Does
|
||||
|
||||
1. Verifies both PostgreSQL pods are running
|
||||
2. Checks that source and destination databases exist
|
||||
3. Dumps the source database using `pg_dump` (custom format)
|
||||
4. Restores to the destination database using `pg_restore`
|
||||
5. Automatically fixes permissions:
|
||||
- Grants USAGE and CREATE on all schemas to the database user
|
||||
- Changes schema ownership to the database user
|
||||
- Grants ALL privileges on all tables and sequences
|
||||
- Sets default privileges for future objects
|
||||
|
||||
#### Default Configuration
|
||||
|
||||
The script uses these defaults:
|
||||
- **Source server**: `prod-postgres-cluster-0` in `homelab` namespace
|
||||
- **Source user**: `homelab`
|
||||
- **Destination server**: `postgres-statefulset-0` in `shared` namespace
|
||||
- **Destination user**: `postgres`
|
||||
|
||||
#### Troubleshooting
|
||||
|
||||
**Error: "role does not exist"**
|
||||
- Check the PostgreSQL user name with: `kubectl exec -n <namespace> <pod> -c <container> -- env | grep POSTGRES_USER`
|
||||
- Use `--source-user` or `--dest-user` flags to specify correct users
|
||||
|
||||
**Error: "database does not exist"**
|
||||
- Create the destination database manually before running the script
|
||||
- Verify database names match the `{namespace}_{name}` convention
|
||||
|
||||
**Error: "permission denied for schema"**
|
||||
- The script should fix this automatically
|
||||
- If issues persist, manually grant permissions:
|
||||
```sql
|
||||
GRANT USAGE ON SCHEMA <schema_name> TO <db_user>;
|
||||
GRANT CREATE ON SCHEMA <schema_name> TO <db_user>;
|
||||
ALTER SCHEMA <schema_name> OWNER TO <db_user>;
|
||||
```
|
||||
|
||||
## Handling Legacy Resources
|
||||
|
||||
Some charts have legacy resources that should be kept as-is:
|
||||
Some charts may still have legacy resources that should be kept as-is:
|
||||
|
||||
- **OidcClient** (legacy) - Keep existing `client.yaml` template
|
||||
- **PostgresDatabase** (legacy) - Keep existing `database.yaml` template
|
||||
- **GenerateSecret** - Keep existing `secret.yaml` template
|
||||
- **OidcClient** (legacy `homelab.mortenolsen.pro/v1`) - Use `common.oidc` for new AuthentikClient instead
|
||||
- **PostgresDatabase** (legacy `homelab.mortenolsen.pro/v1`) - Use `common.database` for new PostgresDatabase instead
|
||||
- **GenerateSecret** (legacy `homelab.mortenolsen.pro/v1`) - Use `common.externalSecrets` for External Secrets instead
|
||||
|
||||
These will be migrated separately when the common library adds support for them.
|
||||
### Migrating from GenerateSecret to External Secrets
|
||||
|
||||
**Before (GenerateSecret):**
|
||||
```yaml
|
||||
# templates/secret.yaml
|
||||
apiVersion: homelab.mortenolsen.pro/v1
|
||||
kind: GenerateSecret
|
||||
metadata:
|
||||
name: '{{ .Release.Name }}-secrets'
|
||||
spec:
|
||||
fields:
|
||||
- name: betterauth
|
||||
encoding: base64
|
||||
length: 64
|
||||
```
|
||||
|
||||
**After (External Secrets):**
|
||||
```yaml
|
||||
# values.yaml
|
||||
externalSecrets:
|
||||
- name: "{release}-secrets"
|
||||
passwords:
|
||||
- name: betterauth
|
||||
length: 64
|
||||
allowRepeat: true # Required for passwords >50 chars
|
||||
noUpper: false
|
||||
encoding: hex # hex, base64, base64url, base32, or raw (default)
|
||||
secretKeys:
|
||||
- betterauth # Required: sets the key name in the secret
|
||||
|
||||
# templates/secret.yaml
|
||||
{{ include "common.externalSecrets" . }}
|
||||
```
|
||||
|
||||
**Note:**
|
||||
- External Secrets generates passwords directly (no encoding option)
|
||||
- The `secretKeys` field is **required** to set the key name in the secret
|
||||
- Without `secretKeys`, the Password generator defaults to using `password` as the key name
|
||||
- The `name` field in the password config is used for the generator name, not the secret key name
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
@@ -398,6 +707,10 @@ After migration, verify:
|
||||
- [ ] VirtualServices route to correct service
|
||||
- [ ] DNS record created (if applicable)
|
||||
- [ ] OIDC client created (if applicable)
|
||||
- [ ] Database resource created (if applicable)
|
||||
- [ ] Database secret references use correct name (`{release}-connection`)
|
||||
- [ ] External Secrets created (if applicable)
|
||||
- [ ] Password generators created for each secret field
|
||||
|
||||
## Post-Migration
|
||||
|
||||
|
||||
@@ -33,6 +33,8 @@ The library provides full resource templates that can be included directly:
|
||||
- `common.virtualService` - Full VirtualService resources (public + private)
|
||||
- `common.dns` - Full DNSRecord resource
|
||||
- `common.oidc` - Full AuthentikClient resource
|
||||
- `common.database` - Full PostgresDatabase resource
|
||||
- `common.externalSecrets` - Full ExternalSecret resources with Password generators
|
||||
|
||||
## Usage Example
|
||||
|
||||
|
||||
Binary file not shown.
@@ -525,3 +525,99 @@ spec:
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Full PostgreSQL Database resource
|
||||
*/}}
|
||||
{{- define "common.database" -}}
|
||||
{{- if and .Values.database.enabled (hasKey .Values.globals "database") (hasKey .Values.globals.database "ref") (hasKey .Values.globals.database.ref "name") (hasKey .Values.globals.database.ref "namespace") (ne .Values.globals.database.ref.name "") (ne .Values.globals.database.ref.namespace "") }}
|
||||
apiVersion: postgres.homelab.mortenolsen.pro/v1
|
||||
kind: PostgresDatabase
|
||||
metadata:
|
||||
name: {{ .Release.Name }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "common.labels" . | nindent 4 }}
|
||||
spec:
|
||||
clusterRef:
|
||||
name: {{ .Values.globals.database.ref.name | quote }}
|
||||
namespace: {{ .Values.globals.database.ref.namespace | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Password generators for External Secrets (create these first)
|
||||
*/}}
|
||||
{{- define "common.externalSecrets.passwordGenerators" -}}
|
||||
{{- if .Values.externalSecrets }}
|
||||
{{- range .Values.externalSecrets }}
|
||||
{{- $secretName := .name | default (printf "%s-%s" $.Release.Name "secrets") }}
|
||||
{{- $secretName = $secretName | replace "{release}" $.Release.Name | replace "{fullname}" (include "common.fullname" $) }}
|
||||
{{- range .passwords }}
|
||||
---
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
metadata:
|
||||
name: {{ $secretName }}-{{ .name }}-generator
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
labels:
|
||||
{{- include "common.labels" $ | nindent 4 }}
|
||||
spec:
|
||||
length: {{ .length | default 32 }}
|
||||
allowRepeat: {{ .allowRepeat | default false }}
|
||||
noUpper: {{ .noUpper | default false }}
|
||||
{{- if .encoding }}
|
||||
encoding: {{ .encoding }}
|
||||
{{- end }}
|
||||
{{- if .secretKeys }}
|
||||
secretKeys:
|
||||
{{- range .secretKeys }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
External Secrets (create these after password generators)
|
||||
*/}}
|
||||
{{- define "common.externalSecrets.externalSecrets" -}}
|
||||
{{- if .Values.externalSecrets }}
|
||||
{{- range .Values.externalSecrets }}
|
||||
{{- $secretName := .name | default (printf "%s-%s" $.Release.Name "secrets") }}
|
||||
{{- $secretName = $secretName | replace "{release}" $.Release.Name | replace "{fullname}" (include "common.fullname" $) }}
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: {{ $secretName }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
labels:
|
||||
{{- include "common.labels" $ | nindent 4 }}
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
target:
|
||||
name: {{ $secretName }}
|
||||
creationPolicy: Owner
|
||||
dataFrom:
|
||||
{{- range .passwords }}
|
||||
- sourceRef:
|
||||
generatorRef:
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
name: {{ $secretName }}-{{ .name }}-generator
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Full External Secrets resources (ExternalSecret + Password generators)
|
||||
Combined helper that outputs generators first, then ExternalSecrets
|
||||
*/}}
|
||||
{{- define "common.externalSecrets" -}}
|
||||
{{- include "common.externalSecrets.passwordGenerators" . }}
|
||||
{{- include "common.externalSecrets.externalSecrets" . }}
|
||||
{{- end }}
|
||||
|
||||
@@ -26,11 +26,7 @@ spec:
|
||||
- values.yaml
|
||||
values: |
|
||||
globals:
|
||||
environment: {{ .Values.globals.environment }}
|
||||
domain: {{ .Values.globals.domain }}
|
||||
timezone: {{ .Values.globals.timezone }}
|
||||
istio:
|
||||
gateway: {{ .Values.globals.istio.gateway }}
|
||||
{{ .Values.globals | toYaml | indent 14 }}
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: prod
|
||||
|
||||
@@ -7,6 +7,10 @@ globals:
|
||||
environment: prod
|
||||
domain: olsen.cloud
|
||||
timezone: Europe/Amsterdam
|
||||
database:
|
||||
ref:
|
||||
name: postgres
|
||||
namespace: shared
|
||||
istio:
|
||||
gateway: shared/private
|
||||
gateways:
|
||||
|
||||
335
scripts/migrate_database.py
Executable file
335
scripts/migrate_database.py
Executable file
@@ -0,0 +1,335 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Database migration script for PostgreSQL databases in Kubernetes.
|
||||
|
||||
Migrates a database from the old PostgreSQL server (prod-postgres-cluster-0 in homelab namespace)
|
||||
to the new PostgreSQL server (postgres-statefulset-0 in shared namespace).
|
||||
|
||||
Usage:
|
||||
python3 migrate_database.py <source_db_name> <dest_db_name> [--clean]
|
||||
|
||||
Examples:
|
||||
python3 migrate_database.py myapp myapp
|
||||
python3 migrate_database.py oldname newname
|
||||
python3 migrate_database.py prod_blinko prod_blinko --clean # Drop existing data first
|
||||
|
||||
WARNING: Without --clean flag, pg_restore will attempt to restore objects. If objects already
|
||||
exist, it may fail with errors or cause data conflicts (duplicate key violations, etc.).
|
||||
Use --clean to drop existing objects before restoring (this will DELETE all existing data).
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import argparse
|
||||
import base64
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
|
||||
# Configuration
|
||||
SOURCE_POD = "prod-postgres-cluster-0"
|
||||
SOURCE_NAMESPACE = "homelab"
|
||||
SOURCE_CONTAINER = "prod-postgres-cluster"
|
||||
SOURCE_DEFAULT_USER = "homelab" # Default user for old server
|
||||
|
||||
DEST_POD = "postgres-statefulset-0"
|
||||
DEST_NAMESPACE = "shared"
|
||||
DEST_CONTAINER = "postgres"
|
||||
DEST_DEFAULT_USER = "postgres" # Default user for new server
|
||||
|
||||
|
||||
def run_kubectl_exec(pod, namespace, container, command, stdin=None, binary=False):
|
||||
"""Execute a command in a Kubernetes pod using kubectl exec."""
|
||||
cmd = [
|
||||
"kubectl", "exec", "-n", namespace, pod,
|
||||
"-c", container, "--"
|
||||
] + command
|
||||
|
||||
try:
|
||||
if stdin:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
input=stdin,
|
||||
capture_output=True,
|
||||
text=not binary,
|
||||
check=True
|
||||
)
|
||||
else:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=not binary,
|
||||
check=True
|
||||
)
|
||||
return result.stdout, result.stderr
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Error executing command: {' '.join(cmd)}", file=sys.stderr)
|
||||
print(f"Exit code: {e.returncode}", file=sys.stderr)
|
||||
if not binary:
|
||||
print(f"stdout: {e.stdout}", file=sys.stderr)
|
||||
print(f"stderr: {e.stderr}", file=sys.stderr)
|
||||
raise
|
||||
|
||||
|
||||
def check_pod_exists(pod, namespace):
|
||||
"""Check if a pod exists and is running."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["kubectl", "get", "pod", pod, "-n", namespace],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True
|
||||
)
|
||||
# Check if pod is running
|
||||
if "Running" in result.stdout:
|
||||
return True
|
||||
return False
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
|
||||
|
||||
def check_database_exists(pod, namespace, container, db_name, pg_user):
|
||||
"""Check if a database exists on a PostgreSQL server."""
|
||||
psql_cmd = [
|
||||
"psql", "-U", pg_user, "-tAc",
|
||||
f"SELECT 1 FROM pg_database WHERE datname='{db_name}'"
|
||||
]
|
||||
|
||||
try:
|
||||
stdout, _ = run_kubectl_exec(pod, namespace, container, psql_cmd)
|
||||
return stdout.strip() == "1"
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
|
||||
|
||||
def dump_database(pod, namespace, container, db_name, pg_user):
|
||||
"""Dump a database from the source server."""
|
||||
print(f"Dumping database '{db_name}' from source server...")
|
||||
|
||||
# Use pg_dump with stdout to avoid file I/O issues
|
||||
dump_cmd = [
|
||||
"pg_dump",
|
||||
"-U", pg_user,
|
||||
"-F", "c", # Custom format (compressed)
|
||||
db_name
|
||||
]
|
||||
|
||||
# Get the dump directly from stdout
|
||||
dump_data, _ = run_kubectl_exec(pod, namespace, container, dump_cmd, binary=True)
|
||||
|
||||
if not dump_data or len(dump_data) == 0:
|
||||
raise RuntimeError(f"Failed to dump database '{db_name}': dump is empty")
|
||||
|
||||
print(f"Successfully dumped database '{db_name}' ({len(dump_data)} bytes).")
|
||||
|
||||
return dump_data # Return as bytes for binary data
|
||||
|
||||
|
||||
def fix_database_permissions(pod, namespace, container, db_name, db_user, pg_user):
|
||||
"""Fix permissions for the database user after migration."""
|
||||
print(f"Fixing permissions for user '{db_user}' on database '{db_name}'...")
|
||||
|
||||
# Get all schemas owned by the database user or that need permissions
|
||||
fix_perms_cmd = [
|
||||
"psql", "-U", pg_user, "-d", db_name, "-tAc",
|
||||
f"""
|
||||
DO $$
|
||||
DECLARE
|
||||
schema_rec RECORD;
|
||||
BEGIN
|
||||
-- Grant permissions on all schemas to the database user
|
||||
FOR schema_rec IN
|
||||
SELECT nspname
|
||||
FROM pg_namespace
|
||||
WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast', 'pg_toast_temp_1')
|
||||
LOOP
|
||||
-- Grant usage and create on schema
|
||||
EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', schema_rec.nspname, '{db_user}');
|
||||
EXECUTE format('GRANT CREATE ON SCHEMA %I TO %I', schema_rec.nspname, '{db_user}');
|
||||
|
||||
-- Grant privileges on all existing tables
|
||||
EXECUTE format('GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I TO %I', schema_rec.nspname, '{db_user}');
|
||||
|
||||
-- Grant privileges on all existing sequences
|
||||
EXECUTE format('GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA %I TO %I', schema_rec.nspname, '{db_user}');
|
||||
|
||||
-- Set default privileges for future objects
|
||||
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON TABLES TO %I', schema_rec.nspname, '{db_user}');
|
||||
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON SEQUENCES TO %I', schema_rec.nspname, '{db_user}');
|
||||
|
||||
-- Change ownership to database user if schema is not system-owned
|
||||
IF schema_rec.nspname != 'public' THEN
|
||||
EXECUTE format('ALTER SCHEMA %I OWNER TO %I', schema_rec.nspname, '{db_user}');
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
-- Grant database connection privilege
|
||||
EXECUTE format('GRANT CONNECT ON DATABASE %I TO %I', '{db_name}', '{db_user}');
|
||||
END $$;
|
||||
"""
|
||||
]
|
||||
|
||||
try:
|
||||
run_kubectl_exec(pod, namespace, container, fix_perms_cmd)
|
||||
print(f"Successfully fixed permissions for user '{db_user}'.")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Warning: Could not fix all permissions automatically. You may need to fix them manually.", file=sys.stderr)
|
||||
print(f"Error: {e.stderr}", file=sys.stderr)
|
||||
|
||||
|
||||
def restore_database(pod, namespace, container, db_name, dump_data, pg_user, clean=False):
|
||||
"""Restore a database dump to the destination server."""
|
||||
print(f"Restoring database '{db_name}' to destination server...")
|
||||
|
||||
# For large dumps, use kubectl cp instead of base64 encoding to avoid command line length limits
|
||||
# Write dump to a temporary file locally, then copy it to the pod
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix='.dump') as tmp_file:
|
||||
tmp_path = tmp_file.name
|
||||
tmp_file.write(dump_data)
|
||||
|
||||
try:
|
||||
# Copy dump file to pod using kubectl cp
|
||||
print("Copying dump file to destination pod...")
|
||||
cp_cmd = [
|
||||
"kubectl", "cp", tmp_path,
|
||||
f"{namespace}/{pod}:/tmp/dump.dump",
|
||||
"-c", container
|
||||
]
|
||||
subprocess.run(cp_cmd, check=True, capture_output=True)
|
||||
print("Dump file copied successfully.")
|
||||
finally:
|
||||
# Clean up local temporary file
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Restore the dump
|
||||
restore_cmd = [
|
||||
"pg_restore",
|
||||
"-U", pg_user,
|
||||
"-d", db_name,
|
||||
"-v", # Verbose
|
||||
"--no-owner", # Don't restore ownership
|
||||
"--no-acl", # Don't restore access privileges
|
||||
]
|
||||
|
||||
# Add --clean flag if requested (drops existing objects before restoring)
|
||||
if clean:
|
||||
restore_cmd.append("--clean")
|
||||
print("WARNING: Using --clean flag. This will drop all existing objects in the destination database!")
|
||||
|
||||
restore_cmd.append("/tmp/dump.dump")
|
||||
|
||||
try:
|
||||
run_kubectl_exec(pod, namespace, container, restore_cmd)
|
||||
print(f"Successfully restored database '{db_name}'.")
|
||||
finally:
|
||||
# Clean up remote dump file
|
||||
rm_cmd = ["rm", "/tmp/dump.dump"]
|
||||
try:
|
||||
run_kubectl_exec(pod, namespace, container, rm_cmd)
|
||||
except subprocess.CalledProcessError:
|
||||
pass # Ignore cleanup errors
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Migrate PostgreSQL database between Kubernetes pods",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s myapp myapp
|
||||
%(prog)s oldname newname
|
||||
%(prog)s source_db dest_db
|
||||
"""
|
||||
)
|
||||
parser.add_argument(
|
||||
"source_db",
|
||||
help="Name of the source database (on old server: prod-postgres-cluster-0)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"dest_db",
|
||||
help="Name of the destination database (on new server: postgres-statefulset-0)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source-user",
|
||||
default=SOURCE_DEFAULT_USER,
|
||||
help=f"PostgreSQL user for source server (default: {SOURCE_DEFAULT_USER})"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dest-user",
|
||||
default=DEST_DEFAULT_USER,
|
||||
help=f"PostgreSQL user for destination server (default: {DEST_DEFAULT_USER})"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--clean",
|
||||
action="store_true",
|
||||
help="Drop existing objects before restoring (WARNING: This will delete all existing data in the destination database)"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
print("=" * 60)
|
||||
print("PostgreSQL Database Migration Script")
|
||||
print("=" * 60)
|
||||
print(f"Source: {SOURCE_POD} ({SOURCE_NAMESPACE}) - Database: {args.source_db}")
|
||||
print(f"Destination: {DEST_POD} ({DEST_NAMESPACE}) - Database: {args.dest_db}")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# Check if pods exist and are running
|
||||
print("Checking source pod...")
|
||||
if not check_pod_exists(SOURCE_POD, SOURCE_NAMESPACE):
|
||||
print(f"ERROR: Source pod '{SOURCE_POD}' not found or not running in namespace '{SOURCE_NAMESPACE}'", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"✓ Source pod '{SOURCE_POD}' is running")
|
||||
|
||||
print("Checking destination pod...")
|
||||
if not check_pod_exists(DEST_POD, DEST_NAMESPACE):
|
||||
print(f"ERROR: Destination pod '{DEST_POD}' not found or not running in namespace '{DEST_NAMESPACE}'", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"✓ Destination pod '{DEST_POD}' is running")
|
||||
print()
|
||||
|
||||
# Check if source database exists
|
||||
print(f"Checking if source database '{args.source_db}' exists...")
|
||||
if not check_database_exists(SOURCE_POD, SOURCE_NAMESPACE, SOURCE_CONTAINER, args.source_db, args.source_user):
|
||||
print(f"ERROR: Source database '{args.source_db}' does not exist", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"✓ Source database '{args.source_db}' exists")
|
||||
|
||||
# Check if destination database exists
|
||||
print(f"Checking if destination database '{args.dest_db}' exists...")
|
||||
if not check_database_exists(DEST_POD, DEST_NAMESPACE, DEST_CONTAINER, args.dest_db, args.dest_user):
|
||||
print(f"ERROR: Destination database '{args.dest_db}' does not exist. Please create it first.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"✓ Destination database '{args.dest_db}' exists")
|
||||
print()
|
||||
|
||||
# Dump source database
|
||||
dump_data = dump_database(SOURCE_POD, SOURCE_NAMESPACE, SOURCE_CONTAINER, args.source_db, args.source_user)
|
||||
print()
|
||||
|
||||
# Restore to destination database
|
||||
restore_database(DEST_POD, DEST_NAMESPACE, DEST_CONTAINER, args.dest_db, dump_data, args.dest_user, clean=args.clean)
|
||||
print()
|
||||
|
||||
# Fix permissions for the database user (assuming db user name matches db name)
|
||||
# This fixes schema ownership and permissions that were lost due to --no-owner and --no-acl
|
||||
fix_database_permissions(DEST_POD, DEST_NAMESPACE, DEST_CONTAINER, args.dest_db, args.dest_db, args.dest_user)
|
||||
print()
|
||||
|
||||
print("=" * 60)
|
||||
print("Migration completed successfully!")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("\n\nMigration interrupted by user.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"\n\nERROR: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user