add database

This commit is contained in:
Morten Olsen
2026-01-01 19:30:51 +01:00
parent 9ac5b61a38
commit d5a0803eee
15 changed files with 805 additions and 35 deletions

3
.gitignore vendored
View File

@@ -3,4 +3,7 @@
**/charts/*.tgz
charts/*.tgz
**/__pycache__/
__pycache__/
**/Chart.lock

View File

@@ -203,20 +203,27 @@ env:
The `PostgresDatabase` resource automatically provisions PostgreSQL databases.
**New Version (Recommended):**
Create `templates/database.yaml`:
```yaml
apiVersion: homelab.mortenolsen.pro/v1
kind: PostgresDatabase
metadata:
name: '{{ .Release.Name }}'
spec:
environment: '{{ .Values.globals.environment }}'
{{ include "common.database" . }}
```
Add to `values.yaml`:
```yaml
database:
enabled: true
```
**Legacy Version (Deprecated):**
The legacy `homelab.mortenolsen.pro/v1` API version is deprecated. For new charts, use the common library template which uses the new `postgres.homelab.mortenolsen.pro/v1` API version.
**What it does:**
- Creates a PostgreSQL database with the same name as your release
- Creates a user with appropriate permissions
- Generates a Kubernetes secret named `{{ .Release.Name }}-database` containing:
- Generates a Kubernetes secret named `{{ .Release.Name }}-connection` containing:
- `url`: Complete PostgreSQL connection URL
- `host`: Database hostname
- `port`: Database port
@@ -230,10 +237,12 @@ env:
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-database"
name: "{{ .Release.Name }}-connection"
key: url
```
**Note:** The secret name changed from `{release}-pg-connection` (legacy) to `{release}-connection` (new version). The common library template handles this automatically.
### 3. Secret Generation
The `GenerateSecret` resource creates secure random secrets.

View File

@@ -1,6 +1,2 @@
apiVersion: homelab.mortenolsen.pro/v1
kind: PostgresDatabase
metadata:
name: '{{ .Release.Name }}'
spec:
environment: '{{ .Values.globals.environment }}'
{{ include "common.database" . }}

View File

@@ -0,0 +1 @@
{{ include "common.oidc" . }}

View File

@@ -0,0 +1 @@
{{ include "common.externalSecrets.externalSecrets" . }}

View File

@@ -0,0 +1 @@
{{ include "common.externalSecrets.passwordGenerators" . }}

View File

@@ -1,9 +0,0 @@
apiVersion: homelab.mortenolsen.pro/v1
kind: GenerateSecret
metadata:
name: '{{ .Release.Name }}-secrets'
spec:
fields:
- name: betterauth
encoding: base64
length: 64

View File

@@ -34,6 +34,28 @@ persistentVolumeClaims:
- name: data
size: 1Gi
# OIDC client
oidc:
enabled: true
redirectUris:
- "/api/auth/callback/authentik"
# Database configuration
database:
enabled: true
# External Secrets configuration
externalSecrets:
- name: "{release}-secrets"
passwords:
- name: betterauth
length: 64
allowRepeat: true # Required for longer passwords
noUpper: false
encoding: hex # hex encoding for the secret
secretKeys:
- betterauth # Use this key name in the secret instead of default "password"
# VirtualService configuration
virtualService:
enabled: true
@@ -56,5 +78,5 @@ env:
DATABASE_URL:
valueFrom:
secretKeyRef:
name: "{release}-pg-connection"
name: "{release}-connection"
key: url

View File

@@ -90,6 +90,16 @@ virtualService:
public: true
private: true
# OIDC client configuration (if applicable)
oidc:
enabled: true
redirectUris:
- "/api/auth/callback/authentik"
# Database configuration (if applicable)
database:
enabled: true
# Environment variables
env:
MY_VAR: "value"
@@ -148,6 +158,36 @@ Replace your template files with simple includes:
{{ include "common.oidc" . }}
```
#### database.yaml (if applicable)
**Before:** ~10 lines
**After:**
```yaml
{{ include "common.database" . }}
```
#### secret.yaml (if using External Secrets)
**Before:** ~10 lines (GenerateSecret)
**After (recommended - split files for correct ordering):**
Create two files:
`templates/secret-password-generators.yaml`:
```yaml
{{ include "common.externalSecrets.passwordGenerators" . }}
```
`templates/secret-external-secrets.yaml`:
```yaml
{{ include "common.externalSecrets.externalSecrets" . }}
```
**Alternative (single file):**
```yaml
{{ include "common.externalSecrets" . }}
```
**Note:** Splitting into separate files ensures Password generators are created before ExternalSecrets, which prevents sync errors.
### Step 4: Update Dependencies
Build the chart dependencies:
@@ -274,6 +314,44 @@ volumes:
persistentVolumeClaim: books # Uses PVC name as-is (not prefixed)
```
### External Secrets (Password Generation)
```yaml
# External Secrets configuration
externalSecrets:
- name: "{release}-secrets" # Secret name (supports placeholders)
passwords:
- name: betterauth # Generator name (used in generator resource name)
length: 64 # Password length (default: 32)
allowRepeat: true # Allow repeated characters (default: false)
# Required for passwords longer than ~50 characters
noUpper: false # Disable uppercase (default: false)
encoding: hex # Encoding format: raw (default), hex, base64, base64url, base32
secretKeys: # Required: sets the key name in the secret
- betterauth # Without this, the key defaults to "password"
- name: apitoken # Generator name
length: 32
allowRepeat: false # Can be false for shorter passwords
secretKeys: # Required: sets the key name in the secret
- apitoken # Without this, the key defaults to "password"
```
**Important:** For passwords longer than approximately 50 characters, you must set `allowRepeat: true`. The default character set (uppercase, lowercase, digits) doesn't have enough unique characters to generate very long passwords without repeats.
**Multiple secrets:**
```yaml
externalSecrets:
- name: "{release}-secrets"
passwords:
- name: password
length: 32
- name: "{release}-api-keys"
passwords:
- name: apikey
length: 64
```
## Available Placeholders
See [TEMPLATING.md](./TEMPLATING.md) for complete placeholder documentation.
@@ -325,15 +403,246 @@ See [TEMPLATING.md](./TEMPLATING.md) for complete placeholder documentation.
- Secret references use `{release}` placeholder
- Cleaner, more maintainable values.yaml
## Database Configuration
The common library supports the new PostgreSQL database resource (API version `postgres.homelab.mortenolsen.pro/v1`).
### Enabling Database Support
Add to your `values.yaml`:
```yaml
# Database configuration
database:
enabled: true
```
### Database Template
Create `templates/database.yaml`:
```yaml
{{ include "common.database" . }}
```
### Generated Secret
The PostgresDatabase resource creates a secret named `{release}-connection` containing:
- `url` - Complete PostgreSQL connection URL
- `host` - Database hostname
- `port` - Database port
- `database` - Database name
- `username` - Database username
- `password` - Database password
### Using Database Secrets
Reference the database secret in your environment variables:
```yaml
env:
DATABASE_URL:
valueFrom:
secretKeyRef:
name: "{release}-connection"
key: url
DB_HOST:
valueFrom:
secretKeyRef:
name: "{release}-connection"
key: host
```
### Global Configuration
The database resource requires global configuration in `apps/root/values.yaml`:
```yaml
globals:
database:
ref:
name: postgres
namespace: shared
```
### Migration from Legacy PostgresDatabase
If migrating from the legacy `homelab.mortenolsen.pro/v1` PostgresDatabase:
1. **Update API version**: Changed from `homelab.mortenolsen.pro/v1` to `postgres.homelab.mortenolsen.pro/v1`
2. **Update spec**: Changed from `environment` to `clusterRef` with `name` and `namespace`
3. **Update secret name**: Changed from `{release}-pg-connection` to `{release}-connection`
4. **Add namespace**: Metadata now includes `namespace: {{ .Release.Namespace }}`
The common library template handles all of this automatically.
### Migrating Database from Old Server to New Server
When migrating databases from the old PostgreSQL server (`prod-postgres-cluster-0` in `homelab` namespace) to the new server (`postgres-statefulset-0` in `shared` namespace), use the migration script.
#### Database Naming Convention
Database names follow the pattern `{namespace}_{name}` where:
- `{namespace}` is the Kubernetes namespace (default: `prod`)
- `{name}` is the application name (release name)
**Examples:**
- `prod_blinko` - blinko app in prod namespace
- `prod_gitea` - gitea app in prod namespace
- `shared_authentik-db` - authentik app in shared namespace
#### Using the Migration Script
The migration script is located at `scripts/migrate_database.py` and handles:
- Dumping the database from the old server
- Restoring to the new server
- Fixing permissions and ownership automatically
**Basic Usage:**
```bash
./scripts/migrate_database.py <source_db_name> <dest_db_name>
```
**Example:**
```bash
# Migrate prod_blinko database (same name on both servers)
./scripts/migrate_database.py prod_blinko prod_blinko
```
**With Different Database Names:**
```bash
# Migrate from old_name to new_name
./scripts/migrate_database.py old_name new_name
```
**With Custom PostgreSQL Users:**
```bash
# If the PostgreSQL users differ from defaults
./scripts/migrate_database.py prod_blinko prod_blinko \
--source-user homelab \
--dest-user postgres
```
**Overwriting Existing Data:**
```bash
# Use --clean flag to drop existing objects before restoring
# WARNING: This will DELETE all existing data in the destination database!
./scripts/migrate_database.py prod_blinko prod_blinko --clean
```
#### Behavior with Existing Databases
**Without `--clean` flag:**
- The script will attempt to restore objects to the destination database
- If tables/objects already exist, `pg_restore` may:
- Fail with errors (e.g., "relation already exists")
- Cause data conflicts (duplicate key violations)
- Partially restore data
- **This will NOT automatically overwrite existing data**
**With `--clean` flag:**
- Drops all existing objects (tables, sequences, functions, etc.) before restoring
- **WARNING: This will DELETE all existing data in the destination database**
- Use this when you want to completely replace the destination database with source data
- Recommended for initial migrations or when you're sure you want to overwrite
**Best Practice:**
- For initial migrations: Use `--clean` to ensure a clean restore
- For updates/re-syncs: Use `--clean` only if you're certain you want to replace all data
- For incremental updates: Consider using application-specific sync mechanisms instead
#### Prerequisites
1. **Destination database must exist** - The script will verify but not create the database
2. **Both pods must be running** - The script checks this automatically
3. **Source database must exist** - The script verifies this before starting
#### What the Script Does
1. Verifies both PostgreSQL pods are running
2. Checks that source and destination databases exist
3. Dumps the source database using `pg_dump` (custom format)
4. Restores to the destination database using `pg_restore`
5. Automatically fixes permissions:
- Grants USAGE and CREATE on all schemas to the database user
- Changes schema ownership to the database user
- Grants ALL privileges on all tables and sequences
- Sets default privileges for future objects
#### Default Configuration
The script uses these defaults:
- **Source server**: `prod-postgres-cluster-0` in `homelab` namespace
- **Source user**: `homelab`
- **Destination server**: `postgres-statefulset-0` in `shared` namespace
- **Destination user**: `postgres`
#### Troubleshooting
**Error: "role does not exist"**
- Check the PostgreSQL user name with: `kubectl exec -n <namespace> <pod> -c <container> -- env | grep POSTGRES_USER`
- Use `--source-user` or `--dest-user` flags to specify correct users
**Error: "database does not exist"**
- Create the destination database manually before running the script
- Verify database names match the `{namespace}_{name}` convention
**Error: "permission denied for schema"**
- The script should fix this automatically
- If issues persist, manually grant permissions:
```sql
GRANT USAGE ON SCHEMA <schema_name> TO <db_user>;
GRANT CREATE ON SCHEMA <schema_name> TO <db_user>;
ALTER SCHEMA <schema_name> OWNER TO <db_user>;
```
## Handling Legacy Resources
Some charts have legacy resources that should be kept as-is:
Some charts may still have legacy resources that should be kept as-is:
- **OidcClient** (legacy) - Keep existing `client.yaml` template
- **PostgresDatabase** (legacy) - Keep existing `database.yaml` template
- **GenerateSecret** - Keep existing `secret.yaml` template
- **OidcClient** (legacy `homelab.mortenolsen.pro/v1`) - Use `common.oidc` for new AuthentikClient instead
- **PostgresDatabase** (legacy `homelab.mortenolsen.pro/v1`) - Use `common.database` for new PostgresDatabase instead
- **GenerateSecret** (legacy `homelab.mortenolsen.pro/v1`) - Use `common.externalSecrets` for External Secrets instead
These will be migrated separately when the common library adds support for them.
### Migrating from GenerateSecret to External Secrets
**Before (GenerateSecret):**
```yaml
# templates/secret.yaml
apiVersion: homelab.mortenolsen.pro/v1
kind: GenerateSecret
metadata:
name: '{{ .Release.Name }}-secrets'
spec:
fields:
- name: betterauth
encoding: base64
length: 64
```
**After (External Secrets):**
```yaml
# values.yaml
externalSecrets:
- name: "{release}-secrets"
passwords:
- name: betterauth
length: 64
allowRepeat: true # Required for passwords >50 chars
noUpper: false
encoding: hex # hex, base64, base64url, base32, or raw (default)
secretKeys:
- betterauth # Required: sets the key name in the secret
# templates/secret.yaml
{{ include "common.externalSecrets" . }}
```
**Note:**
- External Secrets generates passwords directly (no encoding option)
- The `secretKeys` field is **required** to set the key name in the secret
- Without `secretKeys`, the Password generator defaults to using `password` as the key name
- The `name` field in the password config is used for the generator name, not the secret key name
## Troubleshooting
@@ -398,6 +707,10 @@ After migration, verify:
- [ ] VirtualServices route to correct service
- [ ] DNS record created (if applicable)
- [ ] OIDC client created (if applicable)
- [ ] Database resource created (if applicable)
- [ ] Database secret references use correct name (`{release}-connection`)
- [ ] External Secrets created (if applicable)
- [ ] Password generators created for each secret field
## Post-Migration

View File

@@ -33,6 +33,8 @@ The library provides full resource templates that can be included directly:
- `common.virtualService` - Full VirtualService resources (public + private)
- `common.dns` - Full DNSRecord resource
- `common.oidc` - Full AuthentikClient resource
- `common.database` - Full PostgresDatabase resource
- `common.externalSecrets` - Full ExternalSecret resources with Password generators
## Usage Example

Binary file not shown.

View File

@@ -525,3 +525,99 @@ spec:
{{- end }}
{{- end }}
{{- end }}
{{/*
Full PostgreSQL Database resource
*/}}
{{- define "common.database" -}}
{{- if and .Values.database.enabled (hasKey .Values.globals "database") (hasKey .Values.globals.database "ref") (hasKey .Values.globals.database.ref "name") (hasKey .Values.globals.database.ref "namespace") (ne .Values.globals.database.ref.name "") (ne .Values.globals.database.ref.namespace "") }}
apiVersion: postgres.homelab.mortenolsen.pro/v1
kind: PostgresDatabase
metadata:
name: {{ .Release.Name }}
namespace: {{ .Release.Namespace }}
labels:
{{- include "common.labels" . | nindent 4 }}
spec:
clusterRef:
name: {{ .Values.globals.database.ref.name | quote }}
namespace: {{ .Values.globals.database.ref.namespace | quote }}
{{- end }}
{{- end }}
{{/*
Password generators for External Secrets (create these first)
*/}}
{{- define "common.externalSecrets.passwordGenerators" -}}
{{- if .Values.externalSecrets }}
{{- range .Values.externalSecrets }}
{{- $secretName := .name | default (printf "%s-%s" $.Release.Name "secrets") }}
{{- $secretName = $secretName | replace "{release}" $.Release.Name | replace "{fullname}" (include "common.fullname" $) }}
{{- range .passwords }}
---
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
metadata:
name: {{ $secretName }}-{{ .name }}-generator
namespace: {{ $.Release.Namespace }}
labels:
{{- include "common.labels" $ | nindent 4 }}
spec:
length: {{ .length | default 32 }}
allowRepeat: {{ .allowRepeat | default false }}
noUpper: {{ .noUpper | default false }}
{{- if .encoding }}
encoding: {{ .encoding }}
{{- end }}
{{- if .secretKeys }}
secretKeys:
{{- range .secretKeys }}
- {{ . }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{/*
External Secrets (create these after password generators)
*/}}
{{- define "common.externalSecrets.externalSecrets" -}}
{{- if .Values.externalSecrets }}
{{- range .Values.externalSecrets }}
{{- $secretName := .name | default (printf "%s-%s" $.Release.Name "secrets") }}
{{- $secretName = $secretName | replace "{release}" $.Release.Name | replace "{fullname}" (include "common.fullname" $) }}
---
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: {{ $secretName }}
namespace: {{ $.Release.Namespace }}
labels:
{{- include "common.labels" $ | nindent 4 }}
spec:
refreshInterval: 1h
target:
name: {{ $secretName }}
creationPolicy: Owner
dataFrom:
{{- range .passwords }}
- sourceRef:
generatorRef:
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
name: {{ $secretName }}-{{ .name }}-generator
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Full External Secrets resources (ExternalSecret + Password generators)
Combined helper that outputs generators first, then ExternalSecrets
*/}}
{{- define "common.externalSecrets" -}}
{{- include "common.externalSecrets.passwordGenerators" . }}
{{- include "common.externalSecrets.externalSecrets" . }}
{{- end }}

View File

@@ -26,11 +26,7 @@ spec:
- values.yaml
values: |
globals:
environment: {{ .Values.globals.environment }}
domain: {{ .Values.globals.domain }}
timezone: {{ .Values.globals.timezone }}
istio:
gateway: {{ .Values.globals.istio.gateway }}
{{ .Values.globals | toYaml | indent 14 }}
destination:
server: https://kubernetes.default.svc
namespace: prod

View File

@@ -7,6 +7,10 @@ globals:
environment: prod
domain: olsen.cloud
timezone: Europe/Amsterdam
database:
ref:
name: postgres
namespace: shared
istio:
gateway: shared/private
gateways:

335
scripts/migrate_database.py Executable file
View File

@@ -0,0 +1,335 @@
#!/usr/bin/env python3
"""
Database migration script for PostgreSQL databases in Kubernetes.
Migrates a database from the old PostgreSQL server (prod-postgres-cluster-0 in homelab namespace)
to the new PostgreSQL server (postgres-statefulset-0 in shared namespace).
Usage:
python3 migrate_database.py <source_db_name> <dest_db_name> [--clean]
Examples:
python3 migrate_database.py myapp myapp
python3 migrate_database.py oldname newname
python3 migrate_database.py prod_blinko prod_blinko --clean # Drop existing data first
WARNING: Without --clean flag, pg_restore will attempt to restore objects. If objects already
exist, it may fail with errors or cause data conflicts (duplicate key violations, etc.).
Use --clean to drop existing objects before restoring (this will DELETE all existing data).
"""
import subprocess
import sys
import argparse
import base64
import tempfile
import os
# Configuration
SOURCE_POD = "prod-postgres-cluster-0"
SOURCE_NAMESPACE = "homelab"
SOURCE_CONTAINER = "prod-postgres-cluster"
SOURCE_DEFAULT_USER = "homelab" # Default user for old server
DEST_POD = "postgres-statefulset-0"
DEST_NAMESPACE = "shared"
DEST_CONTAINER = "postgres"
DEST_DEFAULT_USER = "postgres" # Default user for new server
def run_kubectl_exec(pod, namespace, container, command, stdin=None, binary=False):
"""Execute a command in a Kubernetes pod using kubectl exec."""
cmd = [
"kubectl", "exec", "-n", namespace, pod,
"-c", container, "--"
] + command
try:
if stdin:
result = subprocess.run(
cmd,
input=stdin,
capture_output=True,
text=not binary,
check=True
)
else:
result = subprocess.run(
cmd,
capture_output=True,
text=not binary,
check=True
)
return result.stdout, result.stderr
except subprocess.CalledProcessError as e:
print(f"Error executing command: {' '.join(cmd)}", file=sys.stderr)
print(f"Exit code: {e.returncode}", file=sys.stderr)
if not binary:
print(f"stdout: {e.stdout}", file=sys.stderr)
print(f"stderr: {e.stderr}", file=sys.stderr)
raise
def check_pod_exists(pod, namespace):
"""Check if a pod exists and is running."""
try:
result = subprocess.run(
["kubectl", "get", "pod", pod, "-n", namespace],
capture_output=True,
text=True,
check=True
)
# Check if pod is running
if "Running" in result.stdout:
return True
return False
except subprocess.CalledProcessError:
return False
def check_database_exists(pod, namespace, container, db_name, pg_user):
"""Check if a database exists on a PostgreSQL server."""
psql_cmd = [
"psql", "-U", pg_user, "-tAc",
f"SELECT 1 FROM pg_database WHERE datname='{db_name}'"
]
try:
stdout, _ = run_kubectl_exec(pod, namespace, container, psql_cmd)
return stdout.strip() == "1"
except subprocess.CalledProcessError:
return False
def dump_database(pod, namespace, container, db_name, pg_user):
"""Dump a database from the source server."""
print(f"Dumping database '{db_name}' from source server...")
# Use pg_dump with stdout to avoid file I/O issues
dump_cmd = [
"pg_dump",
"-U", pg_user,
"-F", "c", # Custom format (compressed)
db_name
]
# Get the dump directly from stdout
dump_data, _ = run_kubectl_exec(pod, namespace, container, dump_cmd, binary=True)
if not dump_data or len(dump_data) == 0:
raise RuntimeError(f"Failed to dump database '{db_name}': dump is empty")
print(f"Successfully dumped database '{db_name}' ({len(dump_data)} bytes).")
return dump_data # Return as bytes for binary data
def fix_database_permissions(pod, namespace, container, db_name, db_user, pg_user):
"""Fix permissions for the database user after migration."""
print(f"Fixing permissions for user '{db_user}' on database '{db_name}'...")
# Get all schemas owned by the database user or that need permissions
fix_perms_cmd = [
"psql", "-U", pg_user, "-d", db_name, "-tAc",
f"""
DO $$
DECLARE
schema_rec RECORD;
BEGIN
-- Grant permissions on all schemas to the database user
FOR schema_rec IN
SELECT nspname
FROM pg_namespace
WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast', 'pg_toast_temp_1')
LOOP
-- Grant usage and create on schema
EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', schema_rec.nspname, '{db_user}');
EXECUTE format('GRANT CREATE ON SCHEMA %I TO %I', schema_rec.nspname, '{db_user}');
-- Grant privileges on all existing tables
EXECUTE format('GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I TO %I', schema_rec.nspname, '{db_user}');
-- Grant privileges on all existing sequences
EXECUTE format('GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA %I TO %I', schema_rec.nspname, '{db_user}');
-- Set default privileges for future objects
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON TABLES TO %I', schema_rec.nspname, '{db_user}');
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON SEQUENCES TO %I', schema_rec.nspname, '{db_user}');
-- Change ownership to database user if schema is not system-owned
IF schema_rec.nspname != 'public' THEN
EXECUTE format('ALTER SCHEMA %I OWNER TO %I', schema_rec.nspname, '{db_user}');
END IF;
END LOOP;
-- Grant database connection privilege
EXECUTE format('GRANT CONNECT ON DATABASE %I TO %I', '{db_name}', '{db_user}');
END $$;
"""
]
try:
run_kubectl_exec(pod, namespace, container, fix_perms_cmd)
print(f"Successfully fixed permissions for user '{db_user}'.")
except subprocess.CalledProcessError as e:
print(f"Warning: Could not fix all permissions automatically. You may need to fix them manually.", file=sys.stderr)
print(f"Error: {e.stderr}", file=sys.stderr)
def restore_database(pod, namespace, container, db_name, dump_data, pg_user, clean=False):
"""Restore a database dump to the destination server."""
print(f"Restoring database '{db_name}' to destination server...")
# For large dumps, use kubectl cp instead of base64 encoding to avoid command line length limits
# Write dump to a temporary file locally, then copy it to the pod
with tempfile.NamedTemporaryFile(delete=False, suffix='.dump') as tmp_file:
tmp_path = tmp_file.name
tmp_file.write(dump_data)
try:
# Copy dump file to pod using kubectl cp
print("Copying dump file to destination pod...")
cp_cmd = [
"kubectl", "cp", tmp_path,
f"{namespace}/{pod}:/tmp/dump.dump",
"-c", container
]
subprocess.run(cp_cmd, check=True, capture_output=True)
print("Dump file copied successfully.")
finally:
# Clean up local temporary file
try:
os.unlink(tmp_path)
except OSError:
pass
# Restore the dump
restore_cmd = [
"pg_restore",
"-U", pg_user,
"-d", db_name,
"-v", # Verbose
"--no-owner", # Don't restore ownership
"--no-acl", # Don't restore access privileges
]
# Add --clean flag if requested (drops existing objects before restoring)
if clean:
restore_cmd.append("--clean")
print("WARNING: Using --clean flag. This will drop all existing objects in the destination database!")
restore_cmd.append("/tmp/dump.dump")
try:
run_kubectl_exec(pod, namespace, container, restore_cmd)
print(f"Successfully restored database '{db_name}'.")
finally:
# Clean up remote dump file
rm_cmd = ["rm", "/tmp/dump.dump"]
try:
run_kubectl_exec(pod, namespace, container, rm_cmd)
except subprocess.CalledProcessError:
pass # Ignore cleanup errors
def main():
parser = argparse.ArgumentParser(
description="Migrate PostgreSQL database between Kubernetes pods",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
%(prog)s myapp myapp
%(prog)s oldname newname
%(prog)s source_db dest_db
"""
)
parser.add_argument(
"source_db",
help="Name of the source database (on old server: prod-postgres-cluster-0)"
)
parser.add_argument(
"dest_db",
help="Name of the destination database (on new server: postgres-statefulset-0)"
)
parser.add_argument(
"--source-user",
default=SOURCE_DEFAULT_USER,
help=f"PostgreSQL user for source server (default: {SOURCE_DEFAULT_USER})"
)
parser.add_argument(
"--dest-user",
default=DEST_DEFAULT_USER,
help=f"PostgreSQL user for destination server (default: {DEST_DEFAULT_USER})"
)
parser.add_argument(
"--clean",
action="store_true",
help="Drop existing objects before restoring (WARNING: This will delete all existing data in the destination database)"
)
args = parser.parse_args()
print("=" * 60)
print("PostgreSQL Database Migration Script")
print("=" * 60)
print(f"Source: {SOURCE_POD} ({SOURCE_NAMESPACE}) - Database: {args.source_db}")
print(f"Destination: {DEST_POD} ({DEST_NAMESPACE}) - Database: {args.dest_db}")
print("=" * 60)
print()
# Check if pods exist and are running
print("Checking source pod...")
if not check_pod_exists(SOURCE_POD, SOURCE_NAMESPACE):
print(f"ERROR: Source pod '{SOURCE_POD}' not found or not running in namespace '{SOURCE_NAMESPACE}'", file=sys.stderr)
sys.exit(1)
print(f"✓ Source pod '{SOURCE_POD}' is running")
print("Checking destination pod...")
if not check_pod_exists(DEST_POD, DEST_NAMESPACE):
print(f"ERROR: Destination pod '{DEST_POD}' not found or not running in namespace '{DEST_NAMESPACE}'", file=sys.stderr)
sys.exit(1)
print(f"✓ Destination pod '{DEST_POD}' is running")
print()
# Check if source database exists
print(f"Checking if source database '{args.source_db}' exists...")
if not check_database_exists(SOURCE_POD, SOURCE_NAMESPACE, SOURCE_CONTAINER, args.source_db, args.source_user):
print(f"ERROR: Source database '{args.source_db}' does not exist", file=sys.stderr)
sys.exit(1)
print(f"✓ Source database '{args.source_db}' exists")
# Check if destination database exists
print(f"Checking if destination database '{args.dest_db}' exists...")
if not check_database_exists(DEST_POD, DEST_NAMESPACE, DEST_CONTAINER, args.dest_db, args.dest_user):
print(f"ERROR: Destination database '{args.dest_db}' does not exist. Please create it first.", file=sys.stderr)
sys.exit(1)
print(f"✓ Destination database '{args.dest_db}' exists")
print()
# Dump source database
dump_data = dump_database(SOURCE_POD, SOURCE_NAMESPACE, SOURCE_CONTAINER, args.source_db, args.source_user)
print()
# Restore to destination database
restore_database(DEST_POD, DEST_NAMESPACE, DEST_CONTAINER, args.dest_db, dump_data, args.dest_user, clean=args.clean)
print()
# Fix permissions for the database user (assuming db user name matches db name)
# This fixes schema ownership and permissions that were lost due to --no-owner and --no-acl
fix_database_permissions(DEST_POD, DEST_NAMESPACE, DEST_CONTAINER, args.dest_db, args.dest_db, args.dest_user)
print()
print("=" * 60)
print("Migration completed successfully!")
print("=" * 60)
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
print("\n\nMigration interrupted by user.", file=sys.stderr)
sys.exit(1)
except Exception as e:
print(f"\n\nERROR: {e}", file=sys.stderr)
sys.exit(1)