From d735f5e31002400ef7eb657d5f99a6920f187b3a Mon Sep 17 00:00:00 2001 From: Peter Smit Date: Sun, 8 Mar 2026 16:05:21 +0100 Subject: [PATCH] Add central postgres instance --- central-database/.env.example | 11 ++ central-database/Dockerfile | 43 ++++++ central-database/MIGRATION_GUIDE.md | 215 ++++++++++++++++++++++++++++ central-database/README.md | 145 +++++++++++++++++++ central-database/docker-compose.yml | 23 +++ central-database/init-postgres.sh | 39 +++++ central-database/migrate_service.sh | 106 ++++++++++++++ 7 files changed, 582 insertions(+) create mode 100644 central-database/.env.example create mode 100644 central-database/Dockerfile create mode 100644 central-database/MIGRATION_GUIDE.md create mode 100644 central-database/README.md create mode 100644 central-database/docker-compose.yml create mode 100755 central-database/init-postgres.sh create mode 100755 central-database/migrate_service.sh diff --git a/central-database/.env.example b/central-database/.env.example new file mode 100644 index 0000000..1431900 --- /dev/null +++ b/central-database/.env.example @@ -0,0 +1,11 @@ +POSTGRES_ADMIN_PASSWORD= + +MATRIX_SYNAPSE_USER_PASSWORD= +MATRIX_MAS_USER_PASSWORD= +MATRIX_SIGNAL_USER_PASSWORD= +MATRIX_WHATSAPP_USER_PASSWORD= +PAPERLESS_USER_PASSWORD= +IMMICH_USER_PASSWORD= +SHLINK_USER_PASSWORD= +SPLIIT_USER_PASSWORD= +AUDIOMUSE_USER_PASSWORD= diff --git a/central-database/Dockerfile b/central-database/Dockerfile new file mode 100644 index 0000000..2620f4c --- /dev/null +++ b/central-database/Dockerfile @@ -0,0 +1,43 @@ +# PostgreSQL 18 with pgvector and vectorchord extensions +FROM postgres:18-alpine + +# Install build dependencies +RUN apk add --no-cache --virtual .build-deps \ + gcc \ + musl-dev \ + postgresql-dev \ + make \ + git \ + cmake \ + clang \ + openssl-dev \ + libxml2-dev \ + libxslt-dev \ + python3 \ + py3-pip \ + build-base + +# Install pgvector extension +RUN git clone --branch v0.7.3 https://github.com/pgvector/pgvector.git /tmp/pgvector \ + && cd /tmp/pgvector \ + && make \ + && make install + +# Install vectorchord extension (for Immich) +RUN git clone --branch v0.3.0 https://github.com/vectorchord/pg_vectorscale.git /tmp/pg_vectorscale \ + && cd /tmp/pg_vectorscale \ + && make \ + && make install + +# Clean up build dependencies +RUN apk del .build-deps \ + && rm -rf /tmp/* \ + && rm -rf /var/lib/apt/lists/* + +# Configure PostgreSQL to load extensions +RUN echo "shared_preload_libraries = 'pg_vectorscale,vector'" >> /var/lib/postgresql/data/postgresql.conf + +COPY init-postgres.sh /docker-entrypoint-initdb.d/ + +EXPOSE 5432 +CMD ["postgres"] diff --git a/central-database/MIGRATION_GUIDE.md b/central-database/MIGRATION_GUIDE.md new file mode 100644 index 0000000..e7482d0 --- /dev/null +++ b/central-database/MIGRATION_GUIDE.md @@ -0,0 +1,215 @@ +# Service Configuration Migration Guide + +This guide shows how to update each service to use the central PostgreSQL database. + +## Service-Specific Configuration Updates + +### 1. Synapse Matrix + +**Current configuration in `synapse/docker-compose.yml`:** +```yaml +synapse_db: + image: docker.io/library/postgres:17 + restart: always + volumes: + - ${SYNAPSE_POSTGRES_DATA_DIR}:/var/lib/postgresql/data + environment: + POSTGRES_DB: ${SYNAPSE_POSTGRES_DB} + POSTGRES_USER: ${SYNAPSE_POSTGRES_USER} + POSTGRES_PASSWORD: ${SYNAPSE_POSTGRES_PASSWORD} + POSTGRES_INITDB_ARGS: '--encoding=UTF-8 --locale=C' + ports: + - 5442:5432 + +synapse: + # ... other config ... + environment: + POSTGRES_HOST: synapse_db + POSTGRES_PORT: 5432 + POSTGRES_DB: ${SYNAPSE_POSTGRES_DB} + POSTGRES_USER: ${SYNAPSE_POSTGRES_USER} + POSTGRES_PASSWORD: ${SYNAPSE_POSTGRES_PASSWORD} +``` + +**Updated configuration:** +```yaml +# Remove the synapse_db service entirely + +synapse: + # ... other config ... + environment: + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + POSTGRES_DB: synapse + POSTGRES_USER: synapse_user + POSTGRES_PASSWORD: ${SYNAPSE_POSTGRES_PASSWORD} + depends_on: + - postgres # Add this dependency + networks: + - default # Ensure same network as postgres container +``` + +### 2. Paperless + +**Current configuration in `paperless/docker-compose.yml`:** +```yaml +db: + image: docker.io/library/postgres:17 + restart: always + volumes: + - ${PAPERLESS_POSTGRES_DATA_DIR}:/var/lib/postgresql/data + environment: + POSTGRES_DB: ${PAPERLESS_POSTGRES_DB} + POSTGRES_USER: ${PAPERLESS_POSTGRES_USER} + POSTGRES_PASSWORD: ${PAPERLESS_POSTGRES_PASSWORD} + ports: + - 5434:5432 + +paperless: + # ... other config ... + environment: + PAPERLESS_DBHOST: db + PAPERLESS_DBNAME: ${PAPERLESS_POSTGRES_DB} + PAPERLESS_DBUSER: ${PAPERLESS_POSTGRES_USER} + PAPERLESS_DBPASS: ${PAPERLESS_POSTGRES_PASSWORD} +``` + +**Updated configuration:** +```yaml +# Remove the db service entirely + +paperless: + # ... other config ... + environment: + PAPERLESS_DBHOST: postgres + PAPERLESS_DBNAME: paperless + PAPERLESS_DBUSER: paperless_user + PAPERLESS_DBPASS: ${PAPERLESS_POSTGRES_PASSWORD} + depends_on: + - postgres # Add this dependency + networks: + - default +``` + +### 3. Immich + +**Current configuration in `immich/docker-compose.yml`:** +```yaml +database: + container_name: immich_postgres + image: ghcr.io/immich-app/postgres:14-vectorchord0.3.0-pgvectors0.2.0 + environment: + POSTGRES_USER: ${IMMICH_POSTGRES_USER} + POSTGRES_PASSWORD: ${IMMICH_POSTGRES_PASSWORD} + POSTGRES_DB: ${IMMICH_POSTGRES_DB} + POSTGRES_INITDB_ARGS: '--data-checksums' + ports: + - 5433:5432 + volumes: + - ${IMMICH_DB_LOCATION}:/var/lib/postgresql/data + +immich-server: + # ... other config ... + environment: + DB_HOSTNAME: ${IMMICH_DB_HOSTNAME} + DB_USERNAME: ${IMMICH_POSTGRES_USER} + DB_PASSWORD: ${IMMICH_POSTGRES_PASSWORD} + DB_DATABASE_NAME: ${IMMICH_POSTGRES_DB} +``` + +**Updated configuration:** +```yaml +# Remove the database service entirely + +immich-server: + # ... other config ... + environment: + DB_HOSTNAME: postgres + DB_USERNAME: immich_user + DB_PASSWORD: ${IMMICH_POSTGRES_PASSWORD} + DB_DATABASE_NAME: immich + depends_on: + - postgres # Add this dependency + networks: + - default +``` + +### 4. Shlink + +**Current configuration in `shlink/docker-compose.yml`:** +```yaml +shlink_db: + image: postgres:17 + container_name: shlink_db + restart: always + volumes: + - ${SHLINK_POSTGRES_DIR}:/var/lib/postgresql/data + environment: + POSTGRES_DB: ${SHLINK_POSTGRES_DB} + POSTGRES_USER: ${SHLINK_POSTGRES_USER} + POSTGRES_PASSWORD: ${SHLINK_POSTGRES_PASSWORD} + ports: + - 5436:5432 + +shlink: + # ... other config ... + environment: + DB_HOST: shlink_db + DB_NAME: ${SHLINK_POSTGRES_DB} + DB_USER: ${SHLINK_POSTGRES_USER} + DB_PASSWORD: ${SHLINK_POSTGRES_PASSWORD} +``` + +**Updated configuration:** +```yaml +# Remove the shlink_db service entirely + +shlink: + # ... other config ... + environment: + DB_HOST: postgres + DB_NAME: shlink + DB_USER: shlink_user + DB_PASSWORD: ${SHLINK_POSTGRES_PASSWORD} + depends_on: + - postgres # Add this dependency + networks: + - default +``` + +## General Pattern + +For each service: + +1. **Remove** the service-specific database container +2. **Update** the main service container's environment variables: + - `POSTGRES_HOST`/`DB_HOST` → `postgres` + - `POSTGRES_PORT`/`DB_PORT` → `5432` + - `POSTGRES_DB`/`DB_NAME` → `` (e.g., `synapse`, `paperless`) + - `POSTGRES_USER`/`DB_USER` → `_user` + - `POSTGRES_PASSWORD`/`DB_PASSWORD` → `${SERVICE_POSTGRES_PASSWORD}` (keep existing) + +3. **Add dependency** on `postgres` service +4. **Update network** configuration if needed (ensure services can reach postgres container) + +## Testing Migration + +After updating each service: + +1. **Stop the service**: `docker-compose down` +2. **Start with new config**: `docker-compose up -d` +3. **Check logs**: `docker-compose logs -f` +4. **Test functionality**: Verify the service works correctly +5. **Rollback plan**: Keep old database container until migration is confirmed successful + +## Network Considerations + +Ensure all services are on the same Docker network or can reach the `central_postgres` container. You may need to: + +1. Add services to the same network +2. Use Docker's internal DNS +3. Configure network aliases if needed + +## Password Management + +Keep using the same password variables but ensure they match the central database user passwords. The migration script preserves existing passwords for seamless transition. \ No newline at end of file diff --git a/central-database/README.md b/central-database/README.md new file mode 100644 index 0000000..dcea471 --- /dev/null +++ b/central-database/README.md @@ -0,0 +1,145 @@ +# Central PostgreSQL Database Consolidation + +This directory contains the configuration for consolidating all PostgreSQL databases into a single PostgreSQL 18 instance with schema isolation. + +## Architecture + +### Before (Multiple Containers) +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Synapse DB │ │ Paperless DB │ │ Immich DB │ +│ PostgreSQL 17 │ │ PostgreSQL 17 │ │ PostgreSQL 14 │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + Port 5442 Port 5434 Port 5433 +``` + +### After (Single Container) +``` +┌─────────────────────────────────────────────────────┐ +│ PostgreSQL 18 (container name: postgres) │ +│ │ +│ ┌─────────────────┐ ┌─────────────────┐ │ +│ │ synapse │ │ paperless │ │ +│ │ database │ │ database │ │ +│ └─────────────────┘ └─────────────────┘ │ +│ │ +│ ┌─────────────────┐ ┌─────────────────┐ │ +│ │ immich │ │ shlink │ │ +│ │ database │ │ database │ │ +│ └─────────────────┘ └─────────────────┘ │ +│ │ +│ Extensions: vector, pg_vectorscale │ +└─────────────────────────────────────────────────────┘ + Port 5432 +``` + +## Setup Instructions + +### 1. Setup Environment +```bash +cd central-database +cp .env.template .env +# Edit .env with your actual passwords (do not commit this file) +``` + +### 2. Build and Start PostgreSQL Container +```bash +docker-compose up -d --build postgres +``` + +### 3. Verify Database is Running +```bash +docker-compose logs -f postgres +docker-compose exec postgres psql -U admin -c "\l" +``` + +### 4. Migrate Services +Use the generic migration script for each service: + +```bash +# Set environment variables (or use .env file) +export SYNAPSE_POSTGRES_PASSWORD="your_actual_password" + +# Example: Migrate Synapse +./migrate_service.sh synapse localhost 5442 synapse synapse_password synapse + +# Example: Migrate Paperless +./migrate_service.sh paperless localhost 5434 paperless paperless_password paperless +``` + +### 5. Update Service Configurations +After migration, update each service's docker-compose.yml to point to the central postgres container: + +```yaml +# Example for Synapse +environment: + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + POSTGRES_DB: synapse + POSTGRES_USER: synapse_user + POSTGRES_PASSWORD: ${SYNAPSE_POSTGRES_PASSWORD} +``` + +## Migration Order Recommendation +1. **Non-critical services first**: shlink, spliit +2. **Document management**: paperless +3. **Media services**: immich, audiomuse +4. **Matrix ecosystem**: mas, signal, whatsapp, synapse + +## Backup Strategy + +### Full Backup (all databases) +```bash +docker-compose exec postgres pg_dumpall -U admin > full_backup_$(date +%Y%m%d).sql +``` + +### Individual Database Backup +```bash +# Backup specific database +docker-compose exec postgres pg_dump -U synapse_user -d synapse > synapse_backup_$(date +%Y%m%d).sql + +# Backup all databases individually +for db in synapse mas signal whatsapp paperless immich shlink spliit audiomuse; do + docker-compose exec postgres pg_dump -U ${db}_user -d $db > ${db}_backup_$(date +%Y%m%d).sql +done +``` + +### Automated Backup (add to cron) +```bash +#!/bin/bash +# Daily backup script +BACKUP_DIR="/backups/postgres" +mkdir -p $BACKUP_DIR + +# Full backup +docker-compose -f /path/to/central-database/docker-compose.yml exec postgres pg_dumpall -U admin | gzip > $BACKUP_DIR/full_backup_$(date +%Y%m%d_%H%M%S).sql.gz + +# Individual database backups +for db in synapse mas signal whatsapp paperless immich shlink spliit audiomuse; do + docker-compose -f /path/to/central-database/docker-compose.yml exec postgres pg_dump -U ${db}_user -d $db | gzip > $BACKUP_DIR/${db}_backup_$(date +%Y%m%d_%H%M%S).sql.gz +done + +# Keep last 7 days +find $BACKUP_DIR -name "*.sql.gz" -mtime +7 -delete +``` + +## Troubleshooting + +### Connection Issues +- Verify central database is running: `docker-compose ps` +- Check logs: `docker-compose logs central_postgres` +- Test connection: `psql -h localhost -p 5432 -U admin -d central_db` + +### Permission Issues +- Ensure service users have correct permissions on their schemas +- Check schema ownership: `\dn+` in psql + +### Extension Issues +- Verify extensions are loaded: `\dx` in psql +- Check PostgreSQL logs for extension errors + +## Security Notes +- Use strong passwords for all service users +- Rotate passwords after migration +- Consider using PostgreSQL role attributes for additional security +- Enable SSL for production environments \ No newline at end of file diff --git a/central-database/docker-compose.yml b/central-database/docker-compose.yml new file mode 100644 index 0000000..0292b11 --- /dev/null +++ b/central-database/docker-compose.yml @@ -0,0 +1,23 @@ +services: + postgres: + build: . + container_name: postgres + restart: always + volumes: + - postgres_data:/var/lib/postgresql/data + environment: + POSTGRES_USER: admin + POSTGRES_PASSWORD: ${POSTGRES_ADMIN_PASSWORD} + ports: + - "5432:5432" + networks: + - default + healthcheck: + test: ["CMD-SHELL", "pg_isready -U admin"] + interval: 5s + timeout: 5s + retries: 5 + +volumes: + postgres_data: + driver: local diff --git a/central-database/init-postgres.sh b/central-database/init-postgres.sh new file mode 100755 index 0000000..6b0e3b0 --- /dev/null +++ b/central-database/init-postgres.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -e + +echo "Waiting for PostgreSQL to start..." +until pg_isready -U admin -h localhost; do + sleep 2 +done + +echo "PostgreSQL is ready. Creating databases and users..." + +# Create databases +for db in matrix_synapse matrix_mas matrix_signal matrix_whatsapp paperless immich shlink spliit audiomuse; do + echo "Creating database: $db" + createdb -U admin "$db" +done + +# Create users and set permissions +for db in matrix_synapse matrix_mas matrix_signal matrix_whatsapp paperless immich shlink spliit audiomuse; do + user="${db}_user" + password_var="${user^^}_PASSWORD" + password=${!password_var} + + if [ -z "$password" ]; then + echo "Warning: Password for $user not set. Using default." + password="default_password" + fi + + echo "Creating user: $user" + psql -U admin -c "CREATE USER $user WITH PASSWORD '$password';" + psql -U admin -c "GRANT ALL PRIVILEGES ON DATABASE $db TO $user;" + psql -U admin -c "ALTER DATABASE $db OWNER TO $user;" +done + +# Enable extensions in immich database +echo "Enabling extensions in immich database..." +psql -U admin -d immich -c "CREATE EXTENSION IF NOT EXISTS vector;" +psql -U admin -d immich -c "CREATE EXTENSION IF NOT EXISTS pg_vectorscale;" + +echo "PostgreSQL initialization complete!" diff --git a/central-database/migrate_service.sh b/central-database/migrate_service.sh new file mode 100755 index 0000000..6d8f032 --- /dev/null +++ b/central-database/migrate_service.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# Generic PostgreSQL service migration script +# Usage: ./migrate_service.sh + +set -e + +if [ "$#" -ne 6 ]; then + echo "Usage: $0 " + exit 1 +fi + +SERVICE_NAME=$1 +OLD_HOST=$2 +OLD_PORT=$3 +OLD_USER=$4 +OLD_PASSWORD=$5 +OLD_DB=$6 + +# Map service names to database names and users +case $SERVICE_NAME in + synapse) + NEW_DB="synapse" + NEW_USER="synapse_user" + NEW_PASSWORD="${SYNAPSE_POSTGRES_PASSWORD}" + ;; + mas) + NEW_DB="mas" + NEW_USER="mas_user" + NEW_PASSWORD="${MAS_POSTGRES_PASSWORD}" + ;; + signal) + NEW_DB="signal" + NEW_USER="signal_user" + NEW_PASSWORD="${MAUTRIX_SIGNAL_POSTGRES_PASSWORD}" + ;; + whatsapp) + NEW_DB="whatsapp" + NEW_USER="whatsapp_user" + NEW_PASSWORD="${MAUTRIX_WHATSAPP_POSTGRES_PASSWORD}" + ;; + paperless) + NEW_DB="paperless" + NEW_USER="paperless_user" + NEW_PASSWORD="${PAPERLESS_POSTGRES_PASSWORD}" + ;; + immich) + NEW_DB="immich" + NEW_USER="immich_user" + NEW_PASSWORD="${IMMICH_POSTGRES_PASSWORD}" + ;; + shlink) + NEW_DB="shlink" + NEW_USER="shlink_user" + NEW_PASSWORD="${SHLINK_POSTGRES_PASSWORD}" + ;; + spliit) + NEW_DB="spliit" + NEW_USER="spliit_user" + NEW_PASSWORD="${SPLIIT_POSTGRES_PASSWORD}" + ;; + audiomuse) + NEW_DB="audiomuse" + NEW_USER="audiomuse_user" + NEW_PASSWORD="${AUDIOMUSE_POSTGRES_PASSWORD}" + ;; + *) + echo "Unknown service: $SERVICE_NAME" + exit 1 + ;; +esac + +echo "Starting migration for $SERVICE_NAME to database $NEW_DB..." + +# Export from old database +echo "Exporting data from $OLD_DB..." +PGPASSWORD="$OLD_PASSWORD" pg_dump \ + -h "$OLD_HOST" \ + -p "$OLD_PORT" \ + -U "$OLD_USER" \ + -d "$OLD_DB" \ + -F custom \ + -f "/tmp/${SERVICE_NAME}_dump.custom" + +# Import to new database +echo "Importing data to $NEW_DB database..." +PGPASSWORD="$NEW_PASSWORD" pg_restore \ + -h localhost \ + -p 5432 \ + -U "$NEW_USER" \ + -d "$NEW_DB" \ + --clean \ + --if-exists \ + "/tmp/${SERVICE_NAME}_dump.custom" + +echo "Migration completed for $SERVICE_NAME!" + +# Clean up +rm -f "/tmp/${SERVICE_NAME}_dump.custom" + +echo "You can now update your $SERVICE_NAME service configuration to use:" +echo " Host: localhost" +echo " Port: 5432" +echo " Database: $NEW_DB" +echo " User: $NEW_USER" +echo " Password: ***" \ No newline at end of file