6 Commits

10 changed files with 589 additions and 7 deletions

View File

@@ -31,7 +31,7 @@ services:
# AudioMuse-AI Flask application service # AudioMuse-AI Flask application service
audiomuse-ai-flask: audiomuse-ai-flask:
image: ghcr.io/neptunehub/audiomuse-ai:0.8.9 # Reflects deployment.yaml image: ghcr.io/neptunehub/audiomuse-ai:0.9.2 # Reflects deployment.yaml
container_name: audiomuse-ai-flask-app container_name: audiomuse-ai-flask-app
ports: ports:
- "8013:8000" - "8013:8000"
@@ -69,7 +69,7 @@ services:
# AudioMuse-AI RQ Worker service # AudioMuse-AI RQ Worker service
audiomuse-ai-worker: audiomuse-ai-worker:
image: ghcr.io/neptunehub/audiomuse-ai:0.8.9 # Reflects deployment.yaml image: ghcr.io/neptunehub/audiomuse-ai:0.9.2 # Reflects deployment.yaml
container_name: audiomuse-ai-worker-instance container_name: audiomuse-ai-worker-instance
env_file: env_file:
- .env - .env

View File

@@ -0,0 +1,11 @@
POSTGRES_ADMIN_PASSWORD=
MATRIX_SYNAPSE_USER_PASSWORD=
MATRIX_MAS_USER_PASSWORD=
MATRIX_SIGNAL_USER_PASSWORD=
MATRIX_WHATSAPP_USER_PASSWORD=
PAPERLESS_USER_PASSWORD=
IMMICH_USER_PASSWORD=
SHLINK_USER_PASSWORD=
SPLIIT_USER_PASSWORD=
AUDIOMUSE_USER_PASSWORD=

View File

@@ -0,0 +1,43 @@
# PostgreSQL 18 with pgvector and vectorchord extensions
FROM postgres:18-alpine
# Install build dependencies
RUN apk add --no-cache --virtual .build-deps \
gcc \
musl-dev \
postgresql-dev \
make \
git \
cmake \
clang \
openssl-dev \
libxml2-dev \
libxslt-dev \
python3 \
py3-pip \
build-base
# Install pgvector extension
RUN git clone --branch v0.7.3 https://github.com/pgvector/pgvector.git /tmp/pgvector \
&& cd /tmp/pgvector \
&& make \
&& make install
# Install vectorchord extension (for Immich)
RUN git clone --branch v0.3.0 https://github.com/vectorchord/pg_vectorscale.git /tmp/pg_vectorscale \
&& cd /tmp/pg_vectorscale \
&& make \
&& make install
# Clean up build dependencies
RUN apk del .build-deps \
&& rm -rf /tmp/* \
&& rm -rf /var/lib/apt/lists/*
# Configure PostgreSQL to load extensions
RUN echo "shared_preload_libraries = 'pg_vectorscale,vector'" >> /var/lib/postgresql/data/postgresql.conf
COPY init-postgres.sh /docker-entrypoint-initdb.d/
EXPOSE 5432
CMD ["postgres"]

View File

@@ -0,0 +1,215 @@
# Service Configuration Migration Guide
This guide shows how to update each service to use the central PostgreSQL database.
## Service-Specific Configuration Updates
### 1. Synapse Matrix
**Current configuration in `synapse/docker-compose.yml`:**
```yaml
synapse_db:
image: docker.io/library/postgres:17
restart: always
volumes:
- ${SYNAPSE_POSTGRES_DATA_DIR}:/var/lib/postgresql/data
environment:
POSTGRES_DB: ${SYNAPSE_POSTGRES_DB}
POSTGRES_USER: ${SYNAPSE_POSTGRES_USER}
POSTGRES_PASSWORD: ${SYNAPSE_POSTGRES_PASSWORD}
POSTGRES_INITDB_ARGS: '--encoding=UTF-8 --locale=C'
ports:
- 5442:5432
synapse:
# ... other config ...
environment:
POSTGRES_HOST: synapse_db
POSTGRES_PORT: 5432
POSTGRES_DB: ${SYNAPSE_POSTGRES_DB}
POSTGRES_USER: ${SYNAPSE_POSTGRES_USER}
POSTGRES_PASSWORD: ${SYNAPSE_POSTGRES_PASSWORD}
```
**Updated configuration:**
```yaml
# Remove the synapse_db service entirely
synapse:
# ... other config ...
environment:
POSTGRES_HOST: postgres
POSTGRES_PORT: 5432
POSTGRES_DB: synapse
POSTGRES_USER: synapse_user
POSTGRES_PASSWORD: ${SYNAPSE_POSTGRES_PASSWORD}
depends_on:
- postgres # Add this dependency
networks:
- default # Ensure same network as postgres container
```
### 2. Paperless
**Current configuration in `paperless/docker-compose.yml`:**
```yaml
db:
image: docker.io/library/postgres:17
restart: always
volumes:
- ${PAPERLESS_POSTGRES_DATA_DIR}:/var/lib/postgresql/data
environment:
POSTGRES_DB: ${PAPERLESS_POSTGRES_DB}
POSTGRES_USER: ${PAPERLESS_POSTGRES_USER}
POSTGRES_PASSWORD: ${PAPERLESS_POSTGRES_PASSWORD}
ports:
- 5434:5432
paperless:
# ... other config ...
environment:
PAPERLESS_DBHOST: db
PAPERLESS_DBNAME: ${PAPERLESS_POSTGRES_DB}
PAPERLESS_DBUSER: ${PAPERLESS_POSTGRES_USER}
PAPERLESS_DBPASS: ${PAPERLESS_POSTGRES_PASSWORD}
```
**Updated configuration:**
```yaml
# Remove the db service entirely
paperless:
# ... other config ...
environment:
PAPERLESS_DBHOST: postgres
PAPERLESS_DBNAME: paperless
PAPERLESS_DBUSER: paperless_user
PAPERLESS_DBPASS: ${PAPERLESS_POSTGRES_PASSWORD}
depends_on:
- postgres # Add this dependency
networks:
- default
```
### 3. Immich
**Current configuration in `immich/docker-compose.yml`:**
```yaml
database:
container_name: immich_postgres
image: ghcr.io/immich-app/postgres:14-vectorchord0.3.0-pgvectors0.2.0
environment:
POSTGRES_USER: ${IMMICH_POSTGRES_USER}
POSTGRES_PASSWORD: ${IMMICH_POSTGRES_PASSWORD}
POSTGRES_DB: ${IMMICH_POSTGRES_DB}
POSTGRES_INITDB_ARGS: '--data-checksums'
ports:
- 5433:5432
volumes:
- ${IMMICH_DB_LOCATION}:/var/lib/postgresql/data
immich-server:
# ... other config ...
environment:
DB_HOSTNAME: ${IMMICH_DB_HOSTNAME}
DB_USERNAME: ${IMMICH_POSTGRES_USER}
DB_PASSWORD: ${IMMICH_POSTGRES_PASSWORD}
DB_DATABASE_NAME: ${IMMICH_POSTGRES_DB}
```
**Updated configuration:**
```yaml
# Remove the database service entirely
immich-server:
# ... other config ...
environment:
DB_HOSTNAME: postgres
DB_USERNAME: immich_user
DB_PASSWORD: ${IMMICH_POSTGRES_PASSWORD}
DB_DATABASE_NAME: immich
depends_on:
- postgres # Add this dependency
networks:
- default
```
### 4. Shlink
**Current configuration in `shlink/docker-compose.yml`:**
```yaml
shlink_db:
image: postgres:17
container_name: shlink_db
restart: always
volumes:
- ${SHLINK_POSTGRES_DIR}:/var/lib/postgresql/data
environment:
POSTGRES_DB: ${SHLINK_POSTGRES_DB}
POSTGRES_USER: ${SHLINK_POSTGRES_USER}
POSTGRES_PASSWORD: ${SHLINK_POSTGRES_PASSWORD}
ports:
- 5436:5432
shlink:
# ... other config ...
environment:
DB_HOST: shlink_db
DB_NAME: ${SHLINK_POSTGRES_DB}
DB_USER: ${SHLINK_POSTGRES_USER}
DB_PASSWORD: ${SHLINK_POSTGRES_PASSWORD}
```
**Updated configuration:**
```yaml
# Remove the shlink_db service entirely
shlink:
# ... other config ...
environment:
DB_HOST: postgres
DB_NAME: shlink
DB_USER: shlink_user
DB_PASSWORD: ${SHLINK_POSTGRES_PASSWORD}
depends_on:
- postgres # Add this dependency
networks:
- default
```
## General Pattern
For each service:
1. **Remove** the service-specific database container
2. **Update** the main service container's environment variables:
- `POSTGRES_HOST`/`DB_HOST``postgres`
- `POSTGRES_PORT`/`DB_PORT``5432`
- `POSTGRES_DB`/`DB_NAME``<service_name>` (e.g., `synapse`, `paperless`)
- `POSTGRES_USER`/`DB_USER``<service_name>_user`
- `POSTGRES_PASSWORD`/`DB_PASSWORD``${SERVICE_POSTGRES_PASSWORD}` (keep existing)
3. **Add dependency** on `postgres` service
4. **Update network** configuration if needed (ensure services can reach postgres container)
## Testing Migration
After updating each service:
1. **Stop the service**: `docker-compose down`
2. **Start with new config**: `docker-compose up -d`
3. **Check logs**: `docker-compose logs -f`
4. **Test functionality**: Verify the service works correctly
5. **Rollback plan**: Keep old database container until migration is confirmed successful
## Network Considerations
Ensure all services are on the same Docker network or can reach the `central_postgres` container. You may need to:
1. Add services to the same network
2. Use Docker's internal DNS
3. Configure network aliases if needed
## Password Management
Keep using the same password variables but ensure they match the central database user passwords. The migration script preserves existing passwords for seamless transition.

145
central-database/README.md Normal file
View File

@@ -0,0 +1,145 @@
# Central PostgreSQL Database Consolidation
This directory contains the configuration for consolidating all PostgreSQL databases into a single PostgreSQL 18 instance with schema isolation.
## Architecture
### Before (Multiple Containers)
```
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
│ Synapse DB │ │ Paperless DB │ │ Immich DB │
│ PostgreSQL 17 │ │ PostgreSQL 17 │ │ PostgreSQL 14 │
└─────────────────┘ └─────────────────┘ └─────────────────┘
Port 5442 Port 5434 Port 5433
```
### After (Single Container)
```
┌─────────────────────────────────────────────────────┐
│ PostgreSQL 18 (container name: postgres) │
│ │
│ ┌─────────────────┐ ┌─────────────────┐ │
│ │ synapse │ │ paperless │ │
│ │ database │ │ database │ │
│ └─────────────────┘ └─────────────────┘ │
│ │
│ ┌─────────────────┐ ┌─────────────────┐ │
│ │ immich │ │ shlink │ │
│ │ database │ │ database │ │
│ └─────────────────┘ └─────────────────┘ │
│ │
│ Extensions: vector, pg_vectorscale │
└─────────────────────────────────────────────────────┘
Port 5432
```
## Setup Instructions
### 1. Setup Environment
```bash
cd central-database
cp .env.template .env
# Edit .env with your actual passwords (do not commit this file)
```
### 2. Build and Start PostgreSQL Container
```bash
docker-compose up -d --build postgres
```
### 3. Verify Database is Running
```bash
docker-compose logs -f postgres
docker-compose exec postgres psql -U admin -c "\l"
```
### 4. Migrate Services
Use the generic migration script for each service:
```bash
# Set environment variables (or use .env file)
export SYNAPSE_POSTGRES_PASSWORD="your_actual_password"
# Example: Migrate Synapse
./migrate_service.sh synapse localhost 5442 synapse synapse_password synapse
# Example: Migrate Paperless
./migrate_service.sh paperless localhost 5434 paperless paperless_password paperless
```
### 5. Update Service Configurations
After migration, update each service's docker-compose.yml to point to the central postgres container:
```yaml
# Example for Synapse
environment:
POSTGRES_HOST: postgres
POSTGRES_PORT: 5432
POSTGRES_DB: synapse
POSTGRES_USER: synapse_user
POSTGRES_PASSWORD: ${SYNAPSE_POSTGRES_PASSWORD}
```
## Migration Order Recommendation
1. **Non-critical services first**: shlink, spliit
2. **Document management**: paperless
3. **Media services**: immich, audiomuse
4. **Matrix ecosystem**: mas, signal, whatsapp, synapse
## Backup Strategy
### Full Backup (all databases)
```bash
docker-compose exec postgres pg_dumpall -U admin > full_backup_$(date +%Y%m%d).sql
```
### Individual Database Backup
```bash
# Backup specific database
docker-compose exec postgres pg_dump -U synapse_user -d synapse > synapse_backup_$(date +%Y%m%d).sql
# Backup all databases individually
for db in synapse mas signal whatsapp paperless immich shlink spliit audiomuse; do
docker-compose exec postgres pg_dump -U ${db}_user -d $db > ${db}_backup_$(date +%Y%m%d).sql
done
```
### Automated Backup (add to cron)
```bash
#!/bin/bash
# Daily backup script
BACKUP_DIR="/backups/postgres"
mkdir -p $BACKUP_DIR
# Full backup
docker-compose -f /path/to/central-database/docker-compose.yml exec postgres pg_dumpall -U admin | gzip > $BACKUP_DIR/full_backup_$(date +%Y%m%d_%H%M%S).sql.gz
# Individual database backups
for db in synapse mas signal whatsapp paperless immich shlink spliit audiomuse; do
docker-compose -f /path/to/central-database/docker-compose.yml exec postgres pg_dump -U ${db}_user -d $db | gzip > $BACKUP_DIR/${db}_backup_$(date +%Y%m%d_%H%M%S).sql.gz
done
# Keep last 7 days
find $BACKUP_DIR -name "*.sql.gz" -mtime +7 -delete
```
## Troubleshooting
### Connection Issues
- Verify central database is running: `docker-compose ps`
- Check logs: `docker-compose logs central_postgres`
- Test connection: `psql -h localhost -p 5432 -U admin -d central_db`
### Permission Issues
- Ensure service users have correct permissions on their schemas
- Check schema ownership: `\dn+` in psql
### Extension Issues
- Verify extensions are loaded: `\dx` in psql
- Check PostgreSQL logs for extension errors
## Security Notes
- Use strong passwords for all service users
- Rotate passwords after migration
- Consider using PostgreSQL role attributes for additional security
- Enable SSL for production environments

View File

@@ -0,0 +1,23 @@
services:
postgres:
build: .
container_name: postgres
restart: always
volumes:
- postgres_data:/var/lib/postgresql/data
environment:
POSTGRES_USER: admin
POSTGRES_PASSWORD: ${POSTGRES_ADMIN_PASSWORD}
ports:
- "5432:5432"
networks:
- default
healthcheck:
test: ["CMD-SHELL", "pg_isready -U admin"]
interval: 5s
timeout: 5s
retries: 5
volumes:
postgres_data:
driver: local

View File

@@ -0,0 +1,39 @@
#!/bin/bash
set -e
echo "Waiting for PostgreSQL to start..."
until pg_isready -U admin -h localhost; do
sleep 2
done
echo "PostgreSQL is ready. Creating databases and users..."
# Create databases
for db in matrix_synapse matrix_mas matrix_signal matrix_whatsapp paperless immich shlink spliit audiomuse; do
echo "Creating database: $db"
createdb -U admin "$db"
done
# Create users and set permissions
for db in matrix_synapse matrix_mas matrix_signal matrix_whatsapp paperless immich shlink spliit audiomuse; do
user="${db}_user"
password_var="${user^^}_PASSWORD"
password=${!password_var}
if [ -z "$password" ]; then
echo "Warning: Password for $user not set. Using default."
password="default_password"
fi
echo "Creating user: $user"
psql -U admin -c "CREATE USER $user WITH PASSWORD '$password';"
psql -U admin -c "GRANT ALL PRIVILEGES ON DATABASE $db TO $user;"
psql -U admin -c "ALTER DATABASE $db OWNER TO $user;"
done
# Enable extensions in immich database
echo "Enabling extensions in immich database..."
psql -U admin -d immich -c "CREATE EXTENSION IF NOT EXISTS vector;"
psql -U admin -d immich -c "CREATE EXTENSION IF NOT EXISTS pg_vectorscale;"
echo "PostgreSQL initialization complete!"

View File

@@ -0,0 +1,106 @@
#!/bin/bash
# Generic PostgreSQL service migration script
# Usage: ./migrate_service.sh <service_name> <old_host> <old_port> <old_user> <old_password> <old_db>
set -e
if [ "$#" -ne 6 ]; then
echo "Usage: $0 <service_name> <old_host> <old_port> <old_user> <old_password> <old_db>"
exit 1
fi
SERVICE_NAME=$1
OLD_HOST=$2
OLD_PORT=$3
OLD_USER=$4
OLD_PASSWORD=$5
OLD_DB=$6
# Map service names to database names and users
case $SERVICE_NAME in
synapse)
NEW_DB="synapse"
NEW_USER="synapse_user"
NEW_PASSWORD="${SYNAPSE_POSTGRES_PASSWORD}"
;;
mas)
NEW_DB="mas"
NEW_USER="mas_user"
NEW_PASSWORD="${MAS_POSTGRES_PASSWORD}"
;;
signal)
NEW_DB="signal"
NEW_USER="signal_user"
NEW_PASSWORD="${MAUTRIX_SIGNAL_POSTGRES_PASSWORD}"
;;
whatsapp)
NEW_DB="whatsapp"
NEW_USER="whatsapp_user"
NEW_PASSWORD="${MAUTRIX_WHATSAPP_POSTGRES_PASSWORD}"
;;
paperless)
NEW_DB="paperless"
NEW_USER="paperless_user"
NEW_PASSWORD="${PAPERLESS_POSTGRES_PASSWORD}"
;;
immich)
NEW_DB="immich"
NEW_USER="immich_user"
NEW_PASSWORD="${IMMICH_POSTGRES_PASSWORD}"
;;
shlink)
NEW_DB="shlink"
NEW_USER="shlink_user"
NEW_PASSWORD="${SHLINK_POSTGRES_PASSWORD}"
;;
spliit)
NEW_DB="spliit"
NEW_USER="spliit_user"
NEW_PASSWORD="${SPLIIT_POSTGRES_PASSWORD}"
;;
audiomuse)
NEW_DB="audiomuse"
NEW_USER="audiomuse_user"
NEW_PASSWORD="${AUDIOMUSE_POSTGRES_PASSWORD}"
;;
*)
echo "Unknown service: $SERVICE_NAME"
exit 1
;;
esac
echo "Starting migration for $SERVICE_NAME to database $NEW_DB..."
# Export from old database
echo "Exporting data from $OLD_DB..."
PGPASSWORD="$OLD_PASSWORD" pg_dump \
-h "$OLD_HOST" \
-p "$OLD_PORT" \
-U "$OLD_USER" \
-d "$OLD_DB" \
-F custom \
-f "/tmp/${SERVICE_NAME}_dump.custom"
# Import to new database
echo "Importing data to $NEW_DB database..."
PGPASSWORD="$NEW_PASSWORD" pg_restore \
-h localhost \
-p 5432 \
-U "$NEW_USER" \
-d "$NEW_DB" \
--clean \
--if-exists \
"/tmp/${SERVICE_NAME}_dump.custom"
echo "Migration completed for $SERVICE_NAME!"
# Clean up
rm -f "/tmp/${SERVICE_NAME}_dump.custom"
echo "You can now update your $SERVICE_NAME service configuration to use:"
echo " Host: localhost"
echo " Port: 5432"
echo " Database: $NEW_DB"
echo " User: $NEW_USER"
echo " Password: ***"

View File

@@ -1,6 +1,6 @@
services: services:
pocket-id: pocket-id:
image: ghcr.io/pocket-id/pocket-id:v2.3.0 image: ghcr.io/pocket-id/pocket-id:v2.4.0
restart: always restart: always
env_file: .env env_file: .env
ports: ports:

View File

@@ -13,7 +13,7 @@ services:
ports: ports:
- 8071:8080 - 8071:8080
element-call-livekit: element-call-livekit:
image: livekit/livekit-server:v1.9.11 image: livekit/livekit-server:v1.9.12
command: --config /etc/livekit.yaml command: --config /etc/livekit.yaml
ports: ports:
- "7880:7880/tcp" - "7880:7880/tcp"
@@ -25,7 +25,7 @@ services:
- ${LIVEKIT_CONFIG_DIR}/config.yaml:/etc/livekit.yaml:ro - ${LIVEKIT_CONFIG_DIR}/config.yaml:/etc/livekit.yaml:ro
mautrix-signal: mautrix-signal:
container_name: mautrix-signal container_name: mautrix-signal
image: dock.mau.dev/mautrix/signal:v0.2602.1 image: dock.mau.dev/mautrix/signal:v0.2602.2
restart: always restart: always
volumes: volumes:
- ${MAUTRIX_SIGNAL_DATA_DIR}:/data - ${MAUTRIX_SIGNAL_DATA_DIR}:/data
@@ -61,7 +61,7 @@ services:
ports: ports:
- 5440:5432 - 5440:5432
mas: mas:
image: ghcr.io/element-hq/matrix-authentication-service:1.12.0 image: ghcr.io/element-hq/matrix-authentication-service:1.13.0
restart: always restart: always
working_dir: /config working_dir: /config
volumes: volumes:
@@ -97,7 +97,7 @@ services:
- 5442:5432 - 5442:5432
synapse: synapse:
container_name: synapse container_name: synapse
image: matrixdotorg/synapse:v1.148.0 image: matrixdotorg/synapse:v1.149.0
restart: always restart: always
volumes: volumes:
- ${SYNAPSE_CONFIG_DIR}:/data - ${SYNAPSE_CONFIG_DIR}:/data