perf(realtime): reduce push update churn

Cache destination lookups and skip empty resource queries during push server updates. Add database indexes and Postgres storage tuning for hot-update tables, and make the realtime entrypoint forward process failures and signals reliably.
2026-06-19 07:35:25 +00:00 · 2026-05-27 19:38:23 +02:00
parent 90aa4e7e73
commit 20f9bb4305
4 changed files with 187 additions and 31 deletions
@@ -101,6 +101,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced

    public bool $foundLogDrainContainer = false;

+    private ?array $cachedDestinationIds = null;
+
    public function middleware(): array
    {
        return [(new WithoutOverlapping('push-server-update-'.$this->server->uuid))->expireAfter(30)->dontRelease()];
@@ -156,6 +158,10 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        $this->serviceApplicationsById ??= collect();
        $this->serviceDatabasesById ??= collect();

+        // Eager-load relations the job touches repeatedly to avoid lazy-load queries
+        // (settings: disk threshold, isProxyShouldRun, isLogDrainEnabled; team: notifications).
+        $this->server->loadMissing(['settings', 'team']);
+
        // TODO: Swarm is not supported yet
        if (! $this->data) {
            throw new \Exception('No data provided');
@@ -327,21 +333,23 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
    {
        [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();

-        $applications = Application::withoutGlobalScope('withRelations')
-            ->select([
-                'id',
-                'uuid',
-                'name',
-                'status',
-                'build_pack',
-                'docker_compose_raw',
-                'destination_id',
-                'destination_type',
-                'last_online_at',
-            ])
-            ->withCount('additional_servers')
-            ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
-            ->get();
+        $applications = ($standaloneDockerIds->isNotEmpty() || $swarmDockerIds->isNotEmpty())
+            ? Application::withoutGlobalScope('withRelations')
+                ->select([
+                    'id',
+                    'uuid',
+                    'name',
+                    'status',
+                    'build_pack',
+                    'docker_compose_raw',
+                    'destination_id',
+                    'destination_type',
+                    'last_online_at',
+                ])
+                ->withCount('additional_servers')
+                ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+                ->get()
+            : collect();

        $additionalApplicationIds = DB::table('additional_destinations')
            ->where('server_id', $this->server->id)
@@ -409,6 +417,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
    private function loadDatabases(): Collection
    {
        [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
+        if ($standaloneDockerIds->isEmpty() && $swarmDockerIds->isEmpty()) {
+            return collect();
+        }
        $databaseColumns = [
            'id',
            'uuid',
@@ -442,7 +453,11 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced

    private function serverDestinationIds(): array
    {
-        return [
+        if ($this->cachedDestinationIds !== null) {
+            return $this->cachedDestinationIds;
+        }
+
+        return $this->cachedDestinationIds = [
            StandaloneDocker::where('server_id', $this->server->id)->pluck('id'),
            SwarmDocker::where('server_id', $this->server->id)->pluck('id'),
        ];
@@ -0,0 +1,27 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Support\Facades\DB;
+
+return new class extends Migration
+{
+    public function up(): void
+    {
+        DB::statement('CREATE INDEX IF NOT EXISTS swarm_dockers_server_id_index ON swarm_dockers (server_id)');
+        DB::statement('CREATE INDEX IF NOT EXISTS services_server_id_index ON services (server_id)');
+        DB::statement('CREATE INDEX IF NOT EXISTS application_previews_application_id_index ON application_previews (application_id)');
+        DB::statement('CREATE INDEX IF NOT EXISTS service_applications_service_id_index ON service_applications (service_id)');
+        DB::statement('CREATE INDEX IF NOT EXISTS service_databases_service_id_index ON service_databases (service_id)');
+        DB::statement('CREATE INDEX IF NOT EXISTS servers_sentinel_updated_at_index ON servers (sentinel_updated_at)');
+    }
+
+    public function down(): void
+    {
+        DB::statement('DROP INDEX IF EXISTS swarm_dockers_server_id_index');
+        DB::statement('DROP INDEX IF EXISTS services_server_id_index');
+        DB::statement('DROP INDEX IF EXISTS application_previews_application_id_index');
+        DB::statement('DROP INDEX IF EXISTS service_applications_service_id_index');
+        DB::statement('DROP INDEX IF EXISTS service_databases_service_id_index');
+        DB::statement('DROP INDEX IF EXISTS servers_sentinel_updated_at_index');
+    }
+};
@@ -0,0 +1,58 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Support\Facades\DB;
+
+return new class extends Migration
+{
+    public function up(): void
+    {
+        // Fillfactor < 100 leaves free space per page so Postgres can do HOT
+        // (Heap-Only Tuple) in-place updates instead of allocating a new tuple
+        // elsewhere. Coolify's hot-update tables churn rows on every Sentinel
+        // push / status change; without page-local headroom, non-HOT updates
+        // accumulate dead tuples and bloat the heap (we've seen up to 50× on
+        // cloud). Lower fillfactor on hot-update tables, default on the rest.
+        DB::statement('ALTER TABLE applications SET (fillfactor = 70)');
+        DB::statement('ALTER TABLE servers SET (fillfactor = 85)');
+        DB::statement('ALTER TABLE services SET (fillfactor = 85)');
+        DB::statement('ALTER TABLE service_applications SET (fillfactor = 85)');
+        DB::statement('ALTER TABLE service_databases SET (fillfactor = 85)');
+        DB::statement('ALTER TABLE standalone_postgresqls SET (fillfactor = 85)');
+        DB::statement('ALTER TABLE standalone_redis SET (fillfactor = 85)');
+        DB::statement('ALTER TABLE standalone_mongodbs SET (fillfactor = 85)');
+        DB::statement('ALTER TABLE standalone_mysqls SET (fillfactor = 85)');
+        DB::statement('ALTER TABLE standalone_mariadbs SET (fillfactor = 85)');
+        DB::statement('ALTER TABLE standalone_keydbs SET (fillfactor = 85)');
+        DB::statement('ALTER TABLE standalone_dragonflies SET (fillfactor = 85)');
+        DB::statement('ALTER TABLE standalone_clickhouses SET (fillfactor = 85)');
+        DB::statement('ALTER TABLE application_deployment_queues SET (fillfactor = 90)');
+
+        // Autovacuum default kicks in at 20% dead tuples — too lazy for our
+        // churn rate. Trigger at 5% on the highest-write tables to keep heap
+        // pages tidy and prevent visibility-map gaps that hurt scan plans.
+        DB::statement('ALTER TABLE applications SET (autovacuum_vacuum_scale_factor = 0.05)');
+        DB::statement('ALTER TABLE servers SET (autovacuum_vacuum_scale_factor = 0.05)');
+        DB::statement('ALTER TABLE service_applications SET (autovacuum_vacuum_scale_factor = 0.05)');
+        DB::statement('ALTER TABLE service_databases SET (autovacuum_vacuum_scale_factor = 0.05)');
+        DB::statement('ALTER TABLE standalone_postgresqls SET (autovacuum_vacuum_scale_factor = 0.05)');
+    }
+
+    public function down(): void
+    {
+        DB::statement('ALTER TABLE applications RESET (fillfactor, autovacuum_vacuum_scale_factor)');
+        DB::statement('ALTER TABLE servers RESET (fillfactor, autovacuum_vacuum_scale_factor)');
+        DB::statement('ALTER TABLE services RESET (fillfactor)');
+        DB::statement('ALTER TABLE service_applications RESET (fillfactor, autovacuum_vacuum_scale_factor)');
+        DB::statement('ALTER TABLE service_databases RESET (fillfactor, autovacuum_vacuum_scale_factor)');
+        DB::statement('ALTER TABLE standalone_postgresqls RESET (fillfactor, autovacuum_vacuum_scale_factor)');
+        DB::statement('ALTER TABLE standalone_redis RESET (fillfactor)');
+        DB::statement('ALTER TABLE standalone_mongodbs RESET (fillfactor)');
+        DB::statement('ALTER TABLE standalone_mysqls RESET (fillfactor)');
+        DB::statement('ALTER TABLE standalone_mariadbs RESET (fillfactor)');
+        DB::statement('ALTER TABLE standalone_keydbs RESET (fillfactor)');
+        DB::statement('ALTER TABLE standalone_dragonflies RESET (fillfactor)');
+        DB::statement('ALTER TABLE standalone_clickhouses RESET (fillfactor)');
+        DB::statement('ALTER TABLE application_deployment_queues RESET (fillfactor)');
+    }
+};
@@ -1,35 +1,91 @@
 #!/bin/sh
-# Function to timestamp logs

-# Check if the first argument is 'watch'
 if [ "$1" = "watch" ]; then
    WATCH_MODE="--watch"
 else
    WATCH_MODE=""
 fi

-timestamp() {
-    date "+%Y-%m-%d %H:%M:%S"
+log() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') [ENTRYPOINT] $*"
 }

-# Start the terminal server in the background with logging
-node $WATCH_MODE /terminal/terminal-server.js > >(while read line; do echo "$(timestamp) [TERMINAL] $line"; done) 2>&1 &
+start_logger() {
+    prefix="$1"
+    fifo_path="$2"
+
+    while read -r line; do
+        echo "$(date '+%Y-%m-%d %H:%M:%S') [$prefix] $line"
+    done < "$fifo_path" &
+}
+
+cleanup() {
+    rm -f "$TERMINAL_LOG_FIFO" "$SOKETI_LOG_FIFO"
+}
+
+TERMINAL_LOG_FIFO="/tmp/coolify-terminal-log.$$"
+SOKETI_LOG_FIFO="/tmp/coolify-soketi-log.$$"
+
+rm -f "$TERMINAL_LOG_FIFO" "$SOKETI_LOG_FIFO"
+mkfifo "$TERMINAL_LOG_FIFO" "$SOKETI_LOG_FIFO"
+
+trap cleanup EXIT
+
+log "Starting realtime container"
+log "WATCH_MODE=${WATCH_MODE:-off}"
+log "SOKETI_DEBUG=${SOKETI_DEBUG:-unset}"
+log "NODE_OPTIONS=${NODE_OPTIONS:-unset}"
+
+start_logger "TERMINAL" "$TERMINAL_LOG_FIFO"
+TERMINAL_LOGGER_PID=$!
+
+start_logger "SOKETI" "$SOKETI_LOG_FIFO"
+SOKETI_LOGGER_PID=$!
+
+node $WATCH_MODE /terminal/terminal-server.js > "$TERMINAL_LOG_FIFO" 2>&1 &
 TERMINAL_PID=$!

-# Start the Soketi process in the background with logging
-node /app/bin/server.js start > >(while read line; do echo "$(timestamp) [SOKETI] $line"; done) 2>&1 &
+log "Terminal server started pid=$TERMINAL_PID logger_pid=$TERMINAL_LOGGER_PID"
+
+node /app/bin/server.js start > "$SOKETI_LOG_FIFO" 2>&1 &
 SOKETI_PID=$!

-# Function to forward signals to child processes
+log "Soketi started pid=$SOKETI_PID logger_pid=$SOKETI_LOGGER_PID"
+
 forward_signal() {
-    kill -$1 $TERMINAL_PID $SOKETI_PID
+    log "Forwarding signal $1 to terminal=$TERMINAL_PID soketi=$SOKETI_PID"
+
+    kill -"$1" "$TERMINAL_PID" 2>/dev/null || true
+    kill -"$1" "$SOKETI_PID" 2>/dev/null || true
 }

-# Forward SIGTERM to child processes
 trap 'forward_signal TERM' TERM
+trap 'forward_signal INT' INT

-# Wait for any process to exit
-wait -n
+while true; do
+    if ! kill -0 "$TERMINAL_PID" 2>/dev/null; then
+        wait "$TERMINAL_PID"
+        EXIT_CODE=$?

-# Exit with status of process that exited first
-exit $?
+        log "Terminal server exited code=$EXIT_CODE; stopping soketi pid=$SOKETI_PID"
+
+        kill "$SOKETI_PID" 2>/dev/null || true
+        wait "$SOKETI_PID" 2>/dev/null || true
+
+        exit "$EXIT_CODE"
+    fi
+
+    if ! kill -0 "$SOKETI_PID" 2>/dev/null; then
+        wait "$SOKETI_PID"
+        EXIT_CODE=$?
+
+        log "Soketi exited code=$EXIT_CODE; stopping terminal pid=$TERMINAL_PID"
+
+        kill "$TERMINAL_PID" 2>/dev/null || true
+        wait "$TERMINAL_PID" 2>/dev/null || true
+
+        exit "$EXIT_CODE"
+    fi
+
+    sleep 1
+done