From bc8a8fbc978bf0c29b23201cf548c128c37f326f Mon Sep 17 00:00:00 2001
From: seonNoh <dark.pearl.nst@gmail.com>
Date: Thu, 8 Oct 2026 02:01:56 +0900
Subject: [PATCH] fix(postgres): compute replication lag without the
 idle-primary artifact
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

upstream replication 컬렉터는 수신 LSN != 재생 LSN이면 now()-마지막 트랜잭션 재생 시각을 지연으로
보고한다. replica 재시작 직후에는 수신 LSN이 세그먼트 시작점(재생 LSN보다 작음)에 머물러 유휴
프라이머리에서 지연이 무한히 커진다(실측 116초→281초, 트랜잭션 1건 뒤 0). DbReplicationLag가
pending으로 올라갔다. 수신 LSN <= 재생 LSN이면 0으로 보는 custom query로 교체하고 upstream
replication 컬렉터를 끈다. pg_replication_is_replica도 같은 쿼리에서 낸다.
---
 vm/postgres-common/config.alloy |  6 ++++--
 vm/postgres-common/queries.yaml | 20 +++++++++++++++++++-
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/vm/postgres-common/config.alloy b/vm/postgres-common/config.alloy
index 68fcf77..1980d26 100644
--- a/vm/postgres-common/config.alloy
+++ b/vm/postgres-common/config.alloy
@@ -1,9 +1,11 @@
 // Alloy on db-primary / db-replica (design 4.1 / 4.3): postgres exporter (built-in collectors: pg_replication_lag_seconds,
 // pg_up, pg_stat_*) + custom pg_is_wal_replay_paused -> Mimir; PostgreSQL logs -> Loki. instance = VM name.
 prometheus.exporter.postgres "db" {
-  data_source_names         = [sys.env("PG_DSN")]
+  data_source_names          = [sys.env("PG_DSN")]
   custom_queries_config_path = "/etc/alloy/queries.yaml"
-  disable_settings_metrics  = true
+  disable_settings_metrics   = true
+  // defaults minus "replication": pg_replication_* come from queries.yaml (idle-primary artifact, see there)
+  enabled_collectors = ["database", "locks", "replication_slot", "roles", "stat_bgwriter", "stat_database", "stat_progress_vacuum", "stat_user_tables", "statio_user_tables", "wal"]
 }
 
 prometheus.scrape "db" {
diff --git a/vm/postgres-common/queries.yaml b/vm/postgres-common/queries.yaml
index 22bbf79..678b325 100644
--- a/vm/postgres-common/queries.yaml
+++ b/vm/postgres-common/queries.yaml
@@ -1,5 +1,23 @@
 # Custom postgres_exporter queries (Alloy prometheus.exporter.postgres custom_queries_config_path).
-# Metric name = <key>_<metric>: pg_is_wal_replay_paused (used by the DbReplicationLag rule, design 4.3).
+# Metric name = <key>_<column>.
+#
+# pg_replication_lag_seconds / pg_replication_is_replica replace the upstream "replication" collector (disabled in
+# config.alloy). The upstream formula reports now() - pg_last_xact_replay_timestamp() whenever receive LSN != replay LSN;
+# right after a replica restart the receive LSN sits at the segment start below the replay LSN until new WAL arrives,
+# so on an idle primary the lag grows without bound (observed 2026-10-08: 116 s -> 281 s, back to 0 after one
+# transaction). Treating receive <= replay as "fully applied" removes that artifact; real lag (replay paused, slow
+# apply under load) still shows because receive runs ahead of replay.
+pg_replication:
+  query: "SELECT CASE WHEN NOT pg_is_in_recovery() THEN 0 WHEN pg_last_wal_receive_lsn() <= pg_last_wal_replay_lsn() THEN 0 ELSE GREATEST(0, EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))) END AS lag_seconds, CASE WHEN pg_is_in_recovery() THEN 1 ELSE 0 END AS is_replica"
+  master: true
+  metrics:
+    - lag_seconds:
+        usage: "GAUGE"
+        description: "Replication lag behind the primary in seconds (0 when every received byte is replayed)"
+    - is_replica:
+        usage: "GAUGE"
+        description: "1 when the server is in recovery (replica)"
+# pg_is_wal_replay_paused (used by the DbReplicationLag rule, design 4.3): not in any upstream collector.
 pg_is_wal_replay:
   query: "SELECT CASE WHEN pg_is_in_recovery() AND pg_is_wal_replay_paused() THEN 1 ELSE 0 END AS paused"
   master: true