Merge pull request #81 from gmr/feature/storage-statistics-rules

Model column attributes, constraint comments, extended statistics and rules

Author: gmr

Cargo.lock

@@ -15,16 +15,16 @@ indicatif = "0.18.4"
 indicatif-log-bridge = "0.2.3"
 jsonschema = { version = "0.46.5", default-features = false }
 libpgdump = "2.3"
-libpgfmt = "1.4"
+libpgfmt = "1.5"
 log = "0.4"
 rpassword = "7"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.150", features = ["preserve_order"] }
 serde_norway = "0.9.42"
 simplelog = "0.12"
 tempfile = "3"
-tree-sitter = "0.26"
-tree-sitter-postgres = "19.0.0-beta.4"
+tree-sitter = "0.27"
+tree-sitter-postgres = "19.0.0-beta.4.1"
 
 [dev-dependencies]
 libpgdump = "2.3"

Cargo.toml

@@ -1,7 +1,7 @@
 # Coverage plan: silent schema loss, RLS, and PostgreSQL 18
 
-Status: Phases 0 to 5 complete, and Phase 6 items 1 to 3. The rest of
-Phase 6, and Phases 7 and 8, proposed.
+Status: Phases 0 to 5 complete, and Phase 6 items 1 to 6. Phase 6
+item 7, and Phases 7 and 8, proposed.
 Written 2026-09-21.
 
 Every claim below was verified against PostgreSQL 18.4 (the version
@@ -548,22 +548,42 @@ enabled state.
 → verify: pull a database containing all seven; `remaining.yaml` is
 absent, and `just round-trip` passes with them in `fixtures/schema.sql`.
 
-### Phase 6 — New object models (~4 days, ranked) — items 1 to 3 **DONE**
+### Phase 6 — New object models (~4 days, ranked) — items 1 to 6 **DONE**
 
-**Done:** `EXCLUDE` constraints (item 1), `REPLICA IDENTITY` (item 2)
-and default privileges (item 3), each pulled, built, deployed and in
-`fixtures/schema.sql`. The coverage gate list is down from 5 entries
-to 4. Items 4 to 7 are still open.
+**Items 4 to 6 done:** column `COMPRESSION` and `STORAGE`, plus the
+statistics target and attribute options, which pg_dump writes the same
+way (item 4); extended `STATISTICS` as a top-level object (item 5); and
+`RULE` on tables and views, with a view's `_RETURN` rule folded into
+its query (item 6). Also comments on primary key, unique, check,
+foreign key and NOT NULL constraints, which the unmatched-comment rule
+from items 1 to 3 otherwise turned into pull failures.
 
-Where the work departs from the items below:
+**Item 7, `SECURITY LABEL`, is deferred.** A label needs a provider
+module preloaded into the server, and the stock `postgres:18` image has
+none, so no gate could verify it. It needs a CI image built with a
+provider (`dummy_seclabel` from the PostgreSQL test modules) first.
+
+Every construct the coverage fixture held is now modeled. It records
+the gaps that remain instead: procedures and operators (which build
+and the models support, but pull cannot parse), and operator classes,
+operator families and access methods (which have no model).
+
+
+**Done (items 1 to 3, historical note):** `EXCLUDE` constraints
+(item 1), `REPLICA IDENTITY` (item 2) and default privileges (item 3),
+each pulled, built, deployed and in `fixtures/schema.sql`. When these
+items were done, items 4 to 7 were still open; see the status above
+for their current state.
+
+Where the work on items 1 to 3 departs from the items below:
 
 - **An unmatched comment now fails the pull.** Pull logged a warning
   for a comment it could not attach and dropped it, which the
   governing invariant forbids. It now keeps the entry in
-  `remaining.yaml`. The model has a comment for an exclusion
-  constraint only, so a comment on a primary key, unique, check or
-  foreign key constraint now fails the pull, and the coverage fixture
-  records that gap.
+  `remaining.yaml`. At that time the model had a comment for an
+  exclusion constraint only, so a comment on another constraint type
+  failed the pull. The `constraint_comments` map on a table (added
+  with items 4 to 6) now closes that gap.
 - **Default privileges are a new top-level object type**,
   `default_privileges/<role>.yaml`, not part of `src/build/acls.rs`:
   they are keyed by the role the defaults are `FOR`, and the role files

PLAN-coverage.md

@@ -98,7 +98,11 @@ echo "==> Gate 2: convergence (mutate, deploy --apply, re-deploy is empty)"
 # row-security drift: turned off and unforced, one policy dropped, one
 # edited, one database-only, and a changed policy comment
 # replica identity reset and changed, an exclusion constraint's comment
-# changed, and another one dropped
+# changed, and another one dropped; column storage, compression,
+# statistics target and options changed; constraint comments changed
+# and cleared, and a commented check replaced by a different one, whose
+# comment deploy has to set again after re-adding it; rules
+# re-enabled, dropped, commented and replaced, on a table and a view
 psql -d "${TARGET_DB}" -q -v ON_ERROR_STOP=1 <<'SQL'
 ALTER TABLE test.users ADD COLUMN scratch text;
 ALTER TABLE test.users ALTER COLUMN display_name SET NOT NULL;
@@ -121,6 +125,20 @@ ALTER TABLE test.replica_index REPLICA IDENTITY FULL;
 COMMENT ON CONSTRAINT room_bookings_no_overlap ON test.room_bookings
     IS 'drift';
 ALTER TABLE test.handles DROP CONSTRAINT handles_unique_lower;
+ALTER TABLE test.documents ALTER COLUMN body SET STORAGE EXTENDED;
+ALTER TABLE test.documents ALTER COLUMN body SET COMPRESSION pglz;
+ALTER TABLE test.documents ALTER COLUMN id SET STATISTICS 100;
+ALTER TABLE test.documents ALTER COLUMN size RESET (n_distinct);
+COMMENT ON CONSTRAINT invoices_pkey ON test.invoices IS 'drift';
+COMMENT ON CONSTRAINT invoices_number_key ON test.invoices IS NULL;
+ALTER TABLE test.invoices DROP CONSTRAINT invoices_total_positive;
+ALTER TABLE test.invoices ADD CONSTRAINT invoices_total_positive
+    CHECK (total > 0);
+ALTER TABLE test.ledger ENABLE RULE ledger_redirect;
+DROP RULE ledger_no_delete ON test.ledger;
+COMMENT ON RULE ledger_audit_insert ON test.ledger IS 'drift';
+CREATE OR REPLACE RULE ledger_view_insert AS ON INSERT TO test.ledger_view
+    DO INSTEAD NOTHING;
 SQL
 ./target/debug/pglifecycle deploy --apply --allow-drop -d "${TARGET_DB}" \
     "${WORKDIR}/project"

bin/deploy-gates

@@ -55,6 +55,7 @@
         'publication',
         'schema',
         'sequence',
+        'statistics',
         'server',
         'subscription',
         'table',
@@ -78,6 +79,7 @@
         'dependencies',
         'foreign_key',
         'policy',
+        'rule',
         'trigger',
     ],
     'Per-schema container files': [

bin/generate-schema-docs.py

@@ -82,8 +82,11 @@ reconciled in place where PostgreSQL can express it:
 - **Tables** — add column, set/drop default, set/drop not-null,
   add/drop check constraints, foreign keys and exclusion constraints,
   primary-key and unique additions, index and trigger create/drop,
-  `REPLICA IDENTITY`, and comment changes. A changed comment on an
-  exclusion constraint is set alone, without rebuilding its index.
+  `REPLICA IDENTITY`, column storage, compression, statistics target
+  and options, rules (`CREATE OR REPLACE RULE`, their state and
+  comment), and comment changes, including comments on constraints. A
+  changed comment on a constraint is set alone, without rebuilding the
+  constraint; a constraint that is re-added gets its comment again.
   Identity columns are added, and their `ALWAYS`/`BY DEFAULT` behavior
   and sequence options changed, with `ALTER COLUMN`; renaming an
   identity's sequence falls back. A `NOT VALID` check, foreign key or
@@ -103,7 +106,8 @@ reconciled in place where PostgreSQL can express it:
   pglifecycle modeled them, leaves the table's row security as the
   database has it.
 - **Functions and views** — `CREATE OR REPLACE` (a function whose
-  return type changed must be dropped first, so it falls back).
+  return type changed must be dropped first, so it falls back). A
+  view's rules are reconciled after it.
 - **Sequences** — a single `ALTER SEQUENCE` of the changed options.
 - **Domains** — set/drop default; a base-type or constraint change
   falls back.
@@ -133,6 +137,12 @@ any are pending. Index, trigger, and constraint drops issued while
 reconciling a table are *not* gated: they lose no data and the project
 is authoritative.
 
+`DROP RULE` is gated although a rule holds no data. A `DO INSTEAD
+NOTHING` rule can block writes, so dropping one can let through changes
+the database refused before. A project pulled with a version of
+pglifecycle that did not model rules has none on any table or view,
+and the gate stops a deploy of that project from dropping every rule.
+
 `DROP IDENTITY` is gated although it keeps every row: the sequence goes
 with it, so adding the identity back restarts the numbering and
 collides with existing keys. A project pulled with a version of
@@ -161,16 +171,17 @@ Ownership is not managed (the script behaves like
 `pg_restore --no-owner`), and roles, users, groups, and tablespaces are
 skipped entirely — they are cluster-level objects a single-database
 dump cannot capture. Aggregates, casts, collations, conversions, event
-triggers, publications, text search objects and default privileges are
-created when
+triggers, publications, text search objects, default privileges and
+extended statistics are created when
 missing but otherwise only existence-checked: `pull` models them, but
 `deploy` does not compare their definitions yet, so a changed one is
 left as the database has it. An aggregate is matched by its name and
 input types, so each overload is checked on its own. Text search
 objects are checked per schema: when a schema has any text search
 object in the database, `deploy` creates none of the project's text
 search objects in that schema. Object types `pull` does not yet model
-(operators, rules, extended statistics, …) are handled the same way. Privileges on
+(operators, operator classes and families, access methods, …) are
+handled the same way. Privileges on
 created objects are emitted (unless `-x`); privilege changes on objects
 that already exist are not yet diffed.
 
@@ -216,14 +227,14 @@ went missing.
 ```console
 $ pglifecycle pull ./project -d mydb
 ...
-error: 3 dump entries could not be modeled (RULE, STATISTICS), so the
+error: 3 dump entries could not be modeled (OPERATOR, OPERATOR CLASS), so the
 generated project would not reproduce the source database.
 The entries were preserved in ./project/remaining.yaml; re-run with
 --allow-unsupported to accept the project as it is.
 ```
 
 A `COMMENT` entry counts as unmodeled when the model has no place for
-it, such as a comment on a primary key or foreign key constraint.
+it, such as a comment on an object type that `pull` does not model.
 
 The project directory is written either way, so `remaining.yaml` is
 there to inspect. `--allow-unsupported` downgrades the failure to a

docs/commands.md

@@ -183,6 +183,48 @@ exclude_constraints:
 replica_identity: FULL
 ```
 
+- Columns may carry `storage`, `compression`, `statistics` (the
+  statistics target) and `options` (such as `n_distinct`), which
+  `build` writes as `ALTER COLUMN ... SET` after `CREATE TABLE`, as
+  `pg_dump` does. Comments on a table's primary key, unique, check,
+  foreign key and NOT NULL constraints live in `constraint_comments`,
+  keyed by constraint name.
+
+- Tables and views carry `rules`. A rule's `commands` are absent for
+  `DO INSTEAD NOTHING`. A view's internal `_RETURN` rule is its query,
+  never a rule.
+
+```yaml
+rules:
+  - name: ledger_audit_insert
+    event: INSERT
+    condition: (new.amount > (0)::numeric)
+    commands:
+      - |-
+        INSERT INTO test.ledger_audit (id)
+          VALUES (new.id)
+  - name: ledger_no_delete
+    event: DELETE
+    instead: true
+    comment: Append only
+```
+
+  Conditions and commands keep the form `pg_dump` writes them in.
+
+- Extended statistics live in `statistics/<schema>/<name>.yaml`, not
+  on the table: the name is schema-qualified, and the owner need not
+  own the table.
+
+```yaml
+---
+name: measurements_ab
+schema: test
+owner: postgres
+table: test.measurements
+kinds: [ndistinct, dependencies]
+elements: [a, b]
+```
+
 - Default privileges live in `default_privileges/<role>.yaml`, one file
   for each role whose new objects they apply to (`ALTER DEFAULT
   PRIVILEGES FOR ROLE`). Global and per-schema declarations compose in

docs/project-format.md

@@ -525,3 +525,63 @@ ALTER TABLE test.replica_index REPLICA IDENTITY USING INDEX replica_index_id;
 ALTER DEFAULT PRIVILEGES IN SCHEMA test GRANT SELECT ON TABLES TO PUBLIC;
 ALTER DEFAULT PRIVILEGES IN SCHEMA test GRANT USAGE ON SEQUENCES TO PUBLIC;
 ALTER DEFAULT PRIVILEGES REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC;
+
+-- Column attributes pg_dump writes as ALTER COLUMN after CREATE TABLE:
+-- compression, storage, the statistics target and attribute options
+CREATE TABLE test.documents (
+    id   INTEGER PRIMARY KEY,
+    body TEXT COMPRESSION lz4,
+    blob BYTEA,
+    size NUMERIC
+);
+ALTER TABLE test.documents ALTER COLUMN body SET STORAGE EXTERNAL;
+ALTER TABLE test.documents ALTER COLUMN blob SET STORAGE MAIN;
+ALTER TABLE test.documents ALTER COLUMN id SET STATISTICS 500;
+ALTER TABLE test.documents ALTER COLUMN size SET (n_distinct = 100);
+
+-- Comments on each kind of constraint: primary key, unique, check,
+-- foreign key, NOT NULL, and a NOT VALID check, whose comment follows
+-- its own entry
+CREATE TABLE test.invoices (
+    id       INTEGER CONSTRAINT invoices_pkey PRIMARY KEY,
+    number   TEXT NOT NULL UNIQUE,
+    total    NUMERIC CONSTRAINT invoices_total_positive CHECK (total >= 0),
+    document INTEGER CONSTRAINT invoices_document REFERENCES test.documents (id)
+);
+ALTER TABLE test.invoices
+    ADD CONSTRAINT invoices_number_short CHECK (length(number) < 20) NOT VALID;
+COMMENT ON CONSTRAINT invoices_pkey ON test.invoices IS 'The invoice id';
+COMMENT ON CONSTRAINT invoices_number_key ON test.invoices IS 'One per number';
+COMMENT ON CONSTRAINT invoices_total_positive ON test.invoices IS 'No credits';
+COMMENT ON CONSTRAINT invoices_document ON test.invoices IS 'Its source';
+COMMENT ON CONSTRAINT invoices_number_not_null ON test.invoices IS 'Required';
+COMMENT ON CONSTRAINT invoices_number_short ON test.invoices IS 'Legacy rows';
+
+-- Extended statistics: chosen kinds, every kind with a target and a
+-- comment, expressions, and one on a materialized view
+CREATE TABLE test.measurements (a INTEGER, b INTEGER, label TEXT);
+CREATE STATISTICS test.measurements_ab (ndistinct, dependencies)
+    ON a, b FROM test.measurements;
+CREATE STATISTICS test.measurements_all ON a, b, label FROM test.measurements;
+ALTER STATISTICS test.measurements_all SET STATISTICS 500;
+COMMENT ON STATISTICS test.measurements_all IS 'Every kind';
+CREATE STATISTICS test.measurements_expr (mcv)
+    ON (a + b), lower(label) FROM test.measurements;
+CREATE STATISTICS test.user_states_stats ON state, total FROM test.user_states;
+
+-- Rules: DO INSTEAD NOTHING with a comment, a conditional DO ALSO with
+-- two commands, a disabled one, and one on a view
+CREATE TABLE test.ledger (id INTEGER, amount NUMERIC);
+CREATE TABLE test.ledger_audit (id INTEGER);
+CREATE RULE ledger_no_delete AS ON DELETE TO test.ledger DO INSTEAD NOTHING;
+COMMENT ON RULE ledger_no_delete ON test.ledger IS 'Append only';
+CREATE RULE ledger_audit_insert AS ON INSERT TO test.ledger
+    WHERE new.amount > 0
+    DO ALSO (INSERT INTO test.ledger_audit VALUES (new.id);
+             INSERT INTO test.ledger_audit VALUES (- new.id));
+CREATE RULE ledger_redirect AS ON UPDATE TO test.ledger
+    DO INSTEAD UPDATE test.ledger_audit SET id = new.id WHERE ledger_audit.id = old.id;
+ALTER TABLE test.ledger DISABLE RULE ledger_redirect;
+CREATE VIEW test.ledger_view AS SELECT id, amount FROM test.ledger;
+CREATE RULE ledger_view_insert AS ON INSERT TO test.ledger_view
+    DO INSTEAD INSERT INTO test.ledger (id, amount) VALUES (new.id, new.amount);

fixtures/schema.sql

@@ -1,4 +1,5 @@
-COMMENT	unsupported.CONSTRAINT payloads_pkey ON payloads
-RULE	unsupported.append_only append_only_no_delete
-STATISTICS	unsupported.measurements_stats
-TABLE	unsupported.payloads
+ACCESS METHOD	pglifecycle_heap
+OPERATOR	unsupported.===
+OPERATOR CLASS	unsupported.int_class
+OPERATOR FAMILY	unsupported.int_family
+PROCEDURE	unsupported.archive(integer)