diff --git a/src/adapter/src/catalog/migrate.rs b/src/adapter/src/catalog/migrate.rs index 0eb128fdaf3e2..3243dd0a22ece 100644 --- a/src/adapter/src/catalog/migrate.rs +++ b/src/adapter/src/catalog/migrate.rs @@ -840,45 +840,57 @@ pub(crate) fn durable_migrate( tx.set_setting(MOCK_AUTHENTICATION_NONCE_KEY.to_string(), Some(nonce))?; } - migrate_builtin_tables_to_mvs(tx)?; + migrate_builtin_item_types(tx)?; Ok(()) } -/// Update system object mappings for builtins whose type changed from table to materialized view. +/// Update system object mappings for builtins whose type changed: from table +/// to materialized view, or from table or materialized view to view. /// /// Required for the work of making builtin tables views over `mz_catalog_raw`. -fn migrate_builtin_tables_to_mvs(tx: &mut Transaction) -> Result<(), anyhow::Error> { - // Collect `(schema, name)` of all builtin MVs. +/// +/// Only the mapping changes. The builtin keeps its ids and its shard, which is +/// left to the step registered for the new description in +/// `builtin_schema_migration`: a `Replacement` gives a materialized view a new +/// shard, a `Drop` leaves a view without one. +fn migrate_builtin_item_types(tx: &mut Transaction) -> Result<(), anyhow::Error> { let expected_mvs: BTreeSet<_> = BUILTINS::materialized_views() .map(|mv| (mv.schema, mv.name)) .collect(); + let expected_views: BTreeSet<_> = BUILTINS::views().map(|v| (v.schema, v.name)).collect(); - // Find persisted mappings for builtin tables that must be migrated. + // Find persisted mappings for builtin tables that must be migrated or builtin materialized + // views that must be dropped. let mut to_remove = BTreeSet::new(); let mut to_add = Vec::new(); for mapping in tx.get_system_object_mappings() { let desc = &mapping.description; - if desc.object_type != CatalogItemType::Table { - continue; - } - let key = (&*desc.schema_name, &*desc.object_name); - if expected_mvs.contains(&key) { - info!( - "migrate: builtin {}.{} changed type from table to MV", - desc.schema_name, desc.object_name, - ); - to_remove.insert(desc.clone()); - to_add.push(SystemObjectMapping { - description: SystemObjectDescription { - schema_name: desc.schema_name.clone(), - object_type: CatalogItemType::MaterializedView, - object_name: desc.object_name.clone(), - }, - unique_identifier: mapping.unique_identifier, - }); - } + let new_type = match desc.object_type { + CatalogItemType::Table if expected_mvs.contains(&key) => { + CatalogItemType::MaterializedView + } + CatalogItemType::Table | CatalogItemType::MaterializedView + if expected_views.contains(&key) => + { + CatalogItemType::View + } + _ => continue, + }; + info!( + "migrate: builtin {}.{} changed type from {} to {}", + desc.schema_name, desc.object_name, desc.object_type, new_type + ); + to_remove.insert(desc.clone()); + to_add.push(SystemObjectMapping { + description: SystemObjectDescription { + schema_name: desc.schema_name.clone(), + object_type: new_type, + object_name: desc.object_name.clone(), + }, + unique_identifier: mapping.unique_identifier, + }); } if !to_remove.is_empty() { @@ -1308,6 +1320,7 @@ fn rewrite_interval_option_floor_1s( #[cfg(test)] mod tests { use super::*; + use crate::catalog::Catalog; fn strip(sql: &str) -> String { let mut stmt = mz_sql::parse::parse(sql) @@ -1370,6 +1383,59 @@ mod tests { ); } + #[mz_ore::test(tokio::test)] + #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `TLS_client_method` on OS `linux` + async fn builtin_converted_to_view_keeps_ids_and_shard() { + Catalog::with_debug(|catalog| async move { + { + let mut storage = catalog.storage().await; + let mut tx = storage.transaction().await.expect("transaction"); + + // Get some existing builtin view and register it as a builtin table. This means + // changing it's system mapping and creating a shard for it. + let view = tx + .get_system_object_mappings() + .find(|m| m.description.object_type == CatalogItemType::View) + .expect("builtin views exist"); + let global_id = view.unique_identifier.global_id; + let shard = ShardId::new(); + tx.remove_system_object_mappings(btreeset! {view.description.clone()}) + .expect("mapping exists"); + tx.set_system_object_mappings(vec![SystemObjectMapping { + description: SystemObjectDescription { + object_type: CatalogItemType::Table, + ..view.description.clone() + }, + unique_identifier: view.unique_identifier.clone(), + }]) + .expect("mapping is unique"); + tx.insert_collection_metadata([(global_id, shard)].into()) + .expect("a view has no shard"); + + migrate_builtin_item_types(&mut tx).expect("migration succeeds"); + // After migrating, expect it to be registered as a view again + // in the system mappings but keep the shard around. This will + // be finalized in a builtin schema migration. + let mappings: Vec<_> = tx + .get_system_object_mappings() + .filter(|m| m.unique_identifier.global_id == global_id) + .collect(); + assert_eq!( + mappings, + vec![view], + "the mapping must become a view under its ids" + ); + assert_eq!( + tx.get_collection_metadata().get(&global_id), + Some(&shard), + "the shard must be left for the `Drop` step" + ); + } + catalog.expire().await; + }) + .await; + } + fn add_doc_on_ids( sql: &str, lookup: impl FnMut(&mz_sql::ast::UnresolvedItemName) -> Option, diff --git a/src/adapter/src/catalog/open.rs b/src/adapter/src/catalog/open.rs index e3db96b64e39d..c3b37f4ef468c 100644 --- a/src/adapter/src/catalog/open.rs +++ b/src/adapter/src/catalog/open.rs @@ -996,9 +996,6 @@ fn add_new_remove_old_builtin_items_migration( // // Objects can be removed from this set after one release. let delete_exceptions: HashSet = [].into(); - // TODO(jkosh44) Technically we could support changing the type of a builtin object outside - // of unstable schemas (i.e. from a table to a view). However, builtin migrations don't currently - // handle that scenario correctly. assert!( deleted_system_objects .iter() diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index 716de42f02a49..f8a0cd251a0ac 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -21,6 +21,8 @@ //! shard's schema in-place. Only works for backward-compatible changes. //! * [`Mechanism::Replacement`] creates a new shard to serve the builtin collection in the new //! version. Works for all schema changes but discards existing data. +//! * [`Mechanism::Drop`] removes an existing shard in the new version. This is used when converting +//! a durable builtin collection to a view. //! //! Which mechanism to use is selected through entries in the `MIGRATIONS` list. In general, the //! `Evolution` mechanism should be used when possible, as it avoids data loss. @@ -76,13 +78,16 @@ use crate::catalog::migrate::get_migration_version; /// can delete all migration steps with versions before `(N-1).0.0`. /// /// Exception: when a builtin's `SystemObjectDescription` changes — e.g. a builtin table is -/// converted to a materialized view (see `migrate_builtin_tables_to_mvs`), or a builtin is +/// converted to a materialized view (see `migrate_builtin_item_types`), or a builtin is /// renamed or removed — existing steps naming the old description must be removed regardless /// of version, because `validate_migration_steps` panics on steps that don't resolve to a /// current builtin. This is safe only if a `Replacement` step for the new description is added /// at the conversion version: every environment that needed the removed steps upgrades from an /// even older version, so the new replacement subsumes them. /// +/// For a builtin converted to a view, that step is a `Drop`, because a view is not a storage +/// collection. `Migration::run` soft-asserts that no view is left with a shard. +/// /// Smallest supported version: 0.147.0 static MIGRATIONS: LazyLock> = LazyLock::new(|| { vec![ @@ -654,6 +659,11 @@ enum Mechanism { /// /// Works for arbitrary schema changes but loses existing contents. Replacement, + /// Shard drop. + /// + /// For a builtin that stopped being a storage collection, such as a table converted to a + /// view. Its shard is finalized and it gets no new one. + Drop, } /// The result of a builtin schema migration. @@ -756,6 +766,8 @@ pub(super) async fn run( /// Result produced by `Migration::run`. struct MigrationRunResult { new_shards: BTreeMap, + /// IDs of items whose shards have been dropped using the `Drop` mechanism. + dropped_collections: BTreeSet, new_fingerprints: BTreeMap, shards_to_finalize: BTreeSet, cleanup_action: BoxFuture<'static, ()>, @@ -765,6 +777,7 @@ impl Default for MigrationRunResult { fn default() -> Self { Self { new_shards: BTreeMap::new(), + dropped_collections: BTreeSet::new(), new_fingerprints: BTreeMap::new(), shards_to_finalize: BTreeSet::new(), cleanup_action: async {}.boxed(), @@ -776,7 +789,8 @@ impl MigrationRunResult { /// Apply this migration result to the given transaction. fn apply(&self, txn: &mut Transaction<'_>) { // Update collection metadata. - let replaced_ids = self.new_shards.keys().copied().collect(); + let mut replaced_ids: BTreeSet<_> = self.new_shards.keys().copied().collect(); + replaced_ids.extend(self.dropped_collections.iter().copied()); let old_metadata = txn.delete_collection_metadata(replaced_ids); txn.insert_collection_metadata(self.new_shards.clone()) .expect("inserting unique shards IDs after deleting existing entries"); @@ -838,6 +852,17 @@ struct Migration { config: BuiltinItemMigrationConfig, } +/// Whether a migration step may use `mechanism` for a `builtin`. +fn supports_migration_step(builtin: &Builtin, mechanism: Mechanism) -> bool { + use Builtin::*; + match builtin { + Table(..) | Source(..) | MaterializedView(..) => mechanism != Mechanism::Drop, + // The only shard a view can have is the one left from when it was a storage collection. + View(..) => mechanism == Mechanism::Drop, + Log(..) | Type(..) | Func(..) | Index(..) | Connection(..) => false, + } +} + /// Whether `builtin` participates in a forced migration using `mechanism`. fn participates_in_forced_migration( builtin: &Builtin, @@ -912,12 +937,25 @@ impl Migration { info!("executing migration plan: {plan:?}"); + // A view cannot use a shard. One that has a shard was a storage collection in the source + // version and must need a `Drop` step. + for (object, info) in &self.system_objects { + let has_stray_shard = + matches!(info.builtin, Builtin::View(..)) && info.shard_id.is_some(); + soft_assert_or_log!( + !has_stray_shard || plan.drop.contains(object), + "builtin {object:?} became a view without a `Drop` step for its shard", + ); + } + self.migrate_evolve(&plan.evolve).await?; let new_shards = self.migrate_replace(&plan.replace).await?; + let dropped_collections = self.migrate_drop(&plan.drop)?; let mut migrated_objects = BTreeSet::new(); migrated_objects.extend(plan.evolve); migrated_objects.extend(plan.replace); + migrated_objects.extend(plan.drop); let new_fingerprints = self.update_fingerprints(&migrated_objects)?; @@ -925,6 +963,7 @@ impl Migration { Ok(MigrationRunResult { new_shards, + dropped_collections, new_fingerprints, shards_to_finalize, cleanup_action, @@ -1011,10 +1050,10 @@ impl Migration { }; let builtin = object_info.builtin; - use Builtin::*; assert!( - matches!(builtin, Table(..) | Source(..) | MaterializedView(..)), - "schema migration not supported for builtin: {builtin:?}", + supports_migration_step(builtin, step.mechanism), + "{:?} migration not supported for builtin: {builtin:?}", + step.mechanism, ); } } @@ -1026,11 +1065,13 @@ impl Migration { // Select a mechanism for each object, according to the requested migrations: // * If any `Replacement` was requested, use `Replacement`. + // * If any `Drop` was requested, use `Drop`. // * Otherwise, (i.e. only `Evolution` was requested), use `Evolution`. let mut by_object = BTreeMap::new(); for step in steps { if let Some(entry) = by_object.get_mut(&step.object) { *entry = match (step.mechanism, *entry) { + (Mechanism::Drop, _) | (_, Mechanism::Drop) => Mechanism::Drop, (Mechanism::Evolution, Mechanism::Evolution) => Mechanism::Evolution, (Mechanism::Replacement, _) | (_, Mechanism::Replacement) => { Mechanism::Replacement @@ -1046,6 +1087,7 @@ impl Migration { match mechanism { Mechanism::Evolution => plan.evolve.push(object), Mechanism::Replacement => plan.replace.push(object), + Mechanism::Drop => plan.drop.push(object), } } @@ -1065,8 +1107,20 @@ impl Migration { match mechanism { Mechanism::Evolution => plan.evolve = objects, Mechanism::Replacement => plan.replace = objects, + Mechanism::Drop => unreachable!("forced migrations use evolution or replacement"), } + // Although there exists no explicit forced migration for Drop, we still want to clear + // any shards that now belong to views. Thus we include them in the plan for drop and + // Migration::run ensures only views with shards are dropped via `drop_shardless`. + let views = self + .system_objects + .iter() + .filter(|(_, info)| matches!(info.builtin, Builtin::View(..))) + .map(|(object, _)| object.clone()) + .collect(); + plan.drop = views; + plan } @@ -1079,6 +1133,8 @@ impl Migration { /// nothing to evolve or replace. Applied to every plan, forced or versioned, so that both /// paths agree; a read-only process that reached `migrate_evolve_one` with such an object /// would otherwise bail out and crash-loop. + /// + /// A view has a shard to drop only if it was a storage collection in the source version. fn drop_shardless(&self, mut plan: Plan) -> Plan { let has_shard = |object: &SystemObjectDescription| { self.system_objects @@ -1087,6 +1143,7 @@ impl Migration { }; plan.evolve.retain(has_shard); plan.replace.retain(has_shard); + plan.drop.retain(has_shard); plan } @@ -1290,6 +1347,30 @@ impl Migration { Ok(replaced_shards) } + /// Migrate the given objects using the `Drop` mechanism, returning their IDs. + /// + /// Nothing happens in persist here. `MigrationRunResult::apply` removes the collection + /// metadata and queues the shards for finalization. + fn migrate_drop( + &self, + objects: &[SystemObjectDescription], + ) -> anyhow::Result> { + let mut ids_to_drop = BTreeSet::new(); + for object in objects { + if let Some(info) = self.system_objects.get(object) { + ids_to_drop.insert(info.global_id); + } else { + bail!("missing id for builtin {object:?}"); + } + } + + if !ids_to_drop.is_empty() { + info!(?objects, ?ids_to_drop, "migrating by drop"); + } + + Ok(ids_to_drop) + } + /// Try to get or insert replacement shards for the given IDs into the migration shard, at /// `target_version` and `deploy_generation`. /// @@ -1602,6 +1683,8 @@ struct Plan { evolve: Vec, /// Objects to migrate using the `Replacement` mechanism. replace: Vec, + /// Objects to migrate using the `Drop` mechanism. + drop: Vec, } /// Types and persist codec impls for the migration shard used by the `Replacement` mechanism. diff --git a/src/adapter/src/catalog/open/builtin_schema_migration_tests.rs b/src/adapter/src/catalog/open/builtin_schema_migration_tests.rs index fbbd32375d689..9fa9ff9750cbd 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration_tests.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration_tests.rs @@ -14,7 +14,7 @@ use std::cell::RefCell; use std::rc::Rc; -use mz_catalog::builtin::{BuiltinSource, BuiltinTable}; +use mz_catalog::builtin::{BuiltinSource, BuiltinTable, BuiltinView}; use mz_persist_client::cache::PersistClientCache; use mz_persist_types::PersistLocation; use mz_repr::{RelationDesc, SqlScalarType}; @@ -26,6 +26,30 @@ use semver::Version; use tracing::info; use super::*; +use crate::catalog::Catalog; + +#[mz_ore::test] +fn drop_is_the_only_mechanism_for_a_view() { + let (_, table) = make_builtin_table("table".into()); + let (_, view) = make_builtin_view("view".into()); + for mechanism in [ + Mechanism::Evolution, + Mechanism::Replacement, + Mechanism::Drop, + ] { + let is_drop = mechanism == Mechanism::Drop; + assert_eq!( + supports_migration_step(view, mechanism), + is_drop, + "{mechanism:?} step naming a view" + ); + assert_eq!( + supports_migration_step(table, mechanism), + !is_drop, + "{mechanism:?} step naming a table" + ); + } +} #[mz_ore::test] fn hydration_history_forced_migration_policy() { @@ -158,6 +182,166 @@ fn test_evolution_skips_builtin_without_shard() { } } +/// A table converted to a view reaches the migration as a view that still has the table's shard. +/// Its `Drop` step must remove that shard without allocating a new one, where the `Replacement` +/// of a table allocates one. A forced migration ignores the steps and must find the view by +/// itself. +#[mz_ore::test(tokio::test)] +#[cfg_attr(miri, ignore)] // too slow +async fn test_drop_leaves_view_without_shard() { + let persist_client = PersistClientCache::new_no_metrics() + .open(PersistLocation::new_in_mem()) + .await + .expect("in-memory persist"); + + let (table_object, table_builtin) = make_builtin_table("table".into()); + let (converted_object, converted_builtin) = make_builtin_view("converted".into()); + let (plain_object, plain_builtin) = make_builtin_view("plain".into()); + let (table_id, converted_id, plain_id) = ( + GlobalId::System(1), + GlobalId::System(2), + GlobalId::System(3), + ); + let mut system_objects = BTreeMap::new(); + system_objects.insert( + table_object.clone(), + ObjectInfo { + global_id: table_id, + shard_id: Some(ShardId::new()), + builtin: evolve_builtin_desc(table_builtin), + fingerprint: table_builtin.fingerprint(), + }, + ); + // `migrate_builtin_item_types` rewrote this mapping from table to view, which keeps the + // table's shard and fingerprint. + system_objects.insert( + converted_object.clone(), + ObjectInfo { + global_id: converted_id, + shard_id: Some(ShardId::new()), + builtin: converted_builtin, + fingerprint: table_builtin.fingerprint(), + }, + ); + // A view that was never a storage collection. + system_objects.insert( + plain_object, + ObjectInfo { + global_id: plain_id, + shard_id: None, + builtin: plain_builtin, + fingerprint: plain_builtin.fingerprint(), + }, + ); + + let target_version = Version::new(0, 3, 0); + let steps = vec![ + MigrationStep { + version: Version::new(0, 2, 0), + object: table_object, + mechanism: Mechanism::Replacement, + }, + MigrationStep { + version: Version::new(0, 2, 0), + object: converted_object, + mechanism: Mechanism::Drop, + }, + ]; + + // Upgrading from a dev version forces an `Evolution` of every object, which evolves the + // table in place. The platform checks force a `Replacement` instead, which replaces it. + let cases = [ + ("versioned", Version::new(0, 1, 0), None, true), + ( + "forced-evolution", + Version::parse("0.1.0-dev.0").unwrap(), + None, + false, + ), + ( + "forced-replacement", + Version::new(0, 1, 0), + Some("replacement"), + true, + ), + ]; + for (case, source_version, force_migration, table_replaced) in cases { + let migration_shard_id = ShardId::new(); + for (mode, read_only) in [("read-only", true), ("leader", false)] { + let name = format!("{case}-{mode}"); + let migration = Migration { + source_version: source_version.clone(), + target_version: target_version.clone(), + deploy_generation: 1, + system_objects: system_objects.clone(), + migration_shard: migration_shard_id, + config: BuiltinItemMigrationConfig { + persist_client: persist_client.clone(), + read_only, + force_migration: force_migration.map(String::from), + }, + }; + let res = migration + .run(&steps) + .await + .unwrap_or_else(|e| panic!("{name}: migration failed: {e}")); + assert_eq!( + res.dropped_collections, + BTreeSet::from([converted_id]), + "{name}: the converted view, and no other object, must lose its shard" + ); + assert!( + !res.new_shards.contains_key(&converted_id), + "{name}: the converted view must not get a new shard" + ); + assert_eq!( + res.new_shards.contains_key(&table_id), + table_replaced, + "{name}: the table must get a new shard only when it is replaced" + ); + } + } +} + +/// The durable effect of a migration: a replaced collection is pointed at its new shard, a +/// dropped one keeps no shard, and both old shards are queued for finalization. +#[mz_ore::test(tokio::test)] +#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `TLS_client_method` on OS `linux` +async fn test_apply_drops_shard_without_replacement() { + Catalog::with_debug(|catalog| async move { + { + let mut storage = catalog.storage().await; + let mut txn = storage.transaction().await.expect("transaction"); + + let replaced_id = GlobalId::System(1_000_001); + let dropped_id = GlobalId::System(1_000_002); + let (replaced_old, replaced_new, dropped_old) = + (ShardId::new(), ShardId::new(), ShardId::new()); + txn.insert_collection_metadata( + [(replaced_id, replaced_old), (dropped_id, dropped_old)].into(), + ) + .expect("ids have no shards yet"); + + let result = MigrationRunResult { + new_shards: [(replaced_id, replaced_new)].into(), + dropped_collections: [dropped_id].into(), + ..Default::default() + }; + result.apply(&mut txn); + + let metadata = txn.get_collection_metadata(); + assert_eq!(metadata.get(&replaced_id), Some(&replaced_new)); + assert_eq!(metadata.get(&dropped_id), None); + let unfinalized = txn.get_unfinalized_shards(); + assert!(unfinalized.contains(&replaced_old)); + assert!(unfinalized.contains(&dropped_old)); + assert!(!unfinalized.contains(&replaced_new)); + } + catalog.expire().await; + }) + .await; +} + #[test] // allow(test-attribute) #[cfg_attr(miri, ignore)] // too slow fn test_builtin_schema_migration() { @@ -423,6 +607,30 @@ fn make_builtin_table(name: String) -> (SystemObjectDescription, &'static Builti (object, builtin) } +fn make_builtin_view(name: String) -> (SystemObjectDescription, &'static Builtin) { + let object = SystemObjectDescription { + schema_name: "schema".into(), + object_type: CatalogItemType::View, + object_name: name.clone(), + }; + + let builtin = BuiltinView { + name: name.leak(), + schema: "schema", + oid: 1, + desc: RelationDesc::builder() + .with_column("a", SqlScalarType::String.nullable(false)) + .finish(), + column_comments: BTreeMap::new(), + sql: "SELECT 'a' AS a", + access: Vec::new(), + ontology: None, + }; + let builtin = leak(Builtin::View(leak(builtin))); + + (object, builtin) +} + fn make_builtin_source(name: String) -> (SystemObjectDescription, &'static Builtin) { let object = SystemObjectDescription { schema_name: "schema".into(), @@ -550,9 +758,13 @@ fn init_persist(sim: &mut turmoil::Sim) -> PersistLocation { #[cfg_attr(miri, ignore)] // can't call foreign function `rust_psm_stack_pointer` on OS `linux` fn test_migration_steps_resolve_to_builtins() { for step in MIGRATIONS.iter() { + let Some((_, builtin)) = BUILTIN_LOOKUP.get(&step.object) else { + panic!("migration step for non-existent builtin: {:?}", step.object); + }; assert!( - BUILTIN_LOOKUP.contains_key(&step.object), - "migration step for non-existent builtin: {:?}", + supports_migration_step(builtin, step.mechanism), + "{:?} migration not supported for builtin: {:?}", + step.mechanism, step.object ); }