Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 90 additions & 24 deletions src/adapter/src/catalog/migrate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -840,45 +840,57 @@ pub(crate) fn durable_migrate(
tx.set_setting(MOCK_AUTHENTICATION_NONCE_KEY.to_string(), Some(nonce))?;
}

migrate_builtin_tables_to_mvs(tx)?;
migrate_builtin_item_types(tx)?;

Ok(())
}

/// Update system object mappings for builtins whose type changed from table to materialized view.
/// Update system object mappings for builtins whose type changed: from table
/// to materialized view, or from table or materialized view to view.
///
/// Required for the work of making builtin tables views over `mz_catalog_raw`.
fn migrate_builtin_tables_to_mvs(tx: &mut Transaction) -> Result<(), anyhow::Error> {
// Collect `(schema, name)` of all builtin MVs.
///
/// Only the mapping changes. The builtin keeps its ids and its shard, which is
/// left to the step registered for the new description in
/// `builtin_schema_migration`: a `Replacement` gives a materialized view a new
/// shard, a `Drop` leaves a view without one.
fn migrate_builtin_item_types(tx: &mut Transaction) -> Result<(), anyhow::Error> {
let expected_mvs: BTreeSet<_> = BUILTINS::materialized_views()
.map(|mv| (mv.schema, mv.name))
.collect();
let expected_views: BTreeSet<_> = BUILTINS::views().map(|v| (v.schema, v.name)).collect();

// Find persisted mappings for builtin tables that must be migrated.
// Find persisted mappings for builtin tables that must be migrated or builtin materialized
// views that must be dropped.
let mut to_remove = BTreeSet::new();
let mut to_add = Vec::new();
for mapping in tx.get_system_object_mappings() {
let desc = &mapping.description;
if desc.object_type != CatalogItemType::Table {
continue;
}

let key = (&*desc.schema_name, &*desc.object_name);
if expected_mvs.contains(&key) {
info!(
"migrate: builtin {}.{} changed type from table to MV",
desc.schema_name, desc.object_name,
);
to_remove.insert(desc.clone());
to_add.push(SystemObjectMapping {
description: SystemObjectDescription {
schema_name: desc.schema_name.clone(),
object_type: CatalogItemType::MaterializedView,
object_name: desc.object_name.clone(),
},
unique_identifier: mapping.unique_identifier,
});
}
let new_type = match desc.object_type {
CatalogItemType::Table if expected_mvs.contains(&key) => {
CatalogItemType::MaterializedView
}
CatalogItemType::Table | CatalogItemType::MaterializedView
if expected_views.contains(&key) =>
{
CatalogItemType::View
}
_ => continue,
};
info!(
"migrate: builtin {}.{} changed type from {} to {}",
desc.schema_name, desc.object_name, desc.object_type, new_type
);
to_remove.insert(desc.clone());
to_add.push(SystemObjectMapping {
description: SystemObjectDescription {
schema_name: desc.schema_name.clone(),
object_type: new_type,
object_name: desc.object_name.clone(),
},
unique_identifier: mapping.unique_identifier,
});
}

if !to_remove.is_empty() {
Expand Down Expand Up @@ -1308,6 +1320,7 @@ fn rewrite_interval_option_floor_1s(
#[cfg(test)]
mod tests {
use super::*;
use crate::catalog::Catalog;

fn strip(sql: &str) -> String {
let mut stmt = mz_sql::parse::parse(sql)
Expand Down Expand Up @@ -1370,6 +1383,59 @@ mod tests {
);
}

#[mz_ore::test(tokio::test)]
#[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `TLS_client_method` on OS `linux`
async fn builtin_converted_to_view_keeps_ids_and_shard() {
Catalog::with_debug(|catalog| async move {
{
let mut storage = catalog.storage().await;
let mut tx = storage.transaction().await.expect("transaction");

// Get some existing builtin view and register it as a builtin table. This means
// changing it's system mapping and creating a shard for it.
let view = tx
.get_system_object_mappings()
.find(|m| m.description.object_type == CatalogItemType::View)
.expect("builtin views exist");
let global_id = view.unique_identifier.global_id;
let shard = ShardId::new();
tx.remove_system_object_mappings(btreeset! {view.description.clone()})
.expect("mapping exists");
tx.set_system_object_mappings(vec![SystemObjectMapping {
description: SystemObjectDescription {
object_type: CatalogItemType::Table,
..view.description.clone()
},
unique_identifier: view.unique_identifier.clone(),
}])
.expect("mapping is unique");
tx.insert_collection_metadata([(global_id, shard)].into())
.expect("a view has no shard");

migrate_builtin_item_types(&mut tx).expect("migration succeeds");
// After migrating, expect it to be registered as a view again
// in the system mappings but keep the shard around. This will
// be finalized in a builtin schema migration.
let mappings: Vec<_> = tx
.get_system_object_mappings()
.filter(|m| m.unique_identifier.global_id == global_id)
.collect();
assert_eq!(
mappings,
vec![view],
"the mapping must become a view under its ids"
);
assert_eq!(
tx.get_collection_metadata().get(&global_id),
Some(&shard),
"the shard must be left for the `Drop` step"
);
}
catalog.expire().await;
})
.await;
}

fn add_doc_on_ids(
sql: &str,
lookup: impl FnMut(&mz_sql::ast::UnresolvedItemName) -> Option<String>,
Expand Down
3 changes: 0 additions & 3 deletions src/adapter/src/catalog/open.rs
Original file line number Diff line number Diff line change
Expand Up @@ -996,9 +996,6 @@ fn add_new_remove_old_builtin_items_migration(
//
// Objects can be removed from this set after one release.
let delete_exceptions: HashSet<SystemObjectDescription> = [].into();
// TODO(jkosh44) Technically we could support changing the type of a builtin object outside
// of unstable schemas (i.e. from a table to a view). However, builtin migrations don't currently
// handle that scenario correctly.
assert!(
deleted_system_objects
.iter()
Expand Down
93 changes: 88 additions & 5 deletions src/adapter/src/catalog/open/builtin_schema_migration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
//! shard's schema in-place. Only works for backward-compatible changes.
//! * [`Mechanism::Replacement`] creates a new shard to serve the builtin collection in the new
//! version. Works for all schema changes but discards existing data.
//! * [`Mechanism::Drop`] removes an existing shard in the new version. This is used when converting
//! a durable builtin collection to a view.
//!
//! Which mechanism to use is selected through entries in the `MIGRATIONS` list. In general, the
//! `Evolution` mechanism should be used when possible, as it avoids data loss.
Expand Down Expand Up @@ -76,13 +78,16 @@ use crate::catalog::migrate::get_migration_version;
/// can delete all migration steps with versions before `(N-1).0.0`.
///
/// Exception: when a builtin's `SystemObjectDescription` changes — e.g. a builtin table is
/// converted to a materialized view (see `migrate_builtin_tables_to_mvs`), or a builtin is
/// converted to a materialized view (see `migrate_builtin_item_types`), or a builtin is
/// renamed or removed — existing steps naming the old description must be removed regardless
/// of version, because `validate_migration_steps` panics on steps that don't resolve to a
/// current builtin. This is safe only if a `Replacement` step for the new description is added
/// at the conversion version: every environment that needed the removed steps upgrades from an
/// even older version, so the new replacement subsumes them.
///
/// For a builtin converted to a view, that step is a `Drop`, because a view is not a storage
/// collection. `Migration::run` soft-asserts that no view is left with a shard.
///
/// Smallest supported version: 0.147.0
static MIGRATIONS: LazyLock<Vec<MigrationStep>> = LazyLock::new(|| {
vec![
Expand Down Expand Up @@ -654,6 +659,11 @@ enum Mechanism {
///
/// Works for arbitrary schema changes but loses existing contents.
Replacement,
/// Shard drop.
///
/// For a builtin that stopped being a storage collection, such as a table converted to a
/// view. Its shard is finalized and it gets no new one.
Drop,
}

/// The result of a builtin schema migration.
Expand Down Expand Up @@ -756,6 +766,8 @@ pub(super) async fn run(
/// Result produced by `Migration::run`.
struct MigrationRunResult {
new_shards: BTreeMap<GlobalId, ShardId>,
/// IDs of items whose shards have been dropped using the `Drop` mechanism.
dropped_collections: BTreeSet<GlobalId>,
new_fingerprints: BTreeMap<SystemObjectDescription, String>,
shards_to_finalize: BTreeSet<ShardId>,
cleanup_action: BoxFuture<'static, ()>,
Expand All @@ -765,6 +777,7 @@ impl Default for MigrationRunResult {
fn default() -> Self {
Self {
new_shards: BTreeMap::new(),
dropped_collections: BTreeSet::new(),
new_fingerprints: BTreeMap::new(),
shards_to_finalize: BTreeSet::new(),
cleanup_action: async {}.boxed(),
Expand All @@ -776,7 +789,8 @@ impl MigrationRunResult {
/// Apply this migration result to the given transaction.
fn apply(&self, txn: &mut Transaction<'_>) {
// Update collection metadata.
let replaced_ids = self.new_shards.keys().copied().collect();
let mut replaced_ids: BTreeSet<_> = self.new_shards.keys().copied().collect();
replaced_ids.extend(self.dropped_collections.iter().copied());
let old_metadata = txn.delete_collection_metadata(replaced_ids);
txn.insert_collection_metadata(self.new_shards.clone())
.expect("inserting unique shards IDs after deleting existing entries");
Expand Down Expand Up @@ -838,6 +852,17 @@ struct Migration {
config: BuiltinItemMigrationConfig,
}

/// Whether a migration step may use `mechanism` for a `builtin`.
fn supports_migration_step(builtin: &Builtin<NameReference>, mechanism: Mechanism) -> bool {
use Builtin::*;
match builtin {
Table(..) | Source(..) | MaterializedView(..) => mechanism != Mechanism::Drop,
// The only shard a view can have is the one left from when it was a storage collection.
View(..) => mechanism == Mechanism::Drop,
Log(..) | Type(..) | Func(..) | Index(..) | Connection(..) => false,
}
}

/// Whether `builtin` participates in a forced migration using `mechanism`.
fn participates_in_forced_migration(
builtin: &Builtin<NameReference>,
Expand Down Expand Up @@ -912,19 +937,33 @@ impl Migration {

info!("executing migration plan: {plan:?}");

// A view cannot use a shard. One that has a shard was a storage collection in the source
// version and must need a `Drop` step.
for (object, info) in &self.system_objects {
let has_stray_shard =
matches!(info.builtin, Builtin::View(..)) && info.shard_id.is_some();
soft_assert_or_log!(
!has_stray_shard || plan.drop.contains(object),
"builtin {object:?} became a view without a `Drop` step for its shard",
);
}

self.migrate_evolve(&plan.evolve).await?;
let new_shards = self.migrate_replace(&plan.replace).await?;
let dropped_collections = self.migrate_drop(&plan.drop)?;

let mut migrated_objects = BTreeSet::new();
migrated_objects.extend(plan.evolve);
migrated_objects.extend(plan.replace);
migrated_objects.extend(plan.drop);

let new_fingerprints = self.update_fingerprints(&migrated_objects)?;

let (shards_to_finalize, cleanup_action) = self.cleanup().await?;

Ok(MigrationRunResult {
new_shards,
dropped_collections,
new_fingerprints,
shards_to_finalize,
cleanup_action,
Expand Down Expand Up @@ -1011,10 +1050,10 @@ impl Migration {
};

let builtin = object_info.builtin;
use Builtin::*;
assert!(
matches!(builtin, Table(..) | Source(..) | MaterializedView(..)),
"schema migration not supported for builtin: {builtin:?}",
supports_migration_step(builtin, step.mechanism),
"{:?} migration not supported for builtin: {builtin:?}",
step.mechanism,
);
}
}
Expand All @@ -1026,11 +1065,13 @@ impl Migration {

// Select a mechanism for each object, according to the requested migrations:
// * If any `Replacement` was requested, use `Replacement`.
// * If any `Drop` was requested, use `Drop`.
// * Otherwise, (i.e. only `Evolution` was requested), use `Evolution`.
let mut by_object = BTreeMap::new();
for step in steps {
if let Some(entry) = by_object.get_mut(&step.object) {
*entry = match (step.mechanism, *entry) {
(Mechanism::Drop, _) | (_, Mechanism::Drop) => Mechanism::Drop,
(Mechanism::Evolution, Mechanism::Evolution) => Mechanism::Evolution,
(Mechanism::Replacement, _) | (_, Mechanism::Replacement) => {
Mechanism::Replacement
Expand All @@ -1046,6 +1087,7 @@ impl Migration {
match mechanism {
Mechanism::Evolution => plan.evolve.push(object),
Mechanism::Replacement => plan.replace.push(object),
Mechanism::Drop => plan.drop.push(object),
}
}

Expand All @@ -1065,8 +1107,20 @@ impl Migration {
match mechanism {
Mechanism::Evolution => plan.evolve = objects,
Mechanism::Replacement => plan.replace = objects,
Mechanism::Drop => unreachable!("forced migrations use evolution or replacement"),
}

// Although there exists no explicit forced migration for Drop, we still want to clear
// any shards that now belong to views. Thus we include them in the plan for drop and
// Migration::run ensures only views with shards are dropped via `drop_shardless`.
let views = self
.system_objects
.iter()
.filter(|(_, info)| matches!(info.builtin, Builtin::View(..)))
.map(|(object, _)| object.clone())
.collect();
plan.drop = views;

plan
}

Expand All @@ -1079,6 +1133,8 @@ impl Migration {
/// nothing to evolve or replace. Applied to every plan, forced or versioned, so that both
/// paths agree; a read-only process that reached `migrate_evolve_one` with such an object
/// would otherwise bail out and crash-loop.
///
/// A view has a shard to drop only if it was a storage collection in the source version.
fn drop_shardless(&self, mut plan: Plan) -> Plan {
let has_shard = |object: &SystemObjectDescription| {
self.system_objects
Expand All @@ -1087,6 +1143,7 @@ impl Migration {
};
plan.evolve.retain(has_shard);
plan.replace.retain(has_shard);
plan.drop.retain(has_shard);
plan
}

Expand Down Expand Up @@ -1290,6 +1347,30 @@ impl Migration {
Ok(replaced_shards)
}

/// Migrate the given objects using the `Drop` mechanism, returning their IDs.
///
/// Nothing happens in persist here. `MigrationRunResult::apply` removes the collection
/// metadata and queues the shards for finalization.
fn migrate_drop(
&self,
objects: &[SystemObjectDescription],
) -> anyhow::Result<BTreeSet<GlobalId>> {
let mut ids_to_drop = BTreeSet::new();
for object in objects {
if let Some(info) = self.system_objects.get(object) {
ids_to_drop.insert(info.global_id);
} else {
bail!("missing id for builtin {object:?}");
}
}

if !ids_to_drop.is_empty() {
info!(?objects, ?ids_to_drop, "migrating by drop");
}

Ok(ids_to_drop)
}

/// Try to get or insert replacement shards for the given IDs into the migration shard, at
/// `target_version` and `deploy_generation`.
///
Expand Down Expand Up @@ -1602,6 +1683,8 @@ struct Plan {
evolve: Vec<SystemObjectDescription>,
/// Objects to migrate using the `Replacement` mechanism.
replace: Vec<SystemObjectDescription>,
/// Objects to migrate using the `Drop` mechanism.
drop: Vec<SystemObjectDescription>,
}

/// Types and persist codec impls for the migration shard used by the `Replacement` mechanism.
Expand Down
Loading
Loading