diff --git a/3d_segmentation/spleen_segmentation_3d_lightning.ipynb b/3d_segmentation/spleen_segmentation_3d_lightning.ipynb index 57738bec1a..81130606a7 100644 --- a/3d_segmentation/spleen_segmentation_3d_lightning.ipynb +++ b/3d_segmentation/spleen_segmentation_3d_lightning.ipynb @@ -326,17 +326,18 @@ " )\n", "\n", " # we use cached datasets - these are 10x faster than regular datasets\n", + " # cache_rate/num_workers reduced (were 1.0/4) to avoid OOM on memory-limited hosts (15 GiB RAM)\n", " self.train_ds = CacheDataset(\n", " data=train_files,\n", " transform=train_transforms,\n", - " cache_rate=1.0,\n", - " num_workers=4,\n", + " cache_rate=0.1,\n", + " num_workers=2,\n", " )\n", " self.val_ds = CacheDataset(\n", " data=val_files,\n", " transform=val_transforms,\n", - " cache_rate=1.0,\n", - " num_workers=4,\n", + " cache_rate=0.1,\n", + " num_workers=2,\n", " )\n", "\n", " # self.train_ds = monai.data.Dataset(\n", @@ -438,6 +439,9 @@ " enable_checkpointing=True,\n", " num_sanity_val_steps=1,\n", " log_every_n_steps=16,\n", + " # disable the rich/tqdm progress bar: its rich-based rendering recurses infinitely\n", + " # (RecursionError in rich.style) under the papermill/ZMQ Jupyter display backend\n", + " enable_progress_bar=False,\n", ")\n", "\n", "# train\n", diff --git a/3d_segmentation/unet_segmentation_3d_ignite.ipynb b/3d_segmentation/unet_segmentation_3d_ignite.ipynb index 9940524b07..a47b39ff8c 100644 --- a/3d_segmentation/unet_segmentation_3d_ignite.ipynb +++ b/3d_segmentation/unet_segmentation_3d_ignite.ipynb @@ -333,7 +333,9 @@ "\n", "# MLFlowHandler plots loss at every iteration on MLFlow web UI\n", "mlflow_dir = os.path.join(log_dir, \"mlruns\")\n", - "train_mlflow_handler = MLFlowHandler(tracking_uri=Path(mlflow_dir).as_uri(), output_transform=lambda x: x)\n", + "train_mlflow_handler = MLFlowHandler(\n", + " tracking_uri=\"sqlite:///\" + os.path.join(mlflow_dir, \"mlruns.db\"), output_transform=lambda x: x\n", + ")\n", "train_mlflow_handler.attach(trainer)" ] }, @@ -419,7 +421,7 @@ "\n", "# add handler to record metrics to MLFlow at every validation epoch\n", "val_mlflow_handler = MLFlowHandler(\n", - " tracking_uri=Path(mlflow_dir).as_uri(),\n", + " tracking_uri=\"sqlite:///\" + os.path.join(mlflow_dir, \"mlruns.db\"),\n", " # no need to plot loss value, so disable per iteration output\n", " output_transform=lambda x: None,\n", " # fetch global epoch number from trainer\n", diff --git a/NOTEBOOK_TEST_LOG.md b/NOTEBOOK_TEST_LOG.md new file mode 100644 index 0000000000..a06751109f --- /dev/null +++ b/NOTEBOOK_TEST_LOG.md @@ -0,0 +1,305 @@ +# MONAI Tutorials — Notebook Execution Test Log + +**Goal:** Verify every notebook in each tutorials folder executes successfully under the +project virtual environment with **MONAI 1.6.1**. Where a notebook fails, make the +*minimal* changes needed to run it (max 5 attempts), and log every change here. If it +still fails after 5 attempts, it is logged as a failure with the final error. + +## Environment + +| Item | Value | +|------|-------| +| Python venv | `/home/ubuntu/tools/MONAI-test/tutorials/.venv` (Python 3.13) | +| MONAI | 1.6.1 | +| PyTorch | 2.10.0+cu128 | +| GPU | NVIDIA A10G, 23 GB (CUDA available) | +| papermill | 2.7.0 | +| Jupyter kernel | `monai-venv` (registered to the venv interpreter) | +| Data cache | a local directory exported via `MONAI_DATA_DIRECTORY` | + +## Methodology + +- Each notebook is executed with **papermill** from its own directory. +- Before execution, long-running loop variables are reduced to 1 (matching the project's + `runner.sh`): `max_epochs`, `val_interval`, `disc_train_interval`, `disc_train_steps`, + `num_batches_for_histogram`. This exercises the full code path quickly. +- A local papermill-based harness runs a reduced copy of each notebook (the original + notebook on disk is not modified) with the venv forced onto `PATH` and a process-group + timeout. This is equivalent to the project's `runner.sh -t `. +- A per-notebook wall-clock **timeout** is applied (default 1800 s). +- **Skipped** notebooks are those the project's `runner.sh` lists under `skip_run_papermill` + — they require external services (3D Slicer, CVAT, QuPath, OHIF), special hardware, + network/data not available in CI, or are known-blocked upstream. These are not failures. +- Each run uses up to **5 attempts**. Fixes applied between attempts are recorded. + +## Status legend + +- ✅ **PASS** — executed end-to-end without error. +- 🔧 **PASS (fixed)** — passed after minimal changes (changes listed). +- ⏭️ **SKIP** — on the project skip list / needs unavailable external resource. +- ❌ **FAIL** — still failing after up to 5 attempts (final error listed). + +--- + +## Results by folder + + + +### 2d_classification + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| monai_101.ipynb | ✅ PASS | 110s | | +| monai_201.ipynb | ✅ PASS | 262s | | +| mednist_tutorial.ipynb | ✅ PASS | 119s | | + +### 2d_registration + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| registration_mednist.ipynb | ✅ PASS | 31s | | + +### 2d_regression + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| image_restoration.ipynb | ⏭️ SKIP | — | On project skip list: `monai.networks.nets.restormer` not yet in released MONAI. | + +### 3d_classification + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| densenet_training_array.ipynb | ✅ PASS | 250s | | + +### 3d_regression + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| densenet_training_array.ipynb | ✅ PASS | 36s | | + +### patch_inferer + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| modular_patch_inferer.ipynb | 🔧 PASS (fixed) | 129s | **Fix:** zarr 3.4.0 was installed, but MONAI 1.6.1's `ZarrAvgMerger` uses the zarr v2 API (`chunks=True` default and `cdata_shape`), which zarr 3.x rejects (`ValueError: True is not a valid chunk input`). Pinned **`zarr<3`** (installed 2.18.7) and updated the notebook's install cell to enforce `zarr<3`. | + +### reconstruction + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| MRI_reconstruction/unet_demo/inference.ipynb | ⏭️ SKIP | — | On project skip list (`MRI_reconstruction`): requires trained model weights / fastMRI data not available in CI. | +| MRI_reconstruction/varnet_demo/inference.ipynb | ⏭️ SKIP | — | On project skip list (`MRI_reconstruction`). | + +### 3d_registration + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| learn2reg_nlst_paired_lung_ct.ipynb | ⏭️ SKIP | — | On project skip list: slow test / large Learn2Reg dataset. | +| learn2reg_oasis_unpaired_brain_mr.ipynb | ⏭️ SKIP | — | On project skip list. | +| paired_lung_ct.ipynb | ✅ PASS | 191s | | + +### hugging_face + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| hugging_face_pipeline_for_monai.ipynb | ✅ PASS | 101s | | +| finetune_vista3d_for_hugging_face_pipeline.ipynb | ⏭️ SKIP | — | On project skip list: requires VISTA3D weights / large data. | + +### acceleration + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| TensorRT_inference_acceleration.ipynb | ⏭️ SKIP | — | On project skip list: requires TensorRT engine build. | +| automatic_mixed_precision.ipynb | 🔧 PASS (fixed) | 258s | **Fix:** process was OOM-killed (host has 15 GiB RAM, no swap) while building the second `CacheDataset(cache_rate=1.0)` at `num_workers=8`. Confirmed via kernel oom-killer log (anon-rss ~13 GB). Reduced the two CacheDataset `num_workers` to 2 to cap peak RAM during caching. Attempts: 1 fail (OOM), 2 fail (OOM), 3 pass. | +| dataset_type_performance.ipynb | ✅ PASS | 393s | | +| fast_training_tutorial.ipynb | 🔧 PASS (fixed) | 144s | **Fix:** missing dependency `nvtx` (`No module named 'nvtx'`). Installed `nvtx` into the venv and added `!python -c "import nvtx" \|\| pip install -q nvtx` to the notebook's install cell. Attempts: 1 fail, 2 pass. | +| threadbuffer_performance.ipynb | ✅ PASS | 66s | | +| transform_speed.ipynb | ✅ PASS | 301s | | + +### bundle + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| 01_bundle_intro.ipynb | 🔧 PASS (fixed) | 93s | **Fix:** `python -m monai.bundle init_bundle` failed with `OptionalImportError: import fire`. Installed `fire` into the venv and added `!python -c "import fire" \|\| pip install -q fire` to the install cell. | +| 02_mednist_classification.ipynb | 🔧 PASS (fixed) | 144s | **Fixes:** (1) added `fire` install (same as above). (2) **Environment:** the notebook's `%%bash` cells call bare `python -m monai.bundle run`, which resolved to a *base conda MONAI 1.4.0* on PATH instead of the venv's 1.6.1 — causing `RuntimeError: Failed to instantiate SupervisedTrainer`. Fixed at the harness level by prepending the venv `bin` to `PATH` so shell cells use the kernel interpreter. See "Environment note" below. Attempts: 1–3 fail, 4 pass. | +| 03_mednist_classification_v2.ipynb | 🔧 PASS (fixed) | 190s | Same two fixes as 02 (fire + PATH to venv for `%%bash` cells). | +| 04_integrating_code.ipynb | 🔧 PASS (fixed) | 147s | **Fix:** added `fire` install line. | +| 05_spleen_segmentation_lightning.ipynb | ⏭️ SKIP | — | On project skip list: hardcoded `.to("cuda")` with no CPU fallback / GPU-only. | +| pythonic_usage_guidance/pythonic_bundle_access.ipynb | 🔧 PASS (fixed) | 191s | **Fix:** OOM (15 GiB host) while the downloaded `spleen_ct_segmentation` bundle built `CacheDataset(cache_rate=1.0)` for the full dataset. Added `train#dataset#cache_rate=0.0` and `validate#dataset#cache_rate=0.0` overrides to the four `create_workflow(...)` calls. Attempts: 1 fail (OOM), 2 pass. | + +> **Environment note (applies to all `%%bash` / `!python` shell cells):** Many notebooks shell +> out with bare `python`/`pip`. Those subshells inherit `$PATH`, where a base conda env with +> **MONAI 1.4.0** was ahead of the project venv. To run these notebooks correctly the venv must be +> the first Python on `PATH` (i.e. activate the venv, or prepend `.venv/bin`). The test harness used +> for this report does this automatically. This is an environment setup requirement, not a defect in +> the notebooks. + +### deepgrow + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| ignite/inference.ipynb | ✅ PASS | 29s | | +| ignite/inference_3d.ipynb | ✅ PASS | 63s | | + +### deepedit + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| ignite/infoANDinference.ipynb | ✅ PASS | 36s | | + +### deep_atlas + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| deep_atlas_tutorial.ipynb | ✅ PASS | 169s | | + +### computer_assisted_intervention + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| endoscopic_inbody_classification.ipynb | ✅ PASS | 36s | | +| video_seg.ipynb | ⏭️ SKIP | — | On project skip list. | + +### multimodal + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| nakaseke_multimodal_early_fusion/multimodal_early_fusion_tutorial.ipynb | ✅ PASS | 28s | | +| openi_multilabel_classification_transchex/transchex_openi_multilabel_classification.ipynb | ⏭️ SKIP | — | On project skip list (`transchex_openi`). | + +### active_learning + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| liver_tumor_al/results_uncertainty_analysis.ipynb | ⏭️ SKIP | — | On project skip list (`active_learning`). | +| tool_tracking_al/results_uncertainty_analysis.ipynb | ⏭️ SKIP | — | On project skip list. | + +### self_supervised_pretraining + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| swinunetr_pretrained/swinunetr_finetune.ipynb | ⏭️ SKIP | — | On project skip list (`swinunetr_finetune`). | +| vit_unetr_ssl/ssl_finetune.ipynb | ⏭️ SKIP | — | On project skip list (`ssl_finetune`). | +| vit_unetr_ssl/ssl_train.ipynb | ⏭️ SKIP | — | On project skip list (`ssl_train`). | + +### experiment_management + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| bundle_integrate_mlflow.ipynb | 🔧 PASS (fixed) | 608s | **Fix:** MONAI 1.6.1's `MLFlowHandler` rejects filesystem (file-store) tracking URIs and requires a `sqlite:///` or remote URI. Changed the two explicit file-store URIs (`./eval/mlruns`) to `sqlite:///./eval/mlruns.db` — in notebook cell with `--tracking_uri`, in the `MLFlowHandler(tracking_uri=...)` cell, and in `experiment_management/mlflow_example.json` (`tracking_uri` now `sqlite:///...mlruns.db`). Attempts: 1 fail, 2 pass. | +| spleen_segmentation_aim.ipynb | ❌ FAIL | — | **Blocked (environment):** requires the `aim` package, which cannot be installed on **Python 3.13**. `aim==3.17.5` (pinned by the notebook) and the latest `aim==3.29.1` both fail to build — their build dep `aimrocks==0.5.*` has no py3.13 wheel/sdist (only 0.2.0 exists) and they pin unavailable Cython alphas. Not fixable via notebook edits; needs an older Python or an `aim` release with py3.13 support. Attempts: 5 (various install strategies: pinned version, latest, `--only-binary`). | +| spleen_segmentation_mlflow.ipynb | ✅ PASS | 93s | | +| unet_segmentation_3d_ignite_clearml.ipynb | ⏭️ SKIP | — | On project skip list (needs ClearML account). | + +### 3d_segmentation + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| brats_segmentation_3d.ipynb | ⏳ RUNS, exceeds time budget | >20min | Dependency fixed (installed `onnxruntime`; the notebook's install cell already handles it). Already uses `cache_rate=0.0` so no OOM. Executes correctly but trains 1 epoch over the full BraTS set (~388 3D volumes) plus ONNX/TensorRT inference, exceeding the 20-min per-notebook budget. Runs correctly but exceeds the 20-min per-notebook budget. BraTS data (7.1 GB) was cached in the local data directory. | +| spleen_segmentation_3d.ipynb | ✅ PASS | 179s | | +| spleen_segmentation_3d_lightning.ipynb | 🔧 PASS (fixed) | 149s | **Fixes (3):** (1) installed `pytorch-lightning` (notebook install cell already requests it). (2) OOM on 15 GiB host → reduced the two `CacheDataset` to `cache_rate=0.1`, `num_workers=2`. (3) `RecursionError` in `rich.style` (Lightning's rich progress bar recurses under the papermill/ZMQ display backend) → added `enable_progress_bar=False` to the `Trainer`. Attempts: 1 fail (dep), 2 fail (OOM), 3 fail (recursion), 4 pass. | +| spleen_segmentation_3d_visualization_basic.ipynb | ⏭️ SKIP | — | On project skip list. | +| swin_unetr_brats21_segmentation_3d.ipynb | ⏭️ SKIP | — | On project skip list (matches `unetr_`). | +| swin_unetr_btcv_segmentation_3d.ipynb | ⏭️ SKIP | — | On project skip list (matches `unetr_`). | +| unet_segmentation_3d_ignite.ipynb | 🔧 PASS (fixed) | 65s | **Fix:** MONAI 1.6.1 `MLFlowHandler` rejects file-store URIs. Changed the two `MLFlowHandler(tracking_uri=Path(mlflow_dir).as_uri())` to `sqlite:////mlruns.db`. Attempts: 1 fail (mlflow), 2 pass. | +| unetr_btcv_segmentation_3d.ipynb | ⏭️ SKIP | — | On project skip list (`unetr_`). | +| unetr_btcv_segmentation_3d_lightning.ipynb | ⏭️ SKIP | — | On project skip list (`unetr_`). | + +### deployment + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| bentoml/mednist_classifier_bentoml.ipynb | ❌ FAIL | — | **Blocked (environment):** notebook pins `bentoml==0.13.1` (2021-era). It cannot work on **Python 3.13** — the pinned version force-downgrades `urllib3`/`sqlalchemy` and then fails to import (`No module named 'urllib3.packages.six.moves'`), and the notebook's entire API (`bentoml.frameworks.pytorch`, `bentoml.adapters`, `@bentoml.env`) was removed in bentoml 1.x. Installing it also *corrupted the shared venv* (broke mlflow); I uninstalled it and restored `urllib3>=2`, `sqlalchemy>=1.4`. Not fixable without rewriting the notebook for bentoml 1.x. Attempts: 5. | +| ray/mednist_classifier_ray.ipynb | ⏭️ SKIP | — | On project skip list (tutorials issue #1307). | + +### microscopy + +| Notebook | Status | Time | Notes | +|----------|--------|------|-------| +| multichannel_microscopy_classification.ipynb | ⏳ (re-run pending) | — | First attempt was aborted by the harness-timeout bug (not a notebook failure). Re-running after the harness fix. | + +### Note on timeouts + +> Blocking notebooks (e.g. ones that start a server) must be killed by process group on timeout — +> otherwise the papermill process and its Jupyter kernel are orphaned and keep running past the cap. +> The harness used for this report enforces a hard per-notebook timeout accordingly. + +--- + +## Summary of progress so far + +As of this commit, the following folders have been fully processed: + +`2d_classification`, `2d_registration`, `2d_regression`, `3d_classification`, +`3d_regression`, `3d_registration`, `patch_inferer`, `reconstruction`, `hugging_face`, +`acceleration`, `bundle`, `deepgrow`, `deepedit`, `deep_atlas`, +`computer_assisted_intervention`, `multimodal`, `active_learning`, +`self_supervised_pretraining`, `experiment_management`, `3d_segmentation`, `deployment`. + +Running totals across processed folders: + +| Outcome | Count | +|---------|-------| +| ✅ / 🔧 PASS (executed successfully) | 30 | +|    of which required a fix (🔧) | 11 | +| ⏭️ SKIP (project `skip_run_papermill` list) | 22 | +| ❌ FAIL (environment-blocked, not fixable by notebook edits) | 2 | +| ⏳ RUNS but exceeds the 20-min per-notebook time budget | 1 | + +### Dependencies added to the venv to make notebooks run + +| Package | Reason | +|---------|--------| +| `zarr<3` (2.18.7) | MONAI 1.6.1 `ZarrAvgMerger` uses the zarr v2 API; zarr 3.x rejects it. | +| `nvtx` | Imported by `acceleration/fast_training_tutorial`. | +| `fire` | Required by the `monai.bundle` CLI (`init_bundle`, `run`). | +| `pytorch-lightning` (2.6.6) | Required by the Lightning-based notebooks. | +| `onnxruntime` (1.30.0) | Imported by `3d_segmentation/brats_segmentation_3d`. | + +> `mlflow` is kept at its original 3.16.1 — the MLflow failures were due to MONAI 1.6.1's +> `MLFlowHandler` rejecting file-store URIs, fixed in-notebook with `sqlite:///` URIs. + +### Environment-blocked failures (not fixable by editing the notebook) + +- `experiment_management/spleen_segmentation_aim.ipynb` — `aim` has no Python 3.13 support. +- `deployment/bentoml/mednist_classifier_bentoml.ipynb` — pins `bentoml==0.13.1` (2021), incompatible + with Python 3.13 and the modern bentoml 1.x API; installing it corrupts the shared env. + +> **Recommendation for both:** they are good candidates to add to `skip_run_papermill` in +> `runner.sh` for the current Python 3.13 / MONAI 1.6.1 environment (or to re-run on an older +> Python). `mednist_classifier_ray` is already skipped for a similar reason. + +--- + +## Remaining work (not yet executed) + +The following folders still need to be run and logged. Counts are runnable notebooks +(before applying the skip list): + +| Folder | Notebooks | Notes | +|--------|-----------|-------| +| `modules` | 51 | Largest folder; mix of CPU demos and GPU training notebooks. | +| `generation` | 31 | GAN / diffusion / MAISI; several are large and GPU-heavy. | +| `auto3dseg` | 9 | AutoML segmentation pipelines; some long-running. | +| `pathology` | 6 | Several already on the skip list (hovernet, nuclei, nuclick). | +| `monailabel` | 10 | Nearly all on the skip list (need 3D Slicer / CVAT / QuPath / OHIF). | +| `competitions` | 3 | All three (`preprocess_*`) are on the skip list. | +| `microscopy` | 1 | `multichannel_microscopy_classification` — in progress. | +| `vista_3d` | 1 | `vista3d_spleen_finetune`. | +| `vista_2d` | 1 | `vista_2d_tutorial_monai` — on the skip list. | +| `model_zoo` | 1 | `TCIA_PROSTATEx...` — on the skip list. | +| `federated_learning` | 1 | On the skip list (needs OpenFL cluster). | +| `full_gpu_inference_pipeline` | 1 | On the skip list (Triton client/server). | + +Folders with **no notebooks** (Python-script tutorials, out of scope for notebook testing): +`detection`, `nnunet`, `automl`, `performance_profiling`. + +### Notes / caveats carried forward + +- **15 GiB RAM ceiling (no swap):** several training notebooks OOM when a `CacheDataset` + caches a full 3D dataset at `cache_rate=1.0`. The minimal fix applied in each case is to lower + `cache_rate`/`num_workers`. Expect the same in `generation`, `auto3dseg`, and parts of `modules`. +- **Shell cells use the venv:** notebooks that shell out with bare `python`/`pip` need the venv to + be first on `PATH` (the harness enforces this). Without it they pick up a base-conda MONAI 1.4.0. +- **Time budget:** per the agreed default, notebooks are capped at ~20 min; anything correct but + slower is logged as "RUNS, exceeds time budget" rather than a failure. diff --git a/acceleration/automatic_mixed_precision.ipynb b/acceleration/automatic_mixed_precision.ipynb index cf9316631e..ec7c6933eb 100644 --- a/acceleration/automatic_mixed_precision.ipynb +++ b/acceleration/automatic_mixed_precision.ipynb @@ -270,8 +270,9 @@ "source": [ "def train_process(amp=False):\n", " train_trans, val_trans = transformations()\n", - " train_ds = CacheDataset(data=train_files, transform=train_trans, cache_rate=1.0, num_workers=8)\n", - " val_ds = CacheDataset(data=val_files, transform=val_trans, cache_rate=1.0, num_workers=5)\n", + " # num_workers reduced (was 8/5) to cap peak RAM during caching on memory-limited hosts (15 GiB).\n", + " train_ds = CacheDataset(data=train_files, transform=train_trans, cache_rate=1.0, num_workers=2)\n", + " val_ds = CacheDataset(data=val_files, transform=val_trans, cache_rate=1.0, num_workers=2)\n", "\n", " # use batch_size=2 to load images and use RandCropByPosNegLabeld\n", " # to generate 2 x 4 images for network training\n", diff --git a/acceleration/fast_training_tutorial.ipynb b/acceleration/fast_training_tutorial.ipynb index 4719ac61e0..020f55a4f7 100644 --- a/acceleration/fast_training_tutorial.ipynb +++ b/acceleration/fast_training_tutorial.ipynb @@ -55,6 +55,7 @@ "source": [ "!python -c \"import monai\" || pip install -q \"monai-weekly[nibabel, tqdm]\"\n", "!python -c \"import matplotlib\" || pip install -q matplotlib\n", + "!python -c \"import nvtx\" || pip install -q nvtx\n", "%matplotlib inline" ] }, diff --git a/bundle/01_bundle_intro.ipynb b/bundle/01_bundle_intro.ipynb index d4f426b0b2..7b3607b18c 100644 --- a/bundle/01_bundle_intro.ipynb +++ b/bundle/01_bundle_intro.ipynb @@ -32,7 +32,9 @@ "metadata": {}, "outputs": [], "source": [ - "!python -c \"import monai\" || pip install -q \"monai-weekly[ignite,pyyaml]\"" + "!python -c \"import monai\" || pip install -q \"monai-weekly[ignite,pyyaml]\"\n", + "# monai.bundle CLI (init_bundle/run) requires the `fire` package\n", + "!python -c \"import fire\" || pip install -q fire" ] }, { diff --git a/bundle/02_mednist_classification.ipynb b/bundle/02_mednist_classification.ipynb index 90fde7f9d4..2d046efd7c 100644 --- a/bundle/02_mednist_classification.ipynb +++ b/bundle/02_mednist_classification.ipynb @@ -32,7 +32,9 @@ "metadata": {}, "outputs": [], "source": [ - "!python -c \"import monai\" || pip install -q \"monai-weekly[ignite,pyyaml]\"" + "!python -c \"import monai\" || pip install -q \"monai-weekly[ignite,pyyaml]\"\n", + "# monai.bundle CLI (init_bundle/run) requires the `fire` package\n", + "!python -c \"import fire\" || pip install -q fire" ] }, { diff --git a/bundle/03_mednist_classification_v2.ipynb b/bundle/03_mednist_classification_v2.ipynb index 78dc1cf30e..be72cd072d 100644 --- a/bundle/03_mednist_classification_v2.ipynb +++ b/bundle/03_mednist_classification_v2.ipynb @@ -32,7 +32,9 @@ "metadata": {}, "outputs": [], "source": [ - "!python -c \"import monai\" || pip install -q \"monai-weekly[ignite,pyyaml]\"" + "!python -c \"import monai\" || pip install -q \"monai-weekly[ignite,pyyaml]\"\n", + "# monai.bundle CLI (init_bundle/run) requires the `fire` package\n", + "!python -c \"import fire\" || pip install -q fire" ] }, { diff --git a/bundle/04_integrating_code.ipynb b/bundle/04_integrating_code.ipynb index 898468c296..ad93fb8448 100644 --- a/bundle/04_integrating_code.ipynb +++ b/bundle/04_integrating_code.ipynb @@ -32,7 +32,9 @@ "metadata": {}, "outputs": [], "source": [ - "!python -c \"import monai\" || pip install -q \"monai-weekly[ignite,pyyaml]\"" + "!python -c \"import monai\" || pip install -q \"monai-weekly[ignite,pyyaml]\"\n", + "# monai.bundle CLI (init_bundle/run) requires the `fire` package\n", + "!python -c \"import fire\" || pip install -q fire" ] }, { diff --git a/bundle/pythonic_usage_guidance/pythonic_bundle_access.ipynb b/bundle/pythonic_usage_guidance/pythonic_bundle_access.ipynb index 8af48d50a2..59101b3530 100644 --- a/bundle/pythonic_usage_guidance/pythonic_bundle_access.ipynb +++ b/bundle/pythonic_usage_guidance/pythonic_bundle_access.ipynb @@ -213,7 +213,12 @@ "source": [ "config_file = Path(root_dir) / \"spleen_ct_segmentation\" / \"configs\" / \"train.json\"\n", "\n", - "train_workflow = create_workflow(config_file=str(config_file), workflow_type=\"train\")" + "# cache_rate overrides added to avoid OOM when caching the full spleen dataset on low-RAM hosts\n", + "train_workflow = create_workflow(\n", + " config_file=str(config_file),\n", + " workflow_type=\"train\",\n", + " **{\"train#dataset#cache_rate\": 0.0, \"validate#dataset#cache_rate\": 0.0},\n", + ")" ] }, { @@ -295,7 +300,13 @@ "source": [ "# 1 override them when you create the workflow\n", "bundle_root = root_dir\n", - "override = {\"epochs\": 1, \"dataset_dir\": str(data_dir), \"bundle_root\": bundle_root}\n", + "override = {\n", + " \"epochs\": 1,\n", + " \"dataset_dir\": str(data_dir),\n", + " \"bundle_root\": bundle_root,\n", + " \"train#dataset#cache_rate\": 0.0,\n", + " \"validate#dataset#cache_rate\": 0.0,\n", + "}\n", "train_workflow = create_workflow(config_file=config_file, workflow_type=\"train\", **override)\n", "print(\"max epochs:\", train_workflow.max_epochs)" ] @@ -467,7 +478,11 @@ "\n", "# Using BundleWorkflow\n", "# config-based\n", - "workflow = create_workflow(config_file=str(config_file), workflow_type=\"train\")\n", + "workflow = create_workflow(\n", + " config_file=str(config_file),\n", + " workflow_type=\"train\",\n", + " **{\"train#dataset#cache_rate\": 0.0, \"validate#dataset#cache_rate\": 0.0},\n", + ")\n", "# python-based\n", "# more details refer to https://github.com/Project-MONAI/tutorials/tree/main/bundle/python_bundle_workflow\n", "# workflow = create_workflow(workflow_name=scripts.train.TrainWorkflow)" @@ -488,7 +503,14 @@ "metadata": {}, "outputs": [], "source": [ - "overrides = {\"network_def#in_channels\": 1, \"lr_scheduler#step_size\": 4000, \"dataset_dir\": str(data_dir), \"epochs\": 1}\n", + "overrides = {\n", + " \"network_def#in_channels\": 1,\n", + " \"lr_scheduler#step_size\": 4000,\n", + " \"dataset_dir\": str(data_dir),\n", + " \"epochs\": 1,\n", + " \"train#dataset#cache_rate\": 0.0,\n", + " \"validate#dataset#cache_rate\": 0.0,\n", + "}\n", "\n", "# Using ConfigParser\n", "# override configuration content\n", diff --git a/experiment_management/bundle_integrate_mlflow.ipynb b/experiment_management/bundle_integrate_mlflow.ipynb index 26da75823b..e4466926a6 100644 --- a/experiment_management/bundle_integrate_mlflow.ipynb +++ b/experiment_management/bundle_integrate_mlflow.ipynb @@ -297,7 +297,7 @@ " --dataset_dir $MONAI_DATA_DIRECTORY/Task09_Spleen \\\n", " --train#trainer#max_epochs 10 \\\n", " --tracking \"mlflow\" \\\n", - " --tracking_uri ./eval/mlruns \\\n", + " --tracking_uri \"sqlite:///./eval/mlruns.db\" \\\n", " --experiment_name \"user_define_name\" \\\n", " --run_name \"user_define_run\"\n", "%cd .." @@ -382,7 +382,8 @@ "outputs": [], "source": [ "%cd spleen_ct_segmentation\n", - "tracking_uri = \"./eval/mlruns\"\n", + "# MONAI 1.6 MLFlowHandler requires a SQLite/remote URI (file store no longer supported)\n", + "tracking_uri = \"sqlite:///./eval/mlruns.db\"\n", "train_mlflow_handler = MLFlowHandler(\n", " tracking_uri=tracking_uri,\n", " experiment_name=\"ConfigWorkflowExperiment\",\n", diff --git a/experiment_management/mlflow_example.json b/experiment_management/mlflow_example.json index fc97bbf5ec..2f44913e5b 100644 --- a/experiment_management/mlflow_example.json +++ b/experiment_management/mlflow_example.json @@ -14,7 +14,7 @@ } }, "configs": { - "tracking_uri": "$@output_dir + '/mlruns'", + "tracking_uri": "$'sqlite:///' + @output_dir + '/mlruns.db'", "experiment_name": "monai_experiment", "run_name": "test1", "is_not_rank0": "$torch.distributed.is_available() and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0", diff --git a/patch_inferer/modular_patch_inferer.ipynb b/patch_inferer/modular_patch_inferer.ipynb index ee439baa56..29988b799e 100644 --- a/patch_inferer/modular_patch_inferer.ipynb +++ b/patch_inferer/modular_patch_inferer.ipynb @@ -48,7 +48,8 @@ "outputs": [], "source": [ "!python -c \"import monai\" || pip install -q \"monai-weekly[pillow,tqdm,cucim,zarr]\"\n", - "!python -c \"import zarr\" || pip install -q zarr\n", + "# MONAI 1.6.1's ZarrAvgMerger uses the zarr v2 API (chunks=True default, cdata_shape); pin zarr<3.\n", + "!python -c \"import zarr, packaging.version as v; assert v.parse(zarr.__version__) < v.parse('3')\" || pip install -q \"zarr<3\"\n", "!python -c \"import matplotlib\" || pip install -q matplotlib" ] },