\n",
" Copyable Markdown
\n",
" | Package | Version |\n",
"| ----------- | --------- |\n",
"| numpy | 1.26.4 |\n",
"| dask | 2024.11.2 |\n",
"| scanpy | 1.11.0rc2 |\n",
"| anndata | 0.11.0rc3 |\n",
"| h5py | 3.12.1 |\n",
"| pandas | 2.2.3 |\n",
"| tqdm | 4.67.1 |\n",
"| distributed | 2024.11.2 |\n",
"\n",
"| Dependency | Version |\n",
"| ---------------------- | -------------------------------- |\n",
"| scikit-learn | 1.5.1 |\n",
"| scipy | 1.14.1 |\n",
"| zstandard | 0.23.0 |\n",
"| Pygments | 2.19.1 |\n",
"| sparse | 0.15.5 |\n",
"| prompt_toolkit | 3.0.50 |\n",
"| wcwidth | 0.2.13 |\n",
"| torch | 2.6.0 (2.6.0+cu124) |\n",
"| setproctitle | 1.3.5 |\n",
"| typing_extensions | 4.12.2 |\n",
"| igraph | 0.11.8 |\n",
"| comm | 0.2.2 |\n",
"| crc32c | 2.7.1 |\n",
"| asciitree | 0.3.3 |\n",
"| louvain | 0.8.2 |\n",
"| ipykernel | 6.29.5 |\n",
"| zarr | 2.18.3 |\n",
"| asttokens | 3.0.0 |\n",
"| tblib | 3.0.0 |\n",
"| cycler | 0.12.1 |\n",
"| numba | 0.60.0 |\n",
"| numcodecs | 0.15.1 |\n",
"| msgpack | 1.1.0 |\n",
"| pyarrow | 17.0.0 |\n",
"| lz4 | 4.3.3 |\n",
"| jedi | 0.19.2 |\n",
"| zict | 3.0.0 |\n",
"| toolz | 1.0.0 (toolz: 1.0.0, tlz: 1.0.1) |\n",
"| jaraco.collections | 5.1.0 |\n",
"| jaraco.text | 3.12.1 |\n",
"| stack_data | 0.6.3 |\n",
"| rapids-dask-dependency | 24.12.0a0 |\n",
"| pickleshare | 0.7.5 |\n",
"| tornado | 6.4.2 |\n",
"| texttable | 1.7.0 |\n",
"| pytz | 2024.1 |\n",
"| pyparsing | 3.2.1 |\n",
"| joblib | 1.4.2 |\n",
"| jupyter_client | 8.6.3 |\n",
"| pynvml | 11.4.1 |\n",
"| omegaconf | 2.3.0 |\n",
"| leidenalg | 0.10.2 |\n",
"| cupy | 13.3.0 |\n",
"| cuda-python | 12.6.0 |\n",
"| traitlets | 5.14.3 |\n",
"| platformdirs | 4.3.6 |\n",
"| rmm | 24.12.1 (24.12.01) |\n",
"| debugpy | 1.8.12 |\n",
"| psutil | 6.1.1 |\n",
"| more-itertools | 10.6.0 |\n",
"| natsort | 8.4.0 |\n",
"| legacy-api-wrap | 1.4.1 |\n",
"| pure_eval | 0.2.3 |\n",
"| Deprecated | 1.2.18 |\n",
"| threadpoolctl | 3.5.0 |\n",
"| pyzmq | 26.2.1 |\n",
"| executing | 2.1.0 |\n",
"| PyYAML | 6.0.2 |\n",
"| colorama | 0.4.6 |\n",
"| session-info2 | 0.1.2 |\n",
"| llvmlite | 0.43.0 |\n",
"| parso | 0.8.4 |\n",
"| locket | 1.0.0 |\n",
"| packaging | 24.2 |\n",
"| fastrlock | 0.8.3 |\n",
"| wrapt | 1.17.2 |\n",
"| jupyter_core | 5.7.2 |\n",
"| jaraco.functools | 4.0.1 |\n",
"| sortedcontainers | 2.4.0 |\n",
"| setuptools | 75.8.0 |\n",
"| click | 8.1.8 |\n",
"| cytoolz | 1.0.1 |\n",
"| ipython | 8.32.0 |\n",
"| jaraco.context | 5.3.0 |\n",
"| Jinja2 | 3.1.5 |\n",
"| cloudpickle | 3.1.1 |\n",
"| decorator | 5.1.1 |\n",
"| defusedxml | 0.7.1 |\n",
"| MarkupSafe | 3.0.2 |\n",
"| kiwisolver | 1.4.8 |\n",
"| pillow | 11.1.0 |\n",
"| six | 1.17.0 |\n",
"| matplotlib | 3.10.0 |\n",
"| python-dateutil | 2.9.0.post0 |\n",
"\n",
"| Component | Info |\n",
"| --------- | ----------------------------------------------------------------------------- |\n",
"| Python | 3.12.8 | packaged by conda-forge | (main, Dec 5 2024, 14:24:40) [GCC 13.3.0] |\n",
"| OS | Linux-5.14.0-427.37.1.el9_4.x86_64-x86_64-with-glibc2.34 |\n",
"| CPU | 128 logical CPU cores, x86_64 |\n",
"| GPU | ID: 0, NVIDIA A100 80GB PCIe, Driver: 560.35.03, Memory: 81920 MiB |\n",
"| Updated | 2025-03-05 16:10 |\n",
" "
],
"text/markdown": [
"| Package | Version |\n",
"| ----------- | --------- |\n",
"| numpy | 1.26.4 |\n",
"| dask | 2024.11.2 |\n",
"| scanpy | 1.11.0rc2 |\n",
"| anndata | 0.11.0rc3 |\n",
"| h5py | 3.12.1 |\n",
"| pandas | 2.2.3 |\n",
"| tqdm | 4.67.1 |\n",
"| distributed | 2024.11.2 |\n",
"\n",
"| Dependency | Version |\n",
"| ---------------------- | -------------------------------- |\n",
"| scikit-learn | 1.5.1 |\n",
"| scipy | 1.14.1 |\n",
"| zstandard | 0.23.0 |\n",
"| Pygments | 2.19.1 |\n",
"| sparse | 0.15.5 |\n",
"| prompt_toolkit | 3.0.50 |\n",
"| wcwidth | 0.2.13 |\n",
"| torch | 2.6.0 (2.6.0+cu124) |\n",
"| setproctitle | 1.3.5 |\n",
"| typing_extensions | 4.12.2 |\n",
"| igraph | 0.11.8 |\n",
"| comm | 0.2.2 |\n",
"| crc32c | 2.7.1 |\n",
"| asciitree | 0.3.3 |\n",
"| louvain | 0.8.2 |\n",
"| ipykernel | 6.29.5 |\n",
"| zarr | 2.18.3 |\n",
"| asttokens | 3.0.0 |\n",
"| tblib | 3.0.0 |\n",
"| cycler | 0.12.1 |\n",
"| numba | 0.60.0 |\n",
"| numcodecs | 0.15.1 |\n",
"| msgpack | 1.1.0 |\n",
"| pyarrow | 17.0.0 |\n",
"| lz4 | 4.3.3 |\n",
"| jedi | 0.19.2 |\n",
"| zict | 3.0.0 |\n",
"| toolz | 1.0.0 (toolz: 1.0.0, tlz: 1.0.1) |\n",
"| jaraco.collections | 5.1.0 |\n",
"| jaraco.text | 3.12.1 |\n",
"| stack_data | 0.6.3 |\n",
"| rapids-dask-dependency | 24.12.0a0 |\n",
"| pickleshare | 0.7.5 |\n",
"| tornado | 6.4.2 |\n",
"| texttable | 1.7.0 |\n",
"| pytz | 2024.1 |\n",
"| pyparsing | 3.2.1 |\n",
"| joblib | 1.4.2 |\n",
"| jupyter_client | 8.6.3 |\n",
"| pynvml | 11.4.1 |\n",
"| omegaconf | 2.3.0 |\n",
"| leidenalg | 0.10.2 |\n",
"| cupy | 13.3.0 |\n",
"| cuda-python | 12.6.0 |\n",
"| traitlets | 5.14.3 |\n",
"| platformdirs | 4.3.6 |\n",
"| rmm | 24.12.1 (24.12.01) |\n",
"| debugpy | 1.8.12 |\n",
"| psutil | 6.1.1 |\n",
"| more-itertools | 10.6.0 |\n",
"| natsort | 8.4.0 |\n",
"| legacy-api-wrap | 1.4.1 |\n",
"| pure_eval | 0.2.3 |\n",
"| Deprecated | 1.2.18 |\n",
"| threadpoolctl | 3.5.0 |\n",
"| pyzmq | 26.2.1 |\n",
"| executing | 2.1.0 |\n",
"| PyYAML | 6.0.2 |\n",
"| colorama | 0.4.6 |\n",
"| session-info2 | 0.1.2 |\n",
"| llvmlite | 0.43.0 |\n",
"| parso | 0.8.4 |\n",
"| locket | 1.0.0 |\n",
"| packaging | 24.2 |\n",
"| fastrlock | 0.8.3 |\n",
"| wrapt | 1.17.2 |\n",
"| jupyter_core | 5.7.2 |\n",
"| jaraco.functools | 4.0.1 |\n",
"| sortedcontainers | 2.4.0 |\n",
"| setuptools | 75.8.0 |\n",
"| click | 8.1.8 |\n",
"| cytoolz | 1.0.1 |\n",
"| ipython | 8.32.0 |\n",
"| jaraco.context | 5.3.0 |\n",
"| Jinja2 | 3.1.5 |\n",
"| cloudpickle | 3.1.1 |\n",
"| decorator | 5.1.1 |\n",
"| defusedxml | 0.7.1 |\n",
"| MarkupSafe | 3.0.2 |\n",
"| kiwisolver | 1.4.8 |\n",
"| pillow | 11.1.0 |\n",
"| six | 1.17.0 |\n",
"| matplotlib | 3.10.0 |\n",
"| python-dateutil | 2.9.0.post0 |\n",
"\n",
"| Component | Info |\n",
"| --------- | ----------------------------------------------------------------------------- |\n",
"| Python | 3.12.8 | packaged by conda-forge | (main, Dec 5 2024, 14:24:40) [GCC 13.3.0] |\n",
"| OS | Linux-5.14.0-427.37.1.el9_4.x86_64-x86_64-with-glibc2.34 |\n",
"| CPU | 128 logical CPU cores, x86_64 |\n",
"| GPU | ID: 0, NVIDIA A100 80GB PCIe, Driver: 560.35.03, Memory: 81920 MiB |\n",
"| Updated | 2025-03-05 16:10 |"
],
"text/plain": [
"numpy\t1.26.4\n",
"dask\t2024.11.2\n",
"scanpy\t1.11.0rc2\n",
"anndata\t0.11.0rc3\n",
"h5py\t3.12.1\n",
"pandas\t2.2.3\n",
"tqdm\t4.67.1\n",
"distributed\t2024.11.2\n",
"----\t----\n",
"scikit-learn\t1.5.1\n",
"scipy\t1.14.1\n",
"zstandard\t0.23.0\n",
"Pygments\t2.19.1\n",
"sparse\t0.15.5\n",
"prompt_toolkit\t3.0.50\n",
"wcwidth\t0.2.13\n",
"torch\t2.6.0 (2.6.0+cu124)\n",
"setproctitle\t1.3.5\n",
"typing_extensions\t4.12.2\n",
"igraph\t0.11.8\n",
"comm\t0.2.2\n",
"crc32c\t2.7.1\n",
"asciitree\t0.3.3\n",
"louvain\t0.8.2\n",
"ipykernel\t6.29.5\n",
"zarr\t2.18.3\n",
"asttokens\t3.0.0\n",
"tblib\t3.0.0\n",
"cycler\t0.12.1\n",
"numba\t0.60.0\n",
"numcodecs\t0.15.1\n",
"msgpack\t1.1.0\n",
"pyarrow\t17.0.0\n",
"lz4\t4.3.3\n",
"jedi\t0.19.2\n",
"zict\t3.0.0\n",
"toolz\t1.0.0 (toolz: 1.0.0, tlz: 1.0.1)\n",
"jaraco.collections\t5.1.0\n",
"jaraco.text\t3.12.1\n",
"stack_data\t0.6.3\n",
"rapids-dask-dependency\t24.12.0a0\n",
"pickleshare\t0.7.5\n",
"tornado\t6.4.2\n",
"texttable\t1.7.0\n",
"pytz\t2024.1\n",
"pyparsing\t3.2.1\n",
"joblib\t1.4.2\n",
"jupyter_client\t8.6.3\n",
"pynvml\t11.4.1\n",
"omegaconf\t2.3.0\n",
"leidenalg\t0.10.2\n",
"cupy\t13.3.0\n",
"cuda-python\t12.6.0\n",
"traitlets\t5.14.3\n",
"platformdirs\t4.3.6\n",
"rmm\t24.12.1 (24.12.01)\n",
"debugpy\t1.8.12\n",
"psutil\t6.1.1\n",
"more-itertools\t10.6.0\n",
"natsort\t8.4.0\n",
"legacy-api-wrap\t1.4.1\n",
"pure_eval\t0.2.3\n",
"Deprecated\t1.2.18\n",
"threadpoolctl\t3.5.0\n",
"pyzmq\t26.2.1\n",
"executing\t2.1.0\n",
"PyYAML\t6.0.2\n",
"colorama\t0.4.6\n",
"session-info2\t0.1.2\n",
"llvmlite\t0.43.0\n",
"parso\t0.8.4\n",
"locket\t1.0.0\n",
"packaging\t24.2\n",
"fastrlock\t0.8.3\n",
"wrapt\t1.17.2\n",
"jupyter_core\t5.7.2\n",
"jaraco.functools\t4.0.1\n",
"sortedcontainers\t2.4.0\n",
"setuptools\t75.8.0\n",
"click\t8.1.8\n",
"cytoolz\t1.0.1\n",
"ipython\t8.32.0\n",
"jaraco.context\t5.3.0\n",
"Jinja2\t3.1.5\n",
"cloudpickle\t3.1.1\n",
"decorator\t5.1.1\n",
"defusedxml\t0.7.1\n",
"MarkupSafe\t3.0.2\n",
"kiwisolver\t1.4.8\n",
"pillow\t11.1.0\n",
"six\t1.17.0\n",
"matplotlib\t3.10.0\n",
"python-dateutil\t2.9.0.post0\n",
"----\t----\n",
"Python\t3.12.8 | packaged by conda-forge | (main, Dec 5 2024, 14:24:40) [GCC 13.3.0]\n",
"OS\tLinux-5.14.0-427.37.1.el9_4.x86_64-x86_64-with-glibc2.34\n",
"CPU\t128 logical CPU cores, x86_64\n",
"GPU\tID: 0, NVIDIA A100 80GB PCIe, Driver: 560.35.03, Memory: 81920 MiB\n",
"Updated\t2025-03-05 16:10"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from pathlib import Path\n",
"\n",
"import numpy as np\n",
"import dask.distributed as dd\n",
"import scanpy as sc\n",
"import anndata as ad\n",
"import h5py\n",
"import dask\n",
"\n",
"from collections import Counter\n",
"import pandas as pd\n",
"from tqdm import tqdm\n",
"import dask\n",
"import time\n",
"from dask.distributed import Client, LocalCluster\n",
"\n",
"import dask\n",
"\n",
"sc.logging.print_header()"
]
},
{
"cell_type": "markdown",
"id": "e1acd85d",
"metadata": {},
"source": [
"## Technical Notes\n",
"\n",
"To turn on one or the other of GPU or CPU, use the flag below. If you are using a GPU, configure the `gpus` flag accordingly to the number of GPUs you have available. Similarly, for the CPU implementation, you can configure the number of workers. We ran this notebook using CPU with a slurm configuration like `srun ... -c 32 --mem=300gb`, using 16 dask workers and the below sparse chunks size while we were only able to get one GPU, although the beauty of `dask` is that one is enough to complete the analysis thanks to the out-of-core capabilities. You can of course use fewer CPU cores and this notebook will still be runnable, such is the beauty of truly lazy, chunked computation. Nothing is loaded into memory until it is needed, and only the amount needed to perform the local computation on a chunk of the data.\n",
"\n",
"The interplay between memory and workers is delicate - too many workers and not enough memory will cause your calculation to crash, so often it is good to have a few fewer workers than you think would be best. For example, because a single GPU has somewhere in the range of 32GB, that is more than the individual 16 workers can have who split the 300GB, thereby allowing for a larger chunk size. Furthermore with [rapids-memory-management](https://github.com/rapidsai/rmm), memory concerns are not so overriding on the GPU.\n",
"\n",
"**Caution**: Out-of-core support is still quite new in both packages, so we are learning more and more daily about performance optimizations. The information here about performance thus represents our best attempt to provide advice on performance. On only one GPU, preprocessing can be quite lengthy, and therefore it is advisable to use rapids only with more than one GPU (be sure to update the `gpus` variable accordingly to reflect this - it should be something like `'0,1,2,3'`, for example, for four GPUs). Otherwise, we have observed that the single-GPU PCA is faster than the multi-core PCA (~12.5 min vs. ~18 min), but the time needed to bring the data back to the CPU from the GPU for writing causes the two to be comparable i.e., PCA + writing the results has the same time between one GPU and multi-core CPU with the above specs."
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "e82a8e5f",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"use_gpu = False"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "f41d220a-5bc4-4824-8c57-c4dd107ba3be",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"if use_gpu:\n",
" import rapids_singlecell as rsc\n",
" SPARSE_CHUNK_SIZE = 1_000_000\n",
" from dask_cuda import LocalCUDACluster\n",
"\n",
" import rmm\n",
" import cupy as cp\n",
" from cupyx.scipy import spx\n",
"\n",
" from rmm.allocators.cupy import rmm_cupy_allocator\n",
"\n",
" def set_mem():\n",
" rmm.reinitialize(managed_memory=True)\n",
" cp.cuda.set_allocator(rmm_cupy_allocator)\n",
" gpus = \"0\" # comma separated like \"0,1,2\" for 3 gpus\n",
" cluster = LocalCUDACluster(CUDA_VISIBLE_DEVICES=gpus)\n",
" dask.array.register_chunk_type(spx.csr_matrix)\n",
"else:\n",
"\n",
" SPARSE_CHUNK_SIZE = 100_000\n",
" cluster = LocalCluster(n_workers=16)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "f0d0954c-07b4-490d-b40d-9c9839d5865c",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"client = Client(cluster)\n",
"if use_gpu:\n",
" client.run(set_mem)\n",
" mod = rsc\n",
"else:\n",
" mod = sc"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "a8077d34-7b5d-4472-9fb6-810b4cdd4a13",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"