diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 2086c20aa4..f0e181f1ba 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -15,8 +15,8 @@ jobs: vmImage: 'ubuntu-22.04' strategy: matrix: - Python3.9: - python.version: '3.9' + Python3.10: + python.version: '3.10' Python3.12: {} minimal_dependencies: TEST_EXTRA: 'test-min' @@ -24,7 +24,7 @@ jobs: DEPENDENCIES_VERSION: "pre-release" TEST_TYPE: "coverage" minimum_versions: - python.version: '3.9' + python.version: '3.10' DEPENDENCIES_VERSION: "minimum-version" TEST_TYPE: "coverage" @@ -103,6 +103,12 @@ jobs: testResultsFormat: NUnit testRunTitle: 'Publish test results for $(Agent.JobName)' + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '.pytest_cache/d/debug' + artifactName: debug-data + condition: eq(variables['TEST_TYPE'], 'coverage') + - script: bash <(curl -s https://codecov.io/bash) displayName: 'Upload to codecov.io' condition: eq(variables['TEST_TYPE'], 'coverage') diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 9b7cac7353..71637016a7 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -1,8 +1,8 @@ name: Bug report description: Scanpy doesn’t do what it should? Please help us fix it! #title: ... +type: Bug labels: -- Bug 🐛 - Triage 🩺 #assignees: [] body: diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 1505f196f5..a0c4b12e00 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,4 +1,4 @@ -blank_issues_enabled: true +blank_issues_enabled: false contact_links: - name: Scanpy Community Forum url: https://discourse.scverse.org/ diff --git a/.github/ISSUE_TEMPLATE/enhancement-request.yml b/.github/ISSUE_TEMPLATE/enhancement-request.yml index 209ee6805a..9e511c592c 100644 --- a/.github/ISSUE_TEMPLATE/enhancement-request.yml +++ b/.github/ISSUE_TEMPLATE/enhancement-request.yml @@ -1,8 +1,8 @@ name: Enhancement request description: Anything you’d like to see in scanpy? #title: ... +type: Enhancement labels: -- Enhancement ✨ - Triage 🩺 #assignees: [] body: @@ -14,6 +14,7 @@ body: - 'Additional function parameters / changed functionality / changed defaults?' - 'New analysis tool: A simple analysis tool you have been using and are missing in `sc.tools`?' - 'New plotting function: A kind of plot you would like to seein `sc.pl`?' + - 'Improved documentation or error message?' - 'Other?' validations: required: true diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f2de5e34df..68e274ad54 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -48,8 +48,7 @@ jobs: key: benchmark-state-${{ hashFiles('benchmarks/**') }} - name: Install dependencies - # TODO: revert once this PR is merged: https://github.com/airspeed-velocity/asv/pull/1397 - run: pip install 'asv @ git+https://github.com/ivirshup/asv@fix-conda-usage' + run: pip install 'asv>=0.6.4' - name: Configure ASV working-directory: ${{ env.ASV_DIR }} diff --git a/.github/workflows/check-pr.yml b/.github/workflows/check-pr.yml index 88d78c9b43..5eaa4dacb3 100644 --- a/.github/workflows/check-pr.yml +++ b/.github/workflows/check-pr.yml @@ -49,13 +49,13 @@ jobs: with: fetch-depth: 0 filter: blob:none - - name: Find out if relevant release notes are modified - uses: dorny/paths-filter@v2 + - name: Find out if a relevant release fragment is added + uses: dorny/paths-filter@v3 id: changes with: filters: | # this is intentionally a string - relnotes: 'docs/release-notes/${{ github.event.pull_request.milestone.title }}.md' - - name: Check if relevant release notes are modified + relnotes: 'docs/release-notes/${{ github.event.pull_request.number }}.*.md' + - name: Check if a relevant release fragment is added uses: flying-sheep/check@v1 with: success: ${{ steps.changes.outputs.relnotes }} diff --git a/.gitignore b/.gitignore index 7aca652727..65f9de7e0a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,6 @@ /tests/**/*failed-diff.png # Environment management -/hatch.toml /Pipfile /Pipfile.lock /requirements*.lock @@ -29,6 +28,7 @@ # Python build files __pycache__/ /src/scanpy/_version.py +/ci/scanpy-min-deps.txt /dist/ /*-env/ /env-*/ @@ -42,7 +42,6 @@ Thumbs.db # IDEs and editors /.idea/ -/.vscode/ # asv benchmark files /benchmarks/.asv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f1fb93af5..c5e0e91d8c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,13 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.9 + rev: v0.9.1 hooks: - id: ruff - types_or: [python, pyi, jupyter] args: ["--fix"] - id: ruff-format - types_or: [python, pyi, jupyter] # The following can be removed once PLR0917 is out of preview - name: ruff preview rules id: ruff - types_or: [python, pyi, jupyter] args: ["--preview", "--select=PLR0917"] - repo: https://github.com/flying-sheep/bibfmt rev: v4.3.0 @@ -19,8 +16,17 @@ repos: args: - --sort-by-bibkey - --drop=abstract +- repo: https://github.com/biomejs/pre-commit + rev: v0.6.1 + hooks: + - id: biome-format + additional_dependencies: ["@biomejs/biome@1.9.4"] +- repo: https://github.com/ComPWA/taplo-pre-commit + rev: v0.9.3 + hooks: + - id: taplo-format - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: trailing-whitespace exclude: tests/_data @@ -34,6 +40,3 @@ repos: - id: detect-private-key - id: no-commit-to-branch args: ["--branch=main"] - -ci: - autofix_prs: false diff --git a/.readthedocs.yml b/.readthedocs.yml index 4a5d3e219f..adcdfb80d7 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -2,9 +2,16 @@ version: 2 submodules: include: all build: - os: ubuntu-20.04 + os: ubuntu-24.04 tools: python: '3.12' + jobs: + post_checkout: + # unshallow so version can be derived from tag + - git fetch --unshallow || true + pre_build: + # run towncrier to preview the next version’s release notes + - ( find docs/release-notes -regex '[^.]+[.][^.]+.md' | grep -q . ) && towncrier build --keep || true sphinx: fail_on_warning: true # do not change or you will be fired configuration: docs/conf.py @@ -14,4 +21,5 @@ python: path: . extra_requirements: - doc + - dev # for towncrier - leiden diff --git a/.taplo.toml b/.taplo.toml new file mode 100644 index 0000000000..41a6cdc5cc --- /dev/null +++ b/.taplo.toml @@ -0,0 +1,5 @@ +[formatting] +array_auto_collapse = false +column_width = 120 +compact_arrays = false +indent_string = ' ' diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000000..d87ef7c54f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Build Documentation", + "type": "debugpy", + "request": "launch", + "module": "sphinx", + "args": ["-M", "html", ".", "_build"], + "cwd": "${workspaceFolder}/docs", + "console": "internalConsole", + "justMyCode": false, + }, + { + "name": "Python: Debug Test", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "purpose": ["debug-test"], + "console": "internalConsole", + "justMyCode": false, + "env": { "PYTEST_ADDOPTS": "--color=yes" }, + "presentation": { "hidden": true }, + }, + ], +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..ae719a4ec8 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,22 @@ +{ + "[python][toml][json][jsonc]": { + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": "explicit", + "source.fixAll": "explicit", + }, + }, + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + }, + "[toml]": { + "editor.defaultFormatter": "tamasfe.even-better-toml", + }, + "[json][jsonc]": { + "editor.defaultFormatter": "biomejs.biome", + }, + "python.analysis.typeCheckingMode": "basic", + "python.testing.pytestArgs": ["-vv", "--color=yes"], + "python.testing.pytestEnabled": true, + "python.terminal.activateEnvironment": true, +} diff --git a/benchmarks/README.md b/benchmarks/README.md index f186cfa876..3546e79eaf 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -8,3 +8,14 @@ Benchmarks are run using the [benchmark bot][]. [asv]: https://asv.readthedocs.io/ [`benchmark.yml`]: ../.github/workflows/benchmark.yml [benchmark bot]: https://github.com/apps/scverse-benchmark + +## Data processing in benchmarks + +Each dataset is processed so it has + +- `.layers['counts']` (containing data in C/row-major format) and `.layers['counts-off-axis']` (containing data in FORTRAN/column-major format) +- `.X` and `.layers['off-axis']` with log-transformed data (formats like above) +- a `.var['mt']` boolean column indicating mitochondrial genes + +The benchmarks are set up so the `layer` parameter indicates the layer that will be moved into `.X` before the benchmark. +That way, we don’t need to add `layer=layer` everywhere. diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index b4fc26a9de..404e0b83dd 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -54,7 +54,7 @@ // The Pythons you'd like to test against. If not provided, defaults // to the current version of Python used to run `asv`. - // "pythons": ["3.9", "3.12"], + // "pythons": ["3.10", "3.12"], // The list of conda channel names to be searched for benchmark // dependency packages in the specified order @@ -78,13 +78,14 @@ "natsort": [""], "pandas": [""], "memory_profiler": [""], - "zarr": [""], + "zarr": ["2.18.4"], "pytest": [""], "scanpy": [""], "python-igraph": [""], // "psutil": [""] "pooch": [""], "scikit-image": [""], + // "scikit-misc": [""], }, // Combinations of libraries/python versions can be excluded/included diff --git a/benchmarks/benchmarks/_utils.py b/benchmarks/benchmarks/_utils.py index 23404ce910..93bb4623f9 100644 --- a/benchmarks/benchmarks/_utils.py +++ b/benchmarks/benchmarks/_utils.py @@ -1,5 +1,6 @@ from __future__ import annotations +import itertools import warnings from functools import cache from typing import TYPE_CHECKING @@ -7,23 +8,39 @@ import numpy as np import pooch from anndata import concat +from asv_runner.benchmarks.mark import skip_for_params +from scipy import sparse import scanpy as sc if TYPE_CHECKING: - from typing import Literal + from collections.abc import Callable, Sequence + from collections.abc import Set as AbstractSet + from typing import Literal, Protocol, TypeVar from anndata import AnnData + C = TypeVar("C", bound=Callable) + + class ParamSkipper(Protocol): + def __call__(self, **skipped: AbstractSet) -> Callable[[C], C]: ... + Dataset = Literal["pbmc68k_reduced", "pbmc3k", "bmmc", "lung93k"] + KeyX = Literal[None, "off-axis"] + KeyCount = Literal["counts", "counts-off-axis"] @cache def _pbmc68k_reduced() -> AnnData: + """A small datasets with a dense `.X`""" adata = sc.datasets.pbmc68k_reduced() + assert isinstance(adata.X, np.ndarray) + assert not np.isfortran(adata.X) + # raw has the same number of genes, so we can use it for counts # it doesn’t actually contain counts for some reason, but close enough - adata.layers["counts"] = adata.raw.X.copy() + assert isinstance(adata.raw.X, sparse.csr_matrix) + adata.layers["counts"] = adata.raw.X.toarray(order="C") mapper = dict( percent_mito="pct_counts_mt", n_counts="total_counts", @@ -39,10 +56,7 @@ def pbmc68k_reduced() -> AnnData: @cache def _pbmc3k() -> AnnData: adata = sc.datasets.pbmc3k() - adata.var["mt"] = adata.var_names.str.startswith("MT-") - sc.pp.calculate_qc_metrics( - adata, qc_vars=["mt"], percent_top=None, log1p=False, inplace=True - ) + assert isinstance(adata.X, sparse.csr_matrix) adata.layers["counts"] = adata.X.astype(np.int32, copy=True) sc.pp.log1p(adata) return adata @@ -76,12 +90,10 @@ def _bmmc(n_obs: int = 4000) -> AnnData: adata = concat(adatas, label="sample") adata.obs_names_make_unique() - adata.var["mt"] = adata.var_names.str.startswith("MT-") - sc.pp.calculate_qc_metrics( - adata, qc_vars=["mt"], percent_top=None, log1p=False, inplace=True - ) - adata.obs["n_counts"] = adata.X.sum(axis=1).A1 - + assert isinstance(adata.X, sparse.csr_matrix) + adata.layers["counts"] = adata.X.astype(np.int32, copy=True) + sc.pp.log1p(adata) + adata.obs["n_counts"] = adata.layers["counts"].sum(axis=1).A1 return adata @@ -95,33 +107,104 @@ def _lung93k() -> AnnData: url="https://figshare.com/ndownloader/files/45788454", known_hash="md5:4f28af5ff226052443e7e0b39f3f9212", ) - return sc.read_h5ad(path) + adata = sc.read_h5ad(path) + assert isinstance(adata.X, sparse.csr_matrix) + adata.layers["counts"] = adata.X.astype(np.int32, copy=True) + sc.pp.log1p(adata) + return adata def lung93k() -> AnnData: return _lung93k().copy() -def get_dataset(dataset: Dataset) -> tuple[AnnData, str | None]: - if dataset == "pbmc68k_reduced": - return pbmc68k_reduced(), None - if dataset == "pbmc3k": - return pbmc3k(), None # can’t use this with batches - if dataset == "bmmc": - # TODO: allow specifying bigger variant - return bmmc(400), "sample" - if dataset == "lung93k": - return lung93k(), "PatientNumber" +def to_off_axis(x: np.ndarray | sparse.csr_matrix) -> np.ndarray | sparse.csc_matrix: + if isinstance(x, sparse.csr_matrix): + return x.tocsc() + if isinstance(x, np.ndarray): + assert not np.isfortran(x) + return x.copy(order="F") + msg = f"Unexpected type {type(x)}" + raise TypeError(msg) + + +def _get_dataset_raw(dataset: Dataset) -> tuple[AnnData, str | None]: + match dataset: + case "pbmc68k_reduced": + adata, batch_key = pbmc68k_reduced(), None + case "pbmc3k": + adata, batch_key = pbmc3k(), None # can’t use this with batches + case "bmmc": + # TODO: allow specifying bigger variant + adata, batch_key = bmmc(400), "sample" + case "lung93k": + adata, batch_key = lung93k(), "PatientNumber" + case _: + msg = f"Unknown dataset {dataset}" + raise AssertionError(msg) + + # add off-axis layers + adata.layers["off-axis"] = to_off_axis(adata.X) + adata.layers["counts-off-axis"] = to_off_axis(adata.layers["counts"]) + + # add mitochondrial gene and pre-compute qc metrics + adata.var["mt"] = adata.var_names.str.startswith("MT-") + assert adata.var["mt"].sum() > 0, "no MT genes in dataset" + sc.pp.calculate_qc_metrics( + adata, qc_vars=["mt"], percent_top=None, log1p=False, inplace=True + ) - msg = f"Unknown dataset {dataset}" - raise AssertionError(msg) + return adata, batch_key + + +def get_dataset(dataset: Dataset, *, layer: KeyX = None) -> tuple[AnnData, str | None]: + adata, batch_key = _get_dataset_raw(dataset) + if layer is not None: + adata.X = adata.layers.pop(layer) + return adata, batch_key -def get_count_dataset(dataset: Dataset) -> tuple[AnnData, str | None]: - adata, batch_key = get_dataset(dataset) +def get_count_dataset( + dataset: Dataset, *, layer: KeyCount = "counts" +) -> tuple[AnnData, str | None]: + adata, batch_key = _get_dataset_raw(dataset) - adata.X = adata.layers.pop("counts") + adata.X = adata.layers.pop(layer) # remove indicators that X was transformed adata.uns.pop("log1p", None) return adata, batch_key + + +def param_skipper( + param_names: Sequence[str], params: tuple[Sequence[object], ...] +) -> ParamSkipper: + """Creates a decorator that will skip all combinations that contain any of the given parameters. + + Examples + -------- + + >>> param_names = ["letters", "numbers"] + >>> params = [["a", "b"], [3, 4, 5]] + >>> skip_when = param_skipper(param_names, params) + + >>> @skip_when(letters={"a"}, numbers={3}) + ... def func(a, b): + ... print(a, b) + >>> run_as_asv_benchmark(func) + b 4 + b 5 + """ + + def skip(**skipped: AbstractSet) -> Callable[[C], C]: + skipped_combs = [ + tuple(record.values()) + for record in ( + dict(zip(param_names, vals)) for vals in itertools.product(*params) + ) + if any(v in skipped.get(n, set()) for n, v in record.items()) + ] + # print(skipped_combs, file=sys.stderr) + return skip_for_params(skipped_combs) + + return skip diff --git a/benchmarks/benchmarks/preprocessing_counts.py b/benchmarks/benchmarks/preprocessing_counts.py index 28ed67934e..587b441e1c 100644 --- a/benchmarks/benchmarks/preprocessing_counts.py +++ b/benchmarks/benchmarks/preprocessing_counts.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: from anndata import AnnData - from ._utils import Dataset + from ._utils import Dataset, KeyCount # setup variables @@ -22,31 +22,20 @@ batch_key: str | None -def setup(dataset: Dataset, *_): +def setup(dataset: Dataset, layer: KeyCount, *_): """Setup global variables before each benchmark.""" global adata, batch_key - adata, batch_key = get_count_dataset(dataset) + adata, batch_key = get_count_dataset(dataset, layer=layer) assert "log1p" not in adata.uns # ASV suite -params: list[Dataset] = ["pbmc68k_reduced", "pbmc3k"] -param_names = ["dataset"] - - -def time_calculate_qc_metrics(*_): - adata.var["mt"] = adata.var_names.str.startswith("MT-") - sc.pp.calculate_qc_metrics( - adata, qc_vars=["mt"], percent_top=None, log1p=False, inplace=True - ) - - -def peakmem_calculate_qc_metrics(*_): - adata.var["mt"] = adata.var_names.str.startswith("MT-") - sc.pp.calculate_qc_metrics( - adata, qc_vars=["mt"], percent_top=None, log1p=False, inplace=True - ) +params: tuple[list[Dataset], list[KeyCount]] = ( + ["pbmc68k_reduced", "pbmc3k"], + ["counts", "counts-off-axis"], +) +param_names = ["dataset", "layer"] def time_filter_cells(*_): @@ -73,19 +62,49 @@ def peakmem_scrublet(*_): sc.pp.scrublet(adata, batch_key=batch_key) -def time_normalize_total(*_): - sc.pp.normalize_total(adata, target_sum=1e4) +# Can’t do seurat v3 yet: https://github.com/conda-forge/scikit-misc-feedstock/issues/17 +""" +def time_hvg_seurat_v3(*_): + # seurat v3 runs on counts + sc.pp.highly_variable_genes(adata, flavor="seurat_v3_paper") + + +def peakmem_hvg_seurat_v3(*_): + sc.pp.highly_variable_genes(adata, flavor="seurat_v3_paper") +""" + + +class FastSuite: + """Suite for fast preprocessing operations.""" + + params: tuple[list[Dataset], list[KeyCount]] = ( + ["pbmc3k", "pbmc68k_reduced", "bmmc", "lung93k"], + ["counts", "counts-off-axis"], + ) + param_names = ["dataset", "layer"] + def time_calculate_qc_metrics(self, *_): + sc.pp.calculate_qc_metrics( + adata, qc_vars=["mt"], percent_top=None, log1p=False, inplace=True + ) -def peakmem_normalize_total(*_): - sc.pp.normalize_total(adata, target_sum=1e4) + def peakmem_calculate_qc_metrics(self, *_): + sc.pp.calculate_qc_metrics( + adata, qc_vars=["mt"], percent_top=None, log1p=False, inplace=True + ) + def time_normalize_total(self, *_): + sc.pp.normalize_total(adata, target_sum=1e4) -def time_log1p(*_): - # TODO: This would fail: assert "log1p" not in adata.uns, "ASV bug?" - # https://github.com/scverse/scanpy/issues/3052 - sc.pp.log1p(adata) + def peakmem_normalize_total(self, *_): + sc.pp.normalize_total(adata, target_sum=1e4) + def time_log1p(self, *_): + # TODO: This would fail: assert "log1p" not in adata.uns, "ASV bug?" + # https://github.com/scverse/scanpy/issues/3052 + adata.uns.pop("log1p", None) + sc.pp.log1p(adata) -def peakmem_log1p(*_): - sc.pp.log1p(adata) + def peakmem_log1p(self, *_): + adata.uns.pop("log1p", None) + sc.pp.log1p(adata) diff --git a/benchmarks/benchmarks/preprocessing_log.py b/benchmarks/benchmarks/preprocessing_log.py index 3819c17f62..6a04d4c7c6 100644 --- a/benchmarks/benchmarks/preprocessing_log.py +++ b/benchmarks/benchmarks/preprocessing_log.py @@ -7,17 +7,15 @@ from typing import TYPE_CHECKING -from asv_runner.benchmarks.mark import skip_for_params - import scanpy as sc from scanpy.preprocessing._utils import _get_mean_var -from ._utils import get_dataset +from ._utils import get_dataset, param_skipper if TYPE_CHECKING: from anndata import AnnData - from ._utils import Dataset + from ._utils import Dataset, KeyX # setup variables @@ -26,16 +24,21 @@ batch_key: str | None -def setup(dataset: Dataset, *_): +def setup(dataset: Dataset, layer: KeyX, *_): """Setup global variables before each benchmark.""" global adata, batch_key - adata, batch_key = get_dataset(dataset) + adata, batch_key = get_dataset(dataset, layer=layer) # ASV suite -params: list[Dataset] = ["pbmc68k_reduced", "pbmc3k"] -param_names = ["dataset"] +params: tuple[list[Dataset], list[KeyX]] = ( + ["pbmc68k_reduced", "pbmc3k"], + [None, "off-axis"], +) +param_names = ["dataset", "layer"] + +skip_when = param_skipper(param_names, params) def time_pca(*_): @@ -47,6 +50,7 @@ def peakmem_pca(*_): def time_highly_variable_genes(*_): + # the default flavor runs on log-transformed data sc.pp.highly_variable_genes(adata, min_mean=0.0125, max_mean=3, min_disp=0.5) @@ -55,12 +59,12 @@ def peakmem_highly_variable_genes(*_): # regress_out is very slow for this dataset -@skip_for_params([("pbmc3k",)]) +@skip_when(dataset={"pbmc3k"}) def time_regress_out(*_): sc.pp.regress_out(adata, ["total_counts", "pct_counts_mt"]) -@skip_for_params([("pbmc3k",)]) +@skip_when(dataset={"pbmc3k"}) def peakmem_regress_out(*_): sc.pp.regress_out(adata, ["total_counts", "pct_counts_mt"]) @@ -76,8 +80,11 @@ def peakmem_scale(*_): class FastSuite: """Suite for fast preprocessing operations.""" - params: list[Dataset] = ["pbmc3k", "pbmc68k_reduced", "bmmc", "lung93k"] - param_names = ["dataset"] + params: tuple[list[Dataset], list[KeyX]] = ( + ["pbmc3k", "pbmc68k_reduced", "bmmc", "lung93k"], + [None, "off-axis"], + ) + param_names = ["dataset", "layer"] def time_mean_var(self, *_): _get_mean_var(adata.X) diff --git a/biome.jsonc b/biome.jsonc new file mode 100644 index 0000000000..cf4a677503 --- /dev/null +++ b/biome.jsonc @@ -0,0 +1,21 @@ +{ + "$schema": "https://biomejs.dev/schemas/1.9.4/schema.json", + "formatter": { + "indentStyle": "space", + "indentWidth": 4, + }, + "overrides": [ + { + "include": ["./.vscode/*.json", "**/*.jsonc", "**/asv.conf.json"], + "json": { + "formatter": { + "trailingCommas": "all", + }, + "parser": { + "allowComments": true, + "allowTrailingCommas": true, + }, + }, + }, + ], +} diff --git a/ci/scripts/min-deps.py b/ci/scripts/min-deps.py index b3f393ea57..4efc304cb6 100755 --- a/ci/scripts/min-deps.py +++ b/ci/scripts/min-deps.py @@ -1,9 +1,18 @@ -#!python3 +#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "tomli; python_version < '3.11'", +# "packaging", +# ] +# /// + from __future__ import annotations import argparse import sys from collections import deque +from contextlib import ExitStack +from functools import cached_property from pathlib import Path from typing import TYPE_CHECKING @@ -16,7 +25,9 @@ from packaging.version import Version if TYPE_CHECKING: - from collections.abc import Generator, Iterable + from collections.abc import Generator, Iterable, Sequence + from collections.abc import Set as AbstractSet + from typing import Any, Self def min_dep(req: Requirement) -> Requirement: @@ -27,18 +38,21 @@ def min_dep(req: Requirement) -> Requirement: ------- >>> min_dep(Requirement("numpy>=1.0")) - "numpy==1.0" + """ req_name = req.name if req.extras: req_name = f"{req_name}[{','.join(req.extras)}]" - if not req.specifier: + filter_specs = [ + spec for spec in req.specifier if spec.operator in {"==", "~=", ">=", ">"} + ] + if not filter_specs: return Requirement(req_name) min_version = Version("0.0.0.a1") - for spec in req.specifier: - if spec.operator in [">", ">=", "~="]: + for spec in filter_specs: + if spec.operator in {">", ">=", "~="}: min_version = max(min_version, Version(spec.version)) elif spec.operator == "==": min_version = Version(spec.version) @@ -57,7 +71,9 @@ def extract_min_deps( # If we are referring to other optional dependency lists, resolve them if req.name == project_name: - assert req.extras, f"Project included itself as dependency, without specifying extras: {req}" + assert req.extras, ( + f"Project included itself as dependency, without specifying extras: {req}" + ) for extra in req.extras: extra_deps = pyproject["project"]["optional-dependencies"][extra] dependencies += map(Requirement, extra_deps) @@ -65,34 +81,92 @@ def extract_min_deps( yield min_dep(req) -def main(): - parser = argparse.ArgumentParser( - prog="min-deps", - description="""Parse a pyproject.toml file and output a list of minimum dependencies. - - Output is directly passable to `pip install`.""", - usage="pip install `python min-deps.py pyproject.toml`", - ) - parser.add_argument( - "path", type=Path, help="pyproject.toml to parse minimum dependencies from" - ) - parser.add_argument( - "--extras", type=str, nargs="*", default=(), help="extras to install" - ) - - args = parser.parse_args() - - pyproject = tomllib.loads(args.path.read_text()) +class Args(argparse.Namespace): + """\ + Parse a pyproject.toml file and output a list of minimum dependencies. + Output is optimized for `[uv] pip install` (see `-o`/`--output` for details). + """ - project_name = pyproject["project"]["name"] + _path: Path + output: Path | None + _extras: list[str] + _all_extras: bool + + @classmethod + def parse(cls, argv: Sequence[str] | None = None) -> Self: + return cls.parser().parse_args(argv, cls()) + + @classmethod + def parser(cls) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="min-deps", + description=cls.__doc__, + usage="pip install `python min-deps.py pyproject.toml`", + ) + parser.add_argument( + "_path", + metavar="pyproject.toml", + type=Path, + help="Path to pyproject.toml to parse minimum dependencies from", + ) + parser.add_argument( + "--extras", + dest="_extras", + metavar="EXTRA", + type=str, + nargs="*", + default=(), + help="extras to install", + ) + parser.add_argument( + "--all-extras", + dest="_all_extras", + action="store_true", + help="get all extras", + ) + parser.add_argument( + *("--output", "-o"), + metavar="FILE", + type=Path, + default=None, + help=( + "output file (default: stdout). " + "Without this option, output is space-separated for direct passing to `pip install`. " + "With this option, output written to a file newline-separated file usable as `requirements.txt` or `constraints.txt`." + ), + ) + return parser + + @cached_property + def pyproject(self) -> dict[str, Any]: + return tomllib.loads(self._path.read_text()) + + @cached_property + def extras(self) -> AbstractSet[str]: + if self._extras: + if self._all_extras: + sys.exit("Cannot specify both --extras and --all-extras") + return dict.fromkeys(self._extras).keys() + if not self._all_extras: + return set() + return self.pyproject["project"]["optional-dependencies"].keys() + + +def main(argv: Sequence[str] | None = None) -> None: + args = Args.parse(argv) + + project_name = args.pyproject["project"]["name"] deps = [ - *map(Requirement, pyproject["project"]["dependencies"]), + *map(Requirement, args.pyproject["project"]["dependencies"]), *(Requirement(f"{project_name}[{extra}]") for extra in args.extras), ] - min_deps = extract_min_deps(deps, pyproject=pyproject) + min_deps = extract_min_deps(deps, pyproject=args.pyproject) - print(" ".join(map(str, min_deps))) + sep = "\n" if args.output else " " + with ExitStack() as stack: + f = stack.enter_context(args.output.open("w")) if args.output else sys.stdout + print(sep.join(map(str, min_deps)), file=f) if __name__ == "__main__": diff --git a/ci/scripts/towncrier_automation.py b/ci/scripts/towncrier_automation.py new file mode 100755 index 0000000000..10a8b0c9dc --- /dev/null +++ b/ci/scripts/towncrier_automation.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [ "towncrier", "packaging" ] +# /// + +from __future__ import annotations + +import argparse +import subprocess +from typing import TYPE_CHECKING + +from packaging.version import Version + +if TYPE_CHECKING: + from collections.abc import Sequence + + +class Args(argparse.Namespace): + version: str + dry_run: bool + + +def parse_args(argv: Sequence[str] | None = None) -> Args: + parser = argparse.ArgumentParser( + prog="towncrier-automation", + description=( + "This script runs towncrier for a given version, " + "creates a branch off of the current one, " + "and then creates a PR into the original branch with the changes. " + "The PR will be backported to main if the current branch is not main." + ), + ) + parser.add_argument( + "version", + type=str, + help=( + "The new version for the release must have at least three parts, like `major.minor.patch` and no `major.minor`. " + "It can have a suffix like `major.minor.patch.dev0` or `major.minor.0rc1`." + ), + ) + parser.add_argument( + "--dry-run", + help="Whether or not to dry-run the actual creation of the pull request", + action="store_true", + ) + args = parser.parse_args(argv, Args()) + # validate the version + if len(Version(args.version).release) != 3: + msg = f"Version argument {args.version} must contain major, minor, and patch version." + raise ValueError(msg) + return args + + +def main(argv: Sequence[str] | None = None) -> None: + args = parse_args(argv) + + # Run towncrier + subprocess.run( + ["towncrier", "build", f"--version={args.version}", "--yes"], check=True + ) + + # Check if we are on the main branch to know if we need to backport + base_branch = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + pr_description = "" if base_branch == "main" else "@meeseeksdev backport to main" + branch_name = f"release_notes_{args.version}" + + # Create a new branch + commit + subprocess.run(["git", "switch", "-c", branch_name], check=True) + subprocess.run(["git", "add", "docs/release-notes"], check=True) + pr_title = f"(chore): generate {args.version} release notes" + subprocess.run(["git", "commit", "-m", pr_title], check=True) + + # push + if not args.dry_run: + subprocess.run( + ["git", "push", "--set-upstream", "origin", branch_name], check=True + ) + else: + print("Dry run, not pushing") + + # Create a PR + subprocess.run( + [ + "gh", + "pr", + "create", + f"--base={base_branch}", + f"--title={pr_title}", + f"--body={pr_description}", + *( + ["--label=no milestone", "--label=Development Process 🚀"] + if base_branch == "main" + else [] + ), + *(["--dry-run"] if args.dry_run else []), + ], + check=True, + ) + + # Enable auto-merge + if not args.dry_run: + subprocess.run( + ["gh", "pr", "merge", branch_name, "--auto", "--squash"], check=True + ) + else: + print("Dry run, not merging") + + +if __name__ == "__main__": + main() diff --git a/docs/_templates/autosummary/class.rst b/docs/_templates/autosummary/class.rst index 618976e3ae..ea2d44832c 100644 --- a/docs/_templates/autosummary/class.rst +++ b/docs/_templates/autosummary/class.rst @@ -14,7 +14,7 @@ :toctree: . {% for item in attributes %} {% if has_member(fullname, item) %} - ~{{ fullname }}.{{ item }} + ~{{ name }}.{{ item }} {% endif %} {%- endfor %} {% endif %} @@ -28,7 +28,7 @@ :toctree: . {% for item in methods %} {%- if item != '__init__' %} - ~{{ fullname }}.{{ item }} + ~{{ name }}.{{ item }} {%- endif -%} {%- endfor %} {% endif %} diff --git a/docs/api/deprecated.md b/docs/api/deprecated.md index 4511f4b3a7..d09c1af405 100644 --- a/docs/api/deprecated.md +++ b/docs/api/deprecated.md @@ -11,4 +11,5 @@ pp.filter_genes_dispersion pp.normalize_per_cell + pp.subsample ``` diff --git a/docs/api/preprocessing.md b/docs/api/preprocessing.md index 4b17567a6b..1834d934a4 100644 --- a/docs/api/preprocessing.md +++ b/docs/api/preprocessing.md @@ -31,7 +31,7 @@ For visual quality control, see {func}`~scanpy.pl.highest_expr_genes` and pp.normalize_total pp.regress_out pp.scale - pp.subsample + pp.sample pp.downsample_counts ``` @@ -49,7 +49,7 @@ For visual quality control, see {func}`~scanpy.pl.highest_expr_genes` and ### Batch effect correction -Also see [Data integration]. Note that a simple batch correction method is available via {func}`pp.regress_out`. Checkout {mod}`scanpy.external` for more. +Also see {ref}`data-integration`. Note that a simple batch correction method is available via {func}`pp.regress_out`. Checkout {mod}`scanpy.external` for more. ```{eval-rst} .. autosummary:: diff --git a/docs/api/tools.md b/docs/api/tools.md index 1d51559e5a..13d82b46c7 100644 --- a/docs/api/tools.md +++ b/docs/api/tools.md @@ -48,6 +48,8 @@ Compute densities on embeddings. tl.paga ``` +(data-integration)= + ### Data integration ```{eval-rst} diff --git a/docs/conf.py b/docs/conf.py index 30dde06a45..e17aa9df0f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,10 +2,12 @@ import sys from datetime import datetime -from pathlib import Path +from functools import partial +from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING import matplotlib # noqa +from docutils import nodes from packaging.version import Version # Don’t use tkinter agg when importing scanpy → … → matplotlib @@ -50,7 +52,14 @@ templates_path = ["_templates"] master_doc = "index" default_role = "literal" -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"] +exclude_patterns = [ + "_build", + "Thumbs.db", + ".DS_Store", + "**.ipynb_checkpoints", + # exclude all 0.x.y.md files, but not index.md + "release-notes/[!i]*.md", +] extensions = [ "myst_nb", @@ -70,6 +79,7 @@ "scanpydoc", # needs to be before sphinx.ext.linkcode "sphinx.ext.linkcode", "sphinx_design", + "sphinx_tabs.tabs", "sphinx_search.extension", "sphinxext.opengraph", *[p.stem for p in (HERE / "extensions").glob("*.py") if p.stem not in {"git_ref"}], @@ -96,6 +106,7 @@ "html_admonition", ] myst_url_schemes = ("http", "https", "mailto", "ftp") +myst_heading_anchors = 3 nb_output_stderr = "remove" nb_execution_mode = "off" nb_merge_streams = True @@ -131,6 +142,8 @@ rapids_singlecell=("https://rapids-singlecell.readthedocs.io/en/latest/", None), scipy=("https://docs.scipy.org/doc/scipy/", None), seaborn=("https://seaborn.pydata.org/", None), + session_info2=("https://session-info2.readthedocs.io/en/stable/", None), + squidpy=("https://squidpy.readthedocs.io/en/stable/", None), sklearn=("https://scikit-learn.org/stable/", None), ) @@ -151,6 +164,8 @@ def setup(app: Sphinx): """App setup hook.""" + app.add_generic_role("small", partial(nodes.inline, classes=["small"])) + app.add_generic_role("smaller", partial(nodes.inline, classes=["smaller"])) app.add_config_value( "recommonmark_config", { @@ -160,7 +175,7 @@ def setup(app: Sphinx): "enable_inline_math": False, "enable_eval_rst": True, }, - True, + True, # noqa: FBT003 ) @@ -225,8 +240,9 @@ def setup(app: Sphinx): plot_html_show_source_link = False plot_working_directory = HERE.parent # Project root -# extlinks config +# link config extlinks = { "issue": ("https://github.com/scverse/scanpy/issues/%s", "issue%s"), "pr": ("https://github.com/scverse/scanpy/pull/%s", "pr%s"), } +rtd_links_prefix = PurePosixPath("src") diff --git a/docs/contributors.md b/docs/contributors.md index a9b2d79e3c..2e9c62d255 100644 --- a/docs/contributors.md +++ b/docs/contributors.md @@ -1,22 +1,25 @@ # Contributors [anndata graph](https://github.com/scverse/anndata/graphs/contributors>) | [scanpy graph](https://github.com/scverse/scanpy/graphs/contributors)| ☀ = maintainer + ## Current developers -- [Isaac Virshup](https://github.com/ivirshup), lead developer since 2019 ☀ -- [Gökcen Eraslan](https://twitter.com/gokcen), developer, diverse contributions ☀ -- [Sergei Rybakov](https://github.com/Koncopd), developer, diverse contributions ☀ -- [Fidel Ramirez](https://github.com/fidelram) developer, plotting ☀ -- [Giovanni Palla](https://twitter.com/g_palla1), developer, spatial data -- [Malte Luecken](https://twitter.com/MDLuecken), developer, community & forum +- [Philipp Angerer](https://github.com/flying-sheep), lead developer since 2023, software quality, initial anndata conception ☀ +- [Ilan Gold](https://github.com/ilan-gold), developer, Dask ☀ +- [Severin Dicks](https://github.com/SeverinDicks), developer, performance ☀ - [Lukas Heumos](https://twitter.com/LukasHeumos), developer, diverse contributions -- [Philipp Angerer](https://github.com/flying-sheep), developer, software quality, initial anndata conception ☀ ## Other roles +- [Isaac Virshup](https://github.com/ivirshup), lead developer 2019-2023 - [Alex Wolf](https://twitter.com/falexwolf): lead developer 2016-2019, initial anndata & scanpy conception - [Fabian Theis](https://twitter.com/fabian_theis) & lab: enabling guidance, support and environment ## Former developers -- Tom White: developer 2018-2019, distributed computing +- [Tom White](https://github.com/tomwhite): developer 2018-2019, distributed computing +- [Gökcen Eraslan](https://twitter.com/gokcen), developer, diverse contributions +- [Sergei Rybakov](https://github.com/Koncopd), developer, diverse contributions +- [Fidel Ramirez](https://github.com/fidelram) developer, plotting +- [Giovanni Palla](https://twitter.com/g_palla1), developer, spatial data +- [Malte Luecken](https://twitter.com/MDLuecken), developer, community & forum diff --git a/docs/dev/code.md b/docs/dev/code.md index 06c267951b..3ca393c8f7 100644 --- a/docs/dev/code.md +++ b/docs/dev/code.md @@ -9,16 +9,16 @@ 5. {ref}`Make sure all tests are passing ` 6. {ref}`Build and visually check any changed documentation ` 7. {ref}`Open a PR back to the main repository ` +8. {ref}`Add a release note to your PR ` ## Code style -New code should follow -[Black](https://black.readthedocs.io/en/stable/the_black_code_style.html) -and -[flake8](https://flake8.pycqa.org). -We ignore a couple of flake8 checks which are documented in the .flake8 file in the root of this repository. -To learn how to ignore checks per line please read -[flake8 violations](https://flake8.pycqa.org/en/latest/user/violations.html). -Additionally, we use Scanpy’s -[EditorConfig](https://github.com/scverse/scanpy/blob/main/.editorconfig), +Code contributions will be formatted and style checked using [Ruff][]. +Ignored checks are configured in the `tool.ruff.lint` section of {file}`pyproject.toml`. +To learn how to ignore checks per line please read about [ignoring errors][]. +Additionally, we use Scanpy’s [EditorConfig][], so using an editor/IDE with support for both is helpful. + +[Ruff]: https://docs.astral.sh/ruff/ +[ignoring errors]: https://docs.astral.sh/ruff/tutorial/#ignoring-errors +[EditorConfig]: https://github.com/scverse/scanpy/blob/main/.editorconfig diff --git a/docs/dev/documentation.md b/docs/dev/documentation.md index 159b533ee3..dcad9533ed 100644 --- a/docs/dev/documentation.md +++ b/docs/dev/documentation.md @@ -4,38 +4,39 @@ ## Building the docs -Dependencies for building the documentation for scanpy can be installed with `pip install -e "scanpy[doc]"` - -To build the docs, enter the `docs` directory and run `make html`. After this process completes you can take a look at the docs by opening `scanpy/docs/_build/html/index.html`. +To build the docs, run `hatch run docs:build`. +Afterwards, you can run `hatch run docs:open` to open {file}`docs/_build/html/index.html`. Your browser and Sphinx cache docs which have been built previously. Sometimes these caches are not invalidated when you've updated the docs. If docs are not updating the way you expect, first try "force reloading" your browser page – e.g. reload the page without using the cache. -Next, if problems persist, clear the sphinx cache and try building them again (`make clean` from `docs` directory). +Next, if problems persist, clear the sphinx cache (`hatch run docs:clean`) and try building them again. -```{note} -If you've cloned the repository pre 1.8.0, you may need to be more thorough in cleaning. -If you run into warnings try removing all untracked files in the docs directory. -``` +(adding-to-the-docs)= ## Adding to the docs -For any user-visible changes, please make sure a note has been added to the release notes for the relevant version so we can credit you! -These files are found in the `docs/release-notes/` directory. -We recommend waiting on this until your PR is close to done since this can often causes merge conflicts. +For any user-visible changes, please make sure a note has been added to the release notes using [`hatch run towncrier:create`][towncrier create]. +When asked for “Issue number (`+` if none)”, enter the *PR number* instead. Once you've added a new function to the documentation, you'll need to make sure there is a link somewhere in the documentation site pointing to it. This should be added to `docs/api.md` under a relevant heading. -For tutorials and more in depth examples, consider adding a notebook to [scanpy-tutorials](https://github.com/scverse/scanpy-tutorials/). +For tutorials and more in depth examples, consider adding a notebook to the [scanpy-tutorials][] repository. + +The tutorials are tied to this repository via a submodule. +To update the submodule, run `git submodule update --remote` from the root of the repository. +Subsequently, commit and push the changes in a PR. +This should be done before each release to ensure the tutorials are up to date. -The tutorials are tied to this repository via a submodule. To update the submodule, run `git submodule update --remote` from the root of the repository. Subsequently, commit and push the changes in a PR. This should be done before each release to ensure the tutorials are up to date. +[towncrier create]: https://towncrier.readthedocs.io/en/stable/tutorial.html#creating-news-fragments +[scanpy-tutorials]: https://github.com/scverse/scanpy-tutorials/ ## docstrings format We use the numpydoc style for writing docstrings. -We'd primarily suggest looking at existing docstrings for examples, but the [napolean guide to numpy style docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_numpy.html#example-numpy) is also a great source. -If you're unfamiliar with the reStructuredText (`rst`) markup format, [Sphinx has a useful primer](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html). +We'd primarily suggest looking at existing docstrings for examples, but the [napolean guide to numpy style docstrings][] is also a great source. +If you're unfamiliar with the reStructuredText (rST) markup format, check out the [Sphinx rST primer][]. Some key points: @@ -46,11 +47,14 @@ Some key points: Look at [sc.tl.louvain](https://github.com/scverse/scanpy/blob/a811fee0ef44fcaecbde0cad6336336bce649484/scanpy/tools/_louvain.py#L22-L90) as an example for everything mentioned here. +[napolean guide to numpy style docstrings]: https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_numpy.html#example-numpy +[sphinx rst primer]: https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html + ### Plots in docstrings One of the most useful things you can include in a docstring is examples of how the function should be used. These are a great way to demonstrate intended usage and give users a template they can copy and modify. -We're able to include the plots produced by these snippets in the rendered docs using [matplotlib's plot directive](https://matplotlib.org/devel/plot_directive.html). +We're able to include the plots produced by these snippets in the rendered docs using [matplotlib's plot directive][]. For examples of this, see the `Examples` sections of {func}`~scanpy.pl.dotplot` or {func}`~scanpy.pp.calculate_qc_metrics`. Note that anything in these sections will need to be run when the docs are built, so please keep them computationally light. @@ -58,6 +62,8 @@ Note that anything in these sections will need to be run when the docs are built - If you need computed features (e.g. an embedding, differential expression results) load data that has this precomputed. - Try to re-use datasets, this reduces the amount of data that needs to be downloaded to the CI server. +[matplotlib's plot directive]: https://matplotlib.org/devel/plot_directive.html + ### `Params` section The `Params` abbreviation is a legit replacement for `Parameters`. @@ -65,7 +71,10 @@ The `Params` abbreviation is a legit replacement for `Parameters`. To document parameter types use type annotations on function parameters. These will automatically populate the docstrings on import, and when the documentation is built. -Use the python standard library types (defined in [collections.abc](https://docs.python.org/3/library/collections.abc.html) and [typing](https://docs.python.org/3/library/typing.html) modules) for containers, e.g. `Sequence`s (like `list`), `Iterable`s (like `set`), and `Mapping`s (like `dict`). +Use the python standard library types (defined in {mod}`collections.abc` and {mod}`typing` modules) for containers, e.g. +{class}`~collections.abc.Sequence`s (like `list`), +{class}`~collections.abc.Iterable`s (like `set`), and +{class}`~collections.abc.Mapping`s (like `dict`). Always specify what these contain, e.g. `{'a': (1, 2)}` → `Mapping[str, Tuple[int, int]]`. If you can’t use one of those, use a concrete class like `AnnData`. If your parameter only accepts an enumeration of strings, specify them like so: `Literal['elem-1', 'elem-2']`. @@ -80,8 +89,7 @@ There are three types of return sections – prose, tuple, and a mix of both. #### Examples -For simple cases, use prose as in -{func}`~scanpy.pp.normalize_total` +For simple cases, use prose as in {func}`~scanpy.pp.normalize_total`: ```rst Returns @@ -110,7 +118,7 @@ def myfunc(...) -> tuple[int, str]: ``` Many functions also just modify parts of the passed AnnData object, like e.g. {func}`~scanpy.tl.dpt`. -You can then combine prose and lists to best describe what happens. +You can then combine prose and lists to best describe what happens: ```rst Returns diff --git a/docs/dev/getting-set-up.md b/docs/dev/getting-set-up.md index 750af53e91..20c6cba63a 100644 --- a/docs/dev/getting-set-up.md +++ b/docs/dev/getting-set-up.md @@ -6,8 +6,11 @@ This section of the docs covers our practices for working with `git` on our code For a more complete git tutorials we recommend checking out: -- [Atlassian's git tutorial](https://www.atlassian.com/git/tutorials) -- Beginner friendly introductions to the git command line interface -- [Setting up git for GitHub](https://docs.github.com/en/free-pro-team@latest/github/getting-started-with-github/set-up-git) -- Configuring git to work with your GitHub user account +[Atlassian's git tutorial](https://www.atlassian.com/git/tutorials) +: Beginner friendly introductions to the git command line interface + +[Setting up git for GitHub](https://docs.github.com/en/free-pro-team@latest/github/getting-started-with-github/set-up-git) +: Configuring git to work with your GitHub user account (forking-and-cloning)= @@ -15,9 +18,9 @@ For a more complete git tutorials we recommend checking out: To get the code, and be able to push changes back to the main project, you'll need to (1) fork the repository on github and (2) clone the repository to your local machine. -This is very straight forward if you're using [GitHub's CLI](https://cli.github.com): +This is very straight forward if you're using [GitHub's CLI][]: -```shell +```console $ gh repo fork scverse/scanpy --clone --remote ``` @@ -25,37 +28,41 @@ This will fork the repo to your github account, create a clone of the repo on yo To do this manually, first make a fork of the repository by clicking the "fork" button on our main github package. Then, on your machine, run: -```shell -# Clone your fork of the repository (substitute in your username) -git clone https://github.com/{your-username}/scanpy.git -# Enter the cloned repository -cd scanpy -# Add our repository as a remote -git remote add upstream https://github.com/scverse/scanpy.git -# git branch --set-upstream-to "upstream/main" +```console +$ # Clone your fork of the repository (substitute in your username) +$ git clone https://github.com/{your-username}/scanpy.git +$ # Enter the cloned repository +$ cd scanpy +$ # Add our repository as a remote +$ git remote add upstream https://github.com/scverse/scanpy.git +$ # git branch --set-upstream-to "upstream/main" ``` +[GitHub's CLI]: https://cli.github.com + ### `pre-commit` -We use [precommit](https://pre-commit.com) to run some styling checks in an automated way. +We use [pre-commit][] to run some styling checks in an automated way. We also test against these checks, so make sure you follow them! You can install pre-commit with: -```shell -pip install pre-commit +```console +$ pip install pre-commit ``` You can then install it to run while developing here with: -```shell -pre-commit install +```console +$ pre-commit install ``` From the root of the repo. If you choose not to run the hooks on each commit, you can run them manually with `pre-commit run --files={your files}`. +[pre-commit]: https://pre-commit.com + (creating-a-branch)= ### Creating a branch for your feature @@ -64,10 +71,10 @@ All development should occur in branches dedicated to the particular work being Additionally, unless you are a maintainer, all changes should be directed at the `main` branch. You can create a branch with: -```shell -git checkout main # Starting from the main branch -git pull # Syncing with the repo -git checkout -b {your-branch-name} # Making and changing to the new branch +```console +$ git checkout main # Starting from the main branch +$ git pull # Syncing with the repo +$ git switch -c {your-branch-name} # Making and changing to the new branch ``` (open-a-pr)= @@ -76,11 +83,11 @@ git checkout -b {your-branch-name} # Making and changing to the new branch When you're ready to have your code reviewed, push your changes up to your fork: -```shell -# The first time you push the branch, you'll need to tell git where -git push --set-upstream origin {your-branch-name} -# After that, just use -git push +```console +$ # The first time you push the branch, you'll need to tell git where +$ git push --set-upstream origin {your-branch-name} +$ # After that, just use +$ git push ``` And open a pull request by going to the main repo and clicking *New pull request*. @@ -93,6 +100,10 @@ We'll try and get back to you soon! ## Development environments It's recommended to do development work in an isolated environment. -There are number of ways to do this, including conda environments, virtual environments, and virtual machines. +There are number of ways to do this, including virtual environments, conda environments, and virtual machines. + +We think the easiest is probably [Hatch environments][]. +Using one of the predefined environments in {file}`hatch.toml` is as simple as running `hatch test` or `hatch run docs:build` (they will be created on demand). +For an in-depth guide, refer to the {ref}`development install instructions ` of `scanpy`. -We think the easiest is probably conda environments. Simply create a new environment with a supported version of python and make a {ref}`development install ` of `scanpy`. +[hatch environments]: https://hatch.pypa.io/latest/tutorials/environment/basic-usage/ diff --git a/docs/dev/release.md b/docs/dev/release.md index f4bcb528a6..f93b73eaa1 100644 --- a/docs/dev/release.md +++ b/docs/dev/release.md @@ -3,13 +3,21 @@ First, check out {doc}`versioning` to see which kind of release you want to make. That page also explains concepts like *pre-releases* and applications thereof. +## Preparing the release + +1. Switch to the `main` branch for a major/minor release and the respective release series branch for a *patch* release (e.g. `1.8.x` when releasing version 1.8.4). +2. Run `hatch towncrier:build` to generate a PR that creates a new release notes file. Wait for the PR to be auto-merged. +3. If it is a *patch* release, merge the backport PR (see {ref}`versioning-tooling`) into the `main` branch. + ## Actually making the release -1. Go to GitHub’s [releases][] page -2. Click “Draft a new release” -3. Click “Choose a tag” and type the version of the tag you want to release, such as `1.9.6` -4. Click “**+ Create new tag: 1.\.\** on publish” -5. If the version is a *pre-release* version, such as `1.7.0rc1` or `1.10.0a1`, tick the “Set as a pre-release” checkbox +1. Go to GitHub’s [releases][] page. +2. Click the “Draft a new release” button. +3. Open the “Choose a tag” dropdown and type the version of the tag you want to release, such as `1.9.6`. +4. Select the dropdown entry “**+ Create new tag: 1.\.\** on publish”. +5. In the second dropdown “Target:”, select the base branch i.e. `main` for a minor/major release, + and e.g. `1.9.x` for our example patch release `1.9.6`. +6. If the version is a *pre-release* version, such as `1.7.0rc1` or `1.10.0a1`, tick the “Set as a pre-release” checkbox. [releases]: https://github.com/scverse/scanpy/releases @@ -17,8 +25,6 @@ That page also explains concepts like *pre-releases* and applications thereof. After *any* release has been made: -- Create a new release notes file for the next bugfix release. - This should be included in both dev and stable branches. - Create a milestone for the next release (in case you made a bugfix release) or releases (in case of a major/minor release). For bugfix releases, this should have `on-merge: backport to 0..x`, so the [meeseeksdev][] bot will create a backport PR. See {doc}`versioning` for more info. @@ -39,27 +45,20 @@ If you changed something about the build process (e.g. [Hatchling’s build conf or something about the package’s structure, you might want to manually check if the build and upload process behaves as expected: -```shell -# Clear out old distributions -rm -r dist - -# Build source distribution and wheel both -python -m build - -# Now check those build artifacts -twine check dist/* - -# List the wheel archive’s contents -bsdtar -tf dist/*.whl - +```console +$ # Clear out old distributions +$ rm -r dist +$ # Build source distribution and wheel both +$ python -m build +$ # Now check those build artifacts +$ twine check dist/* +$ # List the wheel archive’s contents +$ bsdtar -tf dist/*.whl ``` You can also upload the package to ([tutorial][testpypi tutorial]) - -[testpypi tutorial]: https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives - -``` -twine upload --repository testpypi dist/* +```console +$ twine upload --repository testpypi dist/* ``` The above approximates what the [publish workflow][] does automatically for us. @@ -67,4 +66,5 @@ If you want to replicate the process more exactly, make sure you are careful, and create a version tag before building (make sure you delete it after uploading to TestPyPI!). [hatch-build]: https://hatch.pypa.io/latest/config/build/ +[testpypi tutorial]: https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives [publish workflow]: https://github.com/scverse/scanpy/tree/main/.github/workflows/publish.yml diff --git a/docs/dev/testing.md b/docs/dev/testing.md index aeae4ee8da..81eae36c75 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -7,34 +7,43 @@ Implementations may change, but the only way we can know the code is working bef ## Running the tests -We use [pytest](https://docs.pytest.org/en/stable/) to test scanpy. -To run the tests first make sure you have the required dependencies (`pip install -e ".[test,dev]"`), then run `pytest` from the root of the repository. +We use [pytest][] to test scanpy. +To run the tests, simply run `hatch test`. It can take a while to run the whole test suite. There are a few ways to cut down on this while working on a PR: -1. Only run a subset of the tests. This can be done with the `-k` argument from pytest (e.g. `pytest -k test_plotting.py` or `pytest -k "test_umap*"` -2. Run the tests in parallel. If you install the pytest extension [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) you can run tests in parallel with the `--numprocesses` argument to pytest (e.g. `pytest -n 8`). +1. Only run a subset of the tests. + This can be done by specifying paths or test name patterns using the `-k` argument (e.g. `hatch test test_plotting.py` or `hatch test -k "test_umap*"`) +2. Run the tests in parallel using the `-n` argument (e.g. `hatch test -n 8`). + +[pytest]: https://docs.pytest.org/en/stable/ ### Miscellaneous tips -- A lot of warnings can be thrown while running the test suite. It's often easier to read the test results with them hidden via the `--disable-pytest-warnings` argument. +- A lot of warnings can be thrown while running the test suite. + It's often easier to read the test results with them hidden via the `--disable-pytest-warnings` argument. ## Writing tests -You can refer to the [existing test suite](https://github.com/scverse/scanpy/tree/main/scanpy/tests) for examples. -If you haven't written tests before, Software Carpentry has an [in-depth guide](https://katyhuff.github.io/2016-07-11-scipy/testing/01-basics.html) on the topic. +You can refer to the [existing test suite][] for examples. +If you haven't written tests before, Software Carpentry has an [in-depth testing guide][]. -We highly recommend using [Test Driven Development](https://en.wikipedia.org/wiki/Test-driven_development) when contributing code. +We highly recommend using [Test-Driven Development][] when contributing code. This not only ensures you have tests written, it often makes implementation easier since you start out with a specification for your function. Consider parameterizing your tests using the `pytest.mark.parameterize` and `pytest.fixture` decorators. -Documentation on these can be found [here](https://docs.pytest.org/en/stable/fixture.html), but we'd also recommend searching our test suite for existing usage. +You can read more about [fixtures][] in pytest’s documentation, but we’d also recommend searching our test suite for existing usage. + +[existing test suite]: https://github.com/scverse/scanpy/tree/main/scanpy/tests +[in-depth testing guide]: https://katyhuff.github.io/2016-07-11-scipy/testing/ +[test-driven development]: https://en.wikipedia.org/wiki/Test-driven_development +[fixtures]: https://docs.pytest.org/en/stable/fixture.html ### What to test If you're not sure what to tests about your function, some ideas include: -- Are there arguments which conflict with each other? Check that if they are both passed, the function throws an error (see `pytest.raises` [in the pytest docs](https://docs.pytest.org/en/stable/assert.html#assertions-about-expected-exceptions). +- Are there arguments which conflict with each other? Check that if they are both passed, the function throws an error (see [`pytest.raises`][] docs). - Are there input values which should cause your function to error? - Did you add a helpful error message that recommends better outputs? Check that that error message is actually thrown. - Can you place bounds on the values returned by your function? @@ -42,6 +51,8 @@ If you're not sure what to tests about your function, some ideas include: - Do you have arguments which should have orthogonal effects on the output? Check that they are independent. For example, if there is a flag for extended output, the base output should remain the same either way. - Are you optimizing a method? Check that it's results are the same as a gold standard implementation. +[`pytest.raises`]: https://docs.pytest.org/en/stable/assert.html#assertions-about-expected-exceptions + ### Performance It's more important that you're accurately testing the code works than it is that test suite runs quickly. @@ -51,9 +62,11 @@ You can check how long tests take to run by passing `--durations=0` argument to Hopefully your new tests won't show up on top! Some approaches to this include: -- Is there a common setup/ computation happening in each test? Consider caching these in a [scoped test fixture](https://docs.pytest.org/en/stable/fixture.html#sharing-test-data). +- Is there a common setup/ computation happening in each test? Consider caching these in a [scoped test fixture][]. - Is the behaviour you're testing for dependent on the size of the data? If not, consider reducing it. +[scoped test fixture]: https://docs.pytest.org/en/stable/fixture.html#sharing-test-data + ### Plotting tests While computational functions will return arrays and values, it can be harder to work with the output of plotting functions. diff --git a/docs/dev/versioning.md b/docs/dev/versioning.md index c2427f6a01..748b3d2e2c 100644 --- a/docs/dev/versioning.md +++ b/docs/dev/versioning.md @@ -18,22 +18,30 @@ At a `point` release, there should be no changes beyond bug fixes. Valid version numbers are described in [PEP 440](https://peps.python.org/pep-0440/). [Pre-releases](https://peps.python.org/pep-0440/#pre-releases) -: should have versions like `1.7.0rc1` or `1.7.0rc2`. +: should have versions like `1.7.0rc1` or `1.7.0rc2`. + [Development versions](https://peps.python.org/pep-0440/#developmental-releases) -: should look like `1.8.0.dev0`, with a commit hash optionally appended as a local version identifier (e.g. `1.8.0.dev2+g00ad77b`). +: should look like `1.8.0.dev0`, with a commit hash optionally appended as a local version identifier (e.g. `1.8.0.dev2+g00ad77b`). +(versioning-tooling)= ## Tooling To be sure we can follow this scheme and maintain some agility in development, we use some tooling and development practices. When a minor release is made, a release branch should be cut and pushed to the main repo (e.g. `1.7.x` for the `1.7` release series). For PRs which fix an bug in the most recent minor release, the changes will need to added to both the development and release branches. -To accomplish this, PRs which fix bugs must be labelled as such. -After approval, a developer will notify the [meeseeks bot](https://meeseeksbox.github.io) to open a backport PR onto the release branch via a comment saying: +To accomplish this, PRs which fix bugs are assigned a patch version milestone such as `1.7.4`. +Once the PR is approved and merged, the bot will attempt to make a backport and open a PR. +This will sometimes require manual intervention due to merge conflicts or test failures. + +### Technical details + +The [meeseeks bot][] reacts to commands like this, +given as a comment on the PR, or a label or milestone description: > @Meeseeksdev backport \ -Where "\" is the most recent release branch. +In our case, these commands are part of the milestone description, +which causes the merge of a PR assigned to a milestone to trigger the bot. -The bot will attempt to make a backport and open a PR. -This will sometimes require manual intervention due to merge conflicts or test failures. +[meseeks bot]: https://meeseeksbox.github.io diff --git a/docs/ecosystem.md b/docs/ecosystem.md index d78a5bd930..2c7aba9f9b 100644 --- a/docs/ecosystem.md +++ b/docs/ecosystem.md @@ -1,13 +1,5 @@ # Ecosystem -```{eval-rst} -.. role:: small -``` - -```{eval-rst} -.. role:: smaller -``` - ```{warning} We are no longer accepting new tools on this page. Instead, please submit your tool to the [scverse ecosystem package listing](https://scverse.org/packages/#ecosystem). diff --git a/docs/extensions/canonical_tutorial.py b/docs/extensions/canonical_tutorial.py new file mode 100644 index 0000000000..b459fbb059 --- /dev/null +++ b/docs/extensions/canonical_tutorial.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sphinx.util.docutils import SphinxDirective + +if TYPE_CHECKING: + from typing import ClassVar + + from docutils import nodes + from sphinx.application import Sphinx + + +class CanonicalTutorial(SphinxDirective): + """In the scanpy-tutorials repo, this links to the canonical location (here!).""" + + required_arguments: ClassVar = 1 + + def run(self) -> list[nodes.Node]: + return [] + + +def setup(app: Sphinx) -> None: + app.add_directive("canonical-tutorial", CanonicalTutorial) diff --git a/docs/extensions/param_police.py b/docs/extensions/param_police.py index 37942d3687..234ad28e62 100644 --- a/docs/extensions/param_police.py +++ b/docs/extensions/param_police.py @@ -37,7 +37,8 @@ def show_param_warnings(app, exception): line, ) if param_warnings: - raise RuntimeError("Encountered text parameter type. Use annotations.") + msg = "Encountered text parameter type. Use annotations." + raise RuntimeError(msg) def setup(app: Sphinx): diff --git a/docs/how-to/knn-transformers.ipynb b/docs/how-to/knn-transformers.ipynb index 74ae4f265b..f97950c67e 120000 --- a/docs/how-to/knn-transformers.ipynb +++ b/docs/how-to/knn-transformers.ipynb @@ -1 +1 @@ -../../notebooks/knn-transformers.ipynb \ No newline at end of file +../../notebooks/how-to/knn-transformers.ipynb \ No newline at end of file diff --git a/docs/how-to/plotting-with-marsilea.ipynb b/docs/how-to/plotting-with-marsilea.ipynb index 45b62b725a..1deff1bb72 120000 --- a/docs/how-to/plotting-with-marsilea.ipynb +++ b/docs/how-to/plotting-with-marsilea.ipynb @@ -1 +1 @@ -../../notebooks/plotting-with-marsilea.ipynb \ No newline at end of file +../../notebooks/how-to/plotting-with-marsilea.ipynb \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index a8b49c72a4..4d648e8d39 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,14 +2,6 @@ :end-before: '## Citation' ``` -```{eval-rst} -.. role:: small -``` - -```{eval-rst} -.. role:: smaller -``` - ::::{grid} 1 2 3 3 :gutter: 2 diff --git a/docs/installation.md b/docs/installation.md index 815b2a3a15..c28a09d668 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,116 +1,113 @@ # Installation -## Anaconda +To use `scanpy` from another project, install it using your favourite environment manager: -If you do not have a working installation of Python 3.6 (or later), consider -installing [Miniconda] (see [Installing Miniconda]). Then run: +::::{tabs} -```shell -conda install -c conda-forge scanpy python-igraph leidenalg -``` +:::{group-tab} Hatch (recommended) +Adding `scanpy[leiden]` to your dependencies is enough. +See below for how to use Scanpy’s {ref}`dev-install-instructions`. +::: -Pull Scanpy from [PyPI](https://pypi.org/project/scanpy) (consider using `pip3` to access Python 3): +:::{group-tab} Pip/PyPI +If you prefer to exclusively use PyPI run: -```shell -pip install scanpy +```console +$ pip install 'scanpy[leiden]' ``` +::: -## PyPI only +:::{group-tab} Conda +After installing installing e.g. [Miniconda][], run: -If you prefer to exclusively use PyPI run: +```console +$ conda install -c conda-forge scanpy python-igraph leidenalg +``` + +Pull Scanpy [from PyPI][] (consider using `pip3` to access Python 3): -```shell -pip install 'scanpy[leiden]' +```console +$ pip install scanpy ``` -The extra `[leiden]` installs two packages that are needed for popular -parts of scanpy but aren't requirements: [igraph] {cite:p}`Csardi2006` and [leiden] {cite:p}`Traag2019`. +[miniconda]: https://docs.anaconda.com/miniconda/miniconda-install/ +[from pypi]: https://pypi.org/project/scanpy +::: + +:::: + +If you use Hatch or pip, the extra `[leiden]` installs two packages that are needed for popular +parts of scanpy but aren't requirements: [igraph][] {cite:p}`Csardi2006` and [leiden][] {cite:p}`Traag2019`. +If you use conda, you should to add these dependencies to your environment individually. + +[igraph]: https://python.igraph.org/en/stable/ +[leiden]: https://leidenalg.readthedocs.io (dev-install-instructions)= ## Development Version -To work with the latest version [on GitHub]: clone the repository and `cd` into its root directory. +To work with the latest version [on GitHub][]: clone the repository and `cd` into its root directory. -```shell -gh repo clone scverse/scanpy -cd scanpy +```console +$ gh repo clone scverse/scanpy +$ cd scanpy ``` -If you are using `pip>=21.3`, an editable install can be made: +::::{tabs} -```shell -pip install -e '.[dev,test]' -``` +:::{group-tab} Hatch (recommended) +To use one of the predefined [Hatch environments][] in {file}`hatch.toml`, +run either `hatch test [args]` or `hatch run [env:]command [...args]`, e.g.: -If you want to let [conda] handle the installations of dependencies, do: - -```shell -pipx install beni -beni pyproject.toml > environment.yml -conda env create -f environment.yml -conda activate scanpy -pip install -e '.[dev,doc,test]' +```console +$ hatch test -p # run tests in parallel +$ hatch run docs:build # build docs +$ hatch run towncrier:create # create changelog entry ``` -For instructions on how to work with the code, see the {ref}`contribution guide `. +[hatch environments]: https://hatch.pypa.io/latest/tutorials/environment/basic-usage/ +::: -## Docker - -If you're using [Docker], you can use e.g. the image [gcfntnu/scanpy] from Docker Hub. +:::{group-tab} Pip/PyPI +If you are using `pip>=21.3`, an editable install can be made: -## Troubleshooting +```console +$ python -m venv .venv +$ source .venv/bin/activate +$ pip install -e '.[dev,test]' +``` +::: -If you get a `Permission denied` error, never use `sudo pip`. Instead, use virtual environments or: +:::{group-tab} Conda +If you want to let `conda` handle the installations of dependencies, do: -```shell -pip install --user scanpy +```console +$ pipx install beni +$ beni pyproject.toml > environment.yml +$ conda env create -f environment.yml +$ conda activate scanpy +$ pip install -e '.[dev,doc,test]' ``` -**On MacOS**, if **not** using `conda`, you might need to install the C core of igraph via homebrew first - -- `brew install igraph` +For instructions on how to work with the code, see the {ref}`contribution guide `. +::: -- If igraph still fails to install, see the question on [compiling igraph]. - Alternatively consider installing gcc via `brew install gcc --without-multilib` - and exporting the required variables: +:::: - ```shell - export CC="/usr/local/Cellar/gcc/X.x.x/bin/gcc-X" - export CXX="/usr/local/Cellar/gcc/X.x.x/bin/gcc-X" - ``` +[on github]: https://github.com/scverse/scanpy - where `X` and `x` refers to the version of `gcc`; - in my case, the path reads `/usr/local/Cellar/gcc/6.3.0_1/bin/gcc-6`. +## Docker -**On Windows**, there also often problems installing compiled packages such as `igraph`, -but you can find precompiled packages on Christoph Gohlke’s [unofficial binaries]. -Download those and install them using `pip install ./path/to/file.whl` +If you're using [Docker][], you can use e.g. the image [gcfntnu/scanpy][] from Docker Hub. -(conda)= +[docker]: https://en.wikipedia.org/wiki/Docker_(software) +[gcfntnu/scanpy]: https://hub.docker.com/r/gcfntnu/scanpy -## Installing Miniconda +## Troubleshooting -After downloading [Miniconda], in a unix shell (Linux, Mac), run +If you get a `Permission denied` error, never use `sudo pip`. Instead, use virtual environments or: -```shell -cd DOWNLOAD_DIR -chmod +x Miniconda3-latest-VERSION.sh -./Miniconda3-latest-VERSION.sh +```console +$ pip install --user scanpy ``` - -and accept all suggestions. -Either reopen a new terminal or `source ~/.bashrc` on Linux/ `source ~/.bash_profile` on Mac. -The whole process takes just a couple of minutes. - -[bioconda]: https://bioconda.github.io/ -[compiling igraph]: https://stackoverflow.com/q/29589696/247482 -[create symbolic links]: https://docs.microsoft.com/en-us/windows/security/threat-protection/security-policy-settings/create-symbolic-links -[docker]: https://en.wikipedia.org/wiki/Docker_(software) -[from pypi]: https://pypi.org/project/scanpy -[gcfntnu/scanpy]: https://hub.docker.com/r/gcfntnu/scanpy -[leiden]: https://leidenalg.readthedocs.io -[miniconda]: https://docs.conda.io/projects/miniconda/en/latest/ -[on github]: https://github.com/scverse/scanpy -[igraph]: https://python.igraph.org/en/stable/ -[unofficial binaries]: https://www.lfd.uci.edu/~gohlke/pythonlibs/ diff --git a/docs/news.md b/docs/news.md index 5ff7f0b6d5..ac3bcd2856 100644 --- a/docs/news.md +++ b/docs/news.md @@ -1,12 +1,6 @@ (News)= - ## News -```{eval-rst} -.. role:: small - -``` - ### `rapids-singlecell` brings scanpy to the GPU! {small}`2024-03-18` diff --git a/docs/release-notes/0.1.0.md b/docs/release-notes/0.1.0.md index 181907de1a..c2b568ac69 100644 --- a/docs/release-notes/0.1.0.md +++ b/docs/release-notes/0.1.0.md @@ -1,3 +1,4 @@ +(v0.1.0)= ### 0.1.0 {small}`2017-05-17` Scanpy computationally outperforms and allows reproducing both the [Cell Ranger diff --git a/docs/release-notes/0.2.1.md b/docs/release-notes/0.2.1.md index 394a561398..c49fe1f264 100644 --- a/docs/release-notes/0.2.1.md +++ b/docs/release-notes/0.2.1.md @@ -1,3 +1,4 @@ +(v0.2.1)= ### 0.2.1 {small}`2017-07-24` Scanpy includes preprocessing, visualization, clustering, pseudotime and diff --git a/docs/release-notes/0.2.9.md b/docs/release-notes/0.2.9.md index 3d288c6a23..3ac14e9506 100644 --- a/docs/release-notes/0.2.9.md +++ b/docs/release-notes/0.2.9.md @@ -1,7 +1,7 @@ +(v0.2.9)= ### 0.2.9 {small}`2017-10-25` -```{rubric} Initial release of the new trajectory inference method [PAGA](https://github.com/theislab/paga) -``` +#### Initial release of the new trajectory inference method [PAGA](https://github.com/theislab/paga) - {func}`~scanpy.tl.paga` computes an abstracted, coarse-grained (PAGA) graph of the neighborhood graph {smaller}`A Wolf` - {func}`~scanpy.pl.paga_compare` plot this graph next an embedding {smaller}`A Wolf` diff --git a/docs/release-notes/0.3.0.md b/docs/release-notes/0.3.0.md index 9736c1bab0..7b3f48cc44 100644 --- a/docs/release-notes/0.3.0.md +++ b/docs/release-notes/0.3.0.md @@ -1,3 +1,4 @@ +(v0.3.0)= ### 0.3.0 {small}`2017-11-16` - {class}`~anndata.AnnData` gains method {meth}`~anndata.AnnData.concatenate` {smaller}`A Wolf` diff --git a/docs/release-notes/0.3.2.md b/docs/release-notes/0.3.2.md index bcdd791e22..deaa2c2d6f 100644 --- a/docs/release-notes/0.3.2.md +++ b/docs/release-notes/0.3.2.md @@ -1,3 +1,4 @@ +(v0.3.2)= ### 0.3.2 {small}`2017-11-29` - finding marker genes via {func}`~scanpy.pl.rank_genes_groups_violin` improved, diff --git a/docs/release-notes/0.4.0.md b/docs/release-notes/0.4.0.md index 4b5d789da8..870db9457e 100644 --- a/docs/release-notes/0.4.0.md +++ b/docs/release-notes/0.4.0.md @@ -1,3 +1,4 @@ +(v0.4.0)= ### 0.4.0 {small}`2017-12-23` - export to [SPRING] {cite:p}`Weinreb2017` for interactive visualization of data: diff --git a/docs/release-notes/0.4.2.md b/docs/release-notes/0.4.2.md index 2271f01bfb..7b832e6c9b 100644 --- a/docs/release-notes/0.4.2.md +++ b/docs/release-notes/0.4.2.md @@ -1,3 +1,4 @@ +(v0.4.2)= ### 0.4.2 {small}`2018-01-07` - amendments in [PAGA](https://github.com/theislab/paga) and its plotting functions {smaller}`A Wolf` diff --git a/docs/release-notes/0.4.3.md b/docs/release-notes/0.4.3.md index 2a5aedad78..1bff0701d1 100644 --- a/docs/release-notes/0.4.3.md +++ b/docs/release-notes/0.4.3.md @@ -1,3 +1,4 @@ +(v0.4.3)= ### 0.4.3 {small}`2018-02-09` - {func}`~scanpy.pl.clustermap`: heatmap from hierarchical clustering, diff --git a/docs/release-notes/0.4.4.md b/docs/release-notes/0.4.4.md index e69de29bb2..4ebe8392f6 100644 --- a/docs/release-notes/0.4.4.md +++ b/docs/release-notes/0.4.4.md @@ -0,0 +1,6 @@ +(v0.4.4)= +### 0.4.4 {small}`2018-02-26` + +- embed cells using {func}`~scanpy.tl.umap` {cite:p}`McInnes2018` {pr}`92` {smaller}`G Eraslan` +- score sets of genes, e.g. for cell cycle, using {func}`~scanpy.tl.score_genes` {cite:p}`Satija2015`: + [notebook](https://nbviewer.jupyter.org/github/theislab/scanpy_usage/blob/master/180209_cell_cycle/cell_cycle.ipynb) diff --git a/docs/release-notes/1.0.0.md b/docs/release-notes/1.0.0.md index 74ac0ea561..00fa8b43db 100644 --- a/docs/release-notes/1.0.0.md +++ b/docs/release-notes/1.0.0.md @@ -1,7 +1,7 @@ +(v1.0.0)= ### 1.0.0 {small}`2018-03-30` -```{rubric} Major updates -``` +#### Major updates - Scanpy is much faster and more memory efficient: preprocess, cluster and visualize 1.3M cells in [6h], 130K cells in [14min], and 68K cells in [3min] {smaller}`A Wolf` @@ -10,8 +10,7 @@ delegated {smaller}`A Wolf` ```{warning} -```{rubric} Upgrading to 1.0 isn’t fully backwards compatible in the following changes -``` +#### Upgrading to 1.0 isn’t fully backwards compatible in the following changes - the graph-based tools {func}`~scanpy.tl.louvain` {func}`~scanpy.tl.dpt` {func}`~scanpy.tl.draw_graph` @@ -35,8 +34,7 @@ some results might therefore look slightly different ``` -```{rubric} Further updates -``` +#### Further updates - UMAP {cite:p}`McInnes2018` can serve as a first visualization of the data just as tSNE, in contrast to tSNE, UMAP directly embeds the single-cell graph and is faster; diff --git a/docs/release-notes/1.1.0.md b/docs/release-notes/1.1.0.md index 85f22988c3..1d7cfb8a71 100644 --- a/docs/release-notes/1.1.0.md +++ b/docs/release-notes/1.1.0.md @@ -1,3 +1,4 @@ +(v1.1.0)= ### 1.1.0 {small}`2018-06-01` - {func}`~scanpy.set_figure_params` by default passes `vector_friendly=True` and allows you to produce reasonablly sized pdfs by rasterizing large scatter plots {smaller}`A Wolf` diff --git a/docs/release-notes/1.10.0.md b/docs/release-notes/1.10.0.md index a42d409f9f..2e23d24426 100644 --- a/docs/release-notes/1.10.0.md +++ b/docs/release-notes/1.10.0.md @@ -1,3 +1,4 @@ +(v1.10.0)= ### 1.10.0 {small}`2024-03-26` `scanpy` 1.10 brings a large amount of new features, performance improvements, and improved documentation. @@ -10,8 +11,7 @@ Some highlights: * Ability to `mask` observations or variables from a number of methods (see {doc}`/tutorials/plotting/advanced` for an example with plotting embeddings) * A new function {func}`~scanpy.get.aggregate` for computing aggregations of your data, very useful for pseudo bulking! -```{rubric} Features -``` +#### Features * {func}`~scanpy.pp.scrublet` and {func}`~scanpy.pp.scrublet_simulate_doublets` were moved from {mod}`scanpy.external.pp` to {mod}`scanpy.pp`. The `scrublet` implementation is now maintained as part of scanpy {pr}`2703` {smaller}`P Angerer` * {func}`scanpy.pp.pca`, {func}`scanpy.pp.scale`, {func}`scanpy.pl.embedding`, and {func}`scanpy.experimental.pp.normalize_pearson_residuals_pca` now support a `mask` parameter {pr}`2272` {smaller}`C Bright, T Marcella, & P Angerer` @@ -32,8 +32,7 @@ Some highlights: * {func}`scanpy.pp.scale` now clips `np.ndarray` also at `- max_value` for zero-centering {pr}`2913` {smaller}`S Dicks` * Support sparse chunks in dask {func}`~scanpy.pp.scale`, {func}`~scanpy.pp.normalize_total` and {func}`~scanpy.pp.highly_variable_genes` (`seurat` and `cell-ranger` tested) {pr}`2856` {smaller}`ilan-gold` -```{rubric} Docs -``` +#### Documentation * Doc style overhaul {pr}`2220` {smaller}`A Gayoso` * Re-add search-as-you-type, this time via `readthedocs-sphinx-search` {pr}`2805` {smaller}`P Angerer` @@ -44,8 +43,7 @@ Some highlights: * Overhauled {doc}`/tutorials/index` page, and added new {doc}`/how-to/index` section to docs {pr}`2901` {smaller}`I Virshup` * Added a new tutorial on working with dask ({doc}`/tutorials/experimental/dask`) {pr}`2901` {smaller}`I Gold` {smaller}`I Virshup` -```{rubric} Bug fixes -``` +#### Bug fixes * Updated {func}`~scanpy.read_visium` such that it can read spaceranger 2.0 files {smaller}`L Lehner` * Fix {func}`~scanpy.pp.normalize_total` for dask {pr}`2466` {smaller}`P Angerer` @@ -61,14 +59,12 @@ Some highlights: * Fix pytest deprecation warning {pr}`2879` {smaller}`P Angerer` -```{rubric} Development -``` +#### Development Process * Scanpy is now tested against python 3.12 {pr}`2863` {smaller}`ivirshup` * Fix testing package build {pr}`2468` {smaller}`P Angerer` -```{rubric} Deprecations -``` +#### Deprecations * Dropped support for Python 3.8. [More details here](https://numpy.org/neps/nep-0029-deprecation_policy.html). {pr}`2695` {smaller}`P Angerer` * Deprecated specifying large numbers of function parameters by position as opposed to by name/keyword in all public APIs. diff --git a/docs/release-notes/1.10.1.md b/docs/release-notes/1.10.1.md index 6e9414c9bf..859789af5b 100644 --- a/docs/release-notes/1.10.1.md +++ b/docs/release-notes/1.10.1.md @@ -1,16 +1,14 @@ +(v1.10.1)= ### 1.10.1 {small}`2024-04-09` -```{rubric} Docs -``` +#### Documentation * Added {doc}`how-to example ` on plotting with [Marsilea](https://marsilea.readthedocs.io) {pr}`2974` {smaller}`Y Zheng` -```{rubric} Bug fixes -``` +#### Bug fixes * Fix `aggregate` when aggregating by more than two groups {pr}`2965` {smaller}`I Virshup` -```{rubric} Performance -``` +#### Performance * {func}`~scanpy.pp.scale` now uses numba kernels for `sparse.csr_matrix` and `sparse.csc_matrix` when `zero_center==False` and `mask_obs` is provided. This greatly speed up execution {pr}`2942` {smaller}`S Dicks` diff --git a/docs/release-notes/1.10.2.md b/docs/release-notes/1.10.2.md index 586cde1d55..947da0be29 100644 --- a/docs/release-notes/1.10.2.md +++ b/docs/release-notes/1.10.2.md @@ -1,12 +1,11 @@ +(v1.10.2)= ### 1.10.2 {small}`2024-06-25` -```{rubric} Development features -``` +#### Development Process * Add performance benchmarking {pr}`2977` {smaller}`R Shrestha`, {smaller}`P Angerer` -```{rubric} Docs -``` +#### Documentation * Document several missing parameters in docstring {pr}`2888` {smaller}`S Cheney` * Fixed incorrect instructions in "testing" dev docs {pr}`2994` {smaller}`I Virshup` @@ -14,8 +13,7 @@ * Fixed citations {pr}`3032` {smaller}`P Angerer` * Improve dataset documentation {pr}`3060` {smaller}`P Angerer` -```{rubric} Bug fixes -``` +#### Bug fixes * Compatibility with `matplotlib` 3.9 {pr}`2999` {smaller}`I Virshup` * Add clear errors where `backed` mode-like matrices (i.e., from `sparse_dataset`) are not supported {pr}`3048` {smaller}`I gold` @@ -24,8 +22,7 @@ * Fix zappy support {pr}`3089` {smaller}`P Angerer` * Fix dotplot group order with {mod}`pandas` 1.x {pr}`3101` {smaller}`P Angerer` -```{rubric} Performance -``` +#### Performance * `sparse_mean_variance_axis` now uses all cores for the calculations {pr}`3015` {smaller}`S Dicks` * `pp.highly_variable_genes` with `flavor=seurat_v3` now uses a numba kernel {pr}`3017` {smaller}`S Dicks` diff --git a/docs/release-notes/1.10.3.md b/docs/release-notes/1.10.3.md index a0a96b5d3d..f2f06ca94b 100644 --- a/docs/release-notes/1.10.3.md +++ b/docs/release-notes/1.10.3.md @@ -1,16 +1,16 @@ -### 1.10.3 {small}`the future` +(v1.10.3)= +### 1.10.3 {small}`2024-09-17` -```{rubric} Development features -``` +#### Bug fixes -```{rubric} Docs -``` +- Prevent empty control gene set in {func}`~scanpy.tl.score_genes` {smaller}`M Müller` ({pr}`2875`) +- Fix `subset=True` of {func}`~scanpy.pp.highly_variable_genes` when `flavor` is `seurat` or `cell_ranger`, and `batch_key!=None` {smaller}`E Roellin` ({pr}`3042`) +- Add compatibility with {mod}`numpy` 2.0 {smaller}`P Angerer` {pr}`3065` and ({pr}`3115`) +- Fix `legend_loc` argument in {func}`scanpy.pl.embedding` not accepting matplotlib parameters {smaller}`P Angerer` ({pr}`3163`) +- Fix dispersion cutoff in {func}`~scanpy.pp.highly_variable_genes` in presence of `NaN`s {smaller}`P Angerer` ({pr}`3176`) +- Fix axis labeling for swapped axes in {func}`~scanpy.pl.rank_genes_groups_stacked_violin` {smaller}`Ilan Gold` ({pr}`3196`) +- Upper bound dask on account of {issue}`scverse/anndata#1579` {smaller}`Ilan Gold` ({pr}`3217`) +- The [fa2-modified][] package replaces [forceatlas2][] for the latter’s lack of maintenance {smaller}`A Alam` ({pr}`3220`) -```{rubric} Bug fixes -``` - -* Fix `subset=True` of {func}`~scanpy.pp.highly_variable_genes` when `flavor` is `seurat` or `cell_ranger`, and `batch_key!=None` {pr}`3042` {smaller}`E Roellin` - - -```{rubric} Performance -``` + [fa2-modified]: https://github.com/AminAlam/fa2_modified + [forceatlas2]: https://github.com/bhargavchippada/forceatlas2 diff --git a/docs/release-notes/1.10.4.md b/docs/release-notes/1.10.4.md new file mode 100644 index 0000000000..d4ba850a46 --- /dev/null +++ b/docs/release-notes/1.10.4.md @@ -0,0 +1,17 @@ +(v1.10.4)= +### 1.10.4 {small}`2024-11-12` + +### Breaking changes + +- Remove Python 3.9 support {smaller}`P Angerer` ({pr}`3283`) + +### Bug fixes + +- Fix {meth}`scanpy.pl.DotPlot.style`, {meth}`scanpy.pl.MatrixPlot.style`, and {meth}`scanpy.pl.StackedViolin.style` resetting all non-specified parameters {smaller}`P Angerer` ({pr}`3206`) +- Accept `'group'` instead of `'obs'` for `standard_scale` parameter in {func}`~scanpy.pl.stacked_violin` {smaller}`P Angerer` ({pr}`3243`) +- Use `density_norm` instead of of `scale` (cont. from {pr}`2844`) in {func}`~scanpy.pl.violin` and {func}`~scanpy.pl.stacked_violin` {smaller}`P Angerer` ({pr}`3244`) +- Switched all compatibility adapters for positional parameters to {exc}`FutureWarning` {smaller}`P Angerer` ({pr}`3264`) +- Catch `PerfectSeparationWarning` during {func}`~scanpy.pp.regress_out` {smaller}`J Wagner` ({pr}`3275`) +- Fix {func}`scanpy.pp.highly_variable_genes` for batches of size 1 {smaller}`P Angerer` ({pr}`3286`) +- Fix {func}`scanpy.pl.scatter`’s `color` parameter to take collections as advertised {smaller}`P Angerer` ({pr}`3299`) +- Fix {func}`scanpy.pl.highest_expr_genes` when used with a categorical gene symbol column {smaller}`P Angerer` ({pr}`3302`) diff --git a/docs/release-notes/1.11.0.md b/docs/release-notes/1.11.0.md index 8189632632..a41103e0ec 100644 --- a/docs/release-notes/1.11.0.md +++ b/docs/release-notes/1.11.0.md @@ -1,15 +1,38 @@ -### 1.11.0 {small}`the future` +(v1.11.0)= +### 1.11.0rc1 {small}`2024-12-20` -```{rubric} Features -``` +### Features -* Add layer argument to {func}`scanpy.tl.score_genes` and {func}`scanpy.tl.score_genes_cell_cycle` {pr}`2921` {smaller}`L Zappia` +- {func}`~scanpy.pp.sample` supports both upsampling and downsampling of observations and variables. {func}`~scanpy.pp.subsample` is now deprecated. {smaller}`G Eraslan & P Angerer` ({pr}`943`) +- Add `layer` argument to {func}`scanpy.tl.score_genes` and {func}`scanpy.tl.score_genes_cell_cycle` {smaller}`L Zappia` ({pr}`2921`) +- Prevent `raw` conflict with `layer` in {func}`~scanpy.tl.score_genes` {smaller}`S Dicks` ({pr}`3155`) +- Add support for `median` as an aggregation function to {func}`~scanpy.get.aggregate`. This allows for median-based aggregation of data (e.g., pseudobulk), complementing existing methods like mean- and sum-based aggregation {smaller}`M Dehkordi (Farhad)` ({pr}`3180`) +- Add `key_added` argument to {func}`~scanpy.pp.pca`, {func}`~scanpy.tl.tsne` and {func}`~scanpy.tl.umap` {smaller}`P Angerer` ({pr}`3184`) +- Support running {func}`scanpy.pp.pca` on sparse Dask arrays with the `'covariance_eigh'` solver {smaller}`P Angerer` ({pr}`3263`) +- Use upstreamed {class}`~sklearn.decomposition.PCA` implementation for {class}`~scipy.sparse.csr_array` and {class}`~scipy.sparse.csr_matrix` (see scikit-learn {ref}`sklearn:changes_1_4`) {smaller}`P Angerer` ({pr}`3267`) +- Add explicit support to {func}`scanpy.pp.pca` for `svd_solver='covariance_eigh'` {smaller}`P Angerer` ({pr}`3296`) +- Add support for {class}`dask.array.Array` to {func}`scanpy.pp.calculate_qc_metrics` {smaller}`I Gold` ({pr}`3307`) +- Support `layer` parameter in {func}`scanpy.pl.highest_expr_genes` {smaller}`P Angerer` ({pr}`3324`) +- Run numba functions single-threaded when called from inside of a {class}`~multiprocessing.pool.ThreadPool` {smaller}`P Angerer` ({pr}`3335`) +- Switch {func}`~scanpy.logging.print_header` and {func}`~scanpy.logging.print_versions` to {mod}`session_info2` {smaller}`P Angerer` ({pr}`3384`) +- Add sampling probabilities/mask parameter `p` to {func}`~scanpy.pp.sample` {smaller}`P Angerer` ({pr}`3410`) -```{rubric} Docs -``` +### Performance -```{rubric} Bug fixes -``` +- Speed up {func}`~scanpy.pp.regress_out` {smaller}`P Ashish, P Angerer & S Dicks` ({pr}`3284`) -```{rubric} Deprecations -``` +### Documentation + +- Improve {func}`~scanpy.external.pp.harmony_integrate` docs {smaller}`D Kühl` ({pr}`3362`) +- Raise {exc}`FutureWarning` when calling deprecated {mod}`scanpy.pp` functions {smaller}`P Angerer` ({pr}`3380`) +- | Deprecate … | in favor of … | + | --- | --- | + | {func}`scanpy.read_visium` | {func}`squidpy.read.visium` | + | {func}`scanpy.datasets.visium_sge` | {func}`squidpy.datasets.visium` | + | {func}`scanpy.pl.spatial` | {func}`squidpy.pl.spatial_scatter` | + + {smaller}`P Angerer` ({pr}`3407`) + +### Bug fixes + +- Upper-bound {mod}`sklearn` `<1.6.0` due to {issue}`dask/dask-ml#1002` {smaller}`Ilan Gold` ({pr}`3393`) diff --git a/docs/release-notes/1.2.0.md b/docs/release-notes/1.2.0.md index 5bd2ed451e..973b59775c 100644 --- a/docs/release-notes/1.2.0.md +++ b/docs/release-notes/1.2.0.md @@ -1,3 +1,4 @@ +(v1.2.0)= ### 1.2.0 {small}`2018-06-08` - {func}`~scanpy.tl.paga` improved, see [PAGA](https://github.com/theislab/paga); the default model changed, restore the previous default model by passing `model='v1.0'` diff --git a/docs/release-notes/1.2.1.md b/docs/release-notes/1.2.1.md index 3110537aa8..5979029098 100644 --- a/docs/release-notes/1.2.1.md +++ b/docs/release-notes/1.2.1.md @@ -1,6 +1,6 @@ +(v1.2.1)= ### 1.2.1 {small}`2018-06-08` -~~~{rubric} Plotting of {ref}`pl-generic` marker genes and quality control. -~~~ +#### Plotting of {ref}`pl-generic` marker genes and quality control. - {func}`~scanpy.pl.highest_expr_genes` for quality control; plot genes with highest mean fraction of cells, similar to `plotQC` of *Scater* {cite:p}`McCarthy2017` {pr}`169` {smaller}`F Ramirez` diff --git a/docs/release-notes/1.3.1.md b/docs/release-notes/1.3.1.md index 8492d6b9e6..829e56ec26 100644 --- a/docs/release-notes/1.3.1.md +++ b/docs/release-notes/1.3.1.md @@ -1,20 +1,18 @@ +(v1.3.1)= ### 1.3.1 {small}`2018-09-03` -```{rubric} RNA velocity in single cells {cite:p}`LaManno2018` -``` +#### RNA velocity in single cells {cite:p}`LaManno2018` - Scanpy and AnnData support loom’s layers so that computations for single-cell RNA velocity {cite:p}`LaManno2018` become feasible {smaller}`S Rybakov and V Bergen` - [scvelo] harmonizes with Scanpy and is able to process loom files with splicing information produced by Velocyto {cite:p}`LaManno2018`, it runs a lot faster than the count matrix analysis of Velocyto and provides several conceptual developments -~~~{rubric} Plotting ({ref}`pl-generic`) -~~~ +#### Plotting ({ref}`pl-generic`) - {func}`~scanpy.pl.dotplot` for visualizing genes across conditions and clusters, see [here](https://gist.github.com/fidelram/2289b7a8d6da055fb058ac9a79ed485c) {pr}`199` {smaller}`F Ramirez` - {func}`~scanpy.pl.heatmap` for pretty heatmaps {pr}`175` {smaller}`F Ramirez` - {func}`~scanpy.pl.violin` produces very compact overview figures with many panels {pr}`175` {smaller}`F Ramirez` -~~~{rubric} There now is a section on imputation in {doc}`external <../external/index>`: -~~~ +#### There now is a section on imputation in {doc}`external <../external/index>`: - {func}`~scanpy.external.pp.magic` for imputation using data diffusion {cite:p}`vanDijk2018` {pr}`187` {smaller}`S Gigante` - {func}`~scanpy.external.pp.dca` for imputation and latent space construction using an autoencoder {cite:p}`Eraslan2019` {pr}`186` {smaller}`G Eraslan` diff --git a/docs/release-notes/1.3.3.md b/docs/release-notes/1.3.3.md index f94e3c5dec..fd898ec4eb 100644 --- a/docs/release-notes/1.3.3.md +++ b/docs/release-notes/1.3.3.md @@ -1,18 +1,16 @@ +(v1.3.3)= ### 1.3.3 {small}`2018-11-05` -```{rubric} Major updates -``` +#### Major updates - a fully distributed preprocessing backend {smaller}`T White and the Laserson Lab` -```{rubric} Code design -``` +#### Code design - {func}`~scanpy.read_10x_h5` and {func}`~scanpy.read_10x_mtx` read Cell Ranger 3.0 outputs {pr}`334` {smaller}`Q Gong` ```{note} -```{rubric} Also see changes in anndata 0.6. -``` +#### Also see changes in anndata 0.6. - changed default compression to `None` in {meth}`~anndata.AnnData.write_h5ad` to speed up read and write, disk space use is usually less critical - performance gains in {meth}`~anndata.AnnData.write_h5ad` due to better handling of strings and categories {smaller}`S Rybakov` diff --git a/docs/release-notes/1.3.4.md b/docs/release-notes/1.3.4.md index ab4a9b599b..9381bc4647 100644 --- a/docs/release-notes/1.3.4.md +++ b/docs/release-notes/1.3.4.md @@ -1,3 +1,4 @@ +(v1.3.4)= ### 1.3.4 {small}`2018-11-24` - {func}`~scanpy.tl.leiden` wraps the recent graph clustering package by {cite:t}`Traag2019` {smaller}`K Polanski` diff --git a/docs/release-notes/1.3.5.md b/docs/release-notes/1.3.5.md index 198b5e9994..20706e9fca 100644 --- a/docs/release-notes/1.3.5.md +++ b/docs/release-notes/1.3.5.md @@ -1,3 +1,4 @@ +(v1.3.5)= ### 1.3.5 {small}`2018-12-09` - uncountable figure improvements {pr}`369` {smaller}`F Ramirez` diff --git a/docs/release-notes/1.3.6.md b/docs/release-notes/1.3.6.md index 75d5cbbbcc..a8bf3a94a1 100644 --- a/docs/release-notes/1.3.6.md +++ b/docs/release-notes/1.3.6.md @@ -1,19 +1,17 @@ +(v1.3.6)= ### 1.3.6 {small}`2018-12-11` -```{rubric} Major updates -``` +#### Major updates - a new plotting gallery for `visualizing-marker-genes` {smaller}`F Ramirez` - tutorials are integrated on ReadTheDocs, `pbmc3k` and `paga-paul15` {smaller}`A Wolf` -```{rubric} Interactive exploration of analysis results through *manifold viewers* -``` +#### Interactive exploration of analysis results through *manifold viewers* - CZI’s [cellxgene] directly reads `.h5ad` files {smaller}`the cellxgene developers` - the [UCSC Single Cell Browser] requires exporting via {func}`~scanpy.external.exporting.cellbrowser` {smaller}`M Haeussler` -```{rubric} Code design -``` +#### Code design - {func}`~scanpy.pp.highly_variable_genes` supersedes {func}`~scanpy.pp.filter_genes_dispersion`, it gives the same results but, by default, expects logarithmized data and doesn’t subset {smaller}`A Wolf` diff --git a/docs/release-notes/1.3.7.md b/docs/release-notes/1.3.7.md index e69de29bb2..19675fe0e3 100644 --- a/docs/release-notes/1.3.7.md +++ b/docs/release-notes/1.3.7.md @@ -0,0 +1,5 @@ +(v1.3.7)= +### 1.3.7 {small}`2019-01-02` + +- API changed from `import scanpy as sc` to `import scanpy.api as sc`. +- {func}`~scanpy.external.tl.phenograph` wraps the graph clustering package Phenograph {cite:p}`Levine2015` {smaller}`thanks to A Mousa` diff --git a/docs/release-notes/1.3.8.md b/docs/release-notes/1.3.8.md index e69de29bb2..f1f6e01282 100644 --- a/docs/release-notes/1.3.8.md +++ b/docs/release-notes/1.3.8.md @@ -0,0 +1,5 @@ +(v1.3.8)= +### 1.3.8 {small}`2019-02-05` + +- various documentation and dev process improvements +- Added {func}`~scanpy.pp.combat` function for batch effect correction {cite:p}`Johnson2006,Leek2012,Pedersen2012` {pr}`398` {smaller}`M Lange` diff --git a/docs/release-notes/1.4.1.md b/docs/release-notes/1.4.1.md index 18d689a190..ab503085a4 100644 --- a/docs/release-notes/1.4.1.md +++ b/docs/release-notes/1.4.1.md @@ -1,7 +1,7 @@ +(v1.4.1)= ### 1.4.1 {small}`2019-04-26` -```{rubric} New functionality -``` +#### New functionality - Scanpy has a command line interface again. Invoking it with `scanpy somecommand [args]` calls `scanpy-somecommand [args]`, except for builtin commands (currently `scanpy settings`) {pr}`604` {smaller}`P Angerer` - {func}`~scanpy.datasets.ebi_expression_atlas` allows convenient download of EBI expression atlas {smaller}`I Virshup` @@ -12,8 +12,7 @@ - {func}`~scanpy.tl.embedding_density` computes densities on embeddings {pr}`543` {smaller}`M Luecken` - {func}`~scanpy.external.tl.palantir` interfaces Palantir {cite:p}`Setty2019` {pr}`493` {smaller}`A Mousa` -```{rubric} Code design -``` +#### Code design - `.layers` support of scatter plots {smaller}`F Ramirez` - fix double-logarithmization in compute of log fold change in {func}`~scanpy.tl.rank_genes_groups` {smaller}`A Muñoz-Rojas` diff --git a/docs/release-notes/1.4.2.md b/docs/release-notes/1.4.2.md index f08e21de90..fa3ebf345c 100644 --- a/docs/release-notes/1.4.2.md +++ b/docs/release-notes/1.4.2.md @@ -1,19 +1,17 @@ +(v1.4.2)= ### 1.4.2 {small}`2019-05-06` -```{rubric} New functionality -``` +#### New functionality - {func}`~scanpy.pp.combat` supports additional covariates which may include adjustment variables or biological condition {pr}`618` {smaller}`G Eraslan` - {func}`~scanpy.pp.highly_variable_genes` has a `batch_key` option which performs HVG selection in each batch separately to avoid selecting genes that vary strongly across batches {pr}`622` {smaller}`G Eraslan` -```{rubric} Bug fixes -``` +#### Bug fixes - {func}`~scanpy.tl.rank_genes_groups` t-test implementation doesn't return NaN when variance is 0, also changed to scipy's implementation {pr}`621` {smaller}`I Virshup` - {func}`~scanpy.tl.umap` with `init_pos='paga'` detects correct `dtype` {smaller}`A Wolf` - {func}`~scanpy.tl.louvain` and {func}`~scanpy.tl.leiden` auto-generate `key_added=louvain_R` upon passing `restrict_to`, which was temporarily changed in `1.4.1` {smaller}`A Wolf` -```{rubric} Code design -``` +#### Code design - {func}`~scanpy.pp.neighbors` and {func}`~scanpy.tl.umap` got rid of UMAP legacy code and introduced UMAP as a dependency {pr}`576` {smaller}`S Rybakov` diff --git a/docs/release-notes/1.4.3.md b/docs/release-notes/1.4.3.md index d034230132..af60127ded 100644 --- a/docs/release-notes/1.4.3.md +++ b/docs/release-notes/1.4.3.md @@ -1,11 +1,10 @@ +(v1.4.3)= ### 1.4.3 {small}`2019-05-14` -```{rubric} Bug fixes -``` +#### Bug fixes - {func}`~scanpy.pp.neighbors` correctly infers `n_neighbors` again from `params`, which was temporarily broken in `v1.4.2` {smaller}`I Virshup` -```{rubric} Code design -``` +#### Code design - {func}`~scanpy.pp.calculate_qc_metrics` is single threaded by default for datasets under 300,000 cells -- allowing cached compilation {pr}`615` {smaller}`I Virshup` diff --git a/docs/release-notes/1.4.4.md b/docs/release-notes/1.4.4.md index b82bdaa362..b500c11b6d 100644 --- a/docs/release-notes/1.4.4.md +++ b/docs/release-notes/1.4.4.md @@ -1,16 +1,14 @@ +(v1.4.4)= ### 1.4.4 {small}`2019-07-20` -```{rubric} New functionality -``` +#### New functionality - {mod}`scanpy.get` adds helper functions for extracting data in convenient formats {pr}`619` {smaller}`I Virshup` -```{rubric} Bug fixes -``` +#### Bug fixes - Stopped deprecations warnings from AnnData `0.6.22` {smaller}`I Virshup` -```{rubric} Code design -``` +#### Code design - {func}`~scanpy.pp.normalize_total` gains param `exclude_highly_expressed`, and `fraction` is renamed to `max_fraction` with better docs {smaller}`A Wolf` diff --git a/docs/release-notes/1.4.5.md b/docs/release-notes/1.4.5.md index c56c1270da..44a475ad5d 100644 --- a/docs/release-notes/1.4.5.md +++ b/docs/release-notes/1.4.5.md @@ -1,16 +1,15 @@ +(v1.4.5)= ### 1.4.5 {small}`2019-12-30` Please install `scanpy==1.4.5.post3` instead of `scanpy==1.4.5`. -```{rubric} New functionality -``` +#### New functionality - {func}`~scanpy.tl.ingest` maps labels and embeddings of reference data to new data {doc}`/tutorials/basics/integrating-data-using-ingest` {pr}`651` {smaller}`S Rybakov, A Wolf` - {mod}`~scanpy.queries` recieved many updates including enrichment through [gprofiler] and more advanced biomart queries {pr}`467` {smaller}`I Virshup` - {func}`~scanpy.set_figure_params` allows setting `figsize` and accepts `facecolor='white'`, useful for working in dark mode {smaller}`A Wolf` -```{rubric} Code design -``` +#### Code design - {mod}`~scanpy.pp.downsample_counts` now always preserves the dtype of it's input, instead of converting floats to ints {pr}`865` {smaller}`I Virshup` - allow specifying a base for {func}`~scanpy.pp.log1p` {pr}`931` {smaller}`G Eraslan` diff --git a/docs/release-notes/1.4.6.md b/docs/release-notes/1.4.6.md index 51be1d4569..2ea16c184c 100644 --- a/docs/release-notes/1.4.6.md +++ b/docs/release-notes/1.4.6.md @@ -1,19 +1,17 @@ +(v1.4.6)= ### 1.4.6 {small}`2020-03-17` -~~~{rubric} Functionality in `external` -~~~ +#### Functionality in `external` - {func}`~scanpy.external.tl.sam` self-assembling manifolds {cite:p}`Tarashansky2019` {pr}`903` {smaller}`A Tarashansky` - {func}`~scanpy.external.tl.harmony_timeseries` for trajectory inference on discrete time points {pr}`994` {smaller}`A Mousa` - {func}`~scanpy.external.tl.wishbone` for trajectory inference (bifurcations) {pr}`1063` {smaller}`A Mousa` -```{rubric} Code design -``` +#### Code design - {mod}`~scanpy.pl.violin` now reads `.uns['colors_...']` {pr}`1029` {smaller}`michalk8` -```{rubric} Bug fixes -``` +#### Bug fixes - adapt {func}`~scanpy.tl.ingest` for UMAP 0.4 {pr}`1038` {pr}`1106` {smaller}`S Rybakov` - compat with matplotlib 3.1 and 3.2 {pr}`1090` {smaller}`I Virshup, P Angerer` diff --git a/docs/release-notes/1.5.0.md b/docs/release-notes/1.5.0.md index b1fff45e03..956ceb9493 100644 --- a/docs/release-notes/1.5.0.md +++ b/docs/release-notes/1.5.0.md @@ -1,29 +1,26 @@ +(v1.5.0)= ### 1.5.0 {small}`2020-05-15` The `1.5.0` release adds a lot of new functionality, much of which takes advantage of {mod}`anndata` updates `0.7.0 - 0.7.2`. Highlights of this release include support for spatial data, dedicated handling of graphs in AnnData, sparse PCA, an interface with scvi, and others. -```{rubric} Spatial data support -``` +#### Spatial data support -- Basic analysis {doc}`/tutorials/spatial/basic-analysis` and integration with single cell data {doc}`/tutorials/spatial/integration-scanorama` {smaller}`G Palla` +- Tutorials for basic analysis and integration with single cell data {smaller}`G Palla` - {func}`~scanpy.read_visium` read 10x Visium data {pr}`1034` {smaller}`G Palla, P Angerer, I Virshup` - {func}`~scanpy.datasets.visium_sge` load Visium data directly from 10x Genomics {pr}`1013` {smaller}`M Mirkazemi, G Palla, P Angerer` - {func}`~scanpy.pl.spatial` plot spatial data {pr}`1012` {smaller}`G Palla, P Angerer` -```{rubric} New functionality -``` +#### New functionality - Many functions, like {func}`~scanpy.pp.neighbors` and {func}`~scanpy.tl.umap`, now store cell-by-cell graphs in {attr}`~anndata.AnnData.obsp` {pr}`1118` {smaller}`S Rybakov` - {func}`~scanpy.pp.scale` and {func}`~scanpy.pp.log1p` can be used on any element in {attr}`~anndata.AnnData.layers` or {attr}`~anndata.AnnData.obsm` {pr}`1173` {smaller}`I Virshup` -```{rubric} External tools -``` +#### External tools - `scanpy.external.pp.scvi` for preprocessing with scVI {pr}`1085` {smaller}`G Xing` - Guide for using `Scanpy in R` {pr}`1186` {smaller}`L Zappia` -```{rubric} Performance -``` +#### Performance - {func}`~scanpy.pp.pca` now uses efficient implicit centering for sparse matrices. This can lead to signifigantly improved performance for large datasets {pr}`1066` {smaller}`A Tarashansky` - {func}`~scanpy.tl.score_genes` now has an efficient implementation for sparse matrices with missing values {pr}`1196` {smaller}`redst4r`. @@ -32,16 +29,14 @@ The `1.5.0` release adds a lot of new functionality, much of which takes advanta The new {func}`~scanpy.pp.pca` implementation can result in slightly different results for sparse matrices. See the pr ({pr}`1066`) and documentation for more info. ``` -```{rubric} Code design -``` +#### Code design - {func}`~scanpy.pl.stacked_violin` can now be used as a subplot {pr}`1084` {smaller}`P Angerer` - {func}`~scanpy.tl.score_genes` has improved logging {pr}`1119` {smaller}`G Eraslan` - {func}`~scanpy.pp.scale` now saves mean and standard deviation in the {attr}`~anndata.AnnData.var` {pr}`1173` {smaller}`A Wolf` - {func}`~scanpy.external.tl.harmony_timeseries` {pr}`1091` {smaller}`A Mousa` -```{rubric} Bug fixes -``` +#### Bug fixes - {func}`~scanpy.pp.combat` now works when `obs_names` aren't unique. {pr}`1215` {smaller}`I Virshup` - {func}`~scanpy.pp.scale` can now be used on dense arrays without centering {pr}`1160` {smaller}`simonwm` diff --git a/docs/release-notes/1.5.1.md b/docs/release-notes/1.5.1.md index 215fe3b3cc..f7c18bb73f 100644 --- a/docs/release-notes/1.5.1.md +++ b/docs/release-notes/1.5.1.md @@ -1,7 +1,7 @@ +(v1.5.1)= ### 1.5.1 {small}`2020-05-21` -```{rubric} Bug fixes -``` +#### Bug fixes - Fixed a bug in {func}`~scanpy.pp.pca`, where `random_state` did not have an effect for sparse input {pr}`1240` {smaller}`I Virshup` - Fixed docstring in {func}`~scanpy.pp.pca` which included an unused argument {pr}`1240` {smaller}`I Virshup` diff --git a/docs/release-notes/1.6.0.md b/docs/release-notes/1.6.0.md index fe40597a81..19b227fc05 100644 --- a/docs/release-notes/1.6.0.md +++ b/docs/release-notes/1.6.0.md @@ -1,9 +1,9 @@ +(v1.6.0)= ### 1.6.0 {small}`2020-08-15` This release includes an overhaul of {func}`~scanpy.pl.dotplot`, {func}`~scanpy.pl.matrixplot`, and {func}`~scanpy.pl.stacked_violin` ({pr}`1210` {smaller}`F Ramirez`), and of the internals of {func}`~scanpy.tl.rank_genes_groups` ({pr}`1156` {smaller}`S Rybakov`). -~~~{rubric} Overhaul of {func}`~scanpy.pl.dotplot`, {func}`~scanpy.pl.matrixplot`, and {func}`~scanpy.pl.stacked_violin` {pr}`1210` {smaller}`F Ramirez` -~~~ +#### Overhaul of {func}`~scanpy.pl.dotplot`, {func}`~scanpy.pl.matrixplot`, and {func}`~scanpy.pl.stacked_violin` {pr}`1210` {smaller}`F Ramirez` - An overhauled tutorial {doc}`/tutorials/plotting/core`. @@ -38,8 +38,7 @@ This release includes an overhaul of {func}`~scanpy.pl.dotplot`, {func}`~scanpy. > - The linewidth of the violin plots is thinner. > - Removed the tics for the y-axis as they tend to overlap with each other. Using the style method they can be displayed if needed. -```{rubric} Additions -``` +#### Additions - {func}`~anndata.concat` is now exported from scanpy, see {doc}`anndata:concatenation` for more info. {pr}`1338` {smaller}`I Virshup` - Added highly variable gene selection strategy from Seurat v3 {pr}`1204` {smaller}`A Gayoso` @@ -49,8 +48,7 @@ This release includes an overhaul of {func}`~scanpy.pl.dotplot`, {func}`~scanpy. - Optional tie correction for the `'wilcoxon'` method in {func}`~scanpy.tl.rank_genes_groups` {pr}`1330` {smaller}`S Rybakov` - Use `sinfo` for {func}`~scanpy.logging.print_versions` and add {func}`~scanpy.logging.print_header` to do what it previously did. {pr}`1338` {smaller}`I Virshup` {pr}`1373` -```{rubric} Bug fixes -``` +#### Bug fixes - Avoid warning in {func}`~scanpy.tl.rank_genes_groups` if 't-test' is passed {pr}`1303` {smaller}`A Wolf` - Restrict sphinx version to \<3.1, >3.0 {pr}`1297` {smaller}`I Virshup` diff --git a/docs/release-notes/1.7.0.md b/docs/release-notes/1.7.0.md index 7ea8c8c0c5..0c3f77f4ce 100644 --- a/docs/release-notes/1.7.0.md +++ b/docs/release-notes/1.7.0.md @@ -1,7 +1,7 @@ +(v1.7.0)= ### 1.7.0 {small}`2021-02-03` -```{rubric} Features -``` +#### Features - Add new 10x Visium datasets to {func}`~scanpy.datasets.visium_sge` {pr}`1473` {smaller}`G Palla` - Enable download of source image for 10x visium datasets in {func}`~scanpy.datasets.visium_sge` {pr}`1506` {smaller}`H Spitzer` @@ -14,8 +14,7 @@ - Added `na_color` and `na_in_legend` keyword arguments to {func}`~scanpy.pl.embedding` plots. Allows specifying color for missing or filtered values in plots like {func}`~scanpy.pl.umap` or {func}`~scanpy.pl.spatial` {pr}`1356` {smaller}`I Virshup` - {func}`~scanpy.pl.embedding` plots now support passing `dict` of `{cluster_name: cluster_color, ...}` for palette argument {pr}`1392` {smaller}`I Virshup` -```{rubric} External tools (new) -``` +#### External tools (new) - Add [Scanorama](https://github.com/brianhie/scanorama) integration to scanpy external API ({func}`~scanpy.external.pp.scanorama_integrate`, {cite:t}`Hie2019`) {pr}`1332` {smaller}`B Hie` - Scrublet {cite:p}`Wolock2019` integration: {func}`~scanpy.pp.scrublet`, {func}`~scanpy.pp.scrublet_simulate_doublets`, and plotting method {func}`~scanpy.pl.scrublet_score_distribution` {pr}`1476` {smaller}`J Manning` @@ -23,8 +22,7 @@ - Added [scirpy](https://github.com/icbi-lab/scirpy) (sc-AIRR analysis) to ecosystem page {pr}`1453` {smaller}`G Sturm` - Added [scvi-tools](https://scvi-tools.org) to ecosystem page {pr}`1421` {smaller}`A Gayoso` -```{rubric} External tools (changes) -``` +#### External tools (changes) - Updates for {func}`~scanpy.external.tl.palantir` and {func}`~scanpy.external.tl.palantir_results` {pr}`1245` {smaller}`A Mousa` - Fixes to {func}`~scanpy.external.tl.harmony_timeseries` docs {pr}`1248` {smaller}`A Mousa` @@ -32,20 +30,17 @@ - Deprecate `scanpy.external.pp.scvi` {pr}`1554` {smaller}`G Xing` - Updated default params of {func}`~scanpy.external.tl.sam` to work with larger data {pr}`1540` {smaller}`A Tarashansky` -```{rubric} Documentation -``` +#### Documentation - {ref}`New contribution guide ` {pr}`1544` {smaller}`I Virshup` - `zsh` installation instructions {pr}`1444` {smaller}`P Angerer` -```{rubric} Performance -``` +#### Performance - Speed up {func}`~scanpy.read_10x_h5` {pr}`1402` {smaller}`P Weiler` - Speed ups for {func}`~scanpy.get.obs_df` {pr}`1499` {smaller}`F Ramirez` -```{rubric} Bugfixes -``` +#### Bugfixes - Consistent fold-change, fractions calculation for filter_rank_genes_groups {pr}`1391` {smaller}`S Rybakov` - Fixed bug where `score_genes` would error if one gene was passed {pr}`1398` {smaller}`I Virshup` diff --git a/docs/release-notes/1.7.1.md b/docs/release-notes/1.7.1.md index 0e536ca848..c4d28b0455 100644 --- a/docs/release-notes/1.7.1.md +++ b/docs/release-notes/1.7.1.md @@ -1,12 +1,11 @@ +(v1.7.1)= ### 1.7.1 {small}`2021-02-24` -```{rubric} Documentation -``` +#### Documentation - More twitter handles for core devs {pr}`1676` {smaller}`G Eraslan` -```{rubric} Bug fixes -``` +#### Bug fixes - {func}`~scanpy.tl.dendrogram` use `1 - correlation` as distance matrix to compute the dendrogram {pr}`1614` {smaller}`F Ramirez` - Fixed {func}`~scanpy.get.obs_df`/ {func}`~scanpy.get.var_df` erroring when `keys` not passed {pr}`1637` {smaller}`I Virshup` diff --git a/docs/release-notes/1.7.2.md b/docs/release-notes/1.7.2.md index 90e90c5cc7..816819b3d2 100644 --- a/docs/release-notes/1.7.2.md +++ b/docs/release-notes/1.7.2.md @@ -1,15 +1,14 @@ +(v1.7.2)= ### 1.7.2 {small}`2021-04-07` -```{rubric} Bug fixes -``` +#### Bug fixes - {func}`scanpy.logging.print_versions` now works when `python<3.8` {pr}`1691` {smaller}`I Virshup` - {func}`scanpy.pp.regress_out` now uses `joblib` as the parallel backend, and should stop oversubscribing threads {pr}`1694` {smaller}`I Virshup` - {func}`scanpy.pp.highly_variable_genes` with `flavor="seurat_v3"` now returns correct gene means and -variances when used with `batch_key` {pr}`1732` {smaller}`J Lause` - {func}`scanpy.pp.highly_variable_genes` now throws a warning instead of an error when non-integer values are passed for method `"seurat_v3"`. The check can be skipped by passing `check_values=False`. {pr}`1679` {smaller}`G Palla` -```{rubric} Ecosystem -``` +#### Ecosystem - Added `triku` a feature selection method to the ecosystem page {pr}`1722` {smaller}`AM Ascensión` - Added `dorothea` and `progeny` to the ecosystem page {pr}`1767` {smaller}`P Badia-i-Mompel` diff --git a/docs/release-notes/1.8.0.md b/docs/release-notes/1.8.0.md index 2ec0a03496..8bf2c36895 100644 --- a/docs/release-notes/1.8.0.md +++ b/docs/release-notes/1.8.0.md @@ -1,46 +1,42 @@ +(v1.8.0)= ### 1.8.0 {small}`2021-06-28` -```{rubric} Metrics module -``` +#### Metrics module - Added {mod}`scanpy.metrics` module! - > - Added {func}`scanpy.metrics.gearys_c` for spatial autocorrelation {pr}`915` {smaller}`I Virshup` - > - Added {func}`scanpy.metrics.morans_i` for global spatial autocorrelation {pr}`1740` {smaller}`I Virshup, G Palla` - > - Added {func}`scanpy.metrics.confusion_matrix` for comparing labellings {pr}`915` {smaller}`I Virshup` + - Added {func}`scanpy.metrics.gearys_c` for spatial autocorrelation {pr}`915` {smaller}`I Virshup` + - Added {func}`scanpy.metrics.morans_i` for global spatial autocorrelation {pr}`1740` {smaller}`I Virshup, G Palla` + - Added {func}`scanpy.metrics.confusion_matrix` for comparing labellings {pr}`915` {smaller}`I Virshup` -```{rubric} Features -``` +#### Features - Added `layer` and `copy` kwargs to {func}`~scanpy.pp.normalize_total` {pr}`1667` {smaller}`I Virshup` - Added `vcenter` and `norm` arguments to the plotting functions {pr}`1551` {smaller}`G Eraslan` - Standardized and expanded available arguments to the `sc.pl.rank_genes_groups*` family of functions. {pr}`1529` {smaller}`F Ramirez` {smaller}`I Virshup` - \- See examples sections of {func}`~scanpy.pl.rank_genes_groups_dotplot` and {func}`~scanpy.pl.rank_genes_groups_matrixplot` for demonstrations. + - See examples sections of {func}`~scanpy.pl.rank_genes_groups_dotplot` and {func}`~scanpy.pl.rank_genes_groups_matrixplot` for demonstrations. - {func}`scanpy.tl.tsne` now supports the metric argument and records the passed parameters {pr}`1854` {smaller}`I Virshup` - {func}`scanpy.pl.scrublet_score_distribution` now uses same API as other scanpy functions for saving/ showing plots {pr}`1741` {smaller}`J Manning` -```{rubric} Ecosystem -``` +#### Ecosystem - Added [Cubé](https://github.com/connerlambden/Cube) to ecosystem page {pr}`1878` {smaller}`C Lambden` - Added `triku` a feature selection method to the ecosystem page {pr}`1722` {smaller}`AM Ascensión` - Added `dorothea` and `progeny` to the ecosystem page {pr}`1767` {smaller}`P Badia-i-Mompel` -```{rubric} Documentation -``` +#### Documentation - Added {doc}`/community` page to docs {pr}`1856` {smaller}`I Virshup` - Added rendered examples to many plotting functions {issue}`1664` {smaller}`A Schaar` {smaller}`L Zappia` {smaller}`bio-la` {smaller}`L Hetzel` {smaller}`L Dony` {smaller}`M Buttner` {smaller}`K Hrovatin` {smaller}`F Ramirez` {smaller}`I Virshup` {smaller}`LouisK92` {smaller}`mayarali` - Integrated [DocSearch], a find-as-you-type documentation index search. {pr}`1754` {smaller}`P Angerer` -- - Reorganized reference docs {pr}`1753` {smaller}`I Virshup` +- Reorganized reference docs {pr}`1753` {smaller}`I Virshup` - Clarified docs issues for {func}`~scanpy.pp.neighbors`, {func}`~scanpy.tl.diffmap`, {func}`~scanpy.pp.calculate_qc_metrics` {pr}`1680` {smaller}`G Palla` - Fixed typos in grouped plot doc-strings {pr}`1877` {smaller}`C Rands` - Extended examples for differential expression plotting. {pr}`1529` {smaller}`F Ramirez` - \- See {func}`~scanpy.pl.rank_genes_groups_dotplot` or {func}`~scanpy.pl.rank_genes_groups_matrixplot` for examples. + - See {func}`~scanpy.pl.rank_genes_groups_dotplot` or {func}`~scanpy.pl.rank_genes_groups_matrixplot` for examples. -```{rubric} Bug fixes -``` +#### Bug fixes - Fix {func}`scanpy.pl.paga_path` `TypeError` with recent versions of anndata {pr}`1047` {smaller}`P Angerer` - Fix detection of whether IPython is running {pr}`1844` {smaller}`I Virshup` @@ -51,14 +47,12 @@ - {func}`scanpy.pl.rank_genes_groups_violin` now works for `raw=False` {pr}`1669` {smaller}`M van den Beek` - {func}`scanpy.pl.dotplot` now uses `smallest_dot` argument correctly {pr}`1771` {smaller}`S Flemming` -```{rubric} Development processes -``` +#### Development Process - Switched to [flit] for building and deploying the package, a simple tool with an easy to understand command line interface and metadata {pr}`1527` {smaller}`P Angerer` - Use [pre-commit](https://pre-commit.com) for style checks {pr}`1684` {pr}`1848` {smaller}`L Heumos` {smaller}`I Virshup` -```{rubric} Deprecations -``` +#### Deprecations - Dropped support for Python 3.6. [More details here](https://numpy.org/neps/nep-0029-deprecation_policy.html). {pr}`1897` {smaller}`I Virshup` - Deprecated `layers` and `layers_norm` kwargs to {func}`~scanpy.pp.normalize_total` {pr}`1667` {smaller}`I Virshup` diff --git a/docs/release-notes/1.8.1.md b/docs/release-notes/1.8.1.md index 6569748ab4..befef210e5 100644 --- a/docs/release-notes/1.8.1.md +++ b/docs/release-notes/1.8.1.md @@ -1,7 +1,7 @@ +(v1.8.1)= ### 1.8.1 {small}`2021-07-07` -```{rubric} Bug fixes -``` +#### Bug fixes - Fixed reproducibility of {func}`scanpy.tl.score_genes`. Calculation and output is now float64 type. {pr}`1890` {smaller}`I Kucinski` - Workarounds for some changes/ bugs in pandas 1.3 {pr}`1918` {smaller}`I Virshup` diff --git a/docs/release-notes/1.8.2.md b/docs/release-notes/1.8.2.md index 8cae543c03..d26e2e4ac2 100644 --- a/docs/release-notes/1.8.2.md +++ b/docs/release-notes/1.8.2.md @@ -1,12 +1,11 @@ +(v1.8.2)= ### 1.8.2 {small}`2021-11-3` -```{rubric} Docs -``` +#### Documentation - Update conda installation instructions {pr}`1974` {smaller}`L Heumos` -```{rubric} Bug fixes -``` +#### Bug fixes - Fix plotting after {func}`scanpy.tl.filter_rank_genes_groups` {pr}`1942` {smaller}`S Rybakov` - Fix `use_raw=None` using {attr}`anndata.AnnData.var_names` if {attr}`anndata.AnnData.raw` @@ -14,7 +13,6 @@ - Fix compatibility with UMAP 0.5.2 {pr}`2028` {smaller}`L Mcinnes` - Fixed non-determinism in {func}`scanpy.pl.paga` node positions {pr}`1922` {smaller}`I Virshup` -```{rubric} Ecosystem -``` +#### Ecosystem - Added PASTE (a tool to align and integrate spatial transcriptomics data) to scanpy ecosystem. diff --git a/docs/release-notes/1.9.0.md b/docs/release-notes/1.9.0.md index 977489a34b..c89a1b704b 100644 --- a/docs/release-notes/1.9.0.md +++ b/docs/release-notes/1.9.0.md @@ -1,13 +1,12 @@ +(v1.9.0)= ### 1.9.0 {small}`2022-04-01` -```{rubric} Tutorials -``` +#### Tutorials - New tutorial on the usage of Pearson Residuals: {doc}`/tutorials/experimental/pearson_residuals` {smaller}`J Lause, G Palla` - [Materials](https://github.com/scverse/scanpy-tutorials/tree/master/scanpy_workshop) and [recordings](https://www.youtube.com/playlist?list=PL4rcQcNPLZxWQQH7LlRBMkAo5NWuHX1e3) for Scanpy workshops by Maren Büttner -```{rubric} Experimental module -``` +#### Experimental module - Added {mod}`scanpy.experimental` module! Currently contains functionality related to pearson residuals in {mod}`scanpy.experimental.pp` {pr}`1715` {smaller}`J Lause, G Palla, I Virshup`. This includes: @@ -16,8 +15,7 @@ - {func}`~scanpy.experimental.pp.normalize_pearson_residuals_pca` for Pearson Residuals normalization and dimensionality reduction with PCA - {func}`~scanpy.experimental.pp.recipe_pearson_residuals` for Pearson Residuals normalization, HVG selection and dimensionality reduction with PCA -```{rubric} Features -``` +#### Features - {func}`~scanpy.tl.filter_rank_genes_groups` now allows to filter with absolute values of log fold change {pr}`1649` {smaller}`S Rybakov` - `_choose_representation` now subsets the provided representation to n_pcs, regardless of the name of the provided representation (should affect mostly {func}`~scanpy.pp.neighbors`) {pr}`2179` {smaller}`I Virshup` {smaller}`PG Majev` @@ -29,8 +27,7 @@ - Embedding plots now have a `dimensions` argument, which lets users select which dimensions of their embedding to plot and uses the same broadcasting rules as other arguments {pr}`1538` {smaller}`I Virshup` - {func}`~scanpy.logging.print_versions` now uses `session_info` {pr}`2089` {smaller}`P Angerer` {smaller}`I Virshup` -```{rubric} Ecosystem -``` +#### Ecosystem Multiple packages have been added to our ecosystem page, including: @@ -38,8 +35,7 @@ Multiple packages have been added to our ecosystem page, including: - [dandelion](https://github.com/zktuong/dandelion) for B-cell receptor analysis {pr}`1953` {smaller}`Z Tuong` - [CIARA](https://github.com/ScialdoneLab/CIARA_python) a feature selection tools for identifying rare cell types {pr}`2175` {smaller}`M Stock` -```{rubric} Bug fixes -``` +#### Bug fixes - Fixed finding variables with `use_raw=True` and `basis=None` in {func}`scanpy.pl.scatter` {pr}`2027` {smaller}`E Rice` - Fixed {func}`scanpy.pp.scrublet` to address {issue}`1957` {smaller}`FlMai` and ensure raw counts are used for simulation diff --git a/docs/release-notes/1.9.1.md b/docs/release-notes/1.9.1.md index dcaa2a77cb..38bf8922cc 100644 --- a/docs/release-notes/1.9.1.md +++ b/docs/release-notes/1.9.1.md @@ -1,8 +1,7 @@ +(v1.9.1)= ### 1.9.1 {small}`2022-04-05` - -```{rubric} Bug fixes -``` +#### Bug fixes - {func}`~scanpy.pp.normalize_total` works when Dask is not installed {pr}`2209` {smaller}`R Cannoodt` - Fix embedding plots by bumping matplotlib dependency to version 3.4 {pr}`2212` {smaller}`I Virshup` diff --git a/docs/release-notes/1.9.2.md b/docs/release-notes/1.9.2.md index c2f5b647da..6b50147f43 100644 --- a/docs/release-notes/1.9.2.md +++ b/docs/release-notes/1.9.2.md @@ -1,7 +1,7 @@ +(v1.9.2)= ### 1.9.2 {small}`2023-02-16` -```{rubric} Bug fixes -``` +#### Bug fixes * {func}`~scanpy.pp.highly_variable_genes` `layer` argument now works in tandem with `batches` {pr}`2302` {smaller}`D Schaumont` * {func}`~scanpy.pp.highly_variable_genes` with `flavor='cell_ranger'` now handles the case in {issue}`2230` where the number of calculated dispersions is less than `n_top_genes` {pr}`2231` {smaller}`L Zappia` diff --git a/docs/release-notes/1.9.3.md b/docs/release-notes/1.9.3.md index 83dd8c8454..bec9964182 100644 --- a/docs/release-notes/1.9.3.md +++ b/docs/release-notes/1.9.3.md @@ -1,6 +1,6 @@ +(v1.9.3)= ### 1.9.3 {small}`2023-03-02` -```{rubric} Bug fixes -``` +#### Bug fixes * Variety of fixes against pandas 2.0.0rc0 {pr}`2434` {smaller}`I Virshup` diff --git a/docs/release-notes/1.9.4.md b/docs/release-notes/1.9.4.md index 51ca54257b..daf34fa968 100644 --- a/docs/release-notes/1.9.4.md +++ b/docs/release-notes/1.9.4.md @@ -1,7 +1,7 @@ +(v1.9.4)= ### 1.9.4 {small}`2023-08-24` -```{rubric} Bug fixes -``` +#### Bug fixes * Support scikit-learn 1.3 {pr}`2515` {smaller}`P Angerer` * Deal with `None` value vanishing from things like `.uns['log1p']` {pr}`2546` {smaller}`SP Shen` diff --git a/docs/release-notes/1.9.5.md b/docs/release-notes/1.9.5.md index c674da556d..cb96112005 100644 --- a/docs/release-notes/1.9.5.md +++ b/docs/release-notes/1.9.5.md @@ -1,6 +1,6 @@ +(v1.9.5)= ### 1.9.5 {small}`2023-09-08` -```{rubric} Bug fixes -``` +#### Bug fixes - Remove use of deprecated `dtype` argument to AnnData constructor {pr}`2658` {smaller}`Isaac Virshup` diff --git a/docs/release-notes/1.9.6.md b/docs/release-notes/1.9.6.md index ca8d43e498..4e26980a54 100644 --- a/docs/release-notes/1.9.6.md +++ b/docs/release-notes/1.9.6.md @@ -1,7 +1,7 @@ +(v1.9.6)= ### 1.9.6 {small}`2023-10-31` -```{rubric} Bug fixes -``` +#### Bug fixes - Allow {func}`scanpy.pl.scatter` to accept a {class}`str` palette name {pr}`2571` {smaller}`P Angerer` - Make {func}`scanpy.external.tl.palantir` compatible with palantir >=1.3 {pr}`2672` {smaller}`DJ Otto` diff --git a/docs/release-notes/1.9.7.md b/docs/release-notes/1.9.7.md index 26486a1739..0b98ac3d80 100644 --- a/docs/release-notes/1.9.7.md +++ b/docs/release-notes/1.9.7.md @@ -1,7 +1,8 @@ +(v1.9.7)= ### 1.9.7 {small}`2024-01-25` -```{rubric} Bug fixes -``` +#### Bug fixes + - Fix handling of numpy array palettes (e.g. after write-read cycle) {pr}`2734` {smaller}`P Angerer` - Specify correct version of `matplotlib` dependency {pr}`2733` {smaller}`P Fisher` - Fix {func}`scanpy.pl.violin` usage of `seaborn.catplot` {pr}`2739` {smaller}`E Roellin` diff --git a/docs/release-notes/1.9.8.md b/docs/release-notes/1.9.8.md index 4c954a878c..d7e65ec12a 100644 --- a/docs/release-notes/1.9.8.md +++ b/docs/release-notes/1.9.8.md @@ -1,5 +1,6 @@ +(v1.9.8)= ### 1.9.8 {small}`2024-01-26` -```{rubric} Bug fixes -``` +#### Bug fixes + - Fix handling of numpy array palettes for old numpy versions {pr}`2832` {smaller}`P Angerer` diff --git a/docs/release-notes/3418.doc.md b/docs/release-notes/3418.doc.md new file mode 100644 index 0000000000..d46304bf59 --- /dev/null +++ b/docs/release-notes/3418.doc.md @@ -0,0 +1 @@ +Fix reference in {mod}`scanpy.pp` page {smaller}`D Kazemi` diff --git a/docs/release-notes/3426.bugfix.md b/docs/release-notes/3426.bugfix.md new file mode 100644 index 0000000000..4565f1ee35 --- /dev/null +++ b/docs/release-notes/3426.bugfix.md @@ -0,0 +1 @@ +Fix {func}`~scanpy.tl.rank_genes_groups` compatibility with data >10M cells {smaller}`P Angerer` diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 3dc461d0b4..ae08bf99cb 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -2,181 +2,5 @@ # Release notes -## Version 1.11 - -```{include} /release-notes/1.11.0.md -``` - -## Version 1.10 - -```{include} /release-notes/1.10.3.md -``` - -```{include} /release-notes/1.10.2.md -``` - -```{include} /release-notes/1.10.1.md -``` - -```{include} /release-notes/1.10.0.md -``` - -## Version 1.9 - -```{include} /release-notes/1.9.8.md -``` - -```{include} /release-notes/1.9.7.md -``` - -```{include} /release-notes/1.9.6.md -``` - -```{include} /release-notes/1.9.5.md -``` - -```{include} /release-notes/1.9.4.md -``` - -```{include} /release-notes/1.9.3.md -``` - -```{include} /release-notes/1.9.2.md -``` - -```{include} /release-notes/1.9.1.md -``` - -```{include} /release-notes/1.9.0.md -``` - -## Version 1.8 - -```{include} /release-notes/1.8.2.md -``` - -```{include} /release-notes/1.8.1.md -``` - -```{include} /release-notes/1.8.0.md -``` - -## Version 1.7 - -```{include} /release-notes/1.7.2.md -``` - -```{include} /release-notes/1.7.1.md -``` - -```{include} /release-notes/1.7.0.md -``` - -## Version 1.6 - -```{include} 1.6.0.md -``` - -## Version 1.5 - -```{include} 1.5.1.md -``` - -```{include} 1.5.0.md -``` - -## Version 1.4 - -```{include} 1.4.6.md -``` - -```{include} 1.4.5.md -``` - -```{include} 1.4.4.md -``` - -```{include} 1.4.3.md -``` - -```{include} 1.4.2.md -``` - -```{include} 1.4.1.md -``` - -## Version 1.3 - -```{include} 1.3.8.md -``` - -```{include} 1.3.7.md -``` - -```{include} 1.3.6.md -``` - -```{include} 1.3.5.md -``` - -```{include} 1.3.4.md -``` - -```{include} 1.3.3.md -``` - -```{include} 1.3.1.md -``` - -## Version 1.2 - -```{include} 1.2.1.md -``` - -```{include} 1.2.0.md -``` - -## Version 1.1 - -```{include} 1.1.0.md -``` - -## Version 1.0 - -```{include} 1.0.0.md -``` - -## Version 0.4 - -```{include} 0.4.4.md -``` - -```{include} 0.4.3.md -``` - -```{include} 0.4.2.md -``` - -```{include} 0.4.0.md -``` - -## Version 0.3 - -```{include} 0.3.2.md -``` - -```{include} 0.3.0.md -``` - -## Version 0.2 - -```{include} 0.2.9.md -``` - -```{include} 0.2.1.md -``` - -## Version 0.1 - -```{include} 0.1.0.md +```{release-notes} . ``` diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index ee57056a6d..b20ee2b762 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -37,19 +37,6 @@ trajectories/index ## Spatial data -```{seealso} -For more up-to-date tutorials on working with spatial data, see: - -* [SquidPy tutorials](https://squidpy.readthedocs.io/en/stable/notebooks/tutorials/index.html) -* [SpatialData tutorials](https://spatialdata.scverse.org/en/latest/tutorials/notebooks/notebooks.html) -* [Scverse ecosystem spatial tutorials](https://scverse.org/learn/) -``` - -```{toctree} -:maxdepth: 2 - -spatial/index -``` ## Experimental @@ -64,3 +51,12 @@ experimental/index A number of older tutorials can be found at: * The [`scanpy_usage`](https://github.com/scverse/scanpy_usage) repository + +```{seealso} +Scanpy used to have tutorials for its (now deprecated) spatial data functionality.x +For up-to-date tutorials on working with spatial data, see: + +* SquidPy {doc}`squidpy:notebooks/tutorials/index` +* [SpatialData tutorials](https://spatialdata.scverse.org/en/latest/tutorials/notebooks/notebooks.html) +* [Scverse ecosystem spatial tutorials](https://scverse.org/learn/) +``` diff --git a/docs/tutorials/spatial/basic-analysis.ipynb b/docs/tutorials/spatial/basic-analysis.ipynb deleted file mode 120000 index 66d9e48121..0000000000 --- a/docs/tutorials/spatial/basic-analysis.ipynb +++ /dev/null @@ -1 +0,0 @@ -../../../notebooks/spatial/basic-analysis.ipynb \ No newline at end of file diff --git a/docs/tutorials/spatial/index.md b/docs/tutorials/spatial/index.md deleted file mode 100644 index 801b901e53..0000000000 --- a/docs/tutorials/spatial/index.md +++ /dev/null @@ -1,8 +0,0 @@ -## Spatial - -```{toctree} -:maxdepth: 1 - -basic-analysis -integration-scanorama -``` diff --git a/docs/tutorials/spatial/integration-scanorama.ipynb b/docs/tutorials/spatial/integration-scanorama.ipynb deleted file mode 120000 index 5143681577..0000000000 --- a/docs/tutorials/spatial/integration-scanorama.ipynb +++ /dev/null @@ -1 +0,0 @@ -../../../notebooks/spatial/integration-scanorama.ipynb \ No newline at end of file diff --git a/hatch.toml b/hatch.toml new file mode 100644 index 0000000000..b0a1084c61 --- /dev/null +++ b/hatch.toml @@ -0,0 +1,36 @@ +[envs.default] +installer = "uv" +features = [ "dev" ] + +[envs.docs] +features = [ "doc" ] +scripts.build = "sphinx-build -M html docs docs/_build -W --keep-going {args}" +scripts.open = "python3 -m webbrowser -t docs/_build/html/index.html" +scripts.clean = "git clean -fdX -- {args:docs}" + +[envs.towncrier] +scripts.create = "towncrier create {args}" +scripts.build = "python3 ci/scripts/towncrier_automation.py {args}" +scripts.clean = "git restore --source=HEAD --staged --worktree -- docs/release-notes" + +[envs.hatch-test] +default-args = [ ] +features = [ "test", "dask-ml" ] +extra-dependencies = [ "ipykernel" ] +overrides.matrix.deps.env-vars = [ + { if = [ "pre" ], key = "UV_PRERELEASE", value = "allow" }, + { if = [ "min" ], key = "UV_CONSTRAINT", value = "ci/scanpy-min-deps.txt" }, +] +overrides.matrix.deps.pre-install-commands = [ + { if = [ "min" ], value = "uv run ci/scripts/min-deps.py pyproject.toml --all-extras -o ci/scanpy-min-deps.txt" }, +] +overrides.matrix.deps.python = [ + { if = [ "min" ], value = "3.10" }, + { if = [ "stable", "full", "pre" ], value = "3.12" }, +] +overrides.matrix.deps.features = [ + { if = [ "full" ], value = "test-full" }, +] + +[[envs.hatch-test.matrix]] +deps = [ "stable", "full", "pre", "min" ] diff --git a/notebooks b/notebooks index 02c4946e0b..9f6926f87f 160000 --- a/notebooks +++ b/notebooks @@ -1 +1 @@ -Subproject commit 02c4946e0be47e033355ef84b2a4909b302d2513 +Subproject commit 9f6926f87f052603916ee8f222965f654896e0c7 diff --git a/pyproject.toml b/pyproject.toml index 1f8d5a88f8..71f7f1c482 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,30 +1,30 @@ [build-system] build-backend = "hatchling.build" -requires = ["hatchling", "hatch-vcs"] +requires = [ "hatchling", "hatch-vcs" ] [project] name = "scanpy" description = "Single-Cell Analysis in Python." -requires-python = ">=3.9" +requires-python = ">=3.10" license = "BSD-3-clause" authors = [ - {name = "Alex Wolf"}, - {name = "Philipp Angerer"}, - {name = "Fidel Ramirez"}, - {name = "Isaac Virshup"}, - {name = "Sergei Rybakov"}, - {name = "Gokcen Eraslan"}, - {name = "Tom White"}, - {name = "Malte Luecken"}, - {name = "Davide Cittaro"}, - {name = "Tobias Callies"}, - {name = "Marius Lange"}, - {name = "Andrés R. Muñoz-Rojas"}, + { name = "Alex Wolf" }, + { name = "Philipp Angerer" }, + { name = "Fidel Ramirez" }, + { name = "Isaac Virshup" }, + { name = "Sergei Rybakov" }, + { name = "Gokcen Eraslan" }, + { name = "Tom White" }, + { name = "Malte Luecken" }, + { name = "Davide Cittaro" }, + { name = "Tobias Callies" }, + { name = "Marius Lange" }, + { name = "Andrés R. Muñoz-Rojas" }, ] maintainers = [ - {name = "Isaac Virshup", email = "ivirshup@gmail.com"}, - {name = "Philipp Angerer", email = "phil.angerer@gmail.com"}, - {name = "Alex Wolf", email = "f.alex.wolf@gmx.de"}, + { name = "Philipp Angerer", email = "phil.angerer@gmail.com" }, + { name = "Ilan Gold", email = "ilan.gold@helmholtz-munich.de" }, + { name = "Severin Dicks" }, ] readme = "README.md" classifiers = [ @@ -39,7 +39,6 @@ classifiers = [ "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -48,37 +47,37 @@ classifiers = [ ] dependencies = [ "anndata>=0.8", - # TODO: remove <2 requirement once PyNNDescent releases this fix: - # https://github.com/lmcinnes/pynndescent/issues/241 - "numpy>=1.23,<2", + "numpy>=1.24", "matplotlib>=3.6", "pandas >=1.5", "scipy>=1.8", "seaborn>=0.13", - "h5py>=3.1", + "h5py>=3.7", "tqdm", - "scikit-learn>=0.24", + "scikit-learn>=1.1,<1.6.0", "statsmodels>=0.13", - "patsy", + "patsy!=1.0.0", # https://github.com/pydata/patsy/issues/215 "networkx>=2.7", "natsort", "joblib", - "numba>=0.56", + "numba>=0.57", "umap-learn>=0.5,!=0.5.0", "pynndescent>=0.5", "packaging>=21.3", - "session-info", - "legacy-api-wrap>=1.4", # for positional API deprecations - "get-annotations; python_version < '3.10'", + "session-info2", + "legacy-api-wrap>=1.4", # for positional API deprecations + "typing-extensions; python_version < '3.13'", ] -dynamic = ["version"] +dynamic = [ "version" ] +# https://docs.pypi.org/project_metadata/#project-urls [project.urls] Documentation = "https://scanpy.readthedocs.io/" Source = "https://github.com/scverse/scanpy" -Home-page = "https://scanpy.org" +Homepage = "https://scanpy.org" Discourse = "https://discourse.scverse.org/c/help/scanpy/37" -Twitter = "https://twitter.com/scverse_team" +Bluesky = "https://bsky.app/profile/scverse.bsky.social" +Twitter = "https://x.com/scverse_team" [project.scripts] scanpy = "scanpy.cli:console_main" @@ -95,7 +94,7 @@ test = [ "scanpy[test-min]", # Optional but important dependencies "scanpy[leiden]", - "zarr", + "zarr<3", "scanpy[dask]", "scanpy[scrublet]", ] @@ -114,22 +113,22 @@ test-full = [ doc = [ "sphinx>=7", "sphinx-book-theme>=1.1.0", - "scanpydoc>=0.13.4", + "scanpydoc>=0.14.1", "sphinx-autodoc-typehints>=1.25.2", "myst-parser>=2", "myst-nb>=1", "sphinx-design", + "sphinx-tabs", "readthedocs-sphinx-search", - "sphinxext-opengraph", # for nice cards when sharing on social + "sphinxext-opengraph", # for nice cards when sharing on social "sphinx-copybutton", "nbsphinx>=0.9", - "ipython>=7.20", # for nbsphinx code highlighting + "ipython>=7.20", # for nbsphinx code highlighting "matplotlib!=3.6.1", "sphinxcontrib-bibtex", - "setuptools", + "setuptools", # undeclared dependency of sphinxcontrib-bibtex→pybtex # TODO: remove necessity for being able to import doc-linked classes - "dask", - "scanpy[paga]", + "scanpy[paga,dask-ml]", "sam-algorithm", ] dev = [ @@ -137,26 +136,28 @@ dev = [ "setuptools_scm", # static checking "pre-commit", + "towncrier", ] # Algorithms -paga = ["igraph"] -louvain = ["igraph", "louvain>=0.6.0,!=0.6.2"] # Louvain community detection -leiden = ["igraph>=0.10", "leidenalg>=0.9.0"] # Leiden community detection -bbknn = ["bbknn"] # Batch balanced KNN (batch correction) -magic = ["magic-impute>=2.0"] # MAGIC imputation method -skmisc = ["scikit-misc>=0.1.3"] # highly_variable_genes method 'seurat_v3' -harmony = ["harmonypy"] # Harmony dataset integration -scanorama = ["scanorama"] # Scanorama dataset integration -scrublet = ["scikit-image"] # Doublet detection with automatic thresholds +paga = [ "igraph" ] +louvain = [ "igraph", "louvain>=0.6.0,!=0.6.2" ] # Louvain community detection +leiden = [ "igraph>=0.10", "leidenalg>=0.9.0" ] # Leiden community detection +bbknn = [ "bbknn" ] # Batch balanced KNN (batch correction) +magic = [ "magic-impute>=2.0" ] # MAGIC imputation method +skmisc = [ "scikit-misc>=0.1.3" ] # highly_variable_genes method 'seurat_v3' +harmony = [ "harmonypy" ] # Harmony dataset integration +scanorama = [ "scanorama" ] # Scanorama dataset integration +scrublet = [ "scikit-image" ] # Doublet detection with automatic thresholds # Acceleration -rapids = ["cudf>=0.9", "cuml>=0.9", "cugraph>=0.9"] # GPU accelerated calculation of neighbors -dask = ["dask[array]>=2022.09.2"] # Use the Dask parallelization engine -dask-ml = ["dask-ml", "scanpy[dask]"] # Dask-ML for sklearn-like API +rapids = [ "cudf>=0.9", "cuml>=0.9", "cugraph>=0.9" ] # GPU accelerated calculation of neighbors +dask = [ "dask[array]>=2022.09.2,<2024.8.0" ] # Use the Dask parallelization engine +dask-ml = [ "dask-ml", "scanpy[dask]" ] # Dask-ML for sklearn-like API [tool.hatch.build.targets.wheel] -packages = ["src/testing", "src/scanpy"] +packages = [ "src/testing", "src/scanpy" ] [tool.hatch.version] source = "vcs" +raw-options.version_scheme = "release-branch-semver" [tool.hatch.build.hooks.vcs] version-file = "src/scanpy/_version.py" @@ -168,8 +169,8 @@ addopts = [ "-ptesting.scanpy._pytest", "--pyargs", ] -testpaths = ["./tests", "scanpy"] -norecursedirs = ["tests/_images"] +testpaths = [ "./tests", "./ci", "scanpy" ] +norecursedirs = [ "tests/_images" ] xfail_strict = true nunit_attach_on = "fail" markers = [ @@ -202,22 +203,22 @@ filterwarnings = [ [tool.coverage.run] data_file = "test-data/coverage" -source_pkgs = ["scanpy"] -omit = ["tests/*", "src/testing/*"] +source_pkgs = [ "scanpy" ] +omit = [ "tests/*", "src/testing/*" ] [tool.coverage.xml] output = "test-data/coverage.xml" [tool.coverage.paths] -source = [".", "**/site-packages"] +source = [ ".", "**/site-packages" ] [tool.coverage.report] exclude_also = [ "if __name__ == .__main__.:", "if TYPE_CHECKING:", # https://github.com/numba/numba/issues/4268 - "@numba.njit.*", + '@(numba\.|nb\.)njit.*', ] [tool.ruff] -src = ["src"] +src = [ "src" ] [tool.ruff.format] docstring-code-format = true @@ -229,11 +230,16 @@ select = [ "W", # Warning detected by Pycodestyle "UP", # pyupgrade "I", # isort - "TCH", # manage type checking blocks + "TC", # manage type checking blocks "TID251", # Banned imports "ICN", # Follow import conventions "PTH", # Pathlib instead of os.path + "PYI", # Typing "PLR0917", # Ban APIs with too many positional parameters + "FBT", # No positional boolean parameters + "PT", # Pytest style + "SIM", # Simplify control flow + "EM", # Traceback-friendly error messages ] ignore = [ # line too long -> we accept long comment lines; black gets rid of long code lines @@ -244,17 +250,39 @@ ignore = [ "E262", # allow I, O, l as variable names -> I is the identity matrix, i, j, k, l is reasonable indexing notation "E741", + # `Literal["..."] | str` is useful for autocompletion + "PYI051", ] [tool.ruff.lint.per-file-ignores] # Do not assign a lambda expression, use a def -"src/scanpy/tools/_rank_genes_groups.py" = ["E731"] +"src/scanpy/tools/_rank_genes_groups.py" = [ "E731" ] [tool.ruff.lint.isort] -known-first-party = ["scanpy", "testing.scanpy"] -required-imports = ["from __future__ import annotations"] +known-first-party = [ "scanpy", "testing.scanpy" ] +required-imports = [ "from __future__ import annotations" ] [tool.ruff.lint.flake8-tidy-imports.banned-api] "pytest.importorskip".msg = "Use the “@needs” decorator/mark instead" "pandas.api.types.is_categorical_dtype".msg = "Use isinstance(s.dtype, CategoricalDtype) instead" "pandas.value_counts".msg = "Use pd.Series(a).value_counts() instead" +"legacy_api_wrap.legacy_api".msg = "Use scanpy._compat.old_positionals instead" +"numpy.bool".msg = "Use `np.bool_` instead for numpy>=1.24<2 compatibility" +"numba.jit".msg = "Use `scanpy._compat.njit` instead" +"numba.njit".msg = "Use `scanpy._compat.njit` instead" [tool.ruff.lint.flake8-type-checking] -exempt-modules = [] +exempt-modules = [ ] strict = true + +[tool.towncrier] +package = "scanpy" +directory = "docs/release-notes" +filename = "docs/release-notes/{version}.md" +single_file = false +package_dir = "src" +issue_format = "{{pr}}`{issue}`" +title_format = "(v{version})=\n### {version} {{small}}`{project_date}`" +fragment.bugfix.name = "Bug fixes" +fragment.doc.name = "Documentation" +fragment.feature.name = "Features" +fragment.misc.name = "Miscellaneous improvements" +fragment.performance.name = "Performance" +fragment.breaking.name = "Breaking changes" +fragment.dev.name = "Development Process" diff --git a/src/scanpy/__init__.py b/src/scanpy/__init__.py index b9be00f8f3..b844372d1e 100644 --- a/src/scanpy/__init__.py +++ b/src/scanpy/__init__.py @@ -4,6 +4,8 @@ import sys +from packaging.version import Version + try: # See https://github.com/maresb/hatch-vcs-footgun-example from setuptools_scm import get_version @@ -13,9 +15,8 @@ try: from ._version import __version__ except ModuleNotFoundError: - raise RuntimeError( - "scanpy is not correctly installed. Please install it, e.g. with pip." - ) + msg = "scanpy is not correctly installed. Please install it, e.g. with pip." + raise RuntimeError(msg) from ._utils import check_versions @@ -29,18 +30,31 @@ set_figure_params = settings.set_figure_params -from anndata import ( - AnnData, - concat, - read_csv, - read_excel, - read_h5ad, - read_hdf, - read_loom, - read_mtx, - read_text, - read_umi_tools, -) +import anndata + +if Version(anndata.__version__) >= Version("0.11.0rc2"): + from anndata.io import ( + read_csv, + read_excel, + read_h5ad, + read_hdf, + read_loom, + read_mtx, + read_text, + read_umi_tools, + ) +else: + from anndata import ( + read_csv, + read_excel, + read_h5ad, + read_hdf, + read_loom, + read_mtx, + read_text, + read_umi_tools, + ) +from anndata import AnnData, concat from . import datasets, experimental, external, get, logging, metrics, queries from . import plotting as pl diff --git a/src/scanpy/_compat.py b/src/scanpy/_compat.py index 07292dddd9..9ea7780b0d 100644 --- a/src/scanpy/_compat.py +++ b/src/scanpy/_compat.py @@ -1,42 +1,57 @@ from __future__ import annotations +import os +import sys +import warnings from dataclasses import dataclass, field -from functools import partial +from functools import WRAPPER_ASSIGNMENTS, cache, partial, wraps +from importlib.util import find_spec from pathlib import Path +from typing import TYPE_CHECKING, Literal, ParamSpec, TypeVar, cast, overload -from legacy_api_wrap import legacy_api +import numpy as np from packaging.version import Version -try: - from functools import cache -except ImportError: # Python < 3.9 - from functools import lru_cache +if TYPE_CHECKING: + from collections.abc import Callable + from importlib.metadata import PackageMetadata - cache = lru_cache(maxsize=None) -try: +P = ParamSpec("P") +R = TypeVar("R") + +_LegacyRandom = int | np.random.RandomState | None + + +if TYPE_CHECKING: + # type checkers are confused and can only see …core.Array + from dask.array.core import Array as DaskArray +elif find_spec("dask"): from dask.array import Array as DaskArray -except ImportError: +else: class DaskArray: pass -try: +if find_spec("zappy") or TYPE_CHECKING: from zappy.base import ZappyArray -except ImportError: +else: class ZappyArray: pass __all__ = [ - "cache", "DaskArray", "ZappyArray", "fullname", "pkg_metadata", "pkg_version", + "old_positionals", + "deprecated", + "njit", + "_numba_threading_layer", ] @@ -48,9 +63,9 @@ def fullname(typ: type) -> str: return f"{module}.{name}" -try: +if sys.version_info >= (3, 11): from contextlib import chdir -except ImportError: # Python < 3.11 +else: import os from contextlib import AbstractContextManager @@ -67,17 +82,184 @@ def __exit__(self, *_excinfo) -> None: os.chdir(self._old_cwd.pop()) -def pkg_metadata(package): +def pkg_metadata(package: str) -> PackageMetadata: from importlib.metadata import metadata return metadata(package) @cache -def pkg_version(package): +def pkg_version(package: str) -> Version: from importlib.metadata import version return Version(version(package)) -old_positionals = partial(legacy_api, category=FutureWarning) +if find_spec("legacy_api_wrap") or TYPE_CHECKING: + from legacy_api_wrap import legacy_api # noqa: TID251 + + old_positionals = partial(legacy_api, category=FutureWarning) +else: + # legacy_api_wrap is currently a hard dependency, + # but this code makes it possible to run scanpy without it. + def old_positionals(*old_positionals: str): + return lambda func: func + + +if sys.version_info >= (3, 11): + + @wraps(BaseException.add_note) + def add_note(exc: BaseException, note: str) -> None: + exc.add_note(note) +else: + + def add_note(exc: BaseException, note: str) -> None: + if not hasattr(exc, "__notes__"): + exc.__notes__ = [] + exc.__notes__.append(note) + + +if sys.version_info >= (3, 13): + from warnings import deprecated as _deprecated +else: + from typing_extensions import deprecated as _deprecated + + +deprecated = partial(_deprecated, category=FutureWarning) + + +@overload +def njit(fn: Callable[P, R], /) -> Callable[P, R]: ... +@overload +def njit() -> Callable[[Callable[P, R]], Callable[P, R]]: ... +def njit( + fn: Callable[P, R] | None = None, / +) -> Callable[P, R] | Callable[[Callable[P, R]], Callable[P, R]]: + """\ + Jit-compile a function using numba. + + On call, this function dispatches to a parallel or sequential numba function, + depending on if it has been called from a thread pool. + + See + """ + + def decorator(f: Callable[P, R], /) -> Callable[P, R]: + import numba + + fns: dict[bool, Callable[P, R]] = { + parallel: numba.njit(f, cache=True, parallel=parallel) # noqa: TID251 + for parallel in (True, False) + } + + @wraps(f) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + parallel = not _is_in_unsafe_thread_pool() + if not parallel: + msg = ( + "Detected unsupported threading environment. " + f"Trying to run {f.__name__} in serial mode. " + "In case of problems, install `tbb`." + ) + warnings.warn(msg, stacklevel=2) + return fns[parallel](*args, **kwargs) + + return wrapper + + return decorator if fn is None else decorator(fn) + + +LayerType = Literal["default", "safe", "threadsafe", "forksafe"] +Layer = Literal["tbb", "omp", "workqueue"] + + +LAYERS: dict[LayerType, set[Layer]] = { + "default": {"tbb", "omp", "workqueue"}, + "safe": {"tbb"}, + "threadsafe": {"tbb", "omp"}, + "forksafe": {"tbb", "workqueue", *(() if sys.platform == "linux" else {"omp"})}, +} + + +def _is_in_unsafe_thread_pool() -> bool: + import threading + + current_thread = threading.current_thread() + # ThreadPoolExecutor threads typically have names like 'ThreadPoolExecutor-0_1' + return ( + current_thread.name.startswith("ThreadPoolExecutor") + and _numba_threading_layer() not in LAYERS["threadsafe"] + ) + + +@cache +def _numba_threading_layer() -> Layer: + """\ + Get numba’s threading layer. + + This function implements the algorithm as described in + + """ + import importlib + + import numba + + if (available := LAYERS.get(numba.config.THREADING_LAYER)) is None: + # given by direct name + return numba.config.THREADING_LAYER + + # given by layer type (safe, …) + for layer in cast(list[Layer], numba.config.THREADING_LAYER_PRIORITY): + if layer not in available: + continue + if layer != "workqueue": + try: # `importlib.util.find_spec` doesn’t work here + importlib.import_module(f"numba.np.ufunc.{layer}pool") + except ImportError: + continue + # the layer has been found + return layer + msg = ( + f"No loadable threading layer: {numba.config.THREADING_LAYER=} " + f" ({available=}, {numba.config.THREADING_LAYER_PRIORITY=})" + ) + raise ValueError(msg) + + +def _legacy_numpy_gen( + random_state: _LegacyRandom | None = None, +) -> np.random.Generator: + """Return a random generator that behaves like the legacy one.""" + + if random_state is not None: + if isinstance(random_state, np.random.RandomState): + np.random.set_state(random_state.get_state(legacy=False)) + return _FakeRandomGen(random_state) + np.random.seed(random_state) + return _FakeRandomGen(np.random.RandomState(np.random.get_bit_generator())) + + +class _FakeRandomGen(np.random.Generator): + _state: np.random.RandomState + + def __init__(self, random_state: np.random.RandomState) -> None: + self._state = random_state + + @classmethod + def _delegate(cls) -> None: + for name, meth in np.random.Generator.__dict__.items(): + if name.startswith("_") or not callable(meth): + continue + + def mk_wrapper(name: str): + # Old pytest versions try to run the doctests + @wraps(meth, assigned=set(WRAPPER_ASSIGNMENTS) - {"__doc__"}) + def wrapper(self: _FakeRandomGen, *args, **kwargs): + return getattr(self._state, name)(*args, **kwargs) + + return wrapper + + setattr(cls, name, mk_wrapper(name)) + + +_FakeRandomGen._delegate() diff --git a/src/scanpy/_settings.py b/src/scanpy/_settings.py index b090261d1f..fc9ead09b0 100644 --- a/src/scanpy/_settings.py +++ b/src/scanpy/_settings.py @@ -15,14 +15,14 @@ if TYPE_CHECKING: from collections.abc import Generator, Iterable - from typing import Any, Literal, TextIO, Union + from typing import Any, Literal, TextIO # Collected from the print_* functions in matplotlib.backends - _Format = Union[ - Literal["png", "jpg", "tif", "tiff"], - Literal["pdf", "ps", "eps", "svg", "svgz", "pgf"], - Literal["raw", "rgba"], - ] + _Format = ( + Literal["png", "jpg", "tif", "tiff"] # noqa: PYI030 + | Literal["pdf", "ps", "eps", "svg", "svgz", "pgf"] + | Literal["raw", "rgba"] + ) _VERBOSITY_TO_LOGLEVEL = { "error": "ERROR", @@ -82,10 +82,9 @@ def _type_check(var: Any, varname: str, types: type | tuple[type, ...]): possible_types_str = types.__name__ else: type_names = [t.__name__ for t in types] - possible_types_str = "{} or {}".format( - ", ".join(type_names[:-1]), type_names[-1] - ) - raise TypeError(f"{varname} must be of type {possible_types_str}") + possible_types_str = f"{', '.join(type_names[:-1])} or {type_names[-1]}" + msg = f"{varname} must be of type {possible_types_str}" + raise TypeError(msg) class ScanpyConfig: @@ -182,10 +181,11 @@ def verbosity(self, verbosity: Verbosity | int | str): elif isinstance(verbosity, str): verbosity = verbosity.lower() if verbosity not in verbosity_str_options: - raise ValueError( + msg = ( f"Cannot set verbosity to {verbosity}. " f"Accepted string values are: {verbosity_str_options}" ) + raise ValueError(msg) else: self._verbosity = Verbosity(verbosity_str_options.index(verbosity)) else: @@ -216,10 +216,11 @@ def file_format_data(self, file_format: str): _type_check(file_format, "file_format_data", str) file_format_options = {"txt", "csv", "h5ad"} if file_format not in file_format_options: - raise ValueError( + msg = ( f"Cannot set file_format_data to {file_format}. " f"Must be one of {file_format_options}" ) + raise ValueError(msg) self._file_format_data = file_format @property @@ -324,10 +325,11 @@ def cache_compression(self) -> str | None: @cache_compression.setter def cache_compression(self, cache_compression: str | None): if cache_compression not in {"lzf", "gzip", None}: - raise ValueError( + msg = ( f"`cache_compression` ({cache_compression}) " "must be in {'lzf', 'gzip', None}" ) + raise ValueError(msg) self._cache_compression = cache_compression @property @@ -340,7 +342,7 @@ def max_memory(self) -> int | float: return self._max_memory @max_memory.setter - def max_memory(self, max_memory: int | float): + def max_memory(self, max_memory: float): _type_check(max_memory, "max_memory", (int, float)) self._max_memory = max_memory @@ -371,7 +373,7 @@ def logpath(self) -> Path | None: def logpath(self, logpath: Path | str | None): _type_check(logpath, "logfile", (str, Path)) # set via “file object” branch of logfile.setter - self.logfile = Path(logpath).open("a") + self.logfile = Path(logpath).open("a") # noqa: SIM115 self._logpath = Path(logpath) @property @@ -519,7 +521,7 @@ def __str__(self) -> str: return "\n".join( f"{k} = {v!r}" for k, v in inspect.getmembers(self) - if not k.startswith("_") and not k == "getdoc" + if not k.startswith("_") and k != "getdoc" ) diff --git a/src/scanpy/_utils/__init__.py b/src/scanpy/_utils/__init__.py index 0cb0be0e11..326ea216d1 100644 --- a/src/scanpy/_utils/__init__.py +++ b/src/scanpy/_utils/__init__.py @@ -12,14 +12,22 @@ import re import sys import warnings -from collections import namedtuple -from contextlib import contextmanager +from collections.abc import Sequence +from contextlib import contextmanager, suppress from enum import Enum -from functools import partial, singledispatch, wraps -from operator import mul, truediv +from functools import partial, reduce, singledispatch, wraps +from operator import mul, or_, truediv from textwrap import dedent -from types import MethodType, ModuleType -from typing import TYPE_CHECKING, overload +from types import MethodType, ModuleType, UnionType +from typing import ( + TYPE_CHECKING, + Literal, + NamedTuple, + Union, + get_args, + get_origin, + overload, +) from weakref import WeakSet import h5py @@ -42,16 +50,21 @@ from anndata._core.sparse_dataset import SparseDataset if TYPE_CHECKING: - from collections.abc import Mapping + from collections.abc import Callable, Iterable, KeysView, Mapping from pathlib import Path - from typing import Any, Callable, Literal, TypeVar, Union + from typing import Any, TypeVar from anndata import AnnData - from numpy.typing import DTypeLike, NDArray + from numpy.typing import ArrayLike, DTypeLike, NDArray - # e.g. https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html - # maybe in the future random.Generator - AnyRandom = Union[int, np.random.RandomState, None] + from .._compat import _LegacyRandom + from ..neighbors import NeighborsParams, RPForestDict + + +SeedLike = int | np.integer | Sequence[int] | np.random.SeedSequence +RNGLike = np.random.Generator | np.random.BitGenerator + +LegacyUnionType = type(Union[int, str]) # noqa: UP007 class Empty(Enum): @@ -80,11 +93,12 @@ def __getattr__(self, attr: str): def ensure_igraph() -> None: if importlib.util.find_spec("igraph"): return - raise ImportError( + msg = ( "Please install the igraph package: " "`conda install -c conda-forge python-igraph` or " "`pip3 install igraph`." ) + raise ImportError(msg) @contextmanager @@ -107,10 +121,11 @@ def check_versions(): if Version(anndata_version) < Version("0.6.10"): from .. import __version__ - raise ImportError( + msg = ( f"Scanpy {__version__} needs anndata version >=0.6.10, " f"not {anndata_version}.\nRun `pip install anndata -U --no-deps`." ) + raise ImportError(msg) def getdoc(c_or_f: Callable | type) -> str | None: @@ -182,7 +197,8 @@ def _import_name(name: str) -> Any: try: obj = getattr(obj, name) except AttributeError: - raise RuntimeError(f"{parts[:i]}, {parts[i + 1:]}, {obj} {name}") + msg = f"{parts[:i]}, {parts[i + 1 :]}, {obj} {name}" + raise RuntimeError(msg) return obj @@ -242,12 +258,13 @@ def _check_array_function_arguments(**kwargs): # TODO: Figure out a better solution for documenting dispatched functions invalid_args = [k for k, v in kwargs.items() if v is not None] if len(invalid_args) > 0: - raise TypeError( - f"Arguments {invalid_args} are only valid if an AnnData object is passed." - ) + msg = f"Arguments {invalid_args} are only valid if an AnnData object is passed." + raise TypeError(msg) -def _check_use_raw(adata: AnnData, use_raw: None | bool) -> bool: +def _check_use_raw( + adata: AnnData, use_raw: None | bool, *, layer: str | None = None +) -> bool: """ Normalize checking `use_raw`. @@ -255,6 +272,8 @@ def _check_use_raw(adata: AnnData, use_raw: None | bool) -> bool: """ if use_raw is not None: return use_raw + if layer is not None: + return False return adata.raw is not None @@ -274,10 +293,8 @@ def get_igraph_from_adjacency(adjacency, directed=None): g = ig.Graph(directed=directed) g.add_vertices(adjacency.shape[0]) # this adds adjacency.shape[0] vertices g.add_edges(list(zip(sources, targets))) - try: + with suppress(KeyError): g.es["weight"] = weights - except KeyError: - pass if g.vcount() != adjacency.shape[0]: logg.warning( f"The constructed graph has only {g.vcount()} nodes. " @@ -291,6 +308,11 @@ def get_igraph_from_adjacency(adjacency, directed=None): # -------------------------------------------------------------------------------- +class AssoResult(NamedTuple): + asso_names: list[str] + asso_matrix: NDArray[np.floating] + + def compute_association_matrix_of_groups( adata: AnnData, prediction: str, @@ -299,7 +321,7 @@ def compute_association_matrix_of_groups( normalization: Literal["prediction", "reference"] = "prediction", threshold: float = 0.01, max_n_names: int | None = 2, -): +) -> AssoResult: """Compute overlaps between groups. See ``identify_groups`` for identifying the groups. @@ -330,19 +352,17 @@ def compute_association_matrix_of_groups( reference labels, entries are proportional to degree of association. """ if normalization not in {"prediction", "reference"}: - raise ValueError( - '`normalization` needs to be either "prediction" or "reference".' - ) + msg = '`normalization` needs to be either "prediction" or "reference".' + raise ValueError(msg) sanitize_anndata(adata) cats = adata.obs[reference].cat.categories for cat in cats: if cat in settings.categories_to_ignore: logg.info( - f"Ignoring category {cat!r} " - "as it’s in `settings.categories_to_ignore`." + f"Ignoring category {cat!r} as it’s in `settings.categories_to_ignore`." ) - asso_names = [] - asso_matrix = [] + asso_names: list[str] = [] + asso_matrix: list[list[float]] = [] for ipred_group, pred_group in enumerate(adata.obs[prediction].cat.categories): if "?" in pred_group: pred_group = str(ipred_group) @@ -375,13 +395,12 @@ def compute_association_matrix_of_groups( if asso_matrix[-1][i] > threshold ] asso_names += ["\n".join(name_list_pred[:max_n_names])] - Result = namedtuple( - "compute_association_matrix_of_groups", ["asso_names", "asso_matrix"] - ) - return Result(asso_names=asso_names, asso_matrix=np.array(asso_matrix)) + return AssoResult(asso_names=asso_names, asso_matrix=np.array(asso_matrix)) -def get_associated_colors_of_groups(reference_colors, asso_matrix): +def get_associated_colors_of_groups( + reference_colors: Mapping[int, str], asso_matrix: NDArray[np.floating] +) -> list[dict[str, float]]: return [ { reference_colors[i_ref]: asso_matrix[i_pred, i_ref] @@ -391,7 +410,7 @@ def get_associated_colors_of_groups(reference_colors, asso_matrix): ] -def identify_groups(ref_labels, pred_labels, return_overlaps=False): +def identify_groups(ref_labels, pred_labels, *, return_overlaps: bool = False): """Which predicted label explains which reference label? A predicted label explains the reference label which maximizes the minimum @@ -476,7 +495,7 @@ def moving_average(a: np.ndarray, n: int): return ret[n - 1 :] / n -def get_random_state(seed: AnyRandom) -> np.random.RandomState: +def _get_legacy_random(seed: _LegacyRandom) -> np.random.RandomState: if isinstance(seed, np.random.RandomState): return seed return np.random.RandomState(seed) @@ -490,7 +509,8 @@ def get_random_state(seed: AnyRandom) -> np.random.RandomState: def update_params( old_params: Mapping[str, Any], new_params: Mapping[str, Any], - check=False, + *, + check: bool = False, ) -> dict[str, Any]: """\ Update old_params with new_params. @@ -526,15 +546,28 @@ def update_params( return updated_params +# `get_args` returns `tuple[Any]` so I don’t think it’s possible to get the correct type here +def get_literal_vals(typ: UnionType | Any) -> KeysView[Any]: + """Get all literal values from a Literal or Union of … of Literal type.""" + if isinstance(typ, UnionType | LegacyUnionType): + return reduce( + or_, (dict.fromkeys(get_literal_vals(t)) for t in get_args(typ)) + ).keys() + if get_origin(typ) is Literal: + return dict.fromkeys(get_args(typ)).keys() + msg = f"{typ} is not a valid Literal" + raise TypeError(msg) + + # -------------------------------------------------------------------------------- # Others # -------------------------------------------------------------------------------- if TYPE_CHECKING: - _SparseMatrix = Union[sparse.csr_matrix, sparse.csc_matrix] - _MemoryArray = Union[NDArray, _SparseMatrix] - _SupportedArray = Union[_MemoryArray, DaskArray] + _SparseMatrix = sparse.csr_matrix | sparse.csc_matrix + _MemoryArray = NDArray | _SparseMatrix + _SupportedArray = _MemoryArray | DaskArray @singledispatch @@ -571,18 +604,19 @@ def broadcast_axis(divisor: Scaling_T, axis: Literal[0, 1]) -> Scaling_T: def check_op(op): if op not in {truediv, mul}: - raise ValueError(f"{op} not one of truediv or mul") + msg = f"{op} not one of truediv or mul" + raise ValueError(msg) @singledispatch def axis_mul_or_truediv( - X: np.ndarray, + X: ArrayLike, scaling_array: np.ndarray, axis: Literal[0, 1], op: Callable[[Any, Any], Any], *, allow_divide_by_zero: bool = True, - out: np.ndarray | None = None, + out: ArrayLike | None = None, ) -> np.ndarray: check_op(op) scaling_array = broadcast_axis(scaling_array, axis) @@ -605,11 +639,9 @@ def _( out: sparse.csr_matrix | sparse.csc_matrix | None = None, ) -> sparse.csr_matrix | sparse.csc_matrix: check_op(op) - if out is not None: - if X.data is not out.data: - raise ValueError( - "`out` argument provided but not equal to X. This behavior is not supported for sparse matrix scaling." - ) + if out is not None and X.data is not out.data: + msg = "`out` argument provided but not equal to X. This behavior is not supported for sparse matrix scaling." + raise ValueError(msg) if not allow_divide_by_zero and op is truediv: scaling_array = scaling_array.copy() + (scaling_array == 0) @@ -646,7 +678,7 @@ def new_data_op(x): def make_axis_chunks( - X: DaskArray, axis: Literal[0, 1], pad=True + X: DaskArray, axis: Literal[0, 1] ) -> tuple[tuple[int], tuple[int]]: if axis == 0: return (X.chunks[axis], (1,)) @@ -665,9 +697,8 @@ def _( ) -> DaskArray: check_op(op) if out is not None: - raise TypeError( - "`out` is not `None`. Do not do in-place modifications on dask arrays." - ) + msg = "`out` is not `None`. Do not do in-place modifications on dask arrays." + raise TypeError(msg) import dask.array as da @@ -676,7 +707,7 @@ def _( column_scale = axis == 1 if isinstance(scaling_array, DaskArray): - if (row_scale and not X.chunksize[0] == scaling_array.chunksize[0]) or ( + if (row_scale and X.chunksize[0] != scaling_array.chunksize[0]) or ( column_scale and ( ( @@ -708,6 +739,27 @@ def _( ) +@singledispatch +def axis_nnz(X: ArrayLike, axis: Literal[0, 1]) -> np.ndarray: + return np.count_nonzero(X, axis=axis) + + +@axis_nnz.register(sparse.spmatrix) +def _(X: sparse.spmatrix, axis: Literal[0, 1]) -> np.ndarray: + return X.getnnz(axis=axis) + + +@axis_nnz.register(DaskArray) +def _(X: DaskArray, axis: Literal[0, 1]) -> DaskArray: + return X.map_blocks( + partial(axis_nnz, axis=axis), + dtype=np.int64, + meta=np.array([], dtype=np.int64), + drop_axis=0, + chunks=len(X.to_delayed()) * (X.chunksize[int(not axis)],), + ) + + @overload def axis_sum( X: sparse.spmatrix, @@ -745,16 +797,15 @@ def _( def sum_drop_keepdims(*args, **kwargs): kwargs.pop("computing_meta", None) # masked operations on sparse produce which numpy matrices gives the same API issues handled here - if isinstance(X._meta, (sparse.spmatrix, np.matrix)) or isinstance( - args[0], (sparse.spmatrix, np.matrix) + if isinstance(X._meta, sparse.spmatrix | np.matrix) or isinstance( + args[0], sparse.spmatrix | np.matrix ): kwargs.pop("keepdims", None) axis = kwargs["axis"] if isinstance(axis, tuple): if len(axis) != 1: - raise ValueError( - f"`axis_sum` can only sum over one axis when `axis` arg is provided but got {axis} instead" - ) + msg = f"`axis_sum` can only sum over one axis when `axis` arg is provided but got {axis} instead" + raise ValueError(msg) kwargs["axis"] = axis[0] # returns a np.matrix normally, which is undesireable return np.array(np.sum(*args, dtype=dtype, **kwargs)) @@ -800,7 +851,7 @@ def _check_nonnegative_integers_dask(X: DaskArray) -> DaskArray: def select_groups( adata: AnnData, - groups_order_subset: list[str] | Literal["all"] = "all", + groups_order_subset: Iterable[str] | Literal["all"] = "all", key: str = "groups", ) -> tuple[list[str], NDArray[np.bool_]]: """Get subset of groups in adata.obs[key].""" @@ -906,7 +957,8 @@ def subsample( Xsampled = np.array(X[rows]) else: if seed < 0: - raise ValueError(f"Invalid seed value < 0: {seed}") + msg = f"Invalid seed value < 0: {seed}" + raise ValueError(msg) n = int(X.shape[0] / subsample) np.random.seed(seed) Xsampled, rows = subsample_n(X, n=n) @@ -936,7 +988,8 @@ def subsample_n( Indices of rows that are stored in Xsampled. """ if n < 0: - raise ValueError("n must be greater 0") + msg = "n must be greater 0" + raise ValueError(msg) np.random.seed(seed) n = X.shape[0] if (n == 0 or n > X.shape[0]) else n rows = np.random.choice(X.shape[0], size=n, replace=False) @@ -1010,19 +1063,21 @@ class NeighborsView: 'params' in adata.uns[key] """ - def __init__(self, adata, key=None): + def __init__(self, adata: AnnData, key=None): self._connectivities = None self._distances = None if key is None or key == "neighbors": if "neighbors" not in adata.uns: - raise KeyError('No "neighbors" in .uns') + msg = 'No "neighbors" in .uns' + raise KeyError(msg) self._neighbors_dict = adata.uns["neighbors"] self._conns_key = "connectivities" self._dists_key = "distances" else: if key not in adata.uns: - raise KeyError(f'No "{key}" in .uns') + msg = f"No {key!r} in .uns" + raise KeyError(msg) self._neighbors_dict = adata.uns[key] self._conns_key = self._neighbors_dict["connectivities_key"] self._dists_key = self._neighbors_dict["distances_key"] @@ -1041,19 +1096,34 @@ def __init__(self, adata, key=None): self._dists_key, ) - def __getitem__(self, key): + @overload + def __getitem__( + self, key: Literal["distances", "connectivities"] + ) -> sparse.csr_matrix: ... + @overload + def __getitem__(self, key: Literal["params"]) -> NeighborsParams: ... + @overload + def __getitem__(self, key: Literal["rp_forest"]) -> RPForestDict: ... + @overload + def __getitem__(self, key: Literal["connectivities_key"]) -> str: ... + + def __getitem__(self, key: str): if key == "distances": if "distances" not in self: - raise KeyError(f'No "{self._dists_key}" in .obsp') + msg = f"No {self._dists_key!r} in .obsp" + raise KeyError(msg) return self._distances elif key == "connectivities": if "connectivities" not in self: - raise KeyError(f'No "{self._conns_key}" in .obsp') + msg = f"No {self._conns_key!r} in .obsp" + raise KeyError(msg) return self._connectivities + elif key == "connectivities_key": + return self._conns_key else: return self._neighbors_dict[key] - def __contains__(self, key): + def __contains__(self, key: str) -> bool: if key == "distances": return self._distances is not None elif key == "connectivities": @@ -1065,19 +1135,18 @@ def __contains__(self, key): def _choose_graph(adata, obsp, neighbors_key): """Choose connectivities from neighbbors or another obsp column""" if obsp is not None and neighbors_key is not None: - raise ValueError( - "You can't specify both obsp, neighbors_key. " "Please select only one." - ) + msg = "You can't specify both obsp, neighbors_key. Please select only one." + raise ValueError(msg) if obsp is not None: return adata.obsp[obsp] else: neighbors = NeighborsView(adata, neighbors_key) if "connectivities" not in neighbors: - raise ValueError( - "You need to run `pp.neighbors` first " - "to compute a neighborhood graph." + msg = ( + "You need to run `pp.neighbors` first to compute a neighborhood graph." ) + raise ValueError(msg) return neighbors["connectivities"] @@ -1088,15 +1157,15 @@ def _resolve_axis( return (0, "obs") if axis in {1, "var"}: return (1, "var") - raise ValueError(f"`axis` must be either 0, 1, 'obs', or 'var', was {axis!r}") + msg = f"`axis` must be either 0, 1, 'obs', or 'var', was {axis!r}" + raise ValueError(msg) def is_backed_type(X: object) -> bool: - return isinstance(X, (SparseDataset, h5py.File, h5py.Dataset)) + return isinstance(X, SparseDataset | h5py.File | h5py.Dataset) def raise_not_implemented_error_if_backed_type(X: object, method_name: str) -> None: if is_backed_type(X): - raise NotImplementedError( - f"{method_name} is not implemented for matrices of type {type(X)}" - ) + msg = f"{method_name} is not implemented for matrices of type {type(X)}" + raise NotImplementedError(msg) diff --git a/src/scanpy/_utils/_doctests.py b/src/scanpy/_utils/_doctests.py index 6a08099a24..0b3be18bbe 100644 --- a/src/scanpy/_utils/_doctests.py +++ b/src/scanpy/_utils/_doctests.py @@ -19,7 +19,8 @@ def decorator(func: F) -> F: def doctest_skip(reason: str) -> Callable[[F], F]: """Mark function so doctest is skipped.""" if not reason: - raise ValueError("reason must not be empty") + msg = "reason must not be empty" + raise ValueError(msg) def decorator(func: F) -> F: func._doctest_skip_reason = reason diff --git a/src/scanpy/_utils/compute/is_constant.py b/src/scanpy/_utils/compute/is_constant.py index b5ef9c884a..c9fac4abf0 100644 --- a/src/scanpy/_utils/compute/is_constant.py +++ b/src/scanpy/_utils/compute/is_constant.py @@ -5,11 +5,11 @@ from numbers import Integral from typing import TYPE_CHECKING, TypeVar, overload +import numba import numpy as np -from numba import njit from scipy import sparse -from ..._compat import DaskArray +from ..._compat import DaskArray, njit if TYPE_CHECKING: from typing import Literal @@ -24,9 +24,11 @@ def _check_axis_supported(wrapped: C) -> C: def func(a, axis=None): if axis is not None: if not isinstance(axis, Integral): - raise TypeError("axis must be integer or None.") + msg = "axis must be integer or None." + raise TypeError(msg) if axis not in (0, 1): - raise NotImplementedError("We only support axis 0 and 1 at the moment") + msg = "We only support axis 0 and 1 at the moment" + raise NotImplementedError(msg) return wrapped(a, axis) return func @@ -81,7 +83,7 @@ def is_constant( def _(a: NDArray, axis: Literal[0, 1] | None = None) -> bool | NDArray[np.bool_]: # Should eventually support nd, not now. if axis is None: - return (a == a.flat[0]).all() + return bool((a == a.flat[0]).all()) if axis == 0: return _is_constant_rows(a.T) elif axis == 1: @@ -103,28 +105,24 @@ def _( else: return (a.data == 0).all() if axis == 1: - return _is_constant_csr_rows(a.data, a.indices, a.indptr, a.shape) + return _is_constant_csr_rows(a.data, a.indptr, a.shape) elif axis == 0: a = a.T.tocsr() - return _is_constant_csr_rows(a.data, a.indices, a.indptr, a.shape) + return _is_constant_csr_rows(a.data, a.indptr, a.shape) @njit def _is_constant_csr_rows( data: NDArray[np.number], - indices: NDArray[np.integer], indptr: NDArray[np.integer], shape: tuple[int, int], -): - N = len(indptr) - 1 - result = np.ones(N, dtype=np.bool_) - for i in range(N): +) -> NDArray[np.bool_]: + n = len(indptr) - 1 + result = np.ones(n, dtype=np.bool_) + for i in numba.prange(n): start = indptr[i] stop = indptr[i + 1] - if stop - start == shape[1]: - val = data[start] - else: - val = 0 + val = data[start] if stop - start == shape[1] else 0 for j in range(start, stop): if data[j] != val: result[i] = False @@ -142,10 +140,10 @@ def _( else: return (a.data == 0).all() if axis == 0: - return _is_constant_csr_rows(a.data, a.indices, a.indptr, a.shape[::-1]) + return _is_constant_csr_rows(a.data, a.indptr, a.shape[::-1]) elif axis == 1: a = a.T.tocsc() - return _is_constant_csr_rows(a.data, a.indices, a.indptr, a.shape[::-1]) + return _is_constant_csr_rows(a.data, a.indptr, a.shape[::-1]) @is_constant.register(DaskArray) @@ -154,4 +152,8 @@ def _(a: DaskArray, axis: Literal[0, 1] | None = None) -> bool | NDArray[np.bool v = a[tuple(0 for _ in range(a.ndim))].compute() return (a == v).all() # TODO: use overlapping blocks and reduction instead of `drop_axis` - return a.map_blocks(partial(is_constant, axis=axis), drop_axis=axis) + return a.map_blocks( + partial(is_constant, axis=axis), + drop_axis=axis, + meta=np.array([], dtype=a.dtype), + ) diff --git a/src/scanpy/cli.py b/src/scanpy/cli.py index 0ccdb8b33b..c934292dba 100644 --- a/src/scanpy/cli.py +++ b/src/scanpy/cli.py @@ -1,9 +1,9 @@ from __future__ import annotations -import collections.abc as cabc import os import sys from argparse import ArgumentParser, Namespace, _SubParsersAction +from collections.abc import MutableMapping from functools import lru_cache, partial from pathlib import Path from shutil import which @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from collections.abc import Generator, Mapping, Sequence + from collections.abc import Iterator, Mapping, Sequence from subprocess import CompletedProcess from typing import Any @@ -27,7 +27,7 @@ def __init__(self, *args, _command: str, _runargs: dict[str, Any], **kwargs): ) -class _CommandDelegator(cabc.MutableMapping): +class _CommandDelegator(MutableMapping): """\ Provide the ability to delegate, but don’t calculate the whole list until necessary @@ -64,7 +64,7 @@ def __delitem__(self, k: str) -> None: # These methods retrieve the command list or help with doing it - def __iter__(self) -> Generator[str, None, None]: + def __iter__(self) -> Iterator[str]: yield from self.parser_map yield from self.commands @@ -106,9 +106,9 @@ def parse_known_args( args: Sequence[str] | None = None, namespace: Namespace | None = None, ) -> tuple[Namespace, list[str]]: - assert ( - args is not None and namespace is None - ), "Only use DelegatingParser as subparser" + msg = "Only use DelegatingParser as subparser" + assert args is not None, msg + assert namespace is None, msg return Namespace(func=partial(run, [self.prog, *args], **self.cd.runargs)), [] diff --git a/src/scanpy/datasets/_datasets.py b/src/scanpy/datasets/_datasets.py index ccbc9a3bb3..8859de4d74 100644 --- a/src/scanpy/datasets/_datasets.py +++ b/src/scanpy/datasets/_datasets.py @@ -9,7 +9,7 @@ from anndata import AnnData from .. import _utils -from .._compat import old_positionals +from .._compat import deprecated, old_positionals from .._settings import settings from .._utils._doctests import doctest_internet, doctest_needs from ..readwrite import read, read_visium @@ -18,7 +18,7 @@ if TYPE_CHECKING: from typing import Literal - from .._utils import AnyRandom + from .._compat import _LegacyRandom VisiumSampleID = Literal[ "V1_Breast_Cancer_Block_A_Section_1", @@ -63,7 +63,7 @@ def blobs( n_centers: int = 5, cluster_std: float = 1.0, n_observations: int = 640, - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, ) -> AnnData: """\ Gaussian Blobs. @@ -219,7 +219,7 @@ def moignard15() -> AnnData: } # annotate each observation/cell adata.obs["exp_groups"] = [ - next(gname for gname in groups.keys() if sname.startswith(gname)) + next(gname for gname in groups if sname.startswith(gname)) for sname in adata.obs_names ] # fix the order and colors of names in "groups" @@ -509,6 +509,7 @@ def _download_visium_dataset( return sample_dir +@deprecated("Use `squidpy.datasets.visium` instead.") @doctest_internet @check_datasetdir_exists def visium_sge( @@ -519,6 +520,9 @@ def visium_sge( """\ Processed Visium Spatial Gene Expression data from 10x Genomics’ database. + .. deprecated:: 1.11.0 + Use :func:`squidpy.datasets.visium` instead. + The database_ can be browsed online to find the ``sample_id`` you want. .. _database: https://support.10xgenomics.com/spatial-gene-expression/datasets diff --git a/src/scanpy/datasets/_ebi_expression_atlas.py b/src/scanpy/datasets/_ebi_expression_atlas.py index 9f3bcb81ad..b7e1886e71 100644 --- a/src/scanpy/datasets/_ebi_expression_atlas.py +++ b/src/scanpy/datasets/_ebi_expression_atlas.py @@ -65,10 +65,7 @@ def read_mtx_from_stream(stream: BinaryIO) -> sparse.csr_matrix: n, m, _ = (int(x) for x in curline[:-1].split(b" ")) max_int32 = np.iinfo(np.int32).max - if n > max_int32 or m > max_int32: - coord_dtype = np.int64 - else: - coord_dtype = np.int32 + coord_dtype = np.int64 if n > max_int32 or m > max_int32 else np.int32 data = pd.read_csv( stream, diff --git a/src/scanpy/experimental/pp/_highly_variable_genes.py b/src/scanpy/experimental/pp/_highly_variable_genes.py index a8f8929e93..7ad9f36bd7 100644 --- a/src/scanpy/experimental/pp/_highly_variable_genes.py +++ b/src/scanpy/experimental/pp/_highly_variable_genes.py @@ -12,6 +12,7 @@ from anndata import AnnData from scanpy import logging as logg +from scanpy._compat import njit from scanpy._settings import Verbosity, settings from scanpy._utils import _doc_params, check_nonnegative_integers, view_to_actual from scanpy.experimental._docs import ( @@ -32,7 +33,7 @@ from numpy.typing import NDArray -@nb.njit(parallel=True) +@njit def _calculate_res_sparse( indptr: NDArray[np.integer], index: NDArray[np.integer], @@ -92,7 +93,7 @@ def clac_clipped_res_sparse(gene: int, cell: int, value: np.float64) -> np.float return residuals -@nb.njit(parallel=True) +@njit def _calculate_res_dense( matrix, *, @@ -158,7 +159,8 @@ def _highly_variable_pearson_residuals( if theta <= 0: # TODO: would "underdispersion" with negative theta make sense? # then only theta=0 were undefined.. - raise ValueError("Pearson residuals require theta > 0") + msg = "Pearson residuals require theta > 0" + raise ValueError(msg) # prepare clipping if batch_key is None: @@ -184,7 +186,8 @@ def _highly_variable_pearson_residuals( n = X_batch.shape[0] clip = np.sqrt(n) if clip < 0: - raise ValueError("Pearson residuals require `clip>=0` or `clip=None`.") + msg = "Pearson residuals require `clip>=0` or `clip=None`." + raise ValueError(msg) if sp_sparse.issparse(X_batch): X_batch = X_batch.tocsc() @@ -377,17 +380,19 @@ def highly_variable_genes( logg.info("extracting highly variable genes") if not isinstance(adata, AnnData): - raise ValueError( + msg = ( "`pp.highly_variable_genes` expects an `AnnData` argument, " "pass `inplace=False` if you want to return a `pd.DataFrame`." ) + raise ValueError(msg) if flavor == "pearson_residuals": if n_top_genes is None: - raise ValueError( + msg = ( "`pp.highly_variable_genes` requires the argument `n_top_genes`" " for `flavor='pearson_residuals'`" ) + raise ValueError(msg) return _highly_variable_pearson_residuals( adata, layer=layer, @@ -401,6 +406,5 @@ def highly_variable_genes( inplace=inplace, ) else: - raise ValueError( - "This is an experimental API and only `flavor=pearson_residuals` is available." - ) + msg = "This is an experimental API and only `flavor=pearson_residuals` is available." + raise ValueError(msg) diff --git a/src/scanpy/experimental/pp/_normalization.py b/src/scanpy/experimental/pp/_normalization.py index 7041caeb87..ef3d0311d7 100644 --- a/src/scanpy/experimental/pp/_normalization.py +++ b/src/scanpy/experimental/pp/_normalization.py @@ -35,20 +35,22 @@ from ..._utils import Empty -def _pearson_residuals(X, theta, clip, check_values, copy: bool = False): +def _pearson_residuals(X, theta, clip, check_values, *, copy: bool = False): X = X.copy() if copy else X # check theta if theta <= 0: # TODO: would "underdispersion" with negative theta make sense? # then only theta=0 were undefined.. - raise ValueError("Pearson residuals require theta > 0") + msg = "Pearson residuals require theta > 0" + raise ValueError(msg) # prepare clipping if clip is None: n = X.shape[0] clip = np.sqrt(n) if clip < 0: - raise ValueError("Pearson residuals require `clip>=0` or `clip=None`.") + msg = "Pearson residuals require `clip>=0` or `clip=None`." + raise ValueError(msg) if check_values and not check_nonnegative_integers(X): warn( @@ -128,7 +130,8 @@ def normalize_pearson_residuals( if copy: if not inplace: - raise ValueError("`copy=True` cannot be used with `inplace=False`.") + msg = "`copy=True` cannot be used with `inplace=False`." + raise ValueError(msg) adata = adata.copy() view_to_actual(adata) diff --git a/src/scanpy/external/exporting.py b/src/scanpy/external/exporting.py index 8655a7ce9d..8379720ea6 100644 --- a/src/scanpy/external/exporting.py +++ b/src/scanpy/external/exporting.py @@ -86,7 +86,8 @@ def spring_project( neighbors_key = "neighbors" if neighbors_key not in adata.uns: - raise ValueError("Run `sc.pp.neighbors` first.") + msg = "Run `sc.pp.neighbors` first." + raise ValueError(msg) # check that requested 2-D embedding has been generated if embedding_method not in adata.obsm_keys(): @@ -101,9 +102,8 @@ def spring_project( + adata.uns[embedding_method]["params"]["layout"] ) else: - raise ValueError( - f"Run the specified embedding method `{embedding_method}` first." - ) + msg = f"Run the specified embedding method `{embedding_method}` first." + raise ValueError(msg) coords = adata.obsm[embedding_method] @@ -316,8 +316,8 @@ def write_hdf5_cells(E, filename): hf.close() -def write_sparse_npz(E, filename, compressed=False): - '''SPRING standard: filename = main_spring_dir + "/counts_norm.npz"''' +def write_sparse_npz(E, filename, *, compressed: bool = False): + """SPRING standard: filename = f"{main_spring_dir}/counts_norm.npz".""" E = E.tocsc() scipy.sparse.save_npz(filename, E, compressed=compressed) @@ -345,8 +345,8 @@ def _write_color_tracks(ctracks, fname): def _frac_to_hex(frac): - rgb = tuple(np.array(np.array(plt.cm.jet(frac)[:3]) * 255, dtype=int)) - return "#{:02x}{:02x}{:02x}".format(*rgb) + r, g, b = tuple(np.array(np.array(plt.cm.jet(frac)[:3]) * 255, dtype=int)) + return f"#{r:02x}{g:02x}{b:02x}" def _get_color_stats_genes(color_stats, E, gene_list): diff --git a/src/scanpy/external/pl.py b/src/scanpy/external/pl.py index 662bc88eb3..ce305e2f06 100644 --- a/src/scanpy/external/pl.py +++ b/src/scanpy/external/pl.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib from typing import TYPE_CHECKING import matplotlib.pyplot as plt @@ -197,9 +198,8 @@ def sam( try: dt = adata.obsm[projection] except KeyError: - raise ValueError( - "Please create a projection first using run_umap or run_tsne" - ) + msg = "Please create a projection first using run_umap or run_tsne" + raise ValueError(msg) else: dt = projection @@ -214,12 +214,10 @@ def sam( return axes if isinstance(c, str): - try: + with contextlib.suppress(KeyError): c = np.array(list(adata.obs[c])) - except KeyError: - pass - if isinstance(c[0], (str, np.str_)) and isinstance(c, (np.ndarray, list)): + if isinstance(c[0], str | np.str_) and isinstance(c, np.ndarray | list): import samalg.utilities as ut i = ut.convert_annotations(c) @@ -239,7 +237,7 @@ def sam( cbar = plt.colorbar(cax, ax=axes, ticks=ui) cbar.ax.set_yticklabels(c[ai]) else: - if not isinstance(c, (np.ndarray, list)): + if not isinstance(c, np.ndarray | list): colorbar = False i = c diff --git a/src/scanpy/external/pp/_bbknn.py b/src/scanpy/external/pp/_bbknn.py index 4a7d5e7c9b..ee280cc824 100644 --- a/src/scanpy/external/pp/_bbknn.py +++ b/src/scanpy/external/pp/_bbknn.py @@ -6,7 +6,7 @@ from ..._utils._doctests import doctest_needs if TYPE_CHECKING: - from typing import Callable + from collections.abc import Callable from anndata import AnnData from sklearn.metrics import DistanceMetric @@ -133,7 +133,8 @@ def bbknn( try: from bbknn import bbknn except ImportError: - raise ImportError("Please install bbknn: `pip install bbknn`.") + msg = "Please install bbknn: `pip install bbknn`." + raise ImportError(msg) return bbknn( adata=adata, batch_key=batch_key, diff --git a/src/scanpy/external/pp/_dca.py b/src/scanpy/external/pp/_dca.py index 14842c8071..20a97034b8 100644 --- a/src/scanpy/external/pp/_dca.py +++ b/src/scanpy/external/pp/_dca.py @@ -11,7 +11,7 @@ from anndata import AnnData - from ..._utils import AnyRandom + from ..._compat import _LegacyRandom _AEType = Literal["zinb-conddisp", "zinb", "nb-conddisp", "nb"] @@ -62,7 +62,7 @@ def dca( early_stop: int = 15, batch_size: int = 32, optimizer: str = "RMSprop", - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, threads: int | None = None, learning_rate: float | None = None, verbose: bool = False, @@ -181,7 +181,8 @@ def dca( try: from dca.api import dca except ImportError: - raise ImportError("Please install dca package (>= 0.2.1) via `pip install dca`") + msg = "Please install dca package (>= 0.2.1) via `pip install dca`" + raise ImportError(msg) return dca( adata, diff --git a/src/scanpy/external/pp/_harmony_integrate.py b/src/scanpy/external/pp/_harmony_integrate.py index 27c4d2ac8f..824309f817 100644 --- a/src/scanpy/external/pp/_harmony_integrate.py +++ b/src/scanpy/external/pp/_harmony_integrate.py @@ -4,6 +4,7 @@ from __future__ import annotations +from collections.abc import Sequence # noqa: TCH003 from typing import TYPE_CHECKING import numpy as np @@ -19,7 +20,7 @@ @doctest_needs("harmonypy") def harmony_integrate( adata: AnnData, - key: str, + key: str | Sequence[str], *, basis: str = "X_pca", adjusted_basis: str = "X_pca_harmony", @@ -42,7 +43,9 @@ def harmony_integrate( The annotated data matrix. key The name of the column in ``adata.obs`` that differentiates - among experiments/batches. + among experiments/batches. To integrate over two or more covariates, + you can pass multiple column names as a list. See ``vars_use`` + parameter of the ``harmonypy`` package for more details. basis The name of the field in ``adata.obsm`` where the PCA table is stored. Defaults to ``'X_pca'``, which is the default for @@ -88,7 +91,8 @@ def harmony_integrate( try: import harmonypy except ImportError: - raise ImportError("\nplease install harmonypy:\n\n\tpip install harmonypy") + msg = "\nplease install harmonypy:\n\n\tpip install harmonypy" + raise ImportError(msg) X = adata.obsm[basis].astype(np.float64) diff --git a/src/scanpy/external/pp/_hashsolo.py b/src/scanpy/external/pp/_hashsolo.py index 256c863eee..dcb44239b1 100644 --- a/src/scanpy/external/pp/_hashsolo.py +++ b/src/scanpy/external/pp/_hashsolo.py @@ -352,15 +352,15 @@ def hashsolo( adata = adata.copy() if not inplace else adata data = adata.obs[cell_hashing_columns].values if not check_nonnegative_integers(data): - raise ValueError("Cell hashing counts must be non-negative") + msg = "Cell hashing counts must be non-negative" + raise ValueError(msg) if (number_of_noise_barcodes is not None) and ( number_of_noise_barcodes >= len(cell_hashing_columns) ): - raise ValueError( - "number_of_noise_barcodes must be at least one less \ + msg = "number_of_noise_barcodes must be at least one less \ than the number of samples you have as determined by the number of \ cell_hashing_columns you've given as input " - ) + raise ValueError(msg) num_of_cells = adata.shape[0] results = pd.DataFrame( np.zeros((num_of_cells, 6)), diff --git a/src/scanpy/external/pp/_magic.py b/src/scanpy/external/pp/_magic.py index 983db18fcf..12e93f1a8e 100644 --- a/src/scanpy/external/pp/_magic.py +++ b/src/scanpy/external/pp/_magic.py @@ -4,6 +4,7 @@ from __future__ import annotations +from types import NoneType from typing import TYPE_CHECKING from packaging.version import Version @@ -18,7 +19,7 @@ from anndata import AnnData - from ..._utils import AnyRandom + from ..._compat import _LegacyRandom MIN_VERSION = "2.0" @@ -35,7 +36,7 @@ def magic( n_pca: int | None = 100, solver: Literal["exact", "approximate"] = "exact", knn_dist: str = "euclidean", - random_state: AnyRandom = None, + random_state: _LegacyRandom = None, n_jobs: int | None = None, verbose: bool = False, copy: bool | None = None, @@ -141,34 +142,38 @@ def magic( try: from magic import MAGIC, __version__ except ImportError: - raise ImportError( + msg = ( "Please install magic package via `pip install --user " "git+git://github.com/KrishnaswamyLab/MAGIC.git#subdirectory=python`" ) + raise ImportError(msg) else: if Version(__version__) < Version(MIN_VERSION): - raise ImportError( + msg = ( "scanpy requires magic-impute >= " f"v{MIN_VERSION} (detected: v{__version__}). " "Please update magic package via `pip install --user " "--upgrade magic-impute`" ) + raise ImportError(msg) start = logg.info("computing MAGIC") - all_or_pca = isinstance(name_list, (str, type(None))) + all_or_pca = isinstance(name_list, str | NoneType) if all_or_pca and name_list not in {"all_genes", "pca_only", None}: - raise ValueError( + msg = ( "Invalid string value for `name_list`: " "Only `'all_genes'` and `'pca_only'` are allowed." ) + raise ValueError(msg) if copy is None: copy = not all_or_pca elif not all_or_pca and not copy: - raise ValueError( + msg = ( "Can only perform MAGIC in-place with `name_list=='all_genes' or " f"`name_list=='pca_only'` (got {name_list}). Consider setting " "`copy=True`" ) + raise ValueError(msg) adata = adata.copy() if copy else adata n_jobs = settings.n_jobs if n_jobs is None else n_jobs diff --git a/src/scanpy/external/pp/_mnn_correct.py b/src/scanpy/external/pp/_mnn_correct.py index a497189913..518686dc75 100644 --- a/src/scanpy/external/pp/_mnn_correct.py +++ b/src/scanpy/external/pp/_mnn_correct.py @@ -133,10 +133,8 @@ def mnn_correct( import mnnpy from mnnpy import mnn_correct except ImportError: - raise ImportError( - "Please install the package mnnpy " - "(https://github.com/chriscainx/mnnpy). " - ) + msg = "Please install the package mnnpy (https://github.com/chriscainx/mnnpy). " + raise ImportError(msg) n_jobs = settings.n_jobs if n_jobs is None else n_jobs diff --git a/src/scanpy/external/pp/_scanorama_integrate.py b/src/scanpy/external/pp/_scanorama_integrate.py index ca847f8351..c5fb2683b4 100644 --- a/src/scanpy/external/pp/_scanorama_integrate.py +++ b/src/scanpy/external/pp/_scanorama_integrate.py @@ -111,7 +111,8 @@ def scanorama_integrate( try: import scanorama except ImportError: - raise ImportError("\nplease install Scanorama:\n\n\tpip install scanorama") + msg = "\nplease install Scanorama:\n\n\tpip install scanorama" + raise ImportError(msg) # Get batch indices in linear time. curr_batch = None @@ -123,7 +124,8 @@ def scanorama_integrate( curr_batch = batch_name if batch_name in batch_names: # Contiguous batches important for preserving cell order. - raise ValueError("Detected non-contiguous batches.") + msg = "Detected non-contiguous batches." + raise ValueError(msg) batch_names.append(batch_name) # Preserve name order. name2idx[batch_name] = [] name2idx[batch_name].append(idx) diff --git a/src/scanpy/external/tl/_harmony_timeseries.py b/src/scanpy/external/tl/_harmony_timeseries.py index d1746af45a..de3f8cde26 100644 --- a/src/scanpy/external/tl/_harmony_timeseries.py +++ b/src/scanpy/external/tl/_harmony_timeseries.py @@ -140,13 +140,15 @@ def harmony_timeseries( try: import harmony except ImportError: - raise ImportError("\nplease install harmony:\n\n\tpip install harmonyTS") + msg = "\nplease install harmony:\n\n\tpip install harmonyTS" + raise ImportError(msg) adata = adata.copy() if copy else adata logg.info("Harmony augmented affinity matrix") if adata.obs[tp].dtype.name != "category": - raise ValueError(f"{tp!r} column does not contain Categorical data") + msg = f"{tp!r} column does not contain Categorical data" + raise ValueError(msg) timepoints = adata.obs[tp].cat.categories.tolist() timepoint_connections = pd.DataFrame(np.array([timepoints[:-1], timepoints[1:]]).T) diff --git a/src/scanpy/external/tl/_palantir.py b/src/scanpy/external/tl/_palantir.py index 854301466a..eb060bbbe0 100644 --- a/src/scanpy/external/tl/_palantir.py +++ b/src/scanpy/external/tl/_palantir.py @@ -340,4 +340,5 @@ def _check_import(): try: import palantir # noqa: F401 except ImportError: - raise ImportError("\nplease install palantir:\n\tpip install palantir") + msg = "\nplease install palantir:\n\tpip install palantir" + raise ImportError(msg) diff --git a/src/scanpy/external/tl/_phate.py b/src/scanpy/external/tl/_phate.py index 78b50327a9..91d8191e60 100644 --- a/src/scanpy/external/tl/_phate.py +++ b/src/scanpy/external/tl/_phate.py @@ -16,7 +16,7 @@ from anndata import AnnData - from ..._utils import AnyRandom + from ..._compat import _LegacyRandom @old_positionals( @@ -49,7 +49,7 @@ def phate( mds_dist: str = "euclidean", mds: Literal["classic", "metric", "nonmetric"] = "metric", n_jobs: int | None = None, - random_state: AnyRandom = None, + random_state: _LegacyRandom = None, verbose: bool | int | None = None, copy: bool = False, **kwargs, @@ -154,10 +154,11 @@ def phate( try: import phate except ImportError: - raise ImportError( + msg = ( "You need to install the package `phate`: please run `pip install " "--user phate` in a terminal." ) + raise ImportError(msg) X_phate = phate.PHATE( n_components=n_components, k=k, @@ -179,6 +180,6 @@ def phate( logg.info( " finished", time=start, - deep=("added\n" " 'X_phate', PHATE coordinates (adata.obsm)"), + deep=("added\n 'X_phate', PHATE coordinates (adata.obsm)"), ) return adata if copy else None diff --git a/src/scanpy/external/tl/_phenograph.py b/src/scanpy/external/tl/_phenograph.py index 8cecfa7276..fdc3973771 100644 --- a/src/scanpy/external/tl/_phenograph.py +++ b/src/scanpy/external/tl/_phenograph.py @@ -226,17 +226,19 @@ def phenograph( assert phenograph.__version__ >= "1.5.3" except (ImportError, AssertionError, AttributeError): - raise ImportError( + msg = ( "please install the latest release of phenograph:\n\t" "pip install -U PhenoGraph" ) + raise ImportError(msg) if isinstance(data, AnnData): adata = data try: data = data.obsm["X_pca"] except KeyError: - raise KeyError("Please run `sc.pp.pca` on `data` and try again!") + msg = "Please run `sc.pp.pca` on `data` and try again!" + raise KeyError(msg) else: adata = None copy = True @@ -244,8 +246,8 @@ def phenograph( comm_key = ( f"pheno_{clustering_algo}" if clustering_algo in ["louvain", "leiden"] else "" ) - ig_key = "pheno_{}_ig".format("jaccard" if jaccard else "gaussian") - q_key = "pheno_{}_q".format("jaccard" if jaccard else "gaussian") + ig_key = f"pheno_{'jaccard' if jaccard else 'gaussian'}_ig" + q_key = f"pheno_{'jaccard' if jaccard else 'gaussian'}_q" communities, graph, Q = phenograph.cluster( data=data, diff --git a/src/scanpy/external/tl/_pypairs.py b/src/scanpy/external/tl/_pypairs.py index 821a060a4d..2db98ff9a7 100644 --- a/src/scanpy/external/tl/_pypairs.py +++ b/src/scanpy/external/tl/_pypairs.py @@ -13,12 +13,11 @@ if TYPE_CHECKING: from collections.abc import Collection, Mapping - from typing import Union import pandas as pd from anndata import AnnData - Genes = Collection[Union[str, int, bool]] + Genes = Collection[str | int | bool] @doctest_needs("pypairs") @@ -154,8 +153,10 @@ def _check_import(): try: import pypairs except ImportError: - raise ImportError("You need to install the package `pypairs`.") + msg = "You need to install the package `pypairs`." + raise ImportError(msg) min_version = Version("3.0.9") if Version(pypairs.__version__) < min_version: - raise ImportError(f"Please only use `pypairs` >= {min_version}") + msg = f"Please only use `pypairs` >= {min_version}" + raise ImportError(msg) diff --git a/src/scanpy/external/tl/_sam.py b/src/scanpy/external/tl/_sam.py index ebf3156b9a..8daa2c0091 100644 --- a/src/scanpy/external/tl/_sam.py +++ b/src/scanpy/external/tl/_sam.py @@ -211,12 +211,13 @@ def sam( try: from samalg import SAM except ImportError: - raise ImportError( + msg = ( "\nplease install sam-algorithm: \n\n" "\tgit clone git://github.com/atarashansky/self-assembling-manifold.git\n" "\tcd self-assembling-manifold\n" "\tpip install ." ) + raise ImportError(msg) logg.info("Self-assembling manifold") diff --git a/src/scanpy/external/tl/_trimap.py b/src/scanpy/external/tl/_trimap.py index 9146e79b84..122a4792b7 100644 --- a/src/scanpy/external/tl/_trimap.py +++ b/src/scanpy/external/tl/_trimap.py @@ -108,7 +108,8 @@ def trimap( try: from trimap import TRIMAP except ImportError: - raise ImportError("\nplease install trimap: \n\n\tsudo pip install trimap") + msg = "\nplease install trimap: \n\n\tsudo pip install trimap" + raise ImportError(msg) adata = adata.copy() if copy else adata start = logg.info("computing TriMap") adata = adata.copy() if copy else adata @@ -121,10 +122,11 @@ def trimap( else: X = adata.X if scp.issparse(X): - raise ValueError( + msg = ( "trimap currently does not support sparse matrices. Please" "use a dense matrix or apply pca first." ) + raise ValueError(msg) logg.warning("`X_pca` not found. Run `sc.pp.pca` first for speedup.") X_trimap = TRIMAP( n_dims=n_components, diff --git a/src/scanpy/external/tl/_wishbone.py b/src/scanpy/external/tl/_wishbone.py index 7e78d2eec6..3b85ae14a1 100644 --- a/src/scanpy/external/tl/_wishbone.py +++ b/src/scanpy/external/tl/_wishbone.py @@ -1,6 +1,6 @@ from __future__ import annotations -import collections.abc as cabc +from collections.abc import Collection from typing import TYPE_CHECKING import numpy as np @@ -11,7 +11,7 @@ from ..._utils._doctests import doctest_needs if TYPE_CHECKING: - from collections.abc import Collection, Iterable + from collections.abc import Iterable from anndata import AnnData @@ -104,18 +104,18 @@ def wishbone( try: from wishbone.core import wishbone as c_wishbone except ImportError: - raise ImportError( - "\nplease install wishbone:\n\n\thttps://github.com/dpeerlab/wishbone" - ) + msg = "\nplease install wishbone:\n\n\thttps://github.com/dpeerlab/wishbone" + raise ImportError(msg) # Start cell index s = np.where(adata.obs_names == start_cell)[0] if len(s) == 0: - raise RuntimeError( + msg = ( f"Start cell {start_cell} not found in data. " "Please rerun with correct start cell." ) - if isinstance(num_waypoints, cabc.Collection): + raise RuntimeError(msg) + if isinstance(num_waypoints, Collection): diff = np.setdiff1d(num_waypoints, adata.obs.index) if diff.size > 0: logging.warning( @@ -124,10 +124,11 @@ def wishbone( ) num_waypoints = diff.tolist() elif num_waypoints > adata.shape[0]: - raise RuntimeError( + msg = ( "num_waypoints parameter is higher than the number of cells in the " "dataset. Please select a smaller number" ) + raise RuntimeError(msg) s = s[0] # Run the algorithm diff --git a/src/scanpy/get/_aggregated.py b/src/scanpy/get/_aggregated.py index eceaa40fe2..94bf202b69 100644 --- a/src/scanpy/get/_aggregated.py +++ b/src/scanpy/get/_aggregated.py @@ -1,26 +1,26 @@ from __future__ import annotations from functools import singledispatch -from typing import TYPE_CHECKING, Literal, get_args +from typing import TYPE_CHECKING, Literal import numpy as np import pandas as pd from anndata import AnnData, utils from scipy import sparse +from sklearn.utils.sparsefuncs import csc_median_axis_0 -from .._utils import _resolve_axis +from .._utils import _resolve_axis, get_literal_vals from .get import _check_mask if TYPE_CHECKING: from collections.abc import Collection, Iterable - from typing import Union from numpy.typing import NDArray - Array = Union[np.ndarray, sparse.csc_matrix, sparse.csr_matrix] + Array = np.ndarray | sparse.csc_matrix | sparse.csr_matrix -# Used with get_args -AggType = Literal["count_nonzero", "mean", "sum", "var"] +# Used with get_literal_vals +AggType = Literal["count_nonzero", "mean", "sum", "var", "median"] class Aggregate: @@ -138,8 +138,29 @@ def mean_var(self, dof: int = 1) -> tuple[np.ndarray, np.ndarray]: var_ *= (group_counts / (group_counts - dof))[:, np.newaxis] return mean_, var_ + def median(self) -> Array: + """\ + Compute the median per feature per group of observations. + + Returns + ------- + Array of median. + """ + + medians = [] + for group in np.unique(self.groupby.codes): + group_mask = self.groupby.codes == group + group_data = self.data[group_mask] + if sparse.issparse(group_data): + if group_data.format != "csc": + group_data = group_data.tocsc() + medians.append(csc_median_axis_0(group_data)) + else: + medians.append(np.median(group_data, axis=0)) + return np.array(medians) -def _power(X: Array, power: float | int) -> Array: + +def _power(X: Array, power: float) -> Array: """\ Generate elementwise power of a matrix. @@ -235,26 +256,29 @@ def aggregate( Note that this filters out any combination of groups that wasn't present in the original data. """ if not isinstance(adata, AnnData): - raise NotImplementedError( + msg = ( "sc.get.aggregate is currently only implemented for AnnData input, " f"was passed {type(adata)}." ) + raise NotImplementedError(msg) if axis is None: axis = 1 if varm else 0 axis, axis_name = _resolve_axis(axis) - if mask is not None: - mask = _check_mask(adata, mask, axis_name) + mask = _check_mask(adata, mask, axis_name) data = adata.X if sum(p is not None for p in [varm, obsm, layer]) > 1: - raise TypeError("Please only provide one (or none) of varm, obsm, or layer") + msg = "Please only provide one (or none) of varm, obsm, or layer" + raise TypeError(msg) if varm is not None: if axis != 1: - raise ValueError("varm can only be used when axis is 1") + msg = "varm can only be used when axis is 1" + raise ValueError(msg) data = adata.varm[varm] elif obsm is not None: if axis != 0: - raise ValueError("obsm can only be used when axis is 0") + msg = "obsm can only be used when axis is 0" + raise ValueError(msg) data = adata.obsm[obsm] elif layer is not None: data = adata.layers[layer] @@ -304,7 +328,8 @@ def _aggregate( mask: NDArray[np.bool_] | None = None, dof: int = 1, ): - raise NotImplementedError(f"Data type {type(data)} not supported for aggregation") + msg = f"Data type {type(data)} not supported for aggregation" + raise NotImplementedError(msg) @_aggregate.register(pd.DataFrame) @@ -326,8 +351,9 @@ def aggregate_array( result = {} funcs = set([func] if isinstance(func, str) else func) - if unknown := funcs - set(get_args(AggType)): - raise ValueError(f"func {unknown} is not one of {get_args(AggType)}") + if unknown := funcs - get_literal_vals(AggType): + msg = f"func {unknown} is not one of {get_literal_vals(AggType)}" + raise ValueError(msg) if "sum" in funcs: # sum is calculated separately from the rest agg = groupby.sum() @@ -343,7 +369,9 @@ def aggregate_array( result["var"] = var_ if "mean" in funcs: result["mean"] = mean_ - + if "median" in funcs: + agg = groupby.median() + result["median"] = agg return result diff --git a/src/scanpy/get/get.py b/src/scanpy/get/get.py index 936e5c3946..abfa51d1f9 100644 --- a/src/scanpy/get/get.py +++ b/src/scanpy/get/get.py @@ -2,21 +2,26 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypeVar import numpy as np import pandas as pd from anndata import AnnData +from numpy.typing import NDArray from packaging.version import Version from scipy.sparse import spmatrix if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Collection, Iterable from typing import Any, Literal from anndata._core.sparse_dataset import BaseCompressedSparseDataset from anndata._core.views import ArrayView - from numpy.typing import NDArray + from scipy.sparse import csc_matrix, csr_matrix + + from .._compat import DaskArray + + CSMatrix = csr_matrix | csc_matrix # -------------------------------------------------------------------------------- # Plotting data helpers @@ -116,15 +121,12 @@ def _check_indices( alt_index: pd.Index, *, dim: Literal["obs", "var"], - keys: list[str], + keys: Iterable[str], alias_index: pd.Index | None = None, use_raw: bool = False, ) -> tuple[list[str], list[str], list[str]]: """Common logic for checking indices for obs_df and var_df.""" - if use_raw: - alt_repr = "adata.raw" - else: - alt_repr = "adata" + alt_repr = "adata.raw" if use_raw else "adata" alt_dim = ("obs", "var")[dim == "obs"] @@ -147,46 +149,47 @@ def _check_indices( # be further duplicated when selecting them. if not dim_df.columns.is_unique: dup_cols = dim_df.columns[dim_df.columns.duplicated()].tolist() - raise ValueError( + msg = ( f"adata.{dim} contains duplicated columns. Please rename or remove " "these columns first.\n`" f"Duplicated columns {dup_cols}" ) + raise ValueError(msg) if not alt_index.is_unique: - raise ValueError( + msg = ( f"{alt_repr}.{alt_dim}_names contains duplicated items\n" f"Please rename these {alt_dim} names first for example using " f"`adata.{alt_dim}_names_make_unique()`" ) + raise ValueError(msg) # use only unique keys, otherwise duplicated keys will # further duplicate when reordering the keys later in the function - for key in np.unique(keys): + for key in dict.fromkeys(keys): if key in dim_df.columns: col_keys.append(key) if key in alt_names.index: - raise KeyError( - f"The key '{key}' is found in both adata.{dim} and {alt_repr}.{alt_search_repr}." - ) + msg = f"The key {key!r} is found in both adata.{dim} and {alt_repr}.{alt_search_repr}." + raise KeyError(msg) elif key in alt_names.index: val = alt_names[key] if isinstance(val, pd.Series): # while var_names must be unique, adata.var[gene_symbols] does not # It's still ambiguous to refer to a duplicated entry though. assert alias_index is not None - raise KeyError( - f"Found duplicate entries for '{key}' in {alt_repr}.{alt_search_repr}." - ) + msg = f"Found duplicate entries for {key!r} in {alt_repr}.{alt_search_repr}." + raise KeyError(msg) index_keys.append(val) index_aliases.append(key) else: not_found.append(key) if len(not_found) > 0: - raise KeyError( - f"Could not find keys '{not_found}' in columns of `adata.{dim}` or in" + msg = ( + f"Could not find keys {not_found!r} in columns of `adata.{dim}` or in" f" {alt_repr}.{alt_search_repr}." ) + raise KeyError(msg) return col_keys, index_keys, index_aliases @@ -194,7 +197,8 @@ def _check_indices( def _get_array_values( X, dim_names: pd.Index, - keys: list[str], + keys: Iterable[str], + *, axis: Literal[0, 1], backed: bool, ): @@ -223,7 +227,7 @@ def _get_array_values( def obs_df( adata: AnnData, - keys: Iterable[str] = (), + keys: Collection[str] = (), obsm_keys: Iterable[tuple[str, int]] = (), *, layer: str | None = None, @@ -240,7 +244,7 @@ def obs_df( keys Keys from either `.var_names`, `.var[gene_symbols]`, or `.obs.columns`. obsm_keys - Tuple of `(key from obsm, column index of obsm[key])`. + Tuples of `(key from obsm, column index of obsm[key])`. layer Layer of `adata` to use as expression values. gene_symbols @@ -280,17 +284,16 @@ def obs_df( >>> grouped = genedf.groupby("louvain", observed=True) >>> mean, var = grouped.mean(), grouped.var() """ + if isinstance(keys, str): + keys = [keys] if use_raw: - assert ( - layer is None - ), "Cannot specify use_raw=True and a layer at the same time." + assert layer is None, ( + "Cannot specify use_raw=True and a layer at the same time." + ) var = adata.raw.var else: var = adata.var - if gene_symbols is not None: - alias_index = pd.Index(var[gene_symbols]) - else: - alias_index = None + alias_index = pd.Index(var[gene_symbols]) if gene_symbols is not None else None obs_cols, var_idx_keys, var_symbols = _check_indices( adata.obs, @@ -341,7 +344,7 @@ def obs_df( def var_df( adata: AnnData, - keys: Iterable[str] = (), + keys: Collection[str] = (), varm_keys: Iterable[tuple[str, int]] = (), *, layer: str | None = None, @@ -356,7 +359,7 @@ def var_df( keys Keys from either `.obs_names`, or `.var.columns`. varm_keys - Tuple of `(key from varm, column index of varm[key])`. + Tuples of `(key from varm, column index of varm[key])`. layer Layer of `adata` to use as expression values. @@ -366,6 +369,8 @@ def var_df( and `varm_keys`. """ # Argument handling + if isinstance(keys, str): + keys = [keys] var_cols, obs_idx_keys, _ = _check_indices( adata.var, adata.obs_names, dim="var", keys=keys ) @@ -426,7 +431,8 @@ def _get_obs_rep( """ # https://github.com/scverse/scanpy/issues/1546 if not isinstance(use_raw, bool): - raise TypeError(f"use_raw expected to be bool, was {type(use_raw)}.") + msg = f"use_raw expected to be bool, was {type(use_raw)}." + raise TypeError(msg) is_layer = layer is not None is_raw = use_raw is not False @@ -444,10 +450,11 @@ def _get_obs_rep( return adata.obsm[obsm] if is_obsp: return adata.obsp[obsp] - raise AssertionError( + msg = ( "That was unexpected. Please report this bug at:\n\n\t" "https://github.com/scverse/scanpy/issues" ) + raise AssertionError(msg) def _set_obs_rep( @@ -479,17 +486,23 @@ def _set_obs_rep( elif is_obsp: adata.obsp[obsp] = val else: - assert False, ( - "That was unexpected. Please report this bug at:\n\n\t" - " https://github.com/scverse/scanpy/issues" + msg = ( + "That was unexpected. Please report this bug at:\n\n" + "\thttps://github.com/scverse/scanpy/issues" ) + raise AssertionError(msg) + + +M = TypeVar("M", bound=NDArray[np.bool_] | NDArray[np.floating] | pd.Series | None) def _check_mask( - data: AnnData | np.ndarray, - mask: NDArray[np.bool_] | str, + data: AnnData | np.ndarray | CSMatrix | DaskArray, + mask: str | M, dim: Literal["obs", "var"], -) -> NDArray[np.bool_]: # Could also be a series, but should be one or the other + *, + allow_probabilities: bool = False, +) -> M: # Could also be a series, but should be one or the other """ Validate mask argument Params @@ -497,30 +510,45 @@ def _check_mask( data Annotated data matrix or numpy array. mask - The mask. Either an appropriatley sized boolean array, or name of a column which will be used to mask. + Mask (or probabilities if `allow_probabilities=True`). + Either an appropriatley sized array, or name of a column. dim The dimension being masked. + allow_probabilities + Whether to allow probabilities as `mask` """ + if mask is None: + return mask + desc = "mask/probabilities" if allow_probabilities else "mask" + if isinstance(mask, str): if not isinstance(data, AnnData): - msg = "Cannot refer to mask with string without providing anndata object as argument" + msg = f"Cannot refer to {desc} with string without providing anndata object as argument" raise ValueError(msg) annot: pd.DataFrame = getattr(data, dim) if mask not in annot.columns: msg = ( f"Did not find `adata.{dim}[{mask!r}]`. " - f"Either add the mask first to `adata.{dim}`" - "or consider using the mask argument with a boolean array." + f"Either add the {desc} first to `adata.{dim}`" + f"or consider using the {desc} argument with an array." ) raise ValueError(msg) mask_array = annot[mask].to_numpy() else: if len(mask) != data.shape[0 if dim == "obs" else 1]: - raise ValueError("The shape of the mask do not match the data.") + msg = f"The shape of the {desc} do not match the data." + raise ValueError(msg) mask_array = mask - if not pd.api.types.is_bool_dtype(mask_array.dtype): - raise ValueError("Mask array must be boolean.") + is_bool = pd.api.types.is_bool_dtype(mask_array.dtype) + if not allow_probabilities and not is_bool: + msg = "Mask array must be boolean." + raise ValueError(msg) + elif allow_probabilities and not ( + is_bool or pd.api.types.is_float_dtype(mask_array.dtype) + ): + msg = f"{desc} array must be boolean or floating point." + raise ValueError(msg) return mask_array diff --git a/src/scanpy/logging.py b/src/scanpy/logging.py index 168c3b5405..7bd678f568 100644 --- a/src/scanpy/logging.py +++ b/src/scanpy/logging.py @@ -4,17 +4,20 @@ import logging import sys -import warnings from datetime import datetime, timedelta, timezone from functools import partial, update_wrapper from logging import CRITICAL, DEBUG, ERROR, INFO, WARNING -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, overload import anndata.logging +from ._compat import deprecated + if TYPE_CHECKING: from typing import IO + from session_info2 import SessionInfo + from ._settings import ScanpyConfig @@ -127,33 +130,11 @@ def format(self, record: logging.LogRecord): get_memory_usage = anndata.logging.get_memory_usage -_DEPENDENCIES_NUMERICS = [ - "anndata", # anndata actually shouldn't, but as long as it's in development - "umap", - "numpy", - "scipy", - "pandas", - ("sklearn", "scikit-learn"), - "statsmodels", - "igraph", - "louvain", - "leidenalg", - "pynndescent", -] - - -def _versions_dependencies(dependencies): - # this is not the same as the requirements! - for mod in dependencies: - mod_name, dist_name = mod if isinstance(mod, tuple) else (mod, mod) - try: - imp = __import__(mod_name) - yield dist_name, imp.__version__ - except (ImportError, AttributeError): - pass - - -def print_header(*, file=None): +@overload +def print_header(*, file: None = None) -> SessionInfo: ... +@overload +def print_header(*, file: IO[str]) -> None: ... +def print_header(*, file: IO[str] | None = None): """\ Versions that might influence the numerical results. Matplotlib and Seaborn are excluded from this. @@ -163,50 +144,27 @@ def print_header(*, file=None): file Optional path for dependency output. """ + from session_info2 import session_info - modules = ["scanpy"] + _DEPENDENCIES_NUMERICS - print( - " ".join(f"{mod}=={ver}" for mod, ver in _versions_dependencies(modules)), - file=file or sys.stdout, - ) + sinfo = session_info(os=True, cpu=True, gpu=True, dependencies=True) + + if file is not None: + print(sinfo, file=file) + return + + return sinfo -def print_versions(*, file: IO[str] | None = None): +@deprecated("Use `print_header` instead") +def print_versions() -> SessionInfo: """\ - Print versions of imported packages, OS, and jupyter environment. + Alias for `print_header`. - For more options (including rich output) use `session_info.show` directly. + .. deprecated:: 1.11.0 - Parameters - ---------- - file - Optional path for output. + Use :func:`print_header` instead. """ - import session_info - - if file is not None: - from contextlib import redirect_stdout - - warnings.warn( - "Passing argument 'file' to print_versions is deprecated, and will be " - "removed in a future version.", - FutureWarning, - ) - with redirect_stdout(file): - print_versions() - else: - session_info.show( - dependencies=True, - html=False, - excludes=[ - "builtins", - "stdlib_list", - "importlib_metadata", - # Special module present if test coverage being calculated - # https://gitlab.com/joelostblom/session_info/-/issues/10 - "$coverage", - ], - ) + return print_header() def print_version_and_date(*, file=None): @@ -223,7 +181,7 @@ def print_version_and_date(*, file=None): if file is None: file = sys.stdout print( - f"Running Scanpy {__version__}, " f"on {datetime.now():%Y-%m-%d %H:%M}.", + f"Running Scanpy {__version__}, on {datetime.now():%Y-%m-%d %H:%M}.", file=file, ) @@ -235,7 +193,7 @@ def _copy_docs_and_signature(fn): def error( msg: str, *, - time: datetime = None, + time: datetime | None = None, deep: str | None = None, extra: dict | None = None, ) -> datetime: diff --git a/src/scanpy/metrics/_gearys_c.py b/src/scanpy/metrics/_gearys_c.py index 4efb9a14c7..cf4220eb7a 100644 --- a/src/scanpy/metrics/_gearys_c.py +++ b/src/scanpy/metrics/_gearys_c.py @@ -1,3 +1,5 @@ +"""Geary's C autocorrelation.""" + from __future__ import annotations from functools import singledispatch @@ -7,7 +9,7 @@ import numpy as np from scipy import sparse -from .._compat import fullname +from .._compat import fullname, njit from ..get import _get_obs_rep from ._common import _check_vals, _resolve_vals @@ -87,7 +89,7 @@ def gearys_c( Examples -------- - Calculate Gearys C for each components of a dimensionality reduction: + Calculate Geary’s C for each components of a dimensionality reduction: .. code:: python @@ -111,7 +113,8 @@ def gearys_c( elif "neighbors" in adata.uns: g = adata.uns["neighbors"]["connectivities"] else: - raise ValueError("Must run neighbors first.") + msg = "Must run neighbors first." + raise ValueError(msg) else: raise NotImplementedError() if vals is None: @@ -134,30 +137,38 @@ def gearys_c( # tests to fail. -@numba.njit(cache=True, parallel=True) -def _gearys_c_vec(data, indices, indptr, x): +def _gearys_c_vec( + data: np.ndarray, + indices: np.ndarray, + indptr: np.ndarray, + x: np.ndarray, +) -> float: W = data.sum() return _gearys_c_vec_W(data, indices, indptr, x, W) -@numba.njit(cache=True, parallel=True) -def _gearys_c_vec_W(data, indices, indptr, x, W): - N = len(indptr) - 1 - x = x.astype(np.float_) +@njit +def _gearys_c_vec_W( + data: np.ndarray, + indices: np.ndarray, + indptr: np.ndarray, + x: np.ndarray, + W: np.float64, +): + n = len(indptr) - 1 + x = x.astype(np.float64) x_bar = x.mean() total = 0.0 - for i in numba.prange(N): + for i in numba.prange(n): s = slice(indptr[i], indptr[i + 1]) i_indices = indices[s] i_data = data[s] total += np.sum(i_data * ((x[i] - x[i_indices]) ** 2)) - numer = (N - 1) * total + numer = (n - 1) * total denom = 2 * W * ((x - x_bar) ** 2).sum() - C = numer / denom - - return C + return numer / denom # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -171,37 +182,48 @@ def _gearys_c_vec_W(data, indices, indptr, x, W): # https://github.com/numba/numba/issues/6774#issuecomment-788789663 -@numba.njit(cache=True) -def _gearys_c_inner_sparse_x_densevec(g_data, g_indices, g_indptr, x, W): +@numba.njit(cache=True, parallel=False) # noqa: TID251 +def _gearys_c_inner_sparse_x_densevec( + g_data: np.ndarray, + g_indices: np.ndarray, + g_indptr: np.ndarray, + x: np.ndarray, + W: np.float64, +) -> float: x_bar = x.mean() total = 0.0 - N = len(x) - for i in numba.prange(N): + n = len(x) + for i in numba.prange(n): s = slice(g_indptr[i], g_indptr[i + 1]) i_indices = g_indices[s] i_data = g_data[s] total += np.sum(i_data * ((x[i] - x[i_indices]) ** 2)) - numer = (N - 1) * total + numer = (n - 1) * total denom = 2 * W * ((x - x_bar) ** 2).sum() - C = numer / denom - return C + return numer / denom -@numba.njit(cache=True) +@numba.njit(cache=True, parallel=False) # noqa: TID251 def _gearys_c_inner_sparse_x_sparsevec( # noqa: PLR0917 - g_data, g_indices, g_indptr, x_data, x_indices, N, W -): - x = np.zeros(N, dtype=np.float_) + g_data: np.ndarray, + g_indices: np.ndarray, + g_indptr: np.ndarray, + x_data: np.ndarray, + x_indices: np.ndarray, + n: int, + W: np.float64, +) -> float: + x = np.zeros(n, dtype=np.float64) x[x_indices] = x_data - x_bar = np.sum(x_data) / N + x_bar = np.sum(x_data) / n total = 0.0 - N = len(x) - for i in numba.prange(N): + n = len(x) + for i in numba.prange(n): s = slice(g_indptr[i], g_indptr[i + 1]) i_indices = g_indices[s] i_data = g_data[s] total += np.sum(i_data * ((x[i] - x[i_indices]) ** 2)) - numer = (N - 1) * total + numer = (n - 1) * total # Expanded from 2 * W * ((x_k - x_k_bar) ** 2).sum(), but uses sparsity # to skip some calculations # fmt: off @@ -210,43 +232,53 @@ def _gearys_c_inner_sparse_x_sparsevec( # noqa: PLR0917 * ( np.sum(x_data ** 2) - np.sum(x_data * x_bar * 2) - + (x_bar ** 2) * N + + (x_bar ** 2) * n ) ) # fmt: on - C = numer / denom - return C - - -@numba.njit(cache=True, parallel=True) -def _gearys_c_mtx(g_data, g_indices, g_indptr, X): - M, N = X.shape - assert N == len(g_indptr) - 1 + return numer / denom + + +@njit +def _gearys_c_mtx( + g_data: np.ndarray, + g_indices: np.ndarray, + g_indptr: np.ndarray, + X: np.ndarray, +) -> np.ndarray: + m, n = X.shape + assert n == len(g_indptr) - 1 W = g_data.sum() - out = np.zeros(M, dtype=np.float_) - for k in numba.prange(M): - x = X[k, :].astype(np.float_) + out = np.zeros(m, dtype=np.float64) + for k in numba.prange(m): + x = X[k, :].astype(np.float64) out[k] = _gearys_c_inner_sparse_x_densevec(g_data, g_indices, g_indptr, x, W) return out -@numba.njit(cache=True, parallel=True) +@njit def _gearys_c_mtx_csr( # noqa: PLR0917 - g_data, g_indices, g_indptr, x_data, x_indices, x_indptr, x_shape -): - M, N = x_shape + g_data: np.ndarray, + g_indices: np.ndarray, + g_indptr: np.ndarray, + x_data: np.ndarray, + x_indices: np.ndarray, + x_indptr: np.ndarray, + x_shape: tuple, +) -> np.ndarray: + m, n = x_shape W = g_data.sum() - out = np.zeros(M, dtype=np.float_) + out = np.zeros(m, dtype=np.float64) x_data_list = np.split(x_data, x_indptr[1:-1]) x_indices_list = np.split(x_indices, x_indptr[1:-1]) - for k in numba.prange(M): + for k in numba.prange(m): out[k] = _gearys_c_inner_sparse_x_sparsevec( g_data, g_indices, g_indptr, x_data_list[k], x_indices_list[k], - N, + n, W, ) return out @@ -261,7 +293,7 @@ def _gearys_c_mtx_csr( # noqa: PLR0917 def _gearys_c(g: sparse.csr_matrix, vals: np.ndarray | sparse.spmatrix) -> np.ndarray: assert g.shape[0] == g.shape[1], "`g` should be a square adjacency matrix" vals = _resolve_vals(vals) - g_data = g.data.astype(np.float_, copy=False) + g_data = g.data.astype(np.float64, copy=False) if isinstance(vals, sparse.csr_matrix): assert g.shape[0] == vals.shape[1] new_vals, idxer, full_result = _check_vals(vals) @@ -269,7 +301,7 @@ def _gearys_c(g: sparse.csr_matrix, vals: np.ndarray | sparse.spmatrix) -> np.nd g_data, g.indices, g.indptr, - new_vals.data.astype(np.float_, copy=False), + new_vals.data.astype(np.float64, copy=False), new_vals.indices, new_vals.indptr, new_vals.shape, diff --git a/src/scanpy/metrics/_morans_i.py b/src/scanpy/metrics/_morans_i.py index 8497d83d99..c21c455f38 100644 --- a/src/scanpy/metrics/_morans_i.py +++ b/src/scanpy/metrics/_morans_i.py @@ -5,11 +5,11 @@ from functools import singledispatch from typing import TYPE_CHECKING +import numba import numpy as np -from numba import njit, prange from scipy import sparse -from .._compat import fullname +from .._compat import fullname, njit from ..get import _get_obs_rep from ._common import _check_vals, _resolve_vals @@ -88,7 +88,7 @@ def morans_i( Examples -------- - Calculate Morans I for each components of a dimensionality reduction: + Calculate Moran’s I for each components of a dimensionality reduction: .. code:: python @@ -112,7 +112,8 @@ def morans_i( elif "neighbors" in adata.uns: g = adata.uns["neighbors"]["connectivities"] else: - raise ValueError("Must run neighbors first.") + msg = "Must run neighbors first." + raise ValueError(msg) else: raise NotImplementedError() if vals is None: @@ -126,35 +127,31 @@ def morans_i( # This is done in a very similar way to gearys_c. See notes there for details. -@njit(cache=True) -def _morans_i_vec_W_sparse( # noqa: PLR0917 +@njit +def _morans_i_vec( g_data: np.ndarray, g_indices: np.ndarray, g_indptr: np.ndarray, - x_data: np.ndarray, - x_indices: np.ndarray, - N: int, - W: np.float_, + x: np.ndarray, ) -> float: - x = np.zeros(N, dtype=x_data.dtype) - x[x_indices] = x_data + W = g_data.sum() return _morans_i_vec_W(g_data, g_indices, g_indptr, x, W) -@njit(cache=True) +@numba.njit(cache=True, parallel=False) # noqa: TID251 def _morans_i_vec_W( g_data: np.ndarray, g_indices: np.ndarray, g_indptr: np.ndarray, x: np.ndarray, - W: np.float_, + W: np.float64, ) -> float: z = x - x.mean() z2ss = (z * z).sum() - N = len(x) + n = len(x) inum = 0.0 - for i in prange(N): + for i in numba.prange(n): s = slice(g_indptr[i], g_indptr[i + 1]) i_indices = g_indices[s] i_data = g_data[s] @@ -163,57 +160,61 @@ def _morans_i_vec_W( return len(x) / W * inum / z2ss -@njit(cache=True, parallel=True) -def _morans_i_vec( +@numba.njit(cache=True, parallel=False) # noqa: TID251 +def _morans_i_vec_W_sparse( # noqa: PLR0917 g_data: np.ndarray, g_indices: np.ndarray, g_indptr: np.ndarray, - x: np.ndarray, + x_data: np.ndarray, + x_indices: np.ndarray, + n: int, + W: np.float64, ) -> float: - W = g_data.sum() + x = np.zeros(n, dtype=x_data.dtype) + x[x_indices] = x_data return _morans_i_vec_W(g_data, g_indices, g_indptr, x, W) -@njit(cache=True, parallel=True) +@njit def _morans_i_mtx( g_data: np.ndarray, g_indices: np.ndarray, g_indptr: np.ndarray, X: np.ndarray, ) -> np.ndarray: - M, N = X.shape - assert N == len(g_indptr) - 1 + m, n = X.shape + assert n == len(g_indptr) - 1 W = g_data.sum() - out = np.zeros(M, dtype=np.float_) - for k in prange(M): + out = np.zeros(m, dtype=np.float64) + for k in numba.prange(m): x = X[k, :] out[k] = _morans_i_vec_W(g_data, g_indices, g_indptr, x, W) return out -@njit(cache=True, parallel=True) +@njit def _morans_i_mtx_csr( # noqa: PLR0917 g_data: np.ndarray, g_indices: np.ndarray, g_indptr: np.ndarray, - X_data: np.ndarray, - X_indices: np.ndarray, - X_indptr: np.ndarray, - X_shape: tuple, + x_data: np.ndarray, + x_indices: np.ndarray, + x_indptr: np.ndarray, + x_shape: tuple, ) -> np.ndarray: - M, N = X_shape + m, n = x_shape W = g_data.sum() - out = np.zeros(M, dtype=np.float_) - x_data_list = np.split(X_data, X_indptr[1:-1]) - x_indices_list = np.split(X_indices, X_indptr[1:-1]) - for k in prange(M): + out = np.zeros(m, dtype=np.float64) + x_data_list = np.split(x_data, x_indptr[1:-1]) + x_indices_list = np.split(x_indices, x_indptr[1:-1]) + for k in numba.prange(m): out[k] = _morans_i_vec_W_sparse( g_data, g_indices, g_indptr, x_data_list[k], x_indices_list[k], - N, + n, W, ) return out @@ -228,7 +229,7 @@ def _morans_i_mtx_csr( # noqa: PLR0917 def _morans_i(g: sparse.csr_matrix, vals: np.ndarray | sparse.spmatrix) -> np.ndarray: assert g.shape[0] == g.shape[1], "`g` should be a square adjacency matrix" vals = _resolve_vals(vals) - g_data = g.data.astype(np.float_, copy=False) + g_data = g.data.astype(np.float64, copy=False) if isinstance(vals, sparse.csr_matrix): assert g.shape[0] == vals.shape[1] new_vals, idxer, full_result = _check_vals(vals) @@ -236,7 +237,7 @@ def _morans_i(g: sparse.csr_matrix, vals: np.ndarray | sparse.spmatrix) -> np.nd g_data, g.indices, g.indptr, - new_vals.data.astype(np.float_, copy=False), + new_vals.data.astype(np.float64, copy=False), new_vals.indices, new_vals.indptr, new_vals.shape, @@ -250,10 +251,7 @@ def _morans_i(g: sparse.csr_matrix, vals: np.ndarray | sparse.spmatrix) -> np.nd assert g.shape[0] == vals.shape[1] new_vals, idxer, full_result = _check_vals(vals) result = _morans_i_mtx( - g_data, - g.indices, - g.indptr, - new_vals.astype(np.float_, copy=False), + g_data, g.indices, g.indptr, new_vals.astype(np.float64, copy=False) ) full_result[idxer] = result return full_result diff --git a/src/scanpy/neighbors/__init__.py b/src/scanpy/neighbors/__init__.py index 62d8b81d86..214043727b 100644 --- a/src/scanpy/neighbors/__init__.py +++ b/src/scanpy/neighbors/__init__.py @@ -1,8 +1,10 @@ from __future__ import annotations +import contextlib +from collections.abc import Mapping from textwrap import indent from types import MappingProxyType -from typing import TYPE_CHECKING, NamedTuple, TypedDict, get_args +from typing import TYPE_CHECKING, NamedTuple, TypedDict from warnings import warn import numpy as np @@ -14,7 +16,7 @@ from .. import logging as logg from .._compat import old_positionals from .._settings import settings -from .._utils import NeighborsView, _doc_params +from .._utils import NeighborsView, _doc_params, get_literal_vals from . import _connectivity from ._common import ( _get_indices_distances_from_sparse_matrix, @@ -24,18 +26,18 @@ from ._types import _KnownTransformer, _Method if TYPE_CHECKING: - from collections.abc import Callable, Mapping, MutableMapping - from typing import Any, Literal + from collections.abc import Callable, MutableMapping + from typing import Any, Literal, NotRequired from anndata import AnnData from igraph import Graph from scipy.sparse import csr_matrix - from .._utils import AnyRandom + from .._compat import _LegacyRandom from ._types import KnnTransformerLike, _Metric, _MetricFn - RPForestDict = Mapping[str, Mapping[str, np.ndarray]] +RPForestDict = Mapping[str, Mapping[str, np.ndarray]] N_DCS = 15 # default number of diffusion components # Backwards compat, constants should be defined in only one place. @@ -52,7 +54,17 @@ class KwdsForTransformer(TypedDict): n_neighbors: int metric: _Metric | _MetricFn metric_params: Mapping[str, Any] - random_state: AnyRandom + random_state: _LegacyRandom + + +class NeighborsParams(TypedDict): + n_neighbors: int + method: _Method + random_state: _LegacyRandom + metric: _Metric | _MetricFn + metric_kwds: NotRequired[Mapping[str, Any]] + use_rep: NotRequired[str] + n_pcs: NotRequired[int] @_doc_params(n_pcs=doc_n_pcs, use_rep=doc_use_rep) @@ -67,7 +79,7 @@ def neighbors( transformer: KnnTransformerLike | _KnownTransformer | None = None, metric: _Metric | _MetricFn = "euclidean", metric_kwds: Mapping[str, Any] = MappingProxyType({}), - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, key_added: str | None = None, copy: bool = False, ) -> AnnData | None: @@ -203,7 +215,7 @@ def neighbors( neighbors_dict["connectivities_key"] = conns_key neighbors_dict["distances_key"] = dists_key - neighbors_dict["params"] = dict( + neighbors_dict["params"] = NeighborsParams( n_neighbors=neighbors.n_neighbors, method=method, random_state=random_state, @@ -300,7 +312,7 @@ def __init__( self.restrict_array = restrict_array # restrict the array to a subset def __getitem__(self, index): - if isinstance(index, (int, np.integer)): + if isinstance(index, int | np.integer): if self.restrict_array is None: glob_index = index else: @@ -413,10 +425,11 @@ def count_nonzero(a: np.ndarray | csr_matrix) -> int: self._eigen_basis = _backwards_compat_get_full_X_diffmap(adata) if n_dcs is not None: if n_dcs > len(self._eigen_values): - raise ValueError( + msg = ( f"Cannot instantiate using `n_dcs`={n_dcs}. " "Compute diffmap/spectrum with more components first." ) + raise ValueError(msg) self._eigen_values = self._eigen_values[:n_dcs] self._eigen_basis = self._eigen_basis[:, :n_dcs] self.n_dcs = len(self._eigen_values) @@ -457,10 +470,7 @@ def transitions(self) -> np.ndarray | csr_matrix: ----- This has not been tested, in contrast to `transitions_sym`. """ - if issparse(self.Z): - Zinv = self.Z.power(-1) - else: - Zinv = np.diag(1.0 / np.diag(self.Z)) + Zinv = self.Z.power(-1) if issparse(self.Z) else np.diag(1.0 / np.diag(self.Z)) return self.Z @ self.transitions_sym @ Zinv @property @@ -512,7 +522,7 @@ def compute_neighbors( transformer: KnnTransformerLike | _KnownTransformer | None = None, metric: _Metric | _MetricFn = "euclidean", metric_kwds: Mapping[str, Any] = MappingProxyType({}), - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, ) -> None: """\ Compute distances and connectivities of neighbors. @@ -579,10 +589,9 @@ def compute_neighbors( if isinstance(index, NNDescent): # very cautious here - try: + # TODO catch the correct exception + with contextlib.suppress(Exception): self._rp_forest = _make_forest_dict(index) - except Exception: # TODO catch the correct exception - pass start_connect = logg.debug("computed neighbors", time=start_neighbors) if method == "umap": @@ -644,7 +653,9 @@ def _handle_transformer( raise ValueError(msg) method = "umap" transformer = "rapids" - elif method not in (methods := set(get_args(_Method))) and method is not None: + elif ( + method not in (methods := get_literal_vals(_Method)) and method is not None + ): msg = f"`method` needs to be one of {methods}." raise ValueError(msg) @@ -696,13 +707,14 @@ def _handle_transformer( elif isinstance(transformer, str): msg = ( f"Unknown transformer: {transformer}. " - f"Try passing a class or one of {set(get_args(_KnownTransformer))}" + f"Try passing a class or one of {get_literal_vals(_KnownTransformer)}" ) raise ValueError(msg) # else `transformer` is probably an instance return conn_method, transformer, shortcut - def compute_transitions(self, density_normalize: bool = True): + @old_positionals("density_normalize") + def compute_transitions(self, *, density_normalize: bool = True): """\ Compute transition matrix. @@ -746,7 +758,7 @@ def compute_eigen( n_comps: int = 15, sym: bool | None = None, sort: Literal["decrease", "increase"] = "decrease", - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, ): """\ Compute eigen decomposition of transition matrix. @@ -778,7 +790,8 @@ def compute_eigen( """ np.set_printoptions(precision=10) if self._transitions_sym is None: - raise ValueError("Run `.compute_transitions` first.") + msg = "Run `.compute_transitions` first." + raise ValueError(msg) matrix = self._transitions_sym # compute the spectrum if n_comps == 0: @@ -801,9 +814,7 @@ def compute_eigen( if sort == "decrease": evals = evals[::-1] evecs = evecs[:, ::-1] - logg.info( - f" eigenvalues of transition matrix\n" f"{indent(str(evals), ' ')}" - ) + logg.info(f" eigenvalues of transition matrix\n{indent(str(evals), ' ')}") if self._number_connected_components > len(evals) / 2: logg.warning("Transition matrix has many disconnected components!") self._eigen_values = evals @@ -814,10 +825,11 @@ def _init_iroot(self): # set iroot directly if "iroot" in self._adata.uns: if self._adata.uns["iroot"] >= self._adata.n_obs: - logg.warning( - f'Root cell index {self._adata.uns["iroot"]} does not ' + msg = ( + f"Root cell index {self._adata.uns['iroot']} does not " f"exist for {self._adata.n_obs} samples. It’s ignored." ) + logg.warning(msg) else: self.iroot = self._adata.uns["iroot"] return @@ -879,9 +891,8 @@ def _set_iroot_via_xroot(self, xroot: np.ndarray): condition, only relevant for computing pseudotime. """ if self._adata.shape[1] != xroot.size: - raise ValueError( - "The root vector you provided does not have the " "correct dimension." - ) + msg = "The root vector you provided does not have the correct dimension." + raise ValueError(msg) # this is the squared distance dsqroot = 1e10 iroot = 0 diff --git a/src/scanpy/neighbors/_types.py b/src/scanpy/neighbors/_types.py index 473d7378fb..39f50284ec 100644 --- a/src/scanpy/neighbors/_types.py +++ b/src/scanpy/neighbors/_types.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Callable -from typing import TYPE_CHECKING, Literal, Protocol, Union +from typing import TYPE_CHECKING, Literal, Protocol import numpy as np @@ -11,7 +11,7 @@ from scipy.sparse import spmatrix -# These two are used with get_args elsewhere +# These two are used with get_literal_vals elsewhere _Method = Literal["umap", "gauss"] _KnownTransformer = Literal["pynndescent", "sklearn", "rapids"] @@ -42,7 +42,7 @@ "sqeuclidean", "yule", ] -_Metric = Union[_MetricSparseCapable, _MetricScipySpatial] +_Metric = _MetricSparseCapable | _MetricScipySpatial class KnnTransformerLike(Protocol): @@ -55,5 +55,5 @@ def transform(self, X) -> spmatrix: ... def fit_transform(self, X, y: None = None) -> spmatrix: ... # from BaseEstimator - def get_params(self, deep: bool = True) -> dict[str, Any]: ... + def get_params(self, *, deep: bool = True) -> dict[str, Any]: ... def set_params(self, **params: Any) -> Self: ... diff --git a/src/scanpy/plotting/_anndata.py b/src/scanpy/plotting/_anndata.py index 29435a0826..75dd210c0b 100755 --- a/src/scanpy/plotting/_anndata.py +++ b/src/scanpy/plotting/_anndata.py @@ -2,10 +2,11 @@ from __future__ import annotations -import collections.abc as cabc from collections import OrderedDict +from collections.abc import Collection, Mapping, Sequence from itertools import product -from typing import TYPE_CHECKING +from types import NoneType +from typing import TYPE_CHECKING, cast import matplotlib as mpl import numpy as np @@ -21,7 +22,13 @@ from .. import logging as logg from .._compat import old_positionals from .._settings import settings -from .._utils import _check_use_raw, _doc_params, sanitize_anndata +from .._utils import ( + _check_use_raw, + _doc_params, + _empty, + get_literal_vals, + sanitize_anndata, +) from . import _utils from ._docs import ( doc_common_plot_args, @@ -30,6 +37,9 @@ doc_vboundnorm, ) from ._utils import ( + ColorLike, + _deprecated_scale, + _dk, check_colornorm, scatter_base, scatter_group, @@ -37,40 +47,28 @@ ) if TYPE_CHECKING: - from collections.abc import Collection, Iterable, Mapping, Sequence - from typing import Literal, Union + from collections.abc import Iterable + from typing import Literal from anndata import AnnData from cycler import Cycler from matplotlib.axes import Axes from matplotlib.colors import Colormap, ListedColormap, Normalize + from numpy.typing import NDArray from seaborn import FacetGrid from seaborn.matrix import ClusterGrid - from ._utils import ColorLike, _FontSize, _FontWeight + from .._utils import Empty + from ._utils import ( + DensityNorm, + _FontSize, + _FontWeight, + _LegendLoc, + ) # TODO: is that all? _Basis = Literal["pca", "tsne", "umap", "diffmap", "draw_graph_fr"] - _VarNames = Union[str, Sequence[str]] - - -VALID_LEGENDLOCS = { - "none", - "right margin", - "on data", - "on data export", - "best", - "upper right", - "upper left", - "lower left", - "lower right", - "right", - "center left", - "center right", - "lower center", - "upper center", - "center", -} + _VarNames = str | Sequence[str] @old_positionals( @@ -96,7 +94,7 @@ def scatter( x: str | None = None, y: str | None = None, *, - color: str | Collection[str] | None = None, + color: str | ColorLike | Collection[str | ColorLike] | None = None, use_raw: bool | None = None, layers: str | Collection[str] | None = None, sort_order: bool = True, @@ -105,8 +103,8 @@ def scatter( groups: str | Iterable[str] | None = None, components: str | Collection[str] | None = None, projection: Literal["2d", "3d"] = "2d", - legend_loc: str = "right margin", - legend_fontsize: int | float | _FontSize | None = None, + legend_loc: _LegendLoc | None = "right margin", + legend_fontsize: float | _FontSize | None = None, legend_fontweight: int | _FontWeight | None = None, legend_fontoutline: float | None = None, color_map: str | Colormap | None = None, @@ -114,9 +112,9 @@ def scatter( frameon: bool | None = None, right_margin: float | None = None, left_margin: float | None = None, - size: int | float | None = None, + size: float | None = None, marker: str | Sequence[str] = ".", - title: str | None = None, + title: str | Collection[str] | None = None, show: bool | None = None, save: str | bool | None = None, ax: Axes | None = None, @@ -154,69 +152,102 @@ def scatter( ------- If `show==False` a :class:`~matplotlib.axes.Axes` or a list of it. """ + # color can be a obs column name or a matplotlib color specification (or a collection thereof) + if color is not None: + color = cast( + Collection[str | ColorLike], + [color] if isinstance(color, str) or is_color_like(color) else color, + ) args = locals() - if _check_use_raw(adata, use_raw): - var_index = adata.raw.var.index - else: - var_index = adata.var.index + if basis is not None: return _scatter_obs(**args) if x is None or y is None: - raise ValueError("Either provide a `basis` or `x` and `y`.") - if ( - (x in adata.obs.keys() or x in var_index) - and (y in adata.obs.keys() or y in var_index) - and (color is None or color in adata.obs.keys() or color in var_index) - ): + msg = "Either provide a `basis` or `x` and `y`." + raise ValueError(msg) + if _check_if_annotations(adata, "obs", x=x, y=y, colors=color, use_raw=use_raw): return _scatter_obs(**args) - if ( - (x in adata.var.keys() or x in adata.obs.index) - and (y in adata.var.keys() or y in adata.obs.index) - and (color is None or color in adata.var.keys() or color in adata.obs.index) - ): - adata_T = adata.T - axs = _scatter_obs( - adata=adata_T, - **{name: val for name, val in args.items() if name != "adata"}, - ) + if _check_if_annotations(adata, "var", x=x, y=y, colors=color, use_raw=use_raw): + args_t = {**args, "adata": adata.T} + axs = _scatter_obs(**args_t) # store .uns annotations that were added to the new adata object - adata.uns = adata_T.uns + adata.uns = args_t["adata"].uns return axs - raise ValueError( + msg = ( "`x`, `y`, and potential `color` inputs must all " "come from either `.obs` or `.var`" ) + raise ValueError(msg) + + +def _check_if_annotations( + adata: AnnData, + axis_name: Literal["obs", "var"], + *, + x: str | None = None, + y: str | None = None, + colors: Collection[str | ColorLike] | None = None, + use_raw: bool | None = None, +) -> bool: + """Checks if `x`, `y`, and `colors` are annotations of `adata`. + In the case of `colors`, valid matplotlib colors are also accepted. + + If `axis_name` is `obs`, checks in `adata.obs.columns` and `adata.var_names`, + if `axis_name` is `var`, checks in `adata.var.columns` and `adata.obs_names`. + """ + annotations: pd.Index[str] = getattr(adata, axis_name).columns + other_ax_obj = ( + adata.raw if _check_use_raw(adata, use_raw) and axis_name == "obs" else adata + ) + names: pd.Index[str] = getattr( + other_ax_obj, "var" if axis_name == "obs" else "obs" + ).index + + def is_annotation(needle: pd.Index) -> NDArray[np.bool_]: + return needle.isin({None}) | needle.isin(annotations) | needle.isin(names) + + if not is_annotation(pd.Index([x, y])).all(): + return False + + color_idx = pd.Index(colors if colors is not None else []) + # Colors are valid + color_valid: NDArray[np.bool_] = np.fromiter( + map(is_color_like, color_idx), dtype=np.bool_, count=len(color_idx) + ) + # Annotation names are valid too + color_valid[~color_valid] = is_annotation(color_idx[~color_valid]) + return bool(color_valid.all()) def _scatter_obs( *, adata: AnnData, - x=None, - y=None, - color=None, - use_raw=None, - layers=None, - sort_order=True, - alpha=None, - basis=None, - groups=None, - components=None, + x: str | None = None, + y: str | None = None, + color: Collection[str | ColorLike] | None = None, + use_raw: bool | None = None, + layers: str | Collection[str] | None = None, + sort_order: bool = True, + alpha: float | None = None, + basis: _Basis | None = None, + groups: str | Iterable[str] | None = None, + components: str | Collection[str] | None = None, projection: Literal["2d", "3d"] = "2d", - legend_loc="right margin", - legend_fontsize=None, - legend_fontweight=None, - legend_fontoutline=None, - color_map=None, - palette=None, - frameon=None, - right_margin=None, - left_margin=None, - size=None, - marker=".", - title=None, - show=None, - save=None, - ax=None, + legend_loc: _LegendLoc | None = "right margin", + legend_fontsize: float | _FontSize | None = None, + legend_fontweight: int | _FontWeight | None = None, + legend_fontoutline: float | None = None, + color_map: str | Colormap | None = None, + palette: Cycler | ListedColormap | ColorLike | Sequence[ColorLike] | None = None, + frameon: bool | None = None, + right_margin: float | None = None, + left_margin: float | None = None, + size: float | None = None, + marker: str | Sequence[str] = ".", + title: str | Collection[str] | None = None, + show: bool | None = None, + save: str | bool | None = None, + ax: Axes | None = None, ) -> Axes | list[Axes] | None: """See docstring of scatter.""" sanitize_anndata(adata) @@ -224,46 +255,38 @@ def _scatter_obs( use_raw = _check_use_raw(adata, use_raw) # Process layers - if layers in ["X", None] or ( - isinstance(layers, str) and layers in adata.layers.keys() - ): + if layers in ["X", None] or (isinstance(layers, str) and layers in adata.layers): layers = (layers, layers, layers) - elif isinstance(layers, cabc.Collection) and len(layers) == 3: + elif isinstance(layers, Collection) and len(layers) == 3: layers = tuple(layers) for layer in layers: - if layer not in adata.layers.keys() and layer not in ["X", None]: - raise ValueError( + if layer not in adata.layers and layer not in ["X", None]: + msg = ( "`layers` should have elements that are " "either None or in adata.layers.keys()." ) + raise ValueError(msg) else: - raise ValueError( + msg = ( "`layers` should be a string or a collection of strings " f"with length 3, had value '{layers}'" ) + raise ValueError(msg) if use_raw and layers not in [("X", "X", "X"), (None, None, None)]: ValueError("`use_raw` must be `False` if layers are used.") - if legend_loc not in VALID_LEGENDLOCS: - raise ValueError( - f"Invalid `legend_loc`, need to be one of: {VALID_LEGENDLOCS}." - ) + if legend_loc not in (valid_legend_locs := get_literal_vals(_utils._LegendLoc)): + msg = f"Invalid `legend_loc`, need to be one of: {valid_legend_locs}." + raise ValueError(msg) if components is None: components = "1,2" if "2d" in projection else "1,2,3" if isinstance(components, str): components = components.split(",") components = np.array(components).astype(int) - 1 - # color can be a obs column name or a matplotlib color specification - keys = ( - ["grey"] - if color is None - else [color] - if isinstance(color, str) or is_color_like(color) - else color - ) + keys = ["grey"] if color is None else color if title is not None and isinstance(title, str): title = [title] - highlights = adata.uns["highlights"] if "highlights" in adata.uns else [] + highlights = adata.uns.get("highlights", []) if basis is not None: try: # ignore the '0th' diffusion component @@ -274,9 +297,8 @@ def _scatter_obs( if basis == "diffmap": components -= 1 except KeyError: - raise KeyError( - f"compute coordinates using visualization tool {basis} first" - ) + msg = f"compute coordinates using visualization tool {basis} first" + raise KeyError(msg) elif x is not None and y is not None: if use_raw: if x in adata.obs.columns: @@ -293,25 +315,21 @@ def _scatter_obs( Y = np.c_[x_arr, y_arr] else: - raise ValueError("Either provide a `basis` or `x` and `y`.") + msg = "Either provide a `basis` or `x` and `y`." + raise ValueError(msg) if size is None: n = Y.shape[0] size = 120000 / n - if legend_loc.startswith("on data") and legend_fontsize is None: - legend_fontsize = rcParams["legend.fontsize"] - elif legend_fontsize is None: + if legend_fontsize is None: legend_fontsize = rcParams["legend.fontsize"] palette_was_none = False if palette is None: palette_was_none = True - if isinstance(palette, cabc.Sequence) and not isinstance(palette, str): - if not is_color_like(palette[0]): - palettes = palette - else: - palettes = [palette] + if isinstance(palette, Sequence) and not isinstance(palette, str): + palettes = palette if not is_color_like(palette[0]) else [palette] else: palettes = [palette for _ in range(len(keys))] palettes = [_utils.default_palette(palette) for palette in palettes] @@ -335,7 +353,7 @@ def _scatter_obs( else: component_name = None axis_labels = (x, y) if component_name is None else None - show_ticks = True if component_name is None else False + show_ticks = component_name is None # generate the colors color_ids: list[np.ndarray | ColorLike] = [] @@ -360,10 +378,11 @@ def _scatter_obs( c = key colorbar = False else: - raise ValueError( + msg = ( f"key {key!r} is invalid! pass valid observation annotation, " f"one of {adata.obs_keys()} or a gene name {adata.var_names}" ) + raise ValueError(msg) if colorbar is None: colorbar = not categorical colorbars.append(colorbar) @@ -371,15 +390,14 @@ def _scatter_obs( categoricals.append(ikey) color_ids.append(c) - if right_margin is None and len(categoricals) > 0: - if legend_loc == "right margin": - right_margin = 0.5 + if right_margin is None and len(categoricals) > 0 and legend_loc == "right margin": + right_margin = 0.5 if title is None and keys[0] is not None: title = [ key.replace("_", " ") if not is_color_like(key) else "" for key in keys ] - axs = scatter_base( + axs: list[Axes] = scatter_base( Y, title=title, alpha=alpha, @@ -411,7 +429,7 @@ def add_centroid(centroids, name, Y, mask): for ikey, palette in zip(categoricals, palettes): key = keys[ikey] _utils.add_colors_for_categorical_sample_annotation( - adata, key, palette, force_update_colors=not palette_was_none + adata, key, palette=palette, force_update_colors=not palette_was_none ) # actually plot the groups mask_remaining = np.ones(Y.shape[0], dtype=bool) @@ -437,10 +455,11 @@ def add_centroid(centroids, name, Y, mask): groups = [groups] if isinstance(groups, str) else groups for name in groups: if name not in set(adata.obs[key].cat.categories): - raise ValueError( + msg = ( f"{name!r} is invalid! specify valid name, " f"one of {adata.obs[key].cat.categories}" ) + raise ValueError(msg) else: iname = np.flatnonzero( adata.obs[key].cat.categories.values == name @@ -495,10 +514,7 @@ def add_centroid(centroids, name, Y, mask): all_pos = np.zeros((len(adata.obs[key].cat.categories), 2)) for iname, name in enumerate(adata.obs[key].cat.categories): - if name in centroids: - all_pos[iname] = centroids[name] - else: - all_pos[iname] = [np.nan, np.nan] + all_pos[iname] = centroids.get(name, [np.nan, np.nan]) if legend_loc == "on data export": filename = settings.writedir / "pos.csv" logg.warning(f"exporting label positions to {filename}") @@ -704,7 +720,7 @@ def violin( jitter: float | bool = True, size: int = 1, layer: str | None = None, - scale: Literal["area", "count", "width"] = "width", + density_norm: DensityNorm = "width", order: Sequence[str] | None = None, multi_panel: bool | None = None, xlabel: str = "", @@ -713,6 +729,8 @@ def violin( show: bool | None = None, save: bool | str | None = None, ax: Axes | None = None, + # deprecatd + scale: DensityNorm | Empty = _empty, **kwds, ) -> Axes | FacetGrid | None: """\ @@ -745,7 +763,7 @@ def violin( default adata.raw.X is plotted. If `use_raw=False` is set, then `adata.X` is plotted. If `layer` is set to a valid layer name, then the layer is plotted. `layer` takes precedence over `use_raw`. - scale + density_norm The method used to scale the width of each violin. If 'width' (the default), each violin will have the same width. If 'area', each violin will have the same area. @@ -824,28 +842,28 @@ def violin( if isinstance(keys, str): keys = [keys] keys = list(OrderedDict.fromkeys(keys)) # remove duplicates, preserving the order + density_norm = _deprecated_scale(density_norm, scale, default="width") + del scale - if isinstance(ylabel, (str, type(None))): + if isinstance(ylabel, str | NoneType): ylabel = [ylabel] * (1 if groupby is None else len(keys)) if groupby is None: if len(ylabel) != 1: - raise ValueError( - f"Expected number of y-labels to be `1`, found `{len(ylabel)}`." - ) + msg = f"Expected number of y-labels to be `1`, found `{len(ylabel)}`." + raise ValueError(msg) elif len(ylabel) != len(keys): - raise ValueError( - f"Expected number of y-labels to be `{len(keys)}`, " - f"found `{len(ylabel)}`." - ) + msg = f"Expected number of y-labels to be `{len(keys)}`, found `{len(ylabel)}`." + raise ValueError(msg) if groupby is not None: obs_df = get.obs_df(adata, keys=[groupby] + keys, layer=layer, use_raw=use_raw) - if kwds.get("palette", None) is None: + if kwds.get("palette") is None: if not isinstance(adata.obs[groupby].dtype, CategoricalDtype): - raise ValueError( + msg = ( f"The column `adata.obs[{groupby!r}]` needs to be categorical, " f"but is of dtype {adata.obs[groupby].dtype}." ) + raise ValueError(msg) _utils.add_colors_for_categorical_sample_annotation(adata, groupby) kwds["hue"] = groupby kwds["palette"] = dict( @@ -871,7 +889,7 @@ def violin( y=y, data=obs_tidy, kind="violin", - density_norm=scale, + density_norm=density_norm, col=x, col_order=keys, sharey=False, @@ -919,7 +937,7 @@ def violin( data=obs_tidy, order=order, orient="vertical", - density_norm=scale, + density_norm=density_norm, ax=ax, **kwds, ) @@ -1006,8 +1024,9 @@ def clustermap( """ import seaborn as sns # Slow import, only import if called - if not isinstance(obs_keys, (str, type(None))): - raise ValueError("Currently, only a single key is supported.") + if not isinstance(obs_keys, str | NoneType): + msg = "Currently, only a single key is supported." + raise ValueError(msg) sanitize_anndata(adata) use_raw = _check_use_raw(adata, use_raw) X = adata.raw.X if use_raw else adata.X @@ -1193,7 +1212,7 @@ def heatmap( dendro_data = _reorder_categories_after_dendrogram( adata, groupby, - dendrogram, + dendrogram_key=_dk(dendrogram), var_names=var_names, var_group_labels=var_group_labels, var_group_positions=var_group_positions, @@ -1248,10 +1267,7 @@ def heatmap( groupby_width = 0.2 if categorical else 0 if figsize is None: height = 6 - if show_gene_labels: - heatmap_width = len(var_names) * 0.3 - else: - heatmap_width = 8 + heatmap_width = len(var_names) * 0.3 if show_gene_labels else 8 width = heatmap_width + dendro_width + groupby_width else: width, height = figsize @@ -1288,7 +1304,7 @@ def heatmap( heatmap_ax.set_xlim(-0.5, obs_tidy.shape[1] - 0.5) heatmap_ax.tick_params(axis="y", left=False, labelleft=False) heatmap_ax.set_ylabel("") - heatmap_ax.grid(False) + heatmap_ax.grid(visible=False) if show_gene_labels: heatmap_ax.tick_params(axis="x", labelsize="small") @@ -1328,7 +1344,7 @@ def heatmap( if dendrogram: dendro_ax = fig.add_subplot(axs[1, 2], sharey=heatmap_ax) _plot_dendrogram( - dendro_ax, adata, groupby, ticks=ticks, dendrogram_key=dendrogram + dendro_ax, adata, groupby, dendrogram_key=_dk(dendrogram), ticks=ticks ) # plot group legends on top of heatmap_ax (if given) @@ -1355,10 +1371,7 @@ def heatmap( dendro_height = 0.8 if dendrogram else 0 groupby_height = 0.13 if categorical else 0 if figsize is None: - if show_gene_labels: - heatmap_height = len(var_names) * 0.18 - else: - heatmap_height = 4 + heatmap_height = len(var_names) * 0.18 if show_gene_labels else 4 width = 10 height = heatmap_height + dendro_height + groupby_height else: @@ -1392,7 +1405,7 @@ def heatmap( heatmap_ax.set_ylim(obs_tidy.shape[1] - 0.5, -0.5) heatmap_ax.tick_params(axis="x", bottom=False, labelbottom=False) heatmap_ax.set_xlabel("") - heatmap_ax.grid(False) + heatmap_ax.grid(visible=False) if show_gene_labels: heatmap_ax.tick_params(axis="y", labelsize="small", length=1) heatmap_ax.set_yticks(np.arange(len(var_names))) @@ -1431,7 +1444,7 @@ def heatmap( dendro_ax, adata, groupby, - dendrogram_key=dendrogram, + dendrogram_key=_dk(dendrogram), ticks=ticks, orientation="top", ) @@ -1443,10 +1456,7 @@ def heatmap( for idx, (label, pos) in enumerate( zip(var_group_labels, var_group_positions) ): - if var_groups_subset_of_groupby: - label_code = label2code[label] - else: - label_code = idx + label_code = label2code[label] if var_groups_subset_of_groupby else idx arr += [label_code] * (pos[1] + 1 - pos[0]) gene_groups_ax.imshow( np.array([arr]).T, aspect="auto", cmap=groupby_cmap, norm=norm @@ -1549,11 +1559,12 @@ def tracksplot( """ if groupby not in adata.obs_keys() or adata.obs[groupby].dtype.name != "category": - raise ValueError( + msg = ( "groupby has to be a valid categorical observation. " f"Given value: {groupby}, valid categorical observations: " - f'{[x for x in adata.obs_keys() if adata.obs[x].dtype.name == "category"]}' + f"{[x for x in adata.obs_keys() if adata.obs[x].dtype.name == 'category']}" ) + raise ValueError(msg) var_names, var_group_labels, var_group_positions = _check_var_names_type( var_names, var_group_labels, var_group_positions @@ -1583,7 +1594,7 @@ def tracksplot( dendro_data = _reorder_categories_after_dendrogram( adata, groupby, - dendrogram, + dendrogram_key=_dk(dendrogram), var_names=var_names, var_group_labels=var_group_labels, var_group_positions=var_group_positions, @@ -1669,7 +1680,7 @@ def tracksplot( ax.spines["left"].set_visible(False) ax.spines["top"].set_visible(False) ax.spines["bottom"].set_visible(False) - ax.grid(False) + ax.grid(visible=False) ymin, ymax = ax.get_ylim() ymax = int(ymax) ax.set_yticks([ymax]) @@ -1715,7 +1726,7 @@ def tracksplot( dendro_ax, adata, groupby, - dendrogram_key=dendrogram, + dendrogram_key=_dk(dendrogram), orientation="top", ticks=ticks, ) @@ -1876,7 +1887,7 @@ def correlation_matrix( >>> sc.pl.correlation_matrix(adata, 'bulk_labels') """ - dendrogram_key = _get_dendrogram_key(adata, dendrogram, groupby) + dendrogram_key = _get_dendrogram_key(adata, _dk(dendrogram), groupby) index = adata.uns[dendrogram_key]["categories_idx_ordered"] corr_matrix = adata.uns[dendrogram_key]["correlation_matrix"] @@ -1885,7 +1896,8 @@ def correlation_matrix( dendrogram = ax is None if dendrogram: if ax is not None: - raise ValueError("Can only plot dendrogram when not plotting to an axis") + msg = "Can only plot dendrogram when not plotting to an axis" + raise ValueError(msg) assert (len(index)) == corr_matrix.shape[0] corr_matrix = corr_matrix[index, :] corr_matrix = corr_matrix[:, index] @@ -1895,10 +1907,7 @@ def correlation_matrix( labels = adata.obs[groupby].cat.categories num_rows = corr_matrix.shape[0] colorbar_height = 0.2 - if dendrogram: - dendrogram_width = 1.8 - else: - dendrogram_width = 0 + dendrogram_width = 1.8 if dendrogram else 0 if figsize is None: corr_matrix_height = num_rows * 0.6 height = corr_matrix_height + colorbar_height @@ -2036,9 +2045,7 @@ def _prepare_dataframe( """ sanitize_anndata(adata) - use_raw = _check_use_raw(adata, use_raw) - if layer is not None: - use_raw = False + use_raw = _check_use_raw(adata, use_raw, layer=layer) if isinstance(var_names, str): var_names = [var_names] @@ -2057,11 +2064,12 @@ def _prepare_dataframe( "groupby has to be a valid observation. " f"Given {group}, is not in observations: {adata.obs_keys()}" + msg ) - if group in adata.obs.keys() and group == adata.obs.index.name: - raise ValueError( + if group in adata.obs.columns and group == adata.obs.index.name: + msg = ( f"Given group {group} is both and index and a column level, " "which is ambiguous." ) + raise ValueError(msg) if group == adata.obs.index.name: groupby_index = group if groupby_index is not None: @@ -2176,10 +2184,7 @@ def _plot_gene_groups_brackets( if orientation == "top": # rotate labels if any of them is longer than 4 characters if rotation is None and group_labels: - if max([len(x) for x in group_labels]) > 4: - rotation = 90 - else: - rotation = 0 + rotation = 90 if max([len(x) for x in group_labels]) > 4 else 0 for idx in range(len(left)): verts.append((left[idx], 0)) # lower-left verts.append((left[idx], 0.6)) # upper-left @@ -2241,7 +2246,7 @@ def _plot_gene_groups_brackets( patch = patches.PathPatch(path, facecolor="none", lw=1.5) gene_groups_ax.add_patch(patch) - gene_groups_ax.grid(False) + gene_groups_ax.grid(visible=False) gene_groups_ax.axis("off") # remove y ticks gene_groups_ax.tick_params(axis="y", left=False, labelleft=False) @@ -2253,13 +2258,13 @@ def _plot_gene_groups_brackets( def _reorder_categories_after_dendrogram( adata: AnnData, - groupby, - dendrogram, + groupby: str | Sequence[str], *, - var_names=None, - var_group_labels=None, - var_group_positions=None, - categories=None, + dendrogram_key: str | None, + var_names: Sequence[str], + var_group_labels: Sequence[str] | None, + var_group_positions: Sequence[tuple[int, int]] | None, + categories: Sequence[str], ): """\ Function used by plotting functions that need to reorder the the groupby @@ -2279,19 +2284,12 @@ def _reorder_categories_after_dendrogram( 'var_group_labels', and 'var_group_positions' """ - key = _get_dendrogram_key(adata, dendrogram, groupby) - if isinstance(groupby, str): groupby = [groupby] - dendro_info = adata.uns[key] - if groupby != dendro_info["groupby"]: - raise ValueError( - "Incompatible observations. The precomputed dendrogram contains " - f"information for the observation: '{groupby}' while the plot is " - f"made for the observation: '{dendro_info['groupby']}. " - "Please run `sc.tl.dendrogram` using the right observation.'" - ) + dendro_info = adata.uns[ + _get_dendrogram_key(adata, dendrogram_key, groupby, validate_groupby=True) + ] if categories is None: categories = adata.obs[dendro_info["groupby"]].cat.categories @@ -2301,7 +2299,7 @@ def _reorder_categories_after_dendrogram( categories_ordered = dendro_info["categories_ordered"] if len(categories) != len(categories_idx_ordered): - raise ValueError( + msg = ( "Incompatible observations. Dendrogram data has " f"{len(categories_idx_ordered)} categories but current groupby " f"observation {groupby!r} contains {len(categories)} categories. " @@ -2309,38 +2307,38 @@ def _reorder_categories_after_dendrogram( "initial computation of `sc.tl.dendrogram`. " "Please run `sc.tl.dendrogram` again.'" ) + raise ValueError(msg) # reorder var_groups (if any) - if var_names is not None: - var_names_idx_ordered = list(range(len(var_names))) - - if var_group_positions: - if set(var_group_labels) == set(categories): - positions_ordered = [] - labels_ordered = [] - position_start = 0 - var_names_idx_ordered = [] - for cat_name in categories_ordered: - idx = var_group_labels.index(cat_name) - position = var_group_positions[idx] - _var_names = var_names[position[0] : position[1] + 1] - var_names_idx_ordered.extend(range(position[0], position[1] + 1)) - positions_ordered.append( - (position_start, position_start + len(_var_names) - 1) - ) - position_start += len(_var_names) - labels_ordered.append(var_group_labels[idx]) - var_group_labels = labels_ordered - var_group_positions = positions_ordered - else: - logg.warning( - "Groups are not reordered because the `groupby` categories " - "and the `var_group_labels` are different.\n" - f"categories: {_format_first_three_categories(categories)}\n" - f"var_group_labels: {_format_first_three_categories(var_group_labels)}" + if var_group_positions is None or var_group_labels is None: + assert var_group_positions is None + assert var_group_labels is None + var_names_idx_ordered = None + elif set(var_group_labels) == set(categories): + positions_ordered = [] + labels_ordered = [] + position_start = 0 + var_names_idx_ordered = [] + for cat_name in categories_ordered: + idx = var_group_labels.index(cat_name) + position = var_group_positions[idx] + _var_names = var_names[position[0] : position[1] + 1] + var_names_idx_ordered.extend(range(position[0], position[1] + 1)) + positions_ordered.append( + (position_start, position_start + len(_var_names) - 1) ) + position_start += len(_var_names) + labels_ordered.append(var_group_labels[idx]) + var_group_labels = labels_ordered + var_group_positions = positions_ordered else: - var_names_idx_ordered = None + logg.warning( + "Groups are not reordered because the `groupby` categories " + "and the `var_group_labels` are different.\n" + f"categories: {_format_first_three_categories(categories)}\n" + f"var_group_labels: {_format_first_three_categories(var_group_labels)}" + ) + var_names_idx_ordered = list(range(len(var_names))) if var_names_idx_ordered is not None: var_names_ordered = [var_names[x] for x in var_names_idx_ordered] @@ -2364,14 +2362,23 @@ def _format_first_three_categories(categories): return ", ".join(categories) -def _get_dendrogram_key(adata, dendrogram_key, groupby): +def _get_dendrogram_key( + adata: AnnData, + dendrogram_key: str | None, + groupby: str | Sequence[str], + *, + validate_groupby: bool = False, +) -> str: # the `dendrogram_key` can be a bool an NoneType or the name of the # dendrogram key. By default the name of the dendrogram key is 'dendrogram' - if not isinstance(dendrogram_key, str): + if dendrogram_key is None: if isinstance(groupby, str): dendrogram_key = f"dendrogram_{groupby}" - elif isinstance(groupby, list): - dendrogram_key = f'dendrogram_{"_".join(groupby)}' + elif isinstance(groupby, Sequence): + dendrogram_key = f"dendrogram_{'_'.join(groupby)}" + else: + msg = f"groupby has wrong type: {type(groupby).__name__}." + raise AssertionError(msg) if dendrogram_key not in adata.uns: from ..tools._dendrogram import dendrogram @@ -2384,10 +2391,22 @@ def _get_dendrogram_key(adata, dendrogram_key, groupby): dendrogram(adata, groupby, key_added=dendrogram_key) if "dendrogram_info" not in adata.uns[dendrogram_key]: - raise ValueError( + msg = ( f"The given dendrogram key ({dendrogram_key!r}) does not contain " "valid dendrogram information." ) + raise ValueError(msg) + + if validate_groupby: + existing_groupby = adata.uns[dendrogram_key]["groupby"] + if groupby != existing_groupby: + msg = ( + "Incompatible observations. The precomputed dendrogram contains " + f"information for the observation: {groupby!r} while the plot is " + f"made for the observation: {existing_groupby!r}. " + "Please run `sc.tl.dendrogram` using the right observation.'" + ) + raise ValueError(msg) return dendrogram_key @@ -2395,7 +2414,7 @@ def _get_dendrogram_key(adata, dendrogram_key, groupby): def _plot_dendrogram( dendro_ax: Axes, adata: AnnData, - groupby: str, + groupby: str | Sequence[str], *, dendrogram_key: str | None = None, orientation: Literal["top", "bottom", "left", "right"] = "right", @@ -2510,7 +2529,7 @@ def translate_pos(pos_list, new_ticks, old_ticks): labelbottom=False, labeltop=False, labelleft=False, labelright=False ) - dendro_ax.grid(False) + dendro_ax.grid(visible=False) dendro_ax.spines["right"].set_visible(False) dendro_ax.spines["top"].set_visible(False) @@ -2567,7 +2586,7 @@ def _plot_categories_as_colorblocks( value_sum += value label2code[label] = code - groupby_ax.grid(False) + groupby_ax.grid(visible=False) if orientation == "left": groupby_ax.imshow( @@ -2601,11 +2620,8 @@ def _plot_categories_as_colorblocks( ) if len(labels) > 1: groupby_ax.set_xticks(ticks) - if max([len(str(x)) for x in labels]) < 3: - # if the labels are small do not rotate them - rotation = 0 - else: - rotation = 90 + # if the labels are small do not rotate them + rotation = 0 if max(len(str(x)) for x in labels) < 3 else 90 groupby_ax.set_xticklabels(labels, rotation=rotation) # remove x ticks @@ -2671,7 +2687,7 @@ def _check_var_names_type(var_names, var_group_labels, var_group_positions): var_names, var_group_labels, var_group_positions """ - if isinstance(var_names, cabc.Mapping): + if isinstance(var_names, Mapping): if var_group_labels is not None or var_group_positions is not None: logg.warning( "`var_names` is a dictionary. This will reset the current " diff --git a/src/scanpy/plotting/_baseplot_class.py b/src/scanpy/plotting/_baseplot_class.py index 45a95681cc..e14d387f84 100644 --- a/src/scanpy/plotting/_baseplot_class.py +++ b/src/scanpy/plotting/_baseplot_class.py @@ -2,7 +2,7 @@ from __future__ import annotations -import collections.abc as cabc +from collections.abc import Mapping from typing import TYPE_CHECKING, NamedTuple from warnings import warn @@ -12,21 +12,23 @@ from .. import logging as logg from .._compat import old_positionals +from .._utils import _empty from ._anndata import _get_dendrogram_key, _plot_dendrogram, _prepare_dataframe from ._utils import check_colornorm, make_grid_spec if TYPE_CHECKING: - from collections.abc import Iterable, Mapping, Sequence - from typing import Literal, Self, Union + from collections.abc import Iterable, Sequence + from typing import Literal, Self import pandas as pd from anndata import AnnData from matplotlib.axes import Axes - from matplotlib.colors import Normalize + from matplotlib.colors import Colormap, Normalize + from .._utils import Empty from ._utils import ColorLike, _AxesSubplot - _VarNames = Union[str, Sequence[str]] + _VarNames = str | Sequence[str] class VBoundNorm(NamedTuple): @@ -134,9 +136,7 @@ def __init__( self.width, self.height = figsize if figsize is not None else (None, None) self.has_var_groups = ( - True - if var_group_positions is not None and len(var_group_positions) > 0 - else False + var_group_positions is not None and len(var_group_positions) > 0 ) self._update_var_groups() @@ -157,18 +157,19 @@ def __init__( "Plot would be very large." ) - if categories_order is not None: - if set(self.obs_tidy.index.categories) != set(categories_order): - logg.error( - "Please check that the categories given by " - "the `order` parameter match the categories that " - "want to be reordered.\n\n" - "Mismatch: " - f"{set(self.obs_tidy.index.categories).difference(categories_order)}\n\n" - f"Given order categories: {categories_order}\n\n" - f"{groupby} categories: {list(self.obs_tidy.index.categories)}\n" - ) - return + if categories_order is not None and ( + set(self.obs_tidy.index.categories) != set(categories_order) + ): + logg.error( + "Please check that the categories given by " + "the `order` parameter match the categories that " + "want to be reordered.\n\n" + "Mismatch: " + f"{set(self.obs_tidy.index.categories).difference(categories_order)}\n\n" + f"Given order categories: {categories_order}\n\n" + f"{groupby} categories: {list(self.obs_tidy.index.categories)}\n" + ) + return self.adata = adata self.groupby = [groupby] if isinstance(groupby, str) else groupby @@ -204,7 +205,8 @@ def __init__( self.ax_dict = None self.ax = ax - def swap_axes(self, swap_axes: bool | None = True) -> Self: + @old_positionals("swap_axes") + def swap_axes(self, *, swap_axes: bool | None = True) -> Self: """ Plots a transposed image. @@ -231,8 +233,10 @@ def swap_axes(self, swap_axes: bool | None = True) -> Self: self.are_axes_swapped = swap_axes return self + @old_positionals("show", "dendrogram_key", "size") def add_dendrogram( self, + *, show: bool | None = True, dendrogram_key: str | None = None, size: float | None = 0.8, @@ -316,8 +320,10 @@ def add_dendrogram( } return self + @old_positionals("show", "sort", "size", "color") def add_totals( self, + *, show: bool | None = True, sort: Literal["ascending", "descending"] | None = None, size: float | None = 0.8, @@ -380,8 +386,8 @@ def add_totals( self.group_extra_size = 0 return self - _sort = True if sort is not None else False - _ascending = True if sort == "ascending" else False + _sort = sort is not None + _ascending = sort == "ascending" counts_df = self.obs_tidy.index.value_counts(sort=_sort, ascending=_ascending) if _sort: @@ -397,21 +403,23 @@ def add_totals( return self @old_positionals("cmap") - def style(self, *, cmap: str | None = DEFAULT_COLORMAP) -> Self: + def style(self, *, cmap: Colormap | str | None | Empty = _empty) -> Self: """\ Set visual style parameters Parameters ---------- cmap - colormap + Matplotlib color map, specified by name or directly. + If ``None``, use :obj:`matplotlib.rcParams`\\ ``["image.cmap"]`` Returns ------- Returns `self` for method chaining. """ - self.cmap = cmap + if cmap is not _empty: + self.cmap = cmap return self @old_positionals("show", "title", "width") @@ -479,10 +487,7 @@ def _plot_totals( if self.categories_order is not None: counts_df = counts_df.loc[self.categories_order] if params["color"] is None: - if f"{self.groupby}_colors" in self.adata.uns: - color = self.adata.uns[f"{self.groupby}_colors"] - else: - color = "salmon" + color = self.adata.uns.get(f"{self.groupby}_colors", "salmon") else: color = params["color"] @@ -545,7 +550,7 @@ def _plot_totals( ) total_barplot_ax.set_xlim(0, max_x * 1.4) - total_barplot_ax.grid(False) + total_barplot_ax.grid(visible=False) total_barplot_ax.axis("off") def _plot_colorbar(self, color_legend_ax: Axes, normalize) -> None: @@ -597,7 +602,7 @@ def _plot_legend(self, legend_ax, return_ax_dict, normalize): self._plot_colorbar(color_legend_ax, normalize) return_ax_dict["color_legend_ax"] = color_legend_ax - def _mainplot(self, ax): + def _mainplot(self, ax: Axes): y_labels = self.categories x_labels = self.var_names @@ -622,7 +627,7 @@ def _mainplot(self, ax): ax.set_xticklabels(x_labels, rotation=90, ha="center", minor=False) ax.tick_params(axis="both", labelsize="small") - ax.grid(False) + ax.grid(visible=False) # to be consistent with the heatmap plot, is better to # invert the order of the y-axis, such that the first group is on @@ -868,7 +873,7 @@ def savefig(self, filename: str, bbox_inches: str | None = "tight", **kwargs): self.make_figure() plt.savefig(filename, bbox_inches=bbox_inches, **kwargs) - def _reorder_categories_after_dendrogram(self, dendrogram) -> None: + def _reorder_categories_after_dendrogram(self, dendrogram_key: str | None) -> None: """\ Function used by plotting functions that need to reorder the the groupby observations based on the dendrogram results. @@ -894,23 +899,18 @@ def _format_first_three_categories(_categories): _categories = _categories[:3] + ["etc."] return ", ".join(_categories) - key = _get_dendrogram_key(self.adata, dendrogram, self.groupby) - - dendro_info = self.adata.uns[key] - if self.groupby != dendro_info["groupby"]: - raise ValueError( - "Incompatible observations. The precomputed dendrogram contains " - f"information for the observation: '{self.groupby}' while the plot is " - f"made for the observation: '{dendro_info['groupby']}. " - "Please run `sc.tl.dendrogram` using the right observation.'" + dendro_info = self.adata.uns[ + _get_dendrogram_key( + self.adata, dendrogram_key, self.groupby, validate_groupby=True ) + ] # order of groupby categories categories_idx_ordered = dendro_info["categories_idx_ordered"] categories_ordered = dendro_info["categories_ordered"] if len(self.categories) != len(categories_idx_ordered): - raise ValueError( + msg = ( "Incompatible observations. Dendrogram data has " f"{len(categories_idx_ordered)} categories but current groupby " f"observation {self.groupby!r} contains {len(self.categories)} categories. " @@ -918,6 +918,7 @@ def _format_first_three_categories(_categories): "initial computation of `sc.tl.dendrogram`. " "Please run `sc.tl.dendrogram` again.'" ) + raise ValueError(msg) # reorder var_groups (if any) if self.var_names is not None: @@ -1017,10 +1018,7 @@ def _plot_var_groups_brackets( if orientation == "top": # rotate labels if any of them is longer than 4 characters if rotation is None and group_labels: - if max([len(x) for x in group_labels]) > 4: - rotation = 90 - else: - rotation = 0 + rotation = 90 if max([len(x) for x in group_labels]) > 4 else 0 for idx, (left_coor, right_coor) in enumerate(zip(left, right)): verts.append((left_coor, 0)) # lower-left verts.append((left_coor, 0.6)) # upper-left @@ -1075,7 +1073,7 @@ def _plot_var_groups_brackets( patch = patches.PathPatch(path, facecolor="none", lw=1.5) gene_groups_ax.add_patch(patch) - gene_groups_ax.grid(False) + gene_groups_ax.grid(visible=False) gene_groups_ax.axis("off") # remove y ticks gene_groups_ax.tick_params(axis="y", left=False, labelleft=False) @@ -1091,7 +1089,7 @@ def _update_var_groups(self) -> None: updates var_names, var_group_labels, var_group_positions """ - if isinstance(self.var_names, cabc.Mapping): + if isinstance(self.var_names, Mapping): if self.has_var_groups: logg.warning( "`var_names` is a dictionary. This will reset the current " diff --git a/src/scanpy/plotting/_docs.py b/src/scanpy/plotting/_docs.py index 0bf363237e..61f0baa420 100644 --- a/src/scanpy/plotting/_docs.py +++ b/src/scanpy/plotting/_docs.py @@ -80,8 +80,8 @@ projection Projection of plot (default: `'2d'`). legend_loc - Location of legend, either `'on data'`, `'right margin'` or a valid keyword - for the `loc` parameter of :class:`~matplotlib.legend.Legend`. + Location of legend, either `'on data'`, `'right margin'`, `None`, + or a valid keyword for the `loc` parameter of :class:`~matplotlib.legend.Legend`. legend_fontsize Numeric size in pt or string describing the size. See :meth:`~matplotlib.text.Text.set_fontsize`. diff --git a/src/scanpy/plotting/_dotplot.py b/src/scanpy/plotting/_dotplot.py index 4b6d29f0f5..da3d16379b 100644 --- a/src/scanpy/plotting/_dotplot.py +++ b/src/scanpy/plotting/_dotplot.py @@ -8,10 +8,11 @@ from .. import logging as logg from .._compat import old_positionals from .._settings import settings -from .._utils import _doc_params +from .._utils import _doc_params, _empty from ._baseplot_class import BasePlot, doc_common_groupby_plot_args from ._docs import doc_common_plot_args, doc_show_save_ax, doc_vboundnorm from ._utils import ( + _dk, check_colornorm, fix_kwds, make_grid_spec, @@ -25,13 +26,11 @@ import pandas as pd from anndata import AnnData from matplotlib.axes import Axes - from matplotlib.colors import Normalize + from matplotlib.colors import Colormap, Normalize + from .._utils import Empty from ._baseplot_class import _VarNames - from ._utils import ( - ColorLike, - _AxesSubplot, - ) + from ._utils import ColorLike, _AxesSubplot @_doc_params(common_plot_args=doc_common_plot_args) @@ -97,7 +96,7 @@ class DotPlot(BasePlot): DEFAULT_SAVE_PREFIX = "dotplot_" # default style parameters - DEFAULT_COLORMAP = "winter" + DEFAULT_COLORMAP = "Reds" DEFAULT_COLOR_ON = "dot" DEFAULT_DOT_MAX = None DEFAULT_DOT_MIN = None @@ -263,6 +262,7 @@ def __init__( ] for df in (dot_color_df, dot_size_df) ) + self.standard_scale = standard_scale # Set default style parameters self.cmap = self.DEFAULT_COLORMAP @@ -303,18 +303,18 @@ def __init__( def style( self, *, - cmap: str = DEFAULT_COLORMAP, - color_on: Literal["dot", "square"] | None = DEFAULT_COLOR_ON, - dot_max: float | None = DEFAULT_DOT_MAX, - dot_min: float | None = DEFAULT_DOT_MIN, - smallest_dot: float | None = DEFAULT_SMALLEST_DOT, - largest_dot: float | None = DEFAULT_LARGEST_DOT, - dot_edge_color: ColorLike | None = DEFAULT_DOT_EDGECOLOR, - dot_edge_lw: float | None = DEFAULT_DOT_EDGELW, - size_exponent: float | None = DEFAULT_SIZE_EXPONENT, - grid: float | None = False, - x_padding: float | None = DEFAULT_PLOT_X_PADDING, - y_padding: float | None = DEFAULT_PLOT_Y_PADDING, + cmap: Colormap | str | None | Empty = _empty, + color_on: Literal["dot", "square"] | Empty = _empty, + dot_max: float | None | Empty = _empty, + dot_min: float | None | Empty = _empty, + smallest_dot: float | Empty = _empty, + largest_dot: float | Empty = _empty, + dot_edge_color: ColorLike | None | Empty = _empty, + dot_edge_lw: float | None | Empty = _empty, + size_exponent: float | Empty = _empty, + grid: bool | Empty = _empty, + x_padding: float | Empty = _empty, + y_padding: float | Empty = _empty, ) -> Self: r"""\ Modifies plot visual parameters @@ -324,31 +324,30 @@ def style( cmap String denoting matplotlib color map. color_on - Options are 'dot' or 'square'. Be default the colomap is applied to - the color of the dot. Optionally, the colormap can be applied to an - square behind the dot, in which case the dot is transparent and only - the edge is shown. + By default the color map is applied to the color of the ``"dot"``. + Optionally, the colormap can be applied to a ``"square"`` behind the dot, + in which case the dot is transparent and only the edge is shown. dot_max - If none, the maximum dot size is set to the maximum fraction value found - (e.g. 0.6). If given, the value should be a number between 0 and 1. + If ``None``, the maximum dot size is set to the maximum fraction value found (e.g. 0.6). + If given, the value should be a number between 0 and 1. All fractions larger than dot_max are clipped to this value. dot_min - If none, the minimum dot size is set to 0. If given, - the value should be a number between 0 and 1. + If ``None``, the minimum dot size is set to 0. + If given, the value should be a number between 0 and 1. All fractions smaller than dot_min are clipped to this value. smallest_dot - If none, the smallest dot has size 0. All expression fractions with `dot_min` are plotted with this size. largest_dot - If none, the largest dot has size 200. All expression fractions with `dot_max` are plotted with this size. dot_edge_color - Dot edge color. When `color_on='dot'` the default is no edge. When - `color_on='square'`, edge color is white for darker colors and black - for lighter background square colors. + Dot edge color. + When `color_on='dot'`, ``None`` means no edge. + When `color_on='square'`, ``None`` means that + the edge color is white for darker colors and black for lighter background square colors. dot_edge_lw - Dot edge line width. When `color_on='dot'` the default is no edge. When - `color_on='square'`, line width = 1.5. + Dot edge line width. + When `color_on='dot'`, ``None`` means no edge. + When `color_on='square'`, ``None`` means a line width of 1.5. size_exponent Dot size is computed as: fraction ** size exponent and afterwards scaled to match the @@ -388,31 +387,29 @@ def style( ... .style(dot_edge_color='black', dot_edge_lw=1, grid=True) \ ... .show() """ + super().style(cmap=cmap) - # change only the values that had changed - if cmap != self.cmap: - self.cmap = cmap - if dot_max != self.dot_max: + if dot_max is not _empty: self.dot_max = dot_max - if dot_min != self.dot_min: + if dot_min is not _empty: self.dot_min = dot_min - if smallest_dot != self.smallest_dot: + if smallest_dot is not _empty: self.smallest_dot = smallest_dot - if largest_dot != self.largest_dot: + if largest_dot is not _empty: self.largest_dot = largest_dot - if color_on != self.color_on: + if color_on is not _empty: self.color_on = color_on - if size_exponent != self.size_exponent: + if size_exponent is not _empty: self.size_exponent = size_exponent - if dot_edge_color != self.dot_edge_color: + if dot_edge_color is not _empty: self.dot_edge_color = dot_edge_color - if dot_edge_lw != self.dot_edge_lw: + if dot_edge_lw is not _empty: self.dot_edge_lw = dot_edge_lw - if grid != self.grid: + if grid is not _empty: self.grid = grid - if x_padding != self.plot_x_padding: + if x_padding is not _empty: self.plot_x_padding = x_padding - if y_padding != self.plot_y_padding: + if y_padding is not _empty: self.plot_y_padding = y_padding return self @@ -530,7 +527,7 @@ def _plot_size_legend(self, size_legend_ax: Axes): size_legend_ax.spines["top"].set_visible(False) size_legend_ax.spines["left"].set_visible(False) size_legend_ax.spines["bottom"].set_visible(False) - size_legend_ax.grid(False) + size_legend_ax.grid(visible=False) ymax = size_legend_ax.get_ylim()[1] size_legend_ax.set_ylim(-1.05 - self.largest_dot * 0.003, 4) @@ -574,7 +571,7 @@ def _plot_legend(self, legend_ax, return_ax_dict, normalize): self._plot_colorbar(color_legend_ax, normalize) return_ax_dict["color_legend_ax"] = color_legend_ax - def _mainplot(self, ax): + def _mainplot(self, ax: Axes): # work on a copy of the dataframes. This is to avoid changes # on the original data frames after repetitive calls to the # DotPlot object, for example once with swap_axes and other without @@ -599,9 +596,10 @@ def _mainplot(self, ax): _color_df, ax, cmap=self.cmap, + color_on=self.color_on, dot_max=self.dot_max, dot_min=self.dot_min, - color_on=self.color_on, + standard_scale=self.standard_scale, edge_color=self.dot_edge_color, edge_lw=self.dot_edge_lw, smallest_dot=self.smallest_dot, @@ -626,24 +624,23 @@ def _dotplot( dot_color: pd.DataFrame, dot_ax: Axes, *, - cmap: str = "Reds", - color_on: str | None = "dot", - y_label: str | None = None, - dot_max: float | None = None, - dot_min: float | None = None, - standard_scale: Literal["var", "group"] | None = None, - smallest_dot: float | None = 0.0, - largest_dot: float | None = 200, - size_exponent: float | None = 2, - edge_color: ColorLike | None = None, - edge_lw: float | None = None, - grid: bool | None = False, - x_padding: float | None = 0.8, - y_padding: float | None = 1.0, - vmin: float | None = None, - vmax: float | None = None, - vcenter: float | None = None, - norm: Normalize | None = None, + cmap: Colormap | str | None, + color_on: Literal["dot", "square"], + dot_max: float | None, + dot_min: float | None, + standard_scale: Literal["var", "group"] | None, + smallest_dot: float, + largest_dot: float, + size_exponent: float, + edge_color: ColorLike | None, + edge_lw: float | None, + grid: bool, + x_padding: float, + y_padding: float, + vmin: float | None, + vmax: float | None, + vcenter: float | None, + norm: Normalize | None, **kwds, ): """\ @@ -656,47 +653,25 @@ def _dotplot( Parameters ---------- - dot_size: Data frame containing the dot_size. - dot_color: Data frame containing the dot_color, should have the same, - shape, columns and indices as dot_size. - dot_ax: matplotlib axis + dot_size + Data frame containing the dot_size. + dot_color + Data frame containing the dot_color, should have the same, + shape, columns and indices as dot_size. + dot_ax + matplotlib axis cmap - String denoting matplotlib color map. color_on - Options are 'dot' or 'square'. Be default the colomap is applied to - the color of the dot. Optionally, the colormap can be applied to an - square behind the dot, in which case the dot is transparent and only - the edge is shown. - y_label: String. Label for y axis dot_max - If none, the maximum dot size is set to the maximum fraction value found - (e.g. 0.6). If given, the value should be a number between 0 and 1. - All fractions larger than dot_max are clipped to this value. dot_min - If none, the minimum dot size is set to 0. If given, - the value should be a number between 0 and 1. - All fractions smaller than dot_min are clipped to this value. standard_scale - Whether or not to standardize that dimension between 0 and 1, - meaning for each variable or group, - subtract the minimum and divide each by its maximum. smallest_dot - If none, the smallest dot has size 0. - All expression levels with `dot_min` are plotted with this size. edge_color - Dot edge color. When `color_on='dot'` the default is no edge. When - `color_on='square'`, edge color is white edge_lw - Dot edge line width. When `color_on='dot'` the default is no edge. When - `color_on='square'`, line width = 1.5 grid - Adds a grid to the plot - x_paddding - Space between the plot left/right borders and the dots center. A unit - is the distance between the x ticks. Only applied when color_on = dot - y_paddding - Space between the plot top/bottom borders and the dots center. A unit is - the distance between the y ticks. Only applied when color_on = dot + x_padding + y_padding + See `style` kwds Are passed to :func:`matplotlib.pyplot.scatter`. @@ -706,11 +681,11 @@ def _dotplot( """ assert dot_size.shape == dot_color.shape, ( - "please check that dot_size " "and dot_color dataframes have the same shape" + "please check that dot_size and dot_color dataframes have the same shape" ) assert list(dot_size.index) == list(dot_color.index), ( - "please check that dot_size " "and dot_color dataframes have the same index" + "please check that dot_size and dot_color dataframes have the same index" ) assert list(dot_size.columns) == list(dot_color.columns), ( @@ -746,12 +721,14 @@ def _dotplot( dot_max = np.ceil(max(frac) * 10) / 10 else: if dot_max < 0 or dot_max > 1: - raise ValueError("`dot_max` value has to be between 0 and 1") + msg = "`dot_max` value has to be between 0 and 1" + raise ValueError(msg) if dot_min is None: dot_min = 0 else: if dot_min < 0 or dot_min > 1: - raise ValueError("`dot_min` value has to be between 0 and 1") + msg = "`dot_min` value has to be between 0 and 1" + raise ValueError(msg) if dot_min != 0 or dot_max != 1: # clip frac between dot_min and dot_max @@ -805,7 +782,6 @@ def _dotplot( linewidth=edge_lw, edgecolor=edge_color, ) - dot_ax.scatter(x, y, **kwds) y_ticks = np.arange(dot_color.shape[0]) + 0.5 @@ -823,8 +799,7 @@ def _dotplot( minor=False, ) dot_ax.tick_params(axis="both", labelsize="small") - dot_ax.grid(False) - dot_ax.set_ylabel(y_label) + dot_ax.grid(visible=False) # to be consistent with the heatmap plot, is better to # invert the order of the y-axis, such that the first group is on @@ -844,7 +819,7 @@ def _dotplot( dot_ax.set_xlim(-x_padding, dot_color.shape[1] + x_padding) if grid: - dot_ax.grid(True, color="gray", linewidth=0.1) + dot_ax.grid(visible=True, color="gray", linewidth=0.1) dot_ax.set_axisbelow(True) return normalize, dot_min, dot_max @@ -880,13 +855,10 @@ def dotplot( use_raw: bool | None = None, log: bool = False, num_categories: int = 7, + categories_order: Sequence[str] | None = None, expression_cutoff: float = 0.0, mean_only_expressed: bool = False, - cmap: str = "Reds", - dot_max: float | None = DotPlot.DEFAULT_DOT_MAX, - dot_min: float | None = DotPlot.DEFAULT_DOT_MIN, standard_scale: Literal["var", "group"] | None = None, - smallest_dot: float | None = DotPlot.DEFAULT_SMALLEST_DOT, title: str | None = None, colorbar_title: str | None = DotPlot.DEFAULT_COLOR_LEGEND_TITLE, size_title: str | None = DotPlot.DEFAULT_SIZE_LEGEND_TITLE, @@ -907,6 +879,11 @@ def dotplot( vmax: float | None = None, vcenter: float | None = None, norm: Normalize | None = None, + # Style parameters + cmap: Colormap | str | None = DotPlot.DEFAULT_COLORMAP, + dot_max: float | None = DotPlot.DEFAULT_DOT_MAX, + dot_min: float | None = DotPlot.DEFAULT_DOT_MIN, + smallest_dot: float = DotPlot.DEFAULT_SMALLEST_DOT, **kwds, ) -> DotPlot | dict | None: """\ @@ -944,15 +921,14 @@ def dotplot( If True, gene expression is averaged only over the cells expressing the given genes. dot_max - If none, the maximum dot size is set to the maximum fraction value found + If ``None``, the maximum dot size is set to the maximum fraction value found (e.g. 0.6). If given, the value should be a number between 0 and 1. All fractions larger than dot_max are clipped to this value. dot_min - If none, the minimum dot size is set to 0. If given, + If ``None``, the minimum dot size is set to 0. If given, the value should be a number between 0 and 1. All fractions smaller than dot_min are clipped to this value. smallest_dot - If none, the smallest dot has size 0. All expression levels with `dot_min` are plotted with this size. {show_save_ax} {vminmax} @@ -1012,9 +988,7 @@ def dotplot( # backwards compatibility: previous version of dotplot used `color_map` # instead of `cmap` - cmap = kwds.get("color_map", cmap) - if "color_map" in kwds: - del kwds["color_map"] + cmap = kwds.pop("color_map", cmap) dp = DotPlot( adata, @@ -1023,6 +997,7 @@ def dotplot( use_raw=use_raw, log=log, num_categories=num_categories, + categories_order=categories_order, expression_cutoff=expression_cutoff, mean_only_expressed=mean_only_expressed, standard_scale=standard_scale, @@ -1043,7 +1018,7 @@ def dotplot( ) if dendrogram: - dp.add_dendrogram(dendrogram_key=dendrogram) + dp.add_dendrogram(dendrogram_key=_dk(dendrogram)) if swap_axes: dp.swap_axes() @@ -1052,7 +1027,7 @@ def dotplot( dot_max=dot_max, dot_min=dot_min, smallest_dot=smallest_dot, - dot_edge_lw=kwds.pop("linewidth", DotPlot.DEFAULT_DOT_EDGELW), + dot_edge_lw=kwds.pop("linewidth", _empty), ).legend(colorbar_title=colorbar_title, size_title=size_title) if return_fig: diff --git a/src/scanpy/plotting/_matrixplot.py b/src/scanpy/plotting/_matrixplot.py index 011955eb2b..9184f2455b 100644 --- a/src/scanpy/plotting/_matrixplot.py +++ b/src/scanpy/plotting/_matrixplot.py @@ -9,14 +9,14 @@ from .. import logging as logg from .._compat import old_positionals from .._settings import settings -from .._utils import _doc_params +from .._utils import _doc_params, _empty from ._baseplot_class import BasePlot, doc_common_groupby_plot_args from ._docs import ( doc_common_plot_args, doc_show_save_ax, doc_vboundnorm, ) -from ._utils import check_colornorm, fix_kwds, savefig_or_show +from ._utils import _dk, check_colornorm, fix_kwds, savefig_or_show if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -25,8 +25,9 @@ import pandas as pd from anndata import AnnData from matplotlib.axes import Axes - from matplotlib.colors import Normalize + from matplotlib.colors import Colormap, Normalize + from .._utils import Empty from ._baseplot_class import _VarNames from ._utils import ColorLike, _AxesSubplot @@ -134,7 +135,7 @@ def __init__( var_group_labels: Sequence[str] | None = None, var_group_rotation: float | None = None, layer: str | None = None, - standard_scale: Literal["var", "group"] = None, + standard_scale: Literal["var", "group"] | None = None, ax: _AxesSubplot | None = None, values_df: pd.DataFrame | None = None, vmin: float | None = None, @@ -198,9 +199,9 @@ def __init__( def style( self, - cmap: str = DEFAULT_COLORMAP, - edge_color: ColorLike | None = DEFAULT_EDGE_COLOR, - edge_lw: float | None = DEFAULT_EDGE_LW, + cmap: Colormap | str | None | Empty = _empty, + edge_color: ColorLike | None | Empty = _empty, + edge_lw: float | None | Empty = _empty, ) -> Self: """\ Modifies plot visual parameters. @@ -208,11 +209,14 @@ def style( Parameters ---------- cmap - String denoting matplotlib color map. + Matplotlib color map, specified by name or directly. + If ``None``, use :obj:`matplotlib.rcParams`\\ ``["image.cmap"]`` edge_color - Edge color between the squares of matrix plot. Default is gray + Edge color between the squares of matrix plot. + If ``None``, use :obj:`matplotlib.rcParams`\\ ``["patch.edgecolor"]`` edge_lw Edge line width. + If ``None``, use :obj:`matplotlib.rcParams`\\ ``["lines.linewidth"]`` Returns ------- @@ -242,18 +246,16 @@ def style( ) """ + super().style(cmap=cmap) - # change only the values that had changed - if cmap != self.cmap: - self.cmap = cmap - if edge_color != self.edge_color: + if edge_color is not _empty: self.edge_color = edge_color - if edge_lw != self.edge_lw: + if edge_lw is not _empty: self.edge_lw = edge_lw return self - def _mainplot(self, ax): + def _mainplot(self, ax: Axes): # work on a copy of the dataframes. This is to avoid changes # on the original data frames after repetitive calls to the # MatrixPlot object, for example once with swap_axes and other without @@ -301,7 +303,7 @@ def _mainplot(self, ax): ax.set_xticklabels(x_labels, rotation=90, ha="center", minor=False) ax.tick_params(axis="both", labelsize="small") - ax.grid(False) + ax.grid(visible=False) # to be consistent with the heatmap plot, is better to # invert the order of the y-axis, such that the first group is on @@ -343,10 +345,11 @@ def matrixplot( use_raw: bool | None = None, log: bool = False, num_categories: int = 7, + categories_order: Sequence[str] | None = None, figsize: tuple[float, float] | None = None, dendrogram: bool | str = False, title: str | None = None, - cmap: str | None = MatrixPlot.DEFAULT_COLORMAP, + cmap: Colormap | str | None = MatrixPlot.DEFAULT_COLORMAP, colorbar_title: str | None = MatrixPlot.DEFAULT_COLOR_LEGEND_TITLE, gene_symbols: str | None = None, var_group_positions: Sequence[tuple[int, int]] | None = None, @@ -436,6 +439,7 @@ def matrixplot( use_raw=use_raw, log=log, num_categories=num_categories, + categories_order=categories_order, standard_scale=standard_scale, title=title, figsize=figsize, @@ -454,7 +458,7 @@ def matrixplot( ) if dendrogram: - mp.add_dendrogram(dendrogram_key=dendrogram) + mp.add_dendrogram(dendrogram_key=_dk(dendrogram)) if swap_axes: mp.swap_axes() diff --git a/src/scanpy/plotting/_preprocessing.py b/src/scanpy/plotting/_preprocessing.py index 9dce5dca3f..b51688082e 100644 --- a/src/scanpy/plotting/_preprocessing.py +++ b/src/scanpy/plotting/_preprocessing.py @@ -6,7 +6,7 @@ from matplotlib import pyplot as plt from matplotlib import rcParams -from .._compat import old_positionals +from .._compat import deprecated, old_positionals from .._settings import settings from . import _utils @@ -103,8 +103,11 @@ def highly_variable_genes( # backwards compat +@deprecated("Use sc.pl.highly_variable_genes instead") +@old_positionals("log", "show", "save") def filter_genes_dispersion( result: np.recarray, + *, log: bool = False, show: bool | None = None, save: bool | str | None = None, diff --git a/src/scanpy/plotting/_qc.py b/src/scanpy/plotting/_qc.py index e37276f4b6..cd3f764468 100644 --- a/src/scanpy/plotting/_qc.py +++ b/src/scanpy/plotting/_qc.py @@ -24,11 +24,12 @@ def highest_expr_genes( adata: AnnData, n_top: int = 30, *, + layer: str | None = None, + gene_symbols: str | None = None, + log: bool = False, show: bool | None = None, save: str | bool | None = None, ax: Axes | None = None, - gene_symbols: str | None = None, - log: bool = False, **kwds, ): """\ @@ -56,11 +57,13 @@ def highest_expr_genes( Annotated data matrix. n_top Number of top - {show_save_ax} + layer + Layer from which to pull data. gene_symbols Key for field in .var that stores gene symbols if you do not want to use .var_names. log Plot x-axis in log scale + {show_save_ax} **kwds Are passed to :func:`~seaborn.boxplot`. @@ -72,7 +75,7 @@ def highest_expr_genes( from scipy.sparse import issparse # compute the percentage of each gene per cell - norm_dict = normalize_total(adata, target_sum=100, inplace=False) + norm_dict = normalize_total(adata, target_sum=100, layer=layer, inplace=False) # identify the genes with the highest mean if issparse(norm_dict["X"]): @@ -86,7 +89,7 @@ def highest_expr_genes( columns = ( adata.var_names[top_idx] if gene_symbols is None - else adata.var[gene_symbols][top_idx] + else adata.var[gene_symbols].iloc[top_idx].astype("string") ) counts_top_genes = pd.DataFrame( counts_top_genes, index=adata.obs_names, columns=columns diff --git a/src/scanpy/plotting/_scrublet.py b/src/scanpy/plotting/_scrublet.py index 66e33e2e82..050aec6f53 100644 --- a/src/scanpy/plotting/_scrublet.py +++ b/src/scanpy/plotting/_scrublet.py @@ -11,13 +11,13 @@ if TYPE_CHECKING: from collections.abc import Sequence - from typing import Literal, Union + from typing import Literal from anndata import AnnData from matplotlib.axes import Axes from matplotlib.figure import Figure - Scale = Union[Literal["linear", "log", "symlog", "logit"], str] + Scale = Literal["linear", "log", "symlog", "logit"] | str @old_positionals( @@ -72,9 +72,8 @@ def scrublet_score_distribution( """ if "scrublet" not in adata.uns: - raise ValueError( - "Please run scrublet before trying to generate the scrublet plot." - ) + msg = "Please run scrublet before trying to generate the scrublet plot." + raise ValueError(msg) # If batched_by is populated, then we know Scrublet was run over multiple batches diff --git a/src/scanpy/plotting/_stacked_violin.py b/src/scanpy/plotting/_stacked_violin.py index 8745452554..3c58ead35f 100644 --- a/src/scanpy/plotting/_stacked_violin.py +++ b/src/scanpy/plotting/_stacked_violin.py @@ -12,28 +12,28 @@ from .. import logging as logg from .._compat import old_positionals from .._settings import settings -from .._utils import _doc_params +from .._utils import _doc_params, _empty from ._baseplot_class import BasePlot, doc_common_groupby_plot_args from ._docs import doc_common_plot_args, doc_show_save_ax, doc_vboundnorm from ._utils import ( _deprecated_scale, + _dk, check_colornorm, make_grid_spec, savefig_or_show, ) if TYPE_CHECKING: - from collections.abc import ( - Mapping, # Special - Sequence, # ABCs - ) + from collections.abc import Mapping, Sequence from typing import Literal, Self from anndata import AnnData - from matplotlib.colors import Normalize + from matplotlib.axes import Axes + from matplotlib.colors import Colormap, Normalize + from .._utils import Empty from ._baseplot_class import _VarNames - from ._utils import _AxesSubplot + from ._utils import DensityNorm, _AxesSubplot @_doc_params(common_plot_args=doc_common_plot_args) @@ -120,7 +120,7 @@ class StackedViolin(BasePlot): DEFAULT_JITTER_SIZE = 1 DEFAULT_LINE_WIDTH = 0.2 DEFAULT_ROW_PALETTE = None - DEFAULT_DENSITY_NORM: Literal["area", "count", "width"] = "width" + DEFAULT_DENSITY_NORM: DensityNorm = "width" DEFAULT_PLOT_YTICKLABELS = False DEFAULT_YLIM = None DEFAULT_PLOT_X_PADDING = 0.5 # a unit is the distance between two x-axis ticks @@ -224,6 +224,10 @@ def __init__( ) if standard_scale == "obs": + standard_scale = "group" + msg = "`standard_scale='obs'` is deprecated, use `standard_scale='group'` instead" + warnings.warn(msg, FutureWarning) + if standard_scale == "group": self.obs_tidy = self.obs_tidy.sub(self.obs_tidy.min(1), axis=0) self.obs_tidy = self.obs_tidy.div(self.obs_tidy.max(1), axis=0).fillna(0) elif standard_scale == "var": @@ -266,19 +270,19 @@ def __init__( def style( self, *, - cmap: str | None = DEFAULT_COLORMAP, - stripplot: bool | None = DEFAULT_STRIPPLOT, - jitter: float | bool | None = DEFAULT_JITTER, - jitter_size: int | None = DEFAULT_JITTER_SIZE, - linewidth: float | None = DEFAULT_LINE_WIDTH, - row_palette: str | None = DEFAULT_ROW_PALETTE, - density_norm: Literal["area", "count", "width"] = DEFAULT_DENSITY_NORM, - yticklabels: bool | None = DEFAULT_PLOT_YTICKLABELS, - ylim: tuple[float, float] | None = DEFAULT_YLIM, - x_padding: float | None = DEFAULT_PLOT_X_PADDING, - y_padding: float | None = DEFAULT_PLOT_Y_PADDING, + cmap: Colormap | str | None | Empty = _empty, + stripplot: bool | Empty = _empty, + jitter: float | bool | Empty = _empty, + jitter_size: float | Empty = _empty, + linewidth: float | None | Empty = _empty, + row_palette: str | None | Empty = _empty, + density_norm: DensityNorm | Empty = _empty, + yticklabels: bool | Empty = _empty, + ylim: tuple[float, float] | None | Empty = _empty, + x_padding: float | Empty = _empty, + y_padding: float | Empty = _empty, # deprecated - scale: Literal["area", "count", "width"] | None = None, + scale: DensityNorm | Empty = _empty, ) -> Self: r"""\ Modifies plot visual parameters @@ -286,7 +290,8 @@ def style( Parameters ---------- cmap - String denoting matplotlib color map. + Matplotlib color map, specified by name or directly. + If ``None``, use :obj:`matplotlib.rcParams`\ ``["image.cmap"]`` stripplot Add a stripplot on top of the violin plot. See :func:`~seaborn.stripplot`. @@ -296,9 +301,11 @@ def style( jitter_size Size of the jitter points. linewidth - linewidth for the violin plots. + line width for the violin plots. + If None, use :obj:`matplotlib.rcParams`\ ``["lines.linewidth"]`` row_palette The row palette determines the colors to use for the stacked violins. + If ``None``, use :obj:`matplotlib.rcParams`\ ``["axes.prop_cycle"]`` The value should be a valid seaborn or matplotlib palette name (see :func:`~seaborn.color_palette`). Alternatively, a single color name or hex value can be passed, @@ -311,8 +318,9 @@ def style( yticklabels Set to true to view the y tick labels. ylim - minimum and maximum values for the y-axis. If set. All rows will have - the same y-axis range. Example: ylim=(0, 5) + minimum and maximum values for the y-axis. + If not ``None``, all rows will have the same y-axis range. + Example: ``ylim=(0, 5)`` x_padding Space between the plot left/right borders and the violins. A unit is the distance between the x ticks. @@ -335,20 +343,18 @@ def style( >>> sc.pl.StackedViolin(adata, markers, groupby='bulk_labels') \ ... .style(row_palette='Blues', linewidth=0).show() """ + super().style(cmap=cmap) - # modify only values that had changed - if cmap != self.cmap: - self.cmap = cmap - if row_palette != self.row_palette: + if row_palette is not _empty: self.row_palette = row_palette self.kwds["color"] = self.row_palette - if stripplot != self.stripplot: + if stripplot is not _empty: self.stripplot = stripplot - if jitter != self.jitter: + if jitter is not _empty: self.jitter = jitter - if jitter_size != self.jitter_size: + if jitter_size is not _empty: self.jitter_size = jitter_size - if yticklabels != self.plot_yticklabels: + if yticklabels is not _empty: self.plot_yticklabels = yticklabels if self.plot_yticklabels: # space needs to be added to avoid overlapping @@ -356,26 +362,20 @@ def style( self.wspace = 0.3 else: self.wspace = StackedViolin.DEFAULT_WSPACE - if ylim != self.ylim: + if ylim is not _empty: self.ylim = ylim - if x_padding != self.plot_x_padding: + if x_padding is not _empty: self.plot_x_padding = x_padding - if y_padding != self.plot_y_padding: + if y_padding is not _empty: self.plot_y_padding = y_padding - if linewidth != self.kwds["linewidth"] and linewidth != self.DEFAULT_LINE_WIDTH: + if linewidth is not _empty: self.kwds["linewidth"] = linewidth - density_norm = _deprecated_scale( - density_norm, scale, default=self.DEFAULT_DENSITY_NORM - ) - if ( - density_norm != self.kwds["density_norm"] - and density_norm != self.DEFAULT_DENSITY_NORM - ): + if (density_norm := _deprecated_scale(density_norm, scale)) is not _empty: self.kwds["density_norm"] = density_norm return self - def _mainplot(self, ax): + def _mainplot(self, ax: Axes): # to make the stacked violin plots, the # `ax` is subdivided horizontally and in each horizontal sub ax # a seaborn violin plot is added. @@ -412,8 +412,27 @@ def _mainplot(self, ax): colormap_array = cmap(normalize(_color_df.values)) x_spacer_size = self.plot_x_padding y_spacer_size = self.plot_y_padding + + # All columns should have a unique name, yet, frequently + # gene names are repeated in self.var_names, otherwise the + # violin plot will not distinguish those genes + _matrix.columns = [f"{x}_{idx}" for idx, x in enumerate(_matrix.columns)] + + # Ensure the categories axis is always ordered identically. + # If the axes are not swapped, the above _matrix.columns is used in the actual violin plot (i.e., unique names). + # If they are swapped, then use the same as the labels used below. + # Without this, `_make_rows_of_violinplots` does not know about the order of the categories in labels. + labels = _color_df.columns + x_axis_order = labels if self.are_axes_swapped else _matrix.columns + self._make_rows_of_violinplots( - ax, _matrix, colormap_array, _color_df, x_spacer_size, y_spacer_size + ax, + _matrix, + colormap_array, + _color_df, + x_spacer_size, + y_spacer_size, + x_axis_order, ) # turn on axis for `ax` as this is turned off @@ -436,18 +455,24 @@ def _mainplot(self, ax): # 0.5 to position the ticks on the center of the violins x_ticks = np.arange(_color_df.shape[1]) + 0.5 ax.set_xticks(x_ticks) - labels = _color_df.columns ax.set_xticklabels(labels, minor=False, ha="center") # rotate x tick labels if they are longer than 2 characters if max([len(x) for x in labels]) > 2: ax.tick_params(axis="x", labelrotation=90) ax.tick_params(axis="both", labelsize="small") - ax.grid(False) + ax.grid(visible=False) return normalize def _make_rows_of_violinplots( - self, ax, _matrix, colormap_array, _color_df, x_spacer_size, y_spacer_size + self, + ax, + _matrix, + colormap_array, + _color_df, + x_spacer_size: float, + y_spacer_size: float, + x_axis_order, ): import seaborn as sns # Slow import, only import if called @@ -462,11 +487,6 @@ def _make_rows_of_violinplots( else: row_colors = [None] * _color_df.shape[0] - # All columns should have a unique name, yet, frequently - # gene names are repeated in self.var_names, otherwise the - # violin plot will not distinguish those genes - _matrix.columns = [f"{x}_{idx}" for idx, x in enumerate(_matrix.columns)] - # transform the dataframe into a dataframe having three columns: # the categories name (from groupby), # the gene name @@ -545,9 +565,10 @@ def _make_rows_of_violinplots( hue=None if palette_colors is None else x, palette=palette_colors, color=row_colors[idx], + order=x_axis_order, + hue_order=x_axis_order, **self.kwds, ) - if self.stripplot: row_ax = sns.stripplot( x=x, @@ -561,7 +582,7 @@ def _make_rows_of_violinplots( self._setup_violin_axes_ticks(row_ax, num_cols) - def _setup_violin_axes_ticks(self, row_ax, num_cols): + def _setup_violin_axes_ticks(self, row_ax: Axes, num_cols: int): """ Configures each of the violin plot axes ticks like remove or add labels etc. @@ -569,7 +590,7 @@ def _setup_violin_axes_ticks(self, row_ax, num_cols): # remove the default seaborn grids because in such a compact # plot are unnecessary - row_ax.grid(False) + row_ax.grid(visible=False) if self.ylim is not None: row_ax.set_ylim(self.ylim) if self.log: @@ -661,26 +682,30 @@ def stacked_violin( gene_symbols: str | None = None, var_group_positions: Sequence[tuple[int, int]] | None = None, var_group_labels: Sequence[str] | None = None, - standard_scale: Literal["var", "obs"] | None = None, + standard_scale: Literal["var", "group"] | None = None, var_group_rotation: float | None = None, layer: str | None = None, - stripplot: bool = StackedViolin.DEFAULT_STRIPPLOT, - jitter: float | bool = StackedViolin.DEFAULT_JITTER, - size: int = StackedViolin.DEFAULT_JITTER_SIZE, - scale: Literal["area", "count", "width"] = StackedViolin.DEFAULT_DENSITY_NORM, - yticklabels: bool | None = StackedViolin.DEFAULT_PLOT_YTICKLABELS, - order: Sequence[str] | None = None, + categories_order: Sequence[str] | None = None, swap_axes: bool = False, show: bool | None = None, save: bool | str | None = None, return_fig: bool | None = False, - row_palette: str | None = StackedViolin.DEFAULT_ROW_PALETTE, - cmap: str | None = StackedViolin.DEFAULT_COLORMAP, ax: _AxesSubplot | None = None, vmin: float | None = None, vmax: float | None = None, vcenter: float | None = None, norm: Normalize | None = None, + # Style options + cmap: Colormap | str | None = StackedViolin.DEFAULT_COLORMAP, + stripplot: bool = StackedViolin.DEFAULT_STRIPPLOT, + jitter: float | bool = StackedViolin.DEFAULT_JITTER, + size: float = StackedViolin.DEFAULT_JITTER_SIZE, + row_palette: str | None = StackedViolin.DEFAULT_ROW_PALETTE, + density_norm: DensityNorm | Empty = _empty, + yticklabels: bool = StackedViolin.DEFAULT_PLOT_YTICKLABELS, + # deprecated + order: Sequence[str] | None | Empty = _empty, + scale: DensityNorm | Empty = _empty, **kwds, ) -> StackedViolin | dict | None: """\ @@ -708,11 +733,7 @@ def stacked_violin( See :func:`~seaborn.stripplot`. size Size of the jitter points. - order - Order in which to show the categories. Note: if `dendrogram=True` - the categories order will be given by the dendrogram and `order` - will be ignored. - scale + density_norm The method used to scale the width of each violin. If 'width' (the default), each violin will have the same width. If 'area', each violin will have the same area. @@ -729,7 +750,7 @@ def stacked_violin( e.g. `'red'` or `'#cc33ff'`. {show_save_ax} {vminmax} - kwds + **kwds Are passed to :func:`~seaborn.violinplot`. Returns @@ -782,6 +803,13 @@ def stacked_violin( print(axes_dict) """ + if order is not _empty: + msg = ( + "`order` is deprecated (and never worked for `stacked_violin`), " + "use categories_order instead" + ) + warnings.warn(msg, FutureWarning) + # no reason to set `categories_order` here, as `order` never worked. vp = StackedViolin( adata, @@ -790,6 +818,7 @@ def stacked_violin( use_raw=use_raw, log=log, num_categories=num_categories, + categories_order=categories_order, standard_scale=standard_scale, title=title, figsize=figsize, @@ -807,7 +836,7 @@ def stacked_violin( ) if dendrogram: - vp.add_dendrogram(dendrogram_key=dendrogram) + vp.add_dendrogram(dendrogram_key=_dk(dendrogram)) if swap_axes: vp.swap_axes() vp = vp.style( @@ -816,9 +845,9 @@ def stacked_violin( jitter=jitter, jitter_size=size, row_palette=row_palette, - density_norm=kwds.get("density_norm", scale), + density_norm=_deprecated_scale(density_norm, scale), yticklabels=yticklabels, - linewidth=kwds.get("linewidth", StackedViolin.DEFAULT_LINE_WIDTH), + linewidth=kwds.get("linewidth", _empty), ).legend(title=colorbar_title) if return_fig: return vp diff --git a/src/scanpy/plotting/_tools/__init__.py b/src/scanpy/plotting/_tools/__init__.py index dceb779fd8..8f189121e2 100644 --- a/src/scanpy/plotting/_tools/__init__.py +++ b/src/scanpy/plotting/_tools/__init__.py @@ -1,7 +1,6 @@ from __future__ import annotations -import collections.abc as cabc -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from copy import copy from typing import TYPE_CHECKING @@ -15,7 +14,7 @@ from ... import logging as logg from ..._compat import old_positionals from ..._settings import settings -from ..._utils import _doc_params, sanitize_anndata, subsample +from ..._utils import _doc_params, _empty, sanitize_anndata, subsample from ...get import rank_genes_groups_df from .._anndata import ranking from .._docs import ( @@ -38,7 +37,7 @@ from .scatterplots import _panel_grid, embedding, pca if TYPE_CHECKING: - from collections.abc import Iterable, Sequence + from collections.abc import Iterable from typing import Literal from anndata import AnnData @@ -47,6 +46,9 @@ from matplotlib.colors import Colormap, Normalize from matplotlib.figure import Figure + from ..._utils import Empty + from .._utils import DensityNorm + # ------------------------------------------------------------------------------ # PCA # ------------------------------------------------------------------------------ @@ -91,9 +93,7 @@ def pca_overview(adata: AnnData, **params): -------- pp.pca """ - show = params["show"] if "show" in params else None - if "show" in params: - del params["show"] + show = params.pop("show", None) pca(adata, **params, show=False) pca_loadings(adata, show=False) pca_variance_ratio(adata, show=show) @@ -158,14 +158,14 @@ def pca_loadings( components = np.array(components) - 1 if np.any(components < 0): - raise ValueError("Component indices must be greater than zero.") + msg = "Component indices must be greater than zero." + raise ValueError(msg) if n_points is None: n_points = min(30, adata.n_vars) elif adata.n_vars < n_points: - raise ValueError( - f"Tried to plot {n_points} variables, but passed anndata only has {adata.n_vars}." - ) + msg = f"Tried to plot {n_points} variables, but passed anndata only has {adata.n_vars}." + raise ValueError(msg) ranking( adata, @@ -396,15 +396,13 @@ def rank_genes_groups( tl.rank_genes_groups """ - if "n_panels_per_row" in kwds: - n_panels_per_row = kwds["n_panels_per_row"] - else: - n_panels_per_row = ncols + n_panels_per_row = kwds.get("n_panels_per_row", ncols) if n_genes < 1: - raise NotImplementedError( + msg = ( "Specifying a negative number for n_genes has not been implemented for " - f"this plot. Received n_genes={n_genes}." + f"this plot. Received {n_genes=!r}." ) + raise NotImplementedError(msg) reference = str(adata.uns[key]["params"]["reference"]) group_names = adata.uns[key]["names"].dtype.names if groups is None else groups @@ -520,10 +518,11 @@ def _rank_genes_groups_plot( Common function to call the different rank_genes_groups_* plots """ if var_names is not None and n_genes is not None: - raise ValueError( + msg = ( "The arguments n_genes and var_names are mutually exclusive. Please " "select only one." ) + raise ValueError(msg) if var_names is None and n_genes is None: # set n_genes = 10 as default when none of the options is given @@ -565,10 +564,7 @@ def _rank_genes_groups_plot( if len(genes_list) == 0: logg.warning(f"No genes found for group {group}") continue - if n_genes < 0: - genes_list = genes_list[n_genes:] - else: - genes_list = genes_list[:n_genes] + genes_list = genes_list[n_genes:] if n_genes < 0 else genes_list[:n_genes] var_names[group] = genes_list var_names_list.extend(genes_list) @@ -700,7 +696,6 @@ def rank_genes_groups_heatmap( {show_save_ax} **kwds Are passed to :func:`~scanpy.pl.heatmap`. - {show_save_ax} Examples -------- @@ -784,7 +779,6 @@ def rank_genes_groups_tracksplot( {show_save_ax} **kwds Are passed to :func:`~scanpy.pl.tracksplot`. - {show_save_ax} Examples -------- @@ -1213,15 +1207,15 @@ def rank_genes_groups_violin( use_raw: bool | None = None, key: str | None = None, split: bool = True, - density_norm: Literal["area", "count", "width"] = "width", + density_norm: DensityNorm = "width", strip: bool = True, - jitter: int | float | bool = True, + jitter: float | bool = True, size: int = 1, ax: Axes | None = None, show: bool | None = None, save: bool | None = None, # deprecated - scale: Literal["area", "count", "width"] | None = None, + scale: DensityNorm | Empty = _empty, ): """\ Plot ranking of genes for all tested comparisons. @@ -1319,9 +1313,7 @@ def rank_genes_groups_violin( _ax.set_ylabel("expression") _ax.set_xticklabels(new_gene_names, rotation="vertical") writekey = ( - f"rank_genes_groups_" - f"{adata.uns[key]['params']['groupby']}_" - f"{group_name}" + f"rank_genes_groups_{adata.uns[key]['params']['groupby']}_{group_name}" ) savefig_or_show(writekey, show=show, save=save) axs.append(_ax) @@ -1434,7 +1426,7 @@ def embedding_density( *, key: str | None = None, groupby: str | None = None, - group: str | Sequence[str] | None | None = "all", + group: str | Sequence[str] | None = "all", color_map: Colormap | str = "YlOrRd", bg_dotsize: int | None = 80, fg_dotsize: int | None = 180, @@ -1533,7 +1525,8 @@ def embedding_density( basis = "draw_graph_fa" if key is not None and groupby is not None: - raise ValueError("either pass key or groupby but not both") + msg = "either pass key or groupby but not both" + raise ValueError(msg) if key is None: key = "umap_density" @@ -1541,16 +1534,17 @@ def embedding_density( key += f"_{groupby}" if f"X_{basis}" not in adata.obsm_keys(): - raise ValueError( - f"Cannot find the embedded representation `adata.obsm[X_{basis!r}]`. " + msg = ( + f"Cannot find the embedded representation `adata.obsm['X_{basis}']`. " "Compute the embedding first." ) + raise ValueError(msg) if key not in adata.obs or f"{key}_params" not in adata.uns: - raise ValueError( - "Please run `sc.tl.embedding_density()` first " - "and specify the correct key." + msg = ( + "Please run `sc.tl.embedding_density()` first and specify the correct key." ) + raise ValueError(msg) if "components" in kwargs: logg.warning( @@ -1564,18 +1558,16 @@ def embedding_density( # turn group into a list if needed if group == "all": - if groupby is None: - group = None - else: - group = list(adata.obs[groupby].cat.categories) + group = None if groupby is None else list(adata.obs[groupby].cat.categories) elif isinstance(group, str): group = [group] if group is None and groupby is not None: - raise ValueError( + msg = ( "Densities were calculated over an `.obs` covariate. " "Please specify a group from this covariate to plot." ) + raise ValueError(msg) if group is not None and groupby is None: logg.warning( @@ -1585,7 +1577,8 @@ def embedding_density( group = None if np.min(adata.obs[key]) < 0 or np.max(adata.obs[key]) > 1: - raise ValueError("Densities should be scaled between 0 and 1.") + msg = "Densities should be scaled between 0 and 1." + raise ValueError(msg) if wspace is None: # try to set a wspace that is not too large or too small given the @@ -1608,23 +1601,21 @@ def embedding_density( # if group is set, then plot it using multiple panels # (even if only one group is set) - if ( - group is not None - and not isinstance(group, str) - and isinstance(group, cabc.Sequence) - ): + if group is not None and not isinstance(group, str) and isinstance(group, Sequence): if ax is not None: - raise ValueError("Can only specify `ax` if no `group` sequence is given.") + msg = "Can only specify `ax` if no `group` sequence is given." + raise ValueError(msg) fig, gs = _panel_grid(hspace, wspace, ncols, len(group)) axs = [] for count, group_name in enumerate(group): if group_name not in adata.obs[groupby].cat.categories: - raise ValueError( + msg = ( "Please specify a group from the `.obs` category " "over which the density was calculated. " f"Invalid group name: {group_name}" ) + raise ValueError(msg) ax = plt.subplot(gs[count]) # Define plotting data @@ -1635,10 +1626,7 @@ def embedding_density( adata.obs[density_col_name] = dens_values dot_sizes[group_mask] = np.ones(sum(group_mask)) * fg_dotsize - if title is None: - _title = group_name - else: - _title = title + _title = group_name if title is None else title ax = embedding( adata, @@ -1759,9 +1747,8 @@ def _get_values_to_plot( "log10_pvals_adj", ] if values_to_plot not in valid_options: - raise ValueError( - f"given value_to_plot: '{values_to_plot}' is not valid. Valid options are {valid_options}" - ) + msg = f"given value_to_plot: '{values_to_plot}' is not valid. Valid options are {valid_options}" + raise ValueError(msg) values_df = None check_done = False @@ -1775,16 +1762,15 @@ def _get_values_to_plot( df["names"] = df[gene_symbols] # check that all genes are present in the df as sc.tl.rank_genes_groups # can be called with only top genes - if not check_done: - if df.shape[0] < adata.shape[1]: - message = ( - "Please run `sc.tl.rank_genes_groups` with " - "'n_genes=adata.shape[1]' to save all gene " - f"scores. Currently, only {df.shape[0]} " - "are found" - ) - logg.error(message) - raise ValueError(message) + if not check_done and df.shape[0] < adata.shape[1]: + message = ( + "Please run `sc.tl.rank_genes_groups` with " + "'n_genes=adata.shape[1]' to save all gene " + f"scores. Currently, only {df.shape[0]} " + "are found" + ) + logg.error(message) + raise ValueError(message) df["group"] = group df_list.append(df) diff --git a/src/scanpy/plotting/_tools/paga.py b/src/scanpy/plotting/_tools/paga.py index 188221a060..a4b2de3441 100644 --- a/src/scanpy/plotting/_tools/paga.py +++ b/src/scanpy/plotting/_tools/paga.py @@ -1,7 +1,7 @@ from __future__ import annotations -import collections.abc as cabc import warnings +from collections.abc import Collection, Mapping, Sequence from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING @@ -16,6 +16,8 @@ from scipy.sparse import issparse from sklearn.utils import check_random_state +from scanpy.tools._draw_graph import coerce_fa2_layout, fa2_positions + from ... import _utils as _sc_utils from ... import logging as logg from ..._compat import old_positionals @@ -24,14 +26,18 @@ from .._utils import matrix if TYPE_CHECKING: - from collections.abc import Mapping, Sequence from typing import Any, Literal from anndata import AnnData from matplotlib.axes import Axes from matplotlib.colors import Colormap + from scipy.sparse import spmatrix + + from ..._compat import _LegacyRandom + from ...tools._draw_graph import _Layout as _LayoutWithoutEqTree + from .._utils import _FontSize, _FontWeight, _LegendLoc - from .._utils import _FontSize, _FontWeight, _IGraphLayout + _Layout = _LayoutWithoutEqTree | Literal["eq_tree"] @old_positionals( @@ -67,8 +73,8 @@ def paga_compare( groups=None, components=None, projection: Literal["2d", "3d"] = "2d", - legend_loc="on data", - legend_fontsize: int | float | _FontSize | None = None, + legend_loc: _LegendLoc | None = "on data", + legend_fontsize: float | _FontSize | None = None, legend_fontweight: int | _FontWeight = "bold", legend_fontoutline=None, color_map=None, @@ -202,13 +208,13 @@ def paga_compare( def _compute_pos( - adjacency_solid, + adjacency_solid: spmatrix | np.ndarray, *, - layout=None, - random_state=0, - init_pos=None, + layout: _Layout | None = None, + random_state: _LegacyRandom = 0, + init_pos: np.ndarray | None = None, adj_tree=None, - root=0, + root: int = 0, layout_kwds: Mapping[str, Any] = MappingProxyType({}), ): import random @@ -220,58 +226,24 @@ def _compute_pos( nx_g_solid = nx.Graph(adjacency_solid) if layout is None: layout = "fr" - if layout == "fa": - try: - from fa2 import ForceAtlas2 - except ImportError: - logg.warning( - "Package 'fa2' is not installed, falling back to layout 'fr'." - "To use the faster and better ForceAtlas2 layout, " - "install package 'fa2' (`pip install fa2`)." - ) - layout = "fr" + layout = coerce_fa2_layout(layout) if layout == "fa": # np.random.seed(random_state) if init_pos is None: init_coords = random_state.random_sample((adjacency_solid.shape[0], 2)) else: init_coords = init_pos.copy() - forceatlas2 = ForceAtlas2( - # Behavior alternatives - outboundAttractionDistribution=False, # Dissuade hubs - linLogMode=False, # NOT IMPLEMENTED - adjustSizes=False, # Prevent overlap (NOT IMPLEMENTED) - edgeWeightInfluence=1.0, - # Performance - jitterTolerance=1.0, # Tolerance - barnesHutOptimize=True, - barnesHutTheta=1.2, - multiThreaded=False, # NOT IMPLEMENTED - # Tuning - scalingRatio=2.0, - strongGravityMode=False, - gravity=1.0, - # Log - verbose=False, - ) - if "maxiter" in layout_kwds: - iterations = layout_kwds["maxiter"] - elif "iterations" in layout_kwds: - iterations = layout_kwds["iterations"] - else: - iterations = 500 - pos_list = forceatlas2.forceatlas2( - adjacency_solid, pos=init_coords, iterations=iterations - ) - pos = {n: [p[0], -p[1]] for n, p in enumerate(pos_list)} + pos_list = fa2_positions(adjacency_solid, init_coords, **layout_kwds) + pos = {n: (x, -y) for n, (x, y) in enumerate(pos_list)} elif layout == "eq_tree": nx_g_tree = nx.Graph(adj_tree) pos = _utils.hierarchy_pos(nx_g_tree, root) if len(pos) < adjacency_solid.shape[0]: - raise ValueError( + msg = ( "This is a forest and not a single tree. " "Try another `layout`, e.g., {'fr'}." ) + raise ValueError(msg) else: # igraph layouts random.seed(random_state.bytes(8)) @@ -302,7 +274,7 @@ def _compute_pos( ).coords except AttributeError: # hack for empty graphs... pos_list = g.layout(layout, seed=init_coords, **layout_kwds).coords - pos = {n: [p[0], -p[1]] for n, p in enumerate(pos_list)} + pos = {n: (x, -y) for n, (x, y) in enumerate(pos_list)} if len(pos) == 1: pos[0] = (0.5, 0.5) pos_array = np.array([pos[n] for count, n in enumerate(nx_g_solid)]) @@ -333,7 +305,7 @@ def paga( *, threshold: float | None = None, color: str | Mapping[str | int, Mapping[Any, float]] | None = None, - layout: _IGraphLayout | None = None, + layout: _Layout | None = None, layout_kwds: Mapping[str, Any] = MappingProxyType({}), init_pos: np.ndarray | None = None, root: int | str | Sequence[int] | None = 0, @@ -535,14 +507,12 @@ def paga( groups_key = adata.uns["paga"]["groups"] def is_flat(x): - has_one_per_category = isinstance(x, cabc.Collection) and len(x) == len( + has_one_per_category = isinstance(x, Collection) and len(x) == len( adata.obs[groups_key].cat.categories ) return has_one_per_category or x is None or isinstance(x, str) - if isinstance(colors, cabc.Mapping) and isinstance( - colors[next(iter(colors))], cabc.Mapping - ): + if isinstance(colors, Mapping) and isinstance(colors[next(iter(colors))], Mapping): # handle paga pie, remap string keys to integers names_to_ixs = { n: i for i, n in enumerate(adata.obs[groups_key].cat.categories) @@ -578,12 +548,10 @@ def is_flat(x): if isinstance(root, str): if root not in labels: - raise ValueError( - "If `root` is a string, " - f"it needs to be one of {labels} not {root!r}." - ) + msg = f"If `root` is a string, it needs to be one of {labels} not {root!r}." + raise ValueError(msg) root = list(labels).index(root) - if isinstance(root, cabc.Sequence) and root[0] in labels: + if isinstance(root, Sequence) and root[0] in labels: root = [list(labels).index(r) for r in root] # define the adjacency matrices @@ -629,7 +597,7 @@ def is_flat(x): sct = _paga_graph( adata, axs[icolor], - colors=colors if isinstance(colors, cabc.Mapping) else c, + colors=colors if isinstance(colors, Mapping) else c, solid_edges=solid_edges, dashed_edges=dashed_edges, transitions=transitions, @@ -733,11 +701,11 @@ def _paga_graph( and isinstance(node_labels, str) and node_labels != adata.uns["paga"]["groups"] ): - raise ValueError( - "Provide a list of group labels for the PAGA groups {}, not {}.".format( - adata.uns["paga"]["groups"], node_labels - ) + msg = ( + "Provide a list of group labels for the PAGA groups " + f"{adata.uns['paga']['groups']}, not {node_labels}." ) + raise ValueError(msg) groups_key = adata.uns["paga"]["groups"] if node_labels is None: node_labels = adata.obs[groups_key].cat.categories @@ -757,15 +725,16 @@ def _paga_graph( nx_g_dashed = nx.Graph(adjacency_dashed) # convert pos to array and dict - if not isinstance(pos, (Path, str)): + if not isinstance(pos, Path | str): pos_array = pos else: pos = Path(pos) if pos.suffix != ".gdf": - raise ValueError( + msg = ( "Currently only supporting reading positions from .gdf files. " "Consider generating them using, for instance, Gephi." ) + raise ValueError(msg) s = "" # read the node definition from the file with pos.open() as f: f.readline() @@ -793,7 +762,8 @@ def _paga_graph( elif colors == "degree_solid": colors = [d for _, d in nx_g_solid.degree(weight="weight")] else: - raise ValueError('`degree` either "degree_dashed" or "degree_solid".') + msg = '`degree` either "degree_dashed" or "degree_solid".' + raise ValueError(msg) colors = (np.array(colors) - np.min(colors)) / (np.max(colors) - np.min(colors)) # plot gene expression @@ -842,10 +812,11 @@ def _paga_graph( colors = asso_colors if len(colors) != len(node_labels): - raise ValueError( + msg = ( f"Expected `colors` to be of length `{len(node_labels)}`, " f"found `{len(colors)}`." ) + raise ValueError(msg) # count number of connected components n_components, labels = scipy.sparse.csgraph.connected_components(adjacency_solid) @@ -870,7 +841,8 @@ def _paga_graph( ) nx_g_solid = nx.Graph(adjacency_solid) if dashed_edges is not None: - raise ValueError("`single_component` only if `dashed_edges` is `None`.") + msg = "`single_component` only if `dashed_edges` is `None`." + raise ValueError(msg) # edge widths base_edge_width = edge_width_scale * 5 * rcParams["lines.linewidth"] @@ -964,7 +936,7 @@ def _paga_graph( patheffects.withStroke(linewidth=fontoutline, foreground="w") ] # usual scatter plot - if not isinstance(colors[0], cabc.Mapping): + if not isinstance(colors[0], Mapping): n_groups = len(pos_array) sct = ax.scatter( pos_array[:, 0], @@ -988,11 +960,12 @@ def _paga_graph( # else pie chart plot else: for ix, (xx, yy) in enumerate(zip(pos_array[:, 0], pos_array[:, 1])): - if not isinstance(colors[ix], cabc.Mapping): - raise ValueError( + if not isinstance(colors[ix], Mapping): + msg = ( f"{colors[ix]} is neither a dict of valid " "matplotlib colors nor a valid matplotlib color." ) + raise ValueError(msg) color_single = colors[ix].keys() fracs = [colors[ix][c] for c in color_single] total = sum(fracs) @@ -1002,10 +975,11 @@ def _paga_graph( color_single.append("grey") fracs.append(1 - sum(fracs)) elif not np.isclose(total, 1): - raise ValueError( + msg = ( f"Expected fractions for node `{ix}` to be " f"close to 1, found `{total}`." ) + raise ValueError(msg) cumsum = np.cumsum(fracs) cumsum = cumsum / cumsum[-1] @@ -1085,7 +1059,7 @@ def paga_path( show_node_names: bool = True, show_yticks: bool = True, show_colorbar: bool = True, - legend_fontsize: int | float | _FontSize | None = None, + legend_fontsize: float | _FontSize | None = None, legend_fontweight: int | _FontWeight | None = None, normalize_to_zero_one: bool = False, as_heatmap: bool = True, @@ -1156,18 +1130,20 @@ def paga_path( if groups_key is None: if "groups" not in adata.uns["paga"]: - raise KeyError( + msg = ( "Pass the key of the grouping with which you ran PAGA, " "using the parameter `groups_key`." ) + raise KeyError(msg) groups_key = adata.uns["paga"]["groups"] groups_names = adata.obs[groups_key].cat.categories - if "dpt_pseudotime" not in adata.obs.keys(): - raise ValueError( + if "dpt_pseudotime" not in adata.obs.columns: + msg = ( "`pl.paga_path` requires computation of a pseudotime `tl.dpt` " "for ordering at single-cell resolution" ) + raise ValueError(msg) if palette_groups is None: _utils.add_colors_for_categorical_sample_annotation(adata, groups_key) @@ -1188,10 +1164,11 @@ def moving_average(a): groups_names_set = set(groups_names) for node in nodes: if node not in groups_names_set: - raise ValueError( + msg = ( f"Each node/group needs to be in {groups_names.tolist()} " - f"(`groups_key`={groups_key!r}) not {node!r}." + f"({groups_key=!r}) not {node!r}." ) + raise ValueError(msg) nodes_ints.append(groups_names.get_loc(node)) nodes_strs = nodes else: @@ -1209,12 +1186,13 @@ def moving_average(a): adata.obs[groups_key].values == nodes_strs[igroup] ] if len(idcs) == 0: - raise ValueError( + msg = ( "Did not find data points that match " f"`adata.obs[{groups_key!r}].values == {str(group)!r}`. " f"Check whether `adata.obs[{groups_key!r}]` " "actually contains what you expect." ) + raise ValueError(msg) idcs_group = np.argsort( adata.obs["dpt_pseudotime"].values[ adata.obs[groups_key].values == nodes_strs[igroup] @@ -1263,7 +1241,7 @@ def moving_average(a): ax.set_frame_on(False) ax.set_xticks([]) ax.tick_params(axis="both", which="both", length=0) - ax.grid(False) + ax.grid(visible=False) if show_colorbar: plt.colorbar(img, ax=ax) left_margin = 0.2 if left_margin is None else left_margin @@ -1327,7 +1305,7 @@ def moving_average(a): ), ) groups_axis.set_xticks([]) - groups_axis.grid(False) + groups_axis.grid(visible=False) groups_axis.tick_params(axis="both", which="both", length=0) # further annotations y_shift = ax_bounds[3] / len(keys) @@ -1364,7 +1342,7 @@ def moving_average(a): anno_axis.set_yticks([]) anno_axis.set_frame_on(False) anno_axis.set_xticks([]) - anno_axis.grid(False) + anno_axis.grid(visible=False) if title is not None: ax.set_title(title, fontsize=title_fontsize) if show is None and not ax_was_none: diff --git a/src/scanpy/plotting/_tools/scatterplots.py b/src/scanpy/plotting/_tools/scatterplots.py index 51959f66e7..cb3c9d7c66 100644 --- a/src/scanpy/plotting/_tools/scatterplots.py +++ b/src/scanpy/plotting/_tools/scatterplots.py @@ -1,8 +1,6 @@ from __future__ import annotations -import collections.abc as cabc import inspect -import sys from collections.abc import Mapping, Sequence # noqa: TCH003 from copy import copy from functools import partial @@ -30,6 +28,7 @@ from packaging.version import Version from ... import logging as logg +from ..._compat import deprecated from ..._settings import settings from ..._utils import ( Empty, # noqa: TCH001 @@ -38,6 +37,7 @@ sanitize_anndata, ) from ...get import _check_mask +from ...tools._draw_graph import _Layout # noqa: TCH001 from .. import _utils from .._docs import ( doc_adata_color_etc, @@ -51,7 +51,7 @@ VBound, # noqa: TCH001 _FontSize, # noqa: TCH001 _FontWeight, # noqa: TCH001 - _IGraphLayout, # noqa: TCH001 + _LegendLoc, # noqa: TCH001 check_colornorm, check_projection, circles, @@ -95,9 +95,9 @@ def embedding( na_in_legend: bool = True, size: float | Sequence[float] | None = None, frameon: bool | None = None, - legend_fontsize: int | float | _FontSize | None = None, + legend_fontsize: float | _FontSize | None = None, legend_fontweight: int | _FontWeight = "bold", - legend_loc: str = "right margin", + legend_loc: _LegendLoc | None = "right margin", legend_fontoutline: int | None = None, colorbar_loc: str | None = "right", vmax: VBound | Sequence[VBound] | None = None, @@ -149,24 +149,26 @@ def embedding( # Checking the mask format and if used together with groups if groups is not None and mask_obs is not None: - raise ValueError("Groups and mask arguments are incompatible.") - if mask_obs is not None: - mask_obs = _check_mask(adata, mask_obs, "obs") + msg = "Groups and mask arguments are incompatible." + raise ValueError(msg) + mask_obs = _check_mask(adata, mask_obs, "obs") # Figure out if we're using raw if use_raw is None: # check if adata.raw is set use_raw = layer is None and adata.raw is not None if use_raw and layer is not None: - raise ValueError( - "Cannot use both a layer and the raw representation. Was passed:" - f"use_raw={use_raw}, layer={layer}." + msg = ( + "Cannot use both a layer and the raw representation. " + f"Was passed: {use_raw=!r}, {layer=!r}." ) + raise ValueError(msg) if use_raw and adata.raw is None: - raise ValueError( + msg = ( "`use_raw` is set to True but AnnData object does not have raw. " "Please check." ) + raise ValueError(msg) if isinstance(groups, str): groups = [groups] @@ -174,7 +176,8 @@ def embedding( # Color map if color_map is not None: if cmap is not None: - raise ValueError("Cannot specify both `color_map` and `cmap`.") + msg = "Cannot specify both `color_map` and `cmap`." + raise ValueError(msg) else: cmap = color_map cmap = copy(colormaps.get_cmap(cmap)) @@ -182,11 +185,10 @@ def embedding( # Prevents warnings during legend creation na_color = colors.to_hex(na_color, keep_alpha=True) - if "edgecolor" not in kwargs: - # by default turn off edge color. Otherwise, for - # very small sizes the edge will not reduce its size - # (https://github.com/scverse/scanpy/issues/293) - kwargs["edgecolor"] = "none" + # by default turn off edge color. Otherwise, for + # very small sizes the edge will not reduce its size + # (https://github.com/scverse/scanpy/issues/293) + kwargs.setdefault("edgecolor", "none") # Vectorized arguments @@ -201,13 +203,13 @@ def embedding( title = [title] if isinstance(title, str) else list(title) # turn vmax and vmin into a sequence - if isinstance(vmax, str) or not isinstance(vmax, cabc.Sequence): + if isinstance(vmax, str) or not isinstance(vmax, Sequence): vmax = [vmax] - if isinstance(vmin, str) or not isinstance(vmin, cabc.Sequence): + if isinstance(vmin, str) or not isinstance(vmin, Sequence): vmin = [vmin] - if isinstance(vcenter, str) or not isinstance(vcenter, cabc.Sequence): + if isinstance(vcenter, str) or not isinstance(vcenter, Sequence): vcenter = [vcenter] - if isinstance(norm, Normalize) or not isinstance(norm, cabc.Sequence): + if isinstance(norm, Normalize) or not isinstance(norm, Sequence): norm = [norm] # Size @@ -218,7 +220,7 @@ def embedding( # set as ndarray if ( size is not None - and isinstance(size, (cabc.Sequence, pd.Series, np.ndarray)) + and isinstance(size, Sequence | pd.Series | np.ndarray) and len(size) == adata.shape[0] ): size = np.array(size, dtype=float) @@ -244,15 +246,14 @@ def embedding( # Eg. ['Gene1', 'louvain', 'Gene2']. # component_list is a list of components [[0,1], [1,2]] if ( - not isinstance(color, str) - and isinstance(color, cabc.Sequence) - and len(color) > 1 + not isinstance(color, str) and isinstance(color, Sequence) and len(color) > 1 ) or len(dimensions) > 1: if ax is not None: - raise ValueError( + msg = ( "Cannot specify `ax` when plotting multiple panels " "(each for a given value of 'color')." ) + raise ValueError(msg) # each plot needs to be its own panel fig, grid = _panel_grid(hspace, wspace, ncols, len(color)) @@ -596,9 +597,6 @@ def my_vmax(colors): np.percentile(colors, p=80) def _wraps_plot_scatter(wrapper): """Update the wrapper function to use the correct signature.""" - if sys.version_info < (3, 10): - # Python 3.9 does not support `eval_str`, so we only support this in 3.10+ - return wrapper params = inspect.signature(embedding, eval_str=True).parameters.copy() wrapper_sig = inspect.signature(wrapper, eval_str=True) @@ -778,7 +776,7 @@ def diffmap(adata: AnnData, **kwargs) -> Figure | Axes | list[Axes] | None: show_save_ax=doc_show_save_ax, ) def draw_graph( - adata: AnnData, *, layout: _IGraphLayout | None = None, **kwargs + adata: AnnData, *, layout: _Layout | None = None, **kwargs ) -> Figure | Axes | list[Axes] | None: """\ Scatter plot in graph-drawing basis. @@ -817,9 +815,8 @@ def draw_graph( layout = str(adata.uns["draw_graph"]["params"]["layout"]) basis = f"draw_graph_{layout}" if f"X_{basis}" not in adata.obsm_keys(): - raise ValueError( - f"Did not find {basis} in adata.obs. Did you compute layout {layout}?" - ) + msg = f"Did not find {basis} in adata.obs. Did you compute layout {layout}?" + raise ValueError(msg) return embedding(adata, basis, **kwargs) @@ -889,11 +886,12 @@ def pca( return embedding( adata, "pca", show=show, return_fig=return_fig, save=save, **kwargs ) - if "pca" not in adata.obsm.keys() and "X_pca" not in adata.obsm.keys(): - raise KeyError( + if "pca" not in adata.obsm and "X_pca" not in adata.obsm: + msg = ( f"Could not find entry in `obsm` for 'pca'.\n" f"Available keys are: {list(adata.obsm.keys())}." ) + raise KeyError(msg) label_dict = { f"PC{i + 1}": f"PC{i + 1} ({round(v * 100, 2)}%)" @@ -926,6 +924,7 @@ def pca( return axs +@deprecated("Use `squidpy.pl.spatial_scatter` instead.") @_wraps_plot_scatter @_doc_params( adata_color_etc=doc_adata_color_etc, @@ -955,6 +954,9 @@ def spatial( """\ Scatter plot in spatial coordinates. + .. deprecated:: 1.11.0 + Use :func:`squidpy.pl.spatial_scatter` instead. + This function allows overlaying data on top of images. Use the parameter `img_key` to see the image in the background And the parameter `library_id` to select the image. @@ -1001,8 +1003,6 @@ def spatial( -------- :func:`scanpy.datasets.visium_sge` Example visium data. - :doc:`/tutorials/spatial/basic-analysis` - Tutorial on spatial analysis. """ # get default image params if available library_id, spatial_data = _check_spatial_data(adata.uns, library_id) @@ -1014,10 +1014,7 @@ def spatial( crop_coord = _check_crop_coord(crop_coord, scale_factor) na_color = _check_na_color(na_color, img=img) - if bw: - cmap_img = "gray" - else: - cmap_img = None + cmap_img = "gray" if bw else None circle_radius = size * scale_factor * spot_size * 0.5 axs = embedding( @@ -1068,7 +1065,8 @@ def _components_to_dimensions( if components is None and dimensions is None: dimensions = [tuple(i for i in range(ndims))] elif components is not None and dimensions is not None: - raise ValueError("Cannot provide both dimensions and components") + msg = "Cannot provide both dimensions and components" + raise ValueError(msg) # TODO: Consider deprecating this # If components is not None, parse them and set dimensions @@ -1091,11 +1089,11 @@ def _components_to_dimensions( def _add_categorical_legend( - ax, + ax: Axes, color_source_vector, *, palette: dict, - legend_loc: str, + legend_loc: _LegendLoc | None, legend_fontweight, legend_fontsize, legend_fontoutline, @@ -1107,9 +1105,8 @@ def _add_categorical_legend( """Add a legend to the passed Axes.""" if na_in_legend and pd.isnull(color_source_vector).any(): if "NA" in color_source_vector: - raise NotImplementedError( - "No fallback for null labels has been defined if NA already in categories." - ) + msg = "No fallback for null labels has been defined if NA already in categories." + raise NotImplementedError(msg) color_source_vector = color_source_vector.add_categories("NA").fillna("NA") palette = palette.copy() palette["NA"] = na_color @@ -1124,17 +1121,7 @@ def _add_categorical_legend( box = ax.get_position() ax.set_position([box.x0, box.y0, box.width * 0.91, box.height]) - if legend_loc == "right margin": - for label in cats: - ax.scatter([], [], c=palette[label], label=label) - ax.legend( - frameon=False, - loc="center left", - bbox_to_anchor=(1, 0.5), - ncol=(1 if len(cats) <= 14 else 2 if len(cats) <= 30 else 3), - fontsize=legend_fontsize, - ) - elif legend_loc == "on data": + if legend_loc == "on data": # identify centroids to put labels all_pos = ( @@ -1158,6 +1145,19 @@ def _add_categorical_legend( fontsize=legend_fontsize, path_effects=legend_fontoutline, ) + elif legend_loc not in {None, "none"}: + for label in cats: + ax.scatter([], [], c=palette[label], label=label) + if legend_loc == "right margin": + ax.legend( + frameon=False, + loc="center left", + bbox_to_anchor=(1, 0.5), + ncol=(1 if len(cats) <= 14 else 2 if len(cats) <= 30 else 3), + fontsize=legend_fontsize, + ) + else: + ax.legend(loc=legend_loc, fontsize=legend_fontsize) def _get_basis(adata: AnnData, basis: str) -> np.ndarray: @@ -1167,7 +1167,8 @@ def _get_basis(adata: AnnData, basis: str) -> np.ndarray: elif f"X_{basis}" in adata.obsm: return adata.obsm[f"X_{basis}"] else: - raise KeyError(f"Could not find '{basis}' or 'X_{basis}' in .obsm") + msg = f"Could not find {basis!r} or 'X_{basis}' in .obsm" + raise KeyError(msg) def _get_color_source_vector( @@ -1299,10 +1300,11 @@ def _check_spot_size(spatial_data: Mapping | None, spot_size: float | None) -> f This is a required argument for spatial plots. """ if spatial_data is None and spot_size is None: - raise ValueError( + msg = ( "When .uns['spatial'][library_id] does not exist, spot_size must be " "provided directly." ) + raise ValueError(msg) elif spot_size is None: return spatial_data["scalefactors"]["spot_diameter_fullres"] else: @@ -1334,18 +1336,16 @@ def _check_spatial_data( spatial_mapping = uns.get("spatial", {}) if library_id is _empty: if len(spatial_mapping) > 1: - raise ValueError( + msg = ( "Found multiple possible libraries in `.uns['spatial']. Please specify." f" Options are:\n\t{list(spatial_mapping.keys())}" ) + raise ValueError(msg) elif len(spatial_mapping) == 1: library_id = list(spatial_mapping.keys())[0] else: library_id = None - if library_id is not None: - spatial_data = spatial_mapping[library_id] - else: - spatial_data = None + spatial_data = spatial_mapping[library_id] if library_id is not None else None return library_id, spatial_data @@ -1353,6 +1353,7 @@ def _check_img( spatial_data: Mapping | None, img: np.ndarray | None, img_key: None | str | Empty, + *, bw: bool = False, ) -> tuple[np.ndarray | None, str | None]: """ @@ -1377,7 +1378,8 @@ def _check_crop_coord( if crop_coord is None: return None if len(crop_coord) != 4: - raise ValueError("Invalid crop_coord of length {len(crop_coord)}(!=4)") + msg = "Invalid crop_coord of length {len(crop_coord)}(!=4)" + raise ValueError(msg) crop_coord = tuple(c * scale_factor for c in crop_coord) return crop_coord @@ -1386,10 +1388,7 @@ def _check_na_color( na_color: ColorLike | None, *, img: np.ndarray | None = None ) -> ColorLike: if na_color is None: - if img is not None: - na_color = (0.0, 0.0, 0.0, 0.0) - else: - na_color = "lightgray" + na_color = (0.0, 0.0, 0.0, 0.0) if img is not None else "lightgray" return na_color @@ -1399,7 +1398,8 @@ def _broadcast_args(*args): lens = [len(arg) for arg in args] longest = max(lens) if not (set(lens) == {1, longest} or set(lens) == {longest}): - raise ValueError(f"Could not broadast together arguments with shapes: {lens}.") + msg = f"Could not broadcast together arguments with shapes: {lens}." + raise ValueError(msg) return list( [[arg[0] for _ in range(longest)] if len(arg) == 1 else arg for arg in args] ) diff --git a/src/scanpy/plotting/_utils.py b/src/scanpy/plotting/_utils.py index fddafdd973..b6cd920039 100644 --- a/src/scanpy/plotting/_utils.py +++ b/src/scanpy/plotting/_utils.py @@ -1,9 +1,8 @@ from __future__ import annotations -import collections.abc as cabc import warnings -from collections.abc import Sequence -from typing import TYPE_CHECKING, Callable, Literal, Union +from collections.abc import Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Literal, TypedDict, overload import matplotlib as mpl import numpy as np @@ -19,7 +18,7 @@ from .. import logging as logg from .._compat import old_positionals from .._settings import settings -from .._utils import NeighborsView +from .._utils import NeighborsView, _empty from . import palettes if TYPE_CHECKING: @@ -32,17 +31,35 @@ from numpy.typing import ArrayLike from PIL.Image import Image + from .._utils import Empty + # TODO: more DensityNorm = Literal["area", "count", "width"] # These are needed by _wraps_plot_scatter -_IGraphLayout = Literal["fa", "fr", "rt", "rt_circular", "drl", "eq_tree"] -VBound = Union[str, float, Callable[[Sequence[float]], float]] +VBound = str | float | Callable[[Sequence[float]], float] _FontWeight = Literal["light", "normal", "medium", "semibold", "bold", "heavy", "black"] _FontSize = Literal[ "xx-small", "x-small", "small", "medium", "large", "x-large", "xx-large" ] -ColorLike = Union[str, tuple[float, ...]] +_LegendLoc = Literal[ + "none", + "right margin", + "on data", + "on data export", + "best", + "upper right", + "upper left", + "lower left", + "lower right", + "right", + "center left", + "center right", + "lower center", + "upper center", + "center", +] +ColorLike = str | tuple[float, ...] class _AxesSubplot(Axes, axes.SubplotBase): @@ -139,7 +156,7 @@ def timeseries_subplot( """ if color is not None: - use_color_map = isinstance(color[0], (float, np.floating)) + use_color_map = isinstance(color[0], float | np.floating) palette = default_palette(palette) x_range = np.arange(X.shape[0]) if time is None else time if X.ndim == 1: @@ -230,7 +247,7 @@ def timeseries_as_heatmap( _, ax = plt.subplots(figsize=(1.5 * 4, 2 * 4)) img = ax.imshow( - np.array(X, dtype=np.float_), + np.array(X, dtype=np.float64), aspect="auto", interpolation="nearest", cmap=color_map, @@ -354,7 +371,7 @@ def default_palette( ) -> str | Cycler: if palette is None: return rcParams["axes.prop_cycle"] - elif not isinstance(palette, (str, Cycler)): + elif not isinstance(palette, str | Cycler): return cycler(color=palette) else: return palette @@ -381,7 +398,7 @@ def _validate_palette(adata: AnnData, key: str) -> None: else: logg.warning( f"The following color value found in adata.uns['{key}_colors'] " - f"is not valid: '{color}'. Default colors will be used instead." + f"is not valid: {color!r}. Default colors will be used instead." ) _set_default_colors_for_categorical_obs(adata, key) _palette = None @@ -427,12 +444,12 @@ def _set_colors_for_categorical_obs( # this creates a palette from a colormap. E.g. 'Accent, Dark2, tab20' cmap = plt.get_cmap(palette) colors_list = [to_hex(x) for x in cmap(np.linspace(0, 1, len(categories)))] - elif isinstance(palette, cabc.Mapping): + elif isinstance(palette, Mapping): colors_list = [to_hex(palette[k], keep_alpha=True) for k in categories] else: # check if palette is a list and convert it to a cycler, thus # it doesnt matter if the list is shorter than the categories length: - if isinstance(palette, cabc.Sequence): + if isinstance(palette, Sequence): if len(palette) < len(categories): logg.warning( "Length of palette colors is smaller than the number of " @@ -449,21 +466,24 @@ def _set_colors_for_categorical_obs( if color in additional_colors: color = additional_colors[color] else: - raise ValueError( + msg = ( "The following color value of the given palette " f"is not valid: {color}" ) + raise ValueError(msg) _color_list.append(color) palette = cycler(color=_color_list) if not isinstance(palette, Cycler): - raise ValueError( + msg = ( "Please check that the value of 'palette' is a valid " "matplotlib colormap string (eg. Set2), a list of color names " "or a cycler with a 'color' key." ) + raise ValueError(msg) if "color" not in palette.keys: - raise ValueError("Please set the palette key 'color'.") + msg = "Please set the palette key 'color'." + raise ValueError(msg) cc = palette() colors_list = [to_hex(next(cc)["color"]) for x in range(len(categories))] @@ -518,7 +538,7 @@ def _set_default_colors_for_categorical_obs(adata, value_to_plot): def add_colors_for_categorical_sample_annotation( - adata, key, palette=None, force_update_colors=False + adata, key, *, palette=None, force_update_colors=False ): color_key = f"{key}_colors" colors_needed = len(adata.obs[key].cat.categories) @@ -533,13 +553,14 @@ def add_colors_for_categorical_sample_annotation( def plot_edges(axs, adata, basis, edges_width, edges_color, *, neighbors_key=None): import networkx as nx - if not isinstance(axs, cabc.Sequence): + if not isinstance(axs, Sequence): axs = [axs] if neighbors_key is None: neighbors_key = "neighbors" if neighbors_key not in adata.uns: - raise ValueError("`edges=True` requires `pp.neighbors` to be run before.") + msg = "`edges=True` requires `pp.neighbors` to be run before." + raise ValueError(msg) neighbors = NeighborsView(adata, neighbors_key) g = nx.Graph(neighbors["connectivities"]) basis_key = _get_basis(adata, basis) @@ -559,17 +580,18 @@ def plot_edges(axs, adata, basis, edges_width, edges_color, *, neighbors_key=Non def plot_arrows(axs, adata, basis, arrows_kwds=None): - if not isinstance(axs, cabc.Sequence): + if not isinstance(axs, Sequence): axs = [axs] v_prefix = next( (p for p in ["velocity", "Delta"] if f"{p}_{basis}" in adata.obsm), None ) if v_prefix is None: - raise ValueError( + msg = ( "`arrows=True` requires " f"`'velocity_{basis}'` from scvelo or " f"`'Delta_{basis}'` from velocyto." ) + raise ValueError(msg) if v_prefix == "velocity": logg.warning( "The module `scvelo` has improved plotting facilities. " @@ -611,7 +633,8 @@ def scatter_group( color = rgb2hex(adata.uns[key + "_colors"][cat_code]) if not is_color_like(color): - raise ValueError(f'"{color}" is not a valid matplotlib color.') + msg = f"{color!r} is not a valid matplotlib color." + raise ValueError(msg) data = [Y[mask_obs, 0], Y[mask_obs, 1]] if projection == "3d": data.append(Y[mask_obs, 2]) @@ -641,7 +664,8 @@ def setup_axes( """Grid of axes for plotting, legends and colorbars.""" check_projection(projection) if left_margin is not None: - raise NotImplementedError("We currently don’t support `left_margin`.") + msg = "We currently don’t support `left_margin`." + raise NotImplementedError(msg) if np.any(colorbars) and right_margin is None: right_margin = 1 - rcParams["figure.subplot.right"] + 0.21 # 0.25 elif right_margin is None: @@ -706,7 +730,7 @@ def setup_axes( ax = plt.axes([left, bottom, width, height], projection="3d") axs.append(ax) else: - axs = ax if isinstance(ax, cabc.Sequence) else [ax] + axs = ax if isinstance(ax, Sequence) else [ax] return axs, panel_pos, draw_region_width, figure_width @@ -745,7 +769,7 @@ def scatter_base( Depending on whether supplying a single array or a list of arrays, return a single axis or a list of axes. """ - if isinstance(highlights, cabc.Mapping): + if isinstance(highlights, Mapping): highlights_indices = sorted(highlights) highlights_labels = [highlights[i] for i in highlights_indices] else: @@ -784,7 +808,8 @@ def scatter_base( elif projection == "3d": data = Y_sort[:, 0], Y_sort[:, 1], Y_sort[:, 2] else: - raise ValueError(f"Unknown projection {projection!r} not in '2d', '3d'") + msg = f"Unknown projection {projection!r} not in '2d', '3d'" + raise ValueError(msg) if not isinstance(color, str) or color != "white": sct = ax.scatter( *data, @@ -953,7 +978,14 @@ def scale_to_zero_one(x): return xscaled -def hierarchy_pos(G, root, levels=None, width=1.0, height=1.0): +class _Level(TypedDict): + total: int + current: int + + +def hierarchy_pos( + G, root: int, levels_: Mapping[int, int] | None = None, width=1.0, height=1.0 +) -> dict[int, tuple[float, float]]: """Tree layout for networkx graph. See https://stackoverflow.com/questions/29586520/can-one-get-hierarchical-graphs-from-networkx-with-python-3 @@ -972,37 +1004,47 @@ def hierarchy_pos(G, root, levels=None, width=1.0, height=1.0): width: horizontal space allocated for drawing height: vertical space allocated for drawing """ - TOTAL = "total" - CURRENT = "current" - def make_levels(levels, node=root, currentLevel=0, parent=None): + def make_levels( + levels: dict[int, _Level], + node: int = root, + current_level: int = 0, + parent: int | None = None, + ) -> dict[int, _Level]: """Compute the number of nodes for each level""" - if currentLevel not in levels: - levels[currentLevel] = {TOTAL: 0, CURRENT: 0} - levels[currentLevel][TOTAL] += 1 - neighbors = list(G.neighbors(node)) + if current_level not in levels: + levels[current_level] = _Level(total=0, current=0) + levels[current_level]["total"] += 1 + neighbors: list[int] = list(G.neighbors(node)) if parent is not None: neighbors.remove(parent) for neighbor in neighbors: - levels = make_levels(levels, neighbor, currentLevel + 1, node) + levels = make_levels(levels, neighbor, current_level + 1, node) return levels - def make_pos(pos, node=root, currentLevel=0, parent=None, vert_loc=0): - dx = 1 / levels[currentLevel][TOTAL] + if levels_ is None: + levels = make_levels({}) + else: + levels = {k: _Level(total=0, current=0) for k, v in levels_.items()} + + def make_pos( + pos: dict[int, tuple[float, float]], + node: int = root, + current_level: int = 0, + parent: int | None = None, + vert_loc: float = 0.0, + ): + dx = 1 / levels[current_level]["total"] left = dx / 2 - pos[node] = ((left + dx * levels[currentLevel][CURRENT]) * width, vert_loc) - levels[currentLevel][CURRENT] += 1 - neighbors = list(G.neighbors(node)) + pos[node] = ((left + dx * levels[current_level]["current"]) * width, vert_loc) + levels[current_level]["current"] += 1 + neighbors: list[int] = list(G.neighbors(node)) if parent is not None: neighbors.remove(parent) for neighbor in neighbors: - pos = make_pos(pos, neighbor, currentLevel + 1, node, vert_loc - vert_gap) + pos = make_pos(pos, neighbor, current_level + 1, node, vert_loc - vert_gap) return pos - if levels is None: - levels = make_levels({}) - else: - levels = {k: {TOTAL: v, CURRENT: 0} for k, v in levels.items()} vert_gap = height / (max(levels.keys()) + 1) return make_pos({}) @@ -1114,15 +1156,15 @@ def data_to_axis_points(ax: Axes, points_data: np.ndarray): def check_projection(projection): """Validation for projection argument.""" if projection not in {"2d", "3d"}: - raise ValueError(f"Projection must be '2d' or '3d', was '{projection}'.") + msg = f"Projection must be '2d' or '3d', was '{projection}'." + raise ValueError(msg) if projection == "3d": from packaging.version import parse mpl_version = parse(mpl.__version__) if mpl_version < parse("3.3.3"): - raise ImportError( - f"3d plotting requires matplotlib > 3.3.3. Found {mpl.__version__}" - ) + msg = f"3d plotting requires matplotlib > 3.3.3. Found {mpl.__version__}" + raise ImportError(msg) def circles( @@ -1246,10 +1288,10 @@ def fix_kwds(kwds_dict, **kwargs): def _get_basis(adata: AnnData, basis: str): - if basis in adata.obsm.keys(): + if basis in adata.obsm: basis_key = basis - elif f"X_{basis}" in adata.obsm.keys(): + elif f"X_{basis}" in adata.obsm: basis_key = f"X_{basis}" return basis_key @@ -1266,7 +1308,8 @@ def check_colornorm(vmin=None, vmax=None, vcenter=None, norm=None): if norm is not None: if (vmin is not None) or (vmax is not None) or (vcenter is not None): - raise ValueError("Passing both norm and vmin/vmax/vcenter is not allowed.") + msg = "Passing both norm and vmin/vmax/vcenter is not allowed." + raise ValueError(msg) else: if vcenter is not None: norm = DivNorm(vmin=vmin, vmax=vmax, vcenter=vcenter) @@ -1276,10 +1319,31 @@ def check_colornorm(vmin=None, vmax=None, vcenter=None, norm=None): return norm +@overload +def _deprecated_scale( + density_norm: DensityNorm, + scale: DensityNorm | Empty, + *, + default: DensityNorm, +) -> DensityNorm: ... + + +@overload +def _deprecated_scale( + density_norm: DensityNorm | Empty, + scale: DensityNorm | Empty, + *, + default: DensityNorm | Empty = _empty, +) -> DensityNorm | Empty: ... + + def _deprecated_scale( - density_norm: DensityNorm, scale: DensityNorm | None, *, default: DensityNorm -) -> DensityNorm: - if scale is None: + density_norm: DensityNorm | Empty, + scale: DensityNorm | Empty, + *, + default: DensityNorm | Empty = _empty, +) -> DensityNorm | Empty: + if scale is _empty: return density_norm if density_norm != default: msg = "can’t specify both `scale` and `density_norm`" @@ -1287,3 +1351,8 @@ def _deprecated_scale( msg = "`scale` is deprecated, use `density_norm` instead" warnings.warn(msg, FutureWarning) return scale + + +def _dk(dendrogram: bool | str | None) -> str | None: + """Helper to convert the `dendrogram` parameter to a `dendrogram_key` parameter.""" + return None if isinstance(dendrogram, bool) else dendrogram diff --git a/src/scanpy/preprocessing/__init__.py b/src/scanpy/preprocessing/__init__.py index 8c396d8640..4307cbb6c9 100644 --- a/src/scanpy/preprocessing/__init__.py +++ b/src/scanpy/preprocessing/__init__.py @@ -3,6 +3,7 @@ from ..neighbors import neighbors from ._combat import combat from ._deprecated.highly_variable_genes import filter_genes_dispersion +from ._deprecated.sampling import subsample from ._highly_variable_genes import highly_variable_genes from ._normalization import normalize_total from ._pca import pca @@ -17,8 +18,8 @@ log1p, normalize_per_cell, regress_out, + sample, sqrt, - subsample, ) __all__ = [ @@ -40,6 +41,7 @@ "log1p", "normalize_per_cell", "regress_out", + "sample", "scale", "sqrt", "subsample", diff --git a/src/scanpy/preprocessing/_combat.py b/src/scanpy/preprocessing/_combat.py index b8487e4e7e..93052f356c 100644 --- a/src/scanpy/preprocessing/_combat.py +++ b/src/scanpy/preprocessing/_combat.py @@ -171,7 +171,7 @@ def combat( Returns ------- - Returns :class:`numpy.ndarray` if `inplace=True`, else returns `None` and sets the following field in the `adata` object: + Returns :class:`numpy.ndarray` if `inplace=False`, else returns `None` and sets the following field in the `adata` object: `adata.X` : :class:`numpy.ndarray` (dtype `float`) Corrected data matrix. @@ -179,27 +179,26 @@ def combat( # check the input if key not in adata.obs_keys(): - raise ValueError(f"Could not find the key {key!r} in adata.obs") + msg = f"Could not find the key {key!r} in adata.obs" + raise ValueError(msg) if covariates is not None: cov_exist = np.isin(covariates, adata.obs_keys()) if np.any(~cov_exist): missing_cov = np.array(covariates)[~cov_exist].tolist() - raise ValueError( - f"Could not find the covariate(s) {missing_cov!r} in adata.obs" - ) + msg = f"Could not find the covariate(s) {missing_cov!r} in adata.obs" + raise ValueError(msg) if key in covariates: - raise ValueError("Batch key and covariates cannot overlap") + msg = "Batch key and covariates cannot overlap" + raise ValueError(msg) if len(covariates) != len(set(covariates)): - raise ValueError("Covariates must be unique") + msg = "Covariates must be unique" + raise ValueError(msg) # only works on dense matrices so far - if issparse(adata.X): - X = adata.X.toarray().T - else: - X = adata.X.T + X = adata.X.toarray().T if issparse(adata.X) else adata.X.T data = pd.DataFrame(data=X, index=adata.var_names, columns=adata.obs_names) sanitize_anndata(adata) @@ -264,7 +263,7 @@ def combat( # we now apply the parametric adjustment to the standardized data from above # loop over all batches in the data for j, batch_idxs in enumerate(batch_info): - # we basically substract the additive batch effect, rescale by the ratio + # we basically subtract the additive batch effect, rescale by the ratio # of multiplicative batch effect to pooled variance and add the overall gene # wise mean dsq = np.sqrt(delta_star[j, :]) diff --git a/src/scanpy/preprocessing/_deprecated/__init__.py b/src/scanpy/preprocessing/_deprecated/__init__.py index c2c363af01..b821417c0b 100644 --- a/src/scanpy/preprocessing/_deprecated/__init__.py +++ b/src/scanpy/preprocessing/_deprecated/__init__.py @@ -3,9 +3,13 @@ import numpy as np from scipy.sparse import csr_matrix, issparse +from ..._compat import old_positionals + +@old_positionals("max_fraction", "mult_with_mean") def normalize_per_cell_weinreb16_deprecated( - X: np.ndarray, + x: np.ndarray, + *, max_fraction: float = 1, mult_with_mean: bool = False, ) -> np.ndarray: @@ -32,25 +36,26 @@ def normalize_per_cell_weinreb16_deprecated( Normalized version of the original expression matrix. """ if max_fraction < 0 or max_fraction > 1: - raise ValueError("Choose max_fraction between 0 and 1.") + msg = "Choose max_fraction between 0 and 1." + raise ValueError(msg) - counts_per_cell = X.sum(1).A1 if issparse(X) else X.sum(1) - gene_subset = np.all(X <= counts_per_cell[:, None] * max_fraction, axis=0) - if issparse(X): + counts_per_cell = x.sum(1).A1 if issparse(x) else x.sum(1) + gene_subset = np.all(x <= counts_per_cell[:, None] * max_fraction, axis=0) + if issparse(x): gene_subset = gene_subset.A1 tc_include = ( - X[:, gene_subset].sum(1).A1 if issparse(X) else X[:, gene_subset].sum(1) + x[:, gene_subset].sum(1).A1 if issparse(x) else x[:, gene_subset].sum(1) ) - X_norm = ( - X.multiply(csr_matrix(1 / tc_include[:, None])) - if issparse(X) - else X / tc_include[:, None] + x_norm = ( + x.multiply(csr_matrix(1 / tc_include[:, None])) + if issparse(x) + else x / tc_include[:, None] ) if mult_with_mean: - X_norm *= np.mean(counts_per_cell) + x_norm *= np.mean(counts_per_cell) - return X_norm + return x_norm def zscore_deprecated(X: np.ndarray) -> np.ndarray: diff --git a/src/scanpy/preprocessing/_deprecated/highly_variable_genes.py b/src/scanpy/preprocessing/_deprecated/highly_variable_genes.py index ff29536ac8..bba4fb9bbf 100644 --- a/src/scanpy/preprocessing/_deprecated/highly_variable_genes.py +++ b/src/scanpy/preprocessing/_deprecated/highly_variable_genes.py @@ -9,6 +9,7 @@ from scipy.sparse import issparse from ... import logging as logg +from ..._compat import deprecated, old_positionals from .._distributed import materialize_as_ndarray from .._utils import _get_mean_var @@ -18,8 +19,22 @@ from scipy.sparse import spmatrix -def filter_genes_dispersion( # noqa: PLR0917 +@deprecated("Use sc.pp.highly_variable_genes instead") +@old_positionals( + "flavor", + "min_disp", + "max_disp", + "min_mean", + "max_mean", + "n_bins", + "n_top_genes", + "log", + "subset", + "copy", +) +def filter_genes_dispersion( data: AnnData | spmatrix | np.ndarray, + *, flavor: Literal["seurat", "cell_ranger"] = "seurat", min_disp: float | None = None, max_disp: float | None = None, @@ -34,18 +49,17 @@ def filter_genes_dispersion( # noqa: PLR0917 """\ Extract highly variable genes :cite:p:`Satija2015,Zheng2017`. - .. warning:: - .. deprecated:: 1.3.6 - Use :func:`~scanpy.pp.highly_variable_genes` - instead. The new function is equivalent to the present - function, except that + .. deprecated:: 1.3.6 - * the new function always expects logarithmized data - * `subset=False` in the new function, it suffices to - merely annotate the genes, tools like `pp.pca` will - detect the annotation - * you can now call: `sc.pl.highly_variable_genes(adata)` - * `copy` is replaced by `inplace` + Use :func:`~scanpy.pp.highly_variable_genes` instead. + The new function is equivalent to the present function, except that + + * the new function always expects logarithmized data + * `subset=False` in the new function, it suffices to + merely annotate the genes, tools like `pp.pca` will + detect the annotation + * you can now call: `sc.pl.highly_variable_genes(adata)` + * `copy` is replaced by `inplace` If trying out parameters, pass the data matrix instead of AnnData. @@ -200,7 +214,8 @@ def filter_genes_dispersion( # noqa: PLR0917 / disp_mad_bin[df["mean_bin"].values].values ) else: - raise ValueError('`flavor` needs to be "seurat" or "cell_ranger"') + msg = '`flavor` needs to be "seurat" or "cell_ranger"' + raise ValueError(msg) dispersion_norm = df["dispersion_norm"].values.astype("float32") if n_top_genes is not None: dispersion_norm = dispersion_norm[~np.isnan(dispersion_norm)] @@ -254,7 +269,8 @@ def filter_genes_fano_deprecated(X, Ecutoff, Vcutoff): def _filter_genes(X, e_cutoff, v_cutoff, meth): """See `filter_genes_dispersion` :cite:p:`Weinreb2017`.""" if issparse(X): - raise ValueError("Not defined for sparse input. See `filter_genes_dispersion`.") + msg = "Not defined for sparse input. See `filter_genes_dispersion`." + raise ValueError(msg) mean_filter = np.mean(X, axis=0) > e_cutoff var_filter = meth(X, axis=0) / (np.mean(X, axis=0) + 0.0001) > v_cutoff gene_subset = np.nonzero(np.all([mean_filter, var_filter], axis=0))[0] diff --git a/src/scanpy/preprocessing/_deprecated/sampling.py b/src/scanpy/preprocessing/_deprecated/sampling.py new file mode 100644 index 0000000000..02619a2364 --- /dev/null +++ b/src/scanpy/preprocessing/_deprecated/sampling.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ..._compat import _legacy_numpy_gen, old_positionals +from .._simple import sample + +if TYPE_CHECKING: + import numpy as np + from anndata import AnnData + from numpy.typing import NDArray + from scipy.sparse import csc_matrix, csr_matrix + + from ..._compat import _LegacyRandom + + CSMatrix = csr_matrix | csc_matrix + + +@old_positionals("n_obs", "random_state", "copy") +def subsample( + data: AnnData | np.ndarray | CSMatrix, + fraction: float | None = None, + *, + n_obs: int | None = None, + random_state: _LegacyRandom = 0, + copy: bool = False, +) -> AnnData | tuple[np.ndarray | CSMatrix, NDArray[np.int64]] | None: + """\ + Subsample to a fraction of the number of observations. + + .. deprecated:: 1.11.0 + + Use :func:`~scanpy.pp.sample` instead. + + Parameters + ---------- + data + The (annotated) data matrix of shape `n_obs` × `n_vars`. + Rows correspond to cells and columns to genes. + fraction + Subsample to this `fraction` of the number of observations. + n_obs + Subsample to this number of observations. + random_state + Random seed to change subsampling. + copy + If an :class:`~anndata.AnnData` is passed, + determines whether a copy is returned. + + Returns + ------- + Returns `X[obs_indices], obs_indices` if data is array-like, otherwise + subsamples the passed :class:`~anndata.AnnData` (`copy == False`) or + returns a subsampled copy of it (`copy == True`). + """ + + rng = _legacy_numpy_gen(random_state) + return sample( + data=data, fraction=fraction, n=n_obs, rng=rng, copy=copy, replace=False, axis=0 + ) diff --git a/src/scanpy/preprocessing/_highly_variable_genes.py b/src/scanpy/preprocessing/_highly_variable_genes.py index d75faa5af6..356fa8f03f 100644 --- a/src/scanpy/preprocessing/_highly_variable_genes.py +++ b/src/scanpy/preprocessing/_highly_variable_genes.py @@ -65,15 +65,14 @@ def _highly_variable_genes_seurat_v3( try: from skmisc.loess import loess except ImportError: - raise ImportError( - "Please install skmisc package via `pip install --user scikit-misc" - ) + msg = "Please install skmisc package via `pip install --user scikit-misc" + raise ImportError(msg) df = pd.DataFrame(index=adata.var_names) data = _get_obs_rep(adata, layer=layer) if check_values and not check_nonnegative_integers(data): warnings.warn( - f"`flavor='{flavor}'` expects raw count data, but non-integers were found.", + f"`{flavor=!r}` expects raw count data, but non-integers were found.", UserWarning, ) @@ -159,7 +158,8 @@ def _highly_variable_genes_seurat_v3( sort_cols = ["highly_variable_nbatches", "highly_variable_rank"] sort_ascending = [False, True] else: - raise ValueError(f"Did not recognize flavor {flavor}") + msg = f"Did not recognize flavor {flavor}" + raise ValueError(msg) sorted_index = ( df[sort_cols] .sort_values(sort_cols, ascending=sort_ascending, na_position="last") @@ -200,7 +200,8 @@ def _highly_variable_genes_seurat_v3( return df -@numba.njit(cache=True) +# parallel=False needed for accuracy +@numba.njit(cache=True, parallel=False) # noqa: TID251 def _sum_and_sum_squares_clipped( indices: NDArray[np.integer], data: NDArray[np.floating], @@ -211,7 +212,7 @@ def _sum_and_sum_squares_clipped( ) -> tuple[NDArray[np.float64], NDArray[np.float64]]: squared_batch_counts_sum = np.zeros(n_cols, dtype=np.float64) batch_counts_sum = np.zeros(n_cols, dtype=np.float64) - for i in range(nnz): + for i in numba.prange(nnz): idx = indices[i] element = min(np.float64(data[i]), clip_val[idx]) squared_batch_counts_sum[idx] += element**2 @@ -331,7 +332,8 @@ def _get_mean_bins( elif flavor == "cell_ranger": bins = np.r_[-np.inf, np.percentile(means, np.arange(10, 105, 5)), np.inf] else: - raise ValueError('`flavor` needs to be "seurat" or "cell_ranger"') + msg = '`flavor` needs to be "seurat" or "cell_ranger"' + raise ValueError(msg) return pd.cut(means, bins=bins) @@ -346,7 +348,8 @@ def _get_disp_stats( elif flavor == "cell_ranger": disp_bin_stats = disp_grouped.agg(avg="median", dev=_mad) else: - raise ValueError('`flavor` needs to be "seurat" or "cell_ranger"') + msg = '`flavor` needs to be "seurat" or "cell_ranger"' + raise ValueError(msg) return disp_bin_stats.loc[df["mean_bin"]].set_index(df.index) @@ -402,7 +405,7 @@ def _subset_genes( f"the {n_top_genes} top genes correspond to a " f"normalized dispersion cutoff of {disp_cut_off}" ) - return np.nan_to_num(dispersion_norm) >= disp_cut_off + return np.nan_to_num(dispersion_norm, nan=-np.inf) >= disp_cut_off def _nth_highest(x: NDArray[np.float64] | DaskArray, n: int) -> float | DaskArray: @@ -615,7 +618,7 @@ def highly_variable_genes( Returns ------- - Returns a :class:`pandas.DataFrame` with calculated metrics if `inplace=True`, else returns an `AnnData` object where it sets the following field: + Returns a :class:`pandas.DataFrame` with calculated metrics if `inplace=False`, else returns an `AnnData` object where it sets the following field: `adata.var['highly_variable']` : :class:`pandas.Series` (dtype `bool`) boolean indicator of highly-variable genes @@ -646,10 +649,11 @@ def highly_variable_genes( start = logg.info("extracting highly variable genes") if not isinstance(adata, AnnData): - raise ValueError( + msg = ( "`pp.highly_variable_genes` expects an `AnnData` argument, " "pass `inplace=False` if you want to return a `pd.DataFrame`." ) + raise ValueError(msg) if flavor in {"seurat_v3", "seurat_v3_paper"}: if n_top_genes is None: diff --git a/src/scanpy/preprocessing/_normalization.py b/src/scanpy/preprocessing/_normalization.py index de21a18631..e1ee3d4822 100644 --- a/src/scanpy/preprocessing/_normalization.py +++ b/src/scanpy/preprocessing/_normalization.py @@ -26,9 +26,9 @@ from anndata import AnnData -def _normalize_data(X, counts, after=None, copy: bool = False): +def _normalize_data(X, counts, after=None, *, copy: bool = False): X = X.copy() if copy else X - if issubclass(X.dtype.type, (int, np.integer)): + if issubclass(X.dtype.type, int | np.integer): X = X.astype(np.float32) # TODO: Check if float64 should be used if after is None: if isinstance(counts, DaskArray): @@ -175,11 +175,13 @@ def normalize_total( """ if copy: if not inplace: - raise ValueError("`copy=True` cannot be used with `inplace=False`.") + msg = "`copy=True` cannot be used with `inplace=False`." + raise ValueError(msg) adata = adata.copy() if max_fraction < 0 or max_fraction > 1: - raise ValueError("Choose max_fraction between 0 and 1.") + msg = "Choose max_fraction between 0 and 1." + raise ValueError(msg) # Deprecated features if layers is not None: @@ -200,31 +202,30 @@ def normalize_total( if layers == "all": layers = adata.layers.keys() elif isinstance(layers, str): - raise ValueError( - f"`layers` needs to be a list of strings or 'all', not {layers!r}" - ) + msg = f"`layers` needs to be a list of strings or 'all', not {layers!r}" + raise ValueError(msg) view_to_actual(adata) - X = _get_obs_rep(adata, layer=layer) + x = _get_obs_rep(adata, layer=layer) gene_subset = None msg = "normalizing counts per cell" - counts_per_cell = axis_sum(X, axis=1) + counts_per_cell = axis_sum(x, axis=1) if exclude_highly_expressed: counts_per_cell = np.ravel(counts_per_cell) # at least one cell as more than max_fraction of counts per cell - gene_subset = axis_sum((X > counts_per_cell[:, None] * max_fraction), axis=0) + gene_subset = axis_sum((x > counts_per_cell[:, None] * max_fraction), axis=0) gene_subset = np.asarray(np.ravel(gene_subset) == 0) msg += ( ". The following highly-expressed genes are not considered during " f"normalization factor computation:\n{adata.var_names[~gene_subset].tolist()}" ) - counts_per_cell = axis_sum(X[:, gene_subset], axis=1) + counts_per_cell = axis_sum(x[:, gene_subset], axis=1) start = logg.info(msg) counts_per_cell = np.ravel(counts_per_cell) @@ -237,12 +238,12 @@ def normalize_total( if key_added is not None: adata.obs[key_added] = counts_per_cell _set_obs_rep( - adata, _normalize_data(X, counts_per_cell, target_sum), layer=layer + adata, _normalize_data(x, counts_per_cell, target_sum), layer=layer ) else: # not recarray because need to support sparse dat = dict( - X=_normalize_data(X, counts_per_cell, target_sum, copy=True), + X=_normalize_data(x, counts_per_cell, target_sum, copy=True), norm_factor=counts_per_cell, ) @@ -254,7 +255,8 @@ def normalize_total( elif layer_norm is None: after = None else: - raise ValueError('layer_norm should be "after", "X" or None') + msg = 'layer_norm should be "after", "X" or None' + raise ValueError(msg) for layer_to_norm in layers if layers is not None else (): res = normalize_total( diff --git a/src/scanpy/preprocessing/_pca.py b/src/scanpy/preprocessing/_pca/__init__.py similarity index 52% rename from src/scanpy/preprocessing/_pca.py rename to src/scanpy/preprocessing/_pca/__init__.py index 6060bee6f6..db7886a29f 100644 --- a/src/scanpy/preprocessing/_pca.py +++ b/src/scanpy/preprocessing/_pca/__init__.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal, overload from warnings import warn import anndata as ad @@ -9,23 +9,56 @@ from anndata import AnnData from packaging.version import Version from scipy.sparse import issparse -from scipy.sparse.linalg import LinearOperator, svds -from sklearn.utils import check_array, check_random_state -from sklearn.utils.extmath import svd_flip - -from .. import logging as logg -from .._compat import DaskArray, pkg_version -from .._settings import settings -from .._utils import _doc_params, _empty, is_backed_type -from ..get import _check_mask, _get_obs_rep -from ._docs import doc_mask_var_hvg -from ._utils import _get_mean_var +from sklearn.utils import check_random_state + +from ... import logging as logg +from ..._compat import DaskArray, pkg_version +from ..._settings import settings +from ..._utils import _doc_params, _empty, get_literal_vals, is_backed_type +from ...get import _check_mask, _get_obs_rep +from .._docs import doc_mask_var_hvg +from ._compat import _pca_compat_sparse if TYPE_CHECKING: + from collections.abc import Container + from collections.abc import Set as AbstractSet + from typing import LiteralString, TypeVar + + import dask_ml.decomposition as dmld + import sklearn.decomposition as skld from numpy.typing import DTypeLike, NDArray + from scipy import sparse from scipy.sparse import spmatrix - from .._utils import AnyRandom, Empty + from ..._compat import _LegacyRandom + from ..._utils import Empty + + CSMatrix = sparse.csr_matrix | sparse.csc_matrix + + MethodDaskML = type[dmld.PCA | dmld.IncrementalPCA | dmld.TruncatedSVD] + MethodSklearn = type[skld.PCA | skld.TruncatedSVD] + + T = TypeVar("T", bound=LiteralString) + M = TypeVar("M", bound=LiteralString) + + +SvdSolvPCADaskML = Literal["auto", "full", "tsqr", "randomized"] +SvdSolvTruncatedSVDDaskML = Literal["tsqr", "randomized"] +SvdSolvDaskML = SvdSolvPCADaskML | SvdSolvTruncatedSVDDaskML + +if pkg_version("scikit-learn") >= Version("1.5") or TYPE_CHECKING: + SvdSolvPCASparseSklearn = Literal["arpack", "covariance_eigh"] +else: + SvdSolvPCASparseSklearn = Literal["arpack"] +SvdSolvPCADenseSklearn = Literal["auto", "full", "randomized"] | SvdSolvPCASparseSklearn +SvdSolvTruncatedSVDSklearn = Literal["arpack", "randomized"] +SvdSolvSkearn = ( + SvdSolvPCADenseSklearn | SvdSolvPCASparseSklearn | SvdSolvTruncatedSVDSklearn +) + +SvdSolvPCACustom = Literal["covariance_eigh"] + +SvdSolver = SvdSolvDaskML | SvdSolvSkearn | SvdSolvPCACustom @_doc_params( @@ -37,15 +70,16 @@ def pca( *, layer: str | None = None, zero_center: bool | None = True, - svd_solver: str | None = None, - random_state: AnyRandom = 0, + svd_solver: SvdSolver | None = None, + random_state: _LegacyRandom = 0, return_info: bool = False, mask_var: NDArray[np.bool_] | str | None | Empty = _empty, use_highly_variable: bool | None = None, dtype: DTypeLike = "float32", - copy: bool = False, chunked: bool = False, chunk_size: int | None = None, + key_added: str | None = None, + copy: bool = False, ) -> AnnData | np.ndarray | spmatrix | None: """\ Principal component analysis :cite:p:`Pedregosa2011`. @@ -84,6 +118,7 @@ def pca( `None` See `chunked` and `zero_center` descriptions to determine which class will be used. Depending on the class and the type of X different values for default will be set. + For sparse *dask* arrays, will use `'covariance_eigh'`. If *scikit-learn* :class:`~sklearn.decomposition.PCA` is used, will give `'arpack'`, if *scikit-learn* :class:`~sklearn.decomposition.TruncatedSVD` is used, will give `'randomized'`, if *dask-ml* :class:`~dask_ml.decomposition.PCA` or :class:`~dask_ml.decomposition.IncrementalPCA` is used, will give `'auto'`, @@ -91,15 +126,16 @@ def pca( `'arpack'` for the ARPACK wrapper in SciPy (:func:`~scipy.sparse.linalg.svds`) Not available with *dask* arrays. + `'covariance_eigh'` + Classic eigendecomposition of the covariance matrix, suited for tall-and-skinny matrices. + With dask, array must be CSR and chunked as (N, adata.shape[1]). `'randomized'` for the randomized algorithm due to Halko (2009). For *dask* arrays, this will use :func:`~dask.array.linalg.svd_compressed`. `'auto'` chooses automatically depending on the size of the problem. - `'lobpcg'` - An alternative SciPy solver. Not available with dask arrays. `'tsqr'` - Only available with *dask* arrays. "tsqr" + Only available with dense *dask* arrays. "tsqr" algorithm from Benson et. al. (2013). .. versionchanged:: 1.9.3 @@ -108,9 +144,10 @@ def pca( Default value changed from `'auto'` to `'arpack'`. Efficient computation of the principal components of a sparse matrix - currently only works with the `'arpack`' or `'lobpcg'` solvers. + currently only works with the `'arpack`' or `'covariance_eigh`' solver. - If X is a *dask* array, *dask-ml* classes :class:`~dask_ml.decomposition.PCA`, + If X is a sparse *dask* array, a custom `'covariance_eigh'` solver will be used. + If X is a dense *dask* array, *dask-ml* classes :class:`~dask_ml.decomposition.PCA`, :class:`~dask_ml.decomposition.IncrementalPCA`, or :class:`~dask_ml.decomposition.TruncatedSVD` will be used. Otherwise their *scikit-learn* counterparts :class:`~sklearn.decomposition.PCA`, @@ -126,9 +163,6 @@ def pca( Layer of `adata` to use as expression values. dtype Numpy data type string to which to convert the result. - copy - If an :class:`~anndata.AnnData` is passed, determines whether a copy - is returned. Is ignored otherwise. chunked If `True`, perform an incremental PCA on segments of `chunk_size`. The incremental PCA automatically zero centers and ignores settings of @@ -139,6 +173,18 @@ def pca( chunk_size Number of observations to include in each chunk. Required if `chunked=True` was passed. + key_added + If not specified, the embedding is stored as + :attr:`~anndata.AnnData.obsm`\\ `['X_pca']`, the loadings as + :attr:`~anndata.AnnData.varm`\\ `['PCs']`, and the the parameters in + :attr:`~anndata.AnnData.uns`\\ `['pca']`. + If specified, the embedding is stored as + :attr:`~anndata.AnnData.obsm`\\ ``[key_added]``, the loadings as + :attr:`~anndata.AnnData.varm`\\ ``[key_added]``, and the the parameters in + :attr:`~anndata.AnnData.uns`\\ ``[key_added]``. + copy + If an :class:`~anndata.AnnData` is passed, determines whether a copy + is returned. Is ignored otherwise. Returns ------- @@ -149,34 +195,34 @@ def pca( Otherwise, it returns `None` if `copy=False`, else an updated `AnnData` object. Sets the following fields: - `.obsm['X_pca']` : :class:`~scipy.sparse.spmatrix` | :class:`~numpy.ndarray` (shape `(adata.n_obs, n_comps)`) + `.obsm['X_pca' | key_added]` : :class:`~scipy.sparse.spmatrix` | :class:`~numpy.ndarray` (shape `(adata.n_obs, n_comps)`) PCA representation of data. - `.varm['PCs']` : :class:`~numpy.ndarray` (shape `(adata.n_vars, n_comps)`) + `.varm['PCs' | key_added]` : :class:`~numpy.ndarray` (shape `(adata.n_vars, n_comps)`) The principal components containing the loadings. - `.uns['pca']['variance_ratio']` : :class:`~numpy.ndarray` (shape `(n_comps,)`) + `.uns['pca' | key_added]['variance_ratio']` : :class:`~numpy.ndarray` (shape `(n_comps,)`) Ratio of explained variance. - `.uns['pca']['variance']` : :class:`~numpy.ndarray` (shape `(n_comps,)`) + `.uns['pca' | key_added]['variance']` : :class:`~numpy.ndarray` (shape `(n_comps,)`) Explained variance, equivalent to the eigenvalues of the covariance matrix. """ logg_start = logg.info("computing PCA") if layer is not None and chunked: # Current chunking implementation relies on pca being called on X - raise NotImplementedError("Cannot use `layer` and `chunked` at the same time.") + msg = "Cannot use `layer` and `chunked` at the same time." + raise NotImplementedError(msg) # chunked calculation is not randomized, anyways if svd_solver in {"auto", "randomized"} and not chunked: logg.info( "Note that scikit-learn's randomized PCA might not be exactly " "reproducible across different computational platforms. For exact " - "reproducibility, choose `svd_solver='arpack'.`" + "reproducibility, choose `svd_solver='arpack'`." ) data_is_AnnData = isinstance(data, AnnData) if data_is_AnnData: if layer is None and not chunked and is_backed_type(data.X): - raise NotImplementedError( - f"PCA is not implemented for matrices of type {type(data.X)} with chunked as False" - ) + msg = f"PCA is not implemented for matrices of type {type(data.X)} with chunked as False" + raise NotImplementedError(msg) adata = data.copy() if copy else data else: if pkg_version("anndata") < Version("0.8.0rc1"): @@ -191,18 +237,14 @@ def pca( if n_comps is None: min_dim = min(adata_comp.n_vars, adata_comp.n_obs) - if settings.N_PCS >= min_dim: - n_comps = min_dim - 1 - else: - n_comps = settings.N_PCS + n_comps = min_dim - 1 if min_dim <= settings.N_PCS else settings.N_PCS - logg.info(f" with n_comps={n_comps}") + logg.info(f" with {n_comps=}") X = _get_obs_rep(adata_comp, layer=layer) if is_backed_type(X) and layer is not None: - raise NotImplementedError( - f"PCA is not implemented for matrices of type {type(X)} from layers" - ) + msg = f"PCA is not implemented for matrices of type {type(X)} from layers" + raise NotImplementedError(msg) # See: https://github.com/scverse/scanpy/pull/2816#issuecomment-1932650529 if ( Version(ad.__version__) < Version("0.9") @@ -216,11 +258,9 @@ def pca( UserWarning, ) - is_dask = isinstance(X, DaskArray) - # check_random_state returns a numpy RandomState when passed an int but # dask needs an int for random state - if not is_dask: + if not isinstance(X, DaskArray): random_state = check_random_state(random_state) elif not isinstance(random_state, int): msg = f"random_state needs to be an int, not a {type(random_state).__name__} when passing a dask array" @@ -235,12 +275,12 @@ def pca( logg.debug("Ignoring zero_center, random_state, svd_solver") incremental_pca_kwargs = dict() - if is_dask: + if isinstance(X, DaskArray): from dask.array import zeros from dask_ml.decomposition import IncrementalPCA incremental_pca_kwargs["svd_solver"] = _handle_dask_ml_args( - svd_solver, "IncrementalPCA" + svd_solver, IncrementalPCA ) else: from numpy import zeros @@ -257,56 +297,75 @@ def pca( for chunk, start, end in adata_comp.chunked_X(chunk_size): chunk = chunk.toarray() if issparse(chunk) else chunk X_pca[start:end] = pca_.transform(chunk) - elif (not issparse(X) or svd_solver == "randomized") and zero_center: - if is_dask: - from dask_ml.decomposition import PCA - - svd_solver = _handle_dask_ml_args(svd_solver, "PCA") - else: - from sklearn.decomposition import PCA - - svd_solver = _handle_sklearn_args(svd_solver, "PCA") - - if issparse(X) and svd_solver == "randomized": - # This is for backwards compat. Better behaviour would be to either error or use arpack. - warnings.warn( - "svd_solver 'randomized' does not work with sparse input. Densifying the array. " - "This may take a very large amount of memory." + elif zero_center: + if issparse(X) and ( + pkg_version("scikit-learn") < Version("1.4") or svd_solver == "lobpcg" + ): + if svd_solver not in ( + {"lobpcg"} | get_literal_vals(SvdSolvPCASparseSklearn) + ): + if svd_solver is not None: + msg = ( + f"Ignoring {svd_solver=} and using 'arpack', " + "sparse PCA with sklearn < 1.4 only supports 'lobpcg' and 'arpack'." + ) + warnings.warn(msg) + svd_solver = "arpack" + elif svd_solver == "lobpcg": + msg = ( + f"{svd_solver=} for sparse relies on legacy code and will not be supported in the future. " + "Also the lobpcg solver has been observed to be inaccurate. Please use 'arpack' instead." + ) + warnings.warn(msg, FutureWarning) + X_pca, pca_ = _pca_compat_sparse( + X, n_comps, solver=svd_solver, random_state=random_state ) - X = X.toarray() - pca_ = PCA( - n_components=n_comps, svd_solver=svd_solver, random_state=random_state - ) - X_pca = pca_.fit_transform(X) - elif issparse(X) and zero_center: - from sklearn.decomposition import PCA - - svd_solver = _handle_sklearn_args(svd_solver, "PCA (with sparse input)") - - output = _pca_with_sparse( - X, n_comps, solver=svd_solver, random_state=random_state - ) - # this is just a wrapper for the results - X_pca = output["X_pca"] - pca_ = PCA( - n_components=n_comps, svd_solver=svd_solver, random_state=random_state - ) - pca_.components_ = output["components"] - pca_.explained_variance_ = output["variance"] - pca_.explained_variance_ratio_ = output["variance_ratio"] - elif not zero_center: - if is_dask: + else: + if not isinstance(X, DaskArray): + from sklearn.decomposition import PCA + + svd_solver = _handle_sklearn_args(svd_solver, PCA, sparse=issparse(X)) + pca_ = PCA( + n_components=n_comps, + svd_solver=svd_solver, + random_state=random_state, + ) + elif issparse(X._meta): + from ._dask_sparse import PCASparseDask + + if random_state != 0: + msg = f"Ignoring {random_state=} when using a sparse dask array" + warnings.warn(msg) + if svd_solver not in {None, "covariance_eigh"}: + msg = f"Ignoring {svd_solver=} when using a sparse dask array" + warnings.warn(msg) + pca_ = PCASparseDask(n_components=n_comps) + else: + from dask_ml.decomposition import PCA + + svd_solver = _handle_dask_ml_args(svd_solver, PCA) + pca_ = PCA( + n_components=n_comps, + svd_solver=svd_solver, + random_state=random_state, + ) + X_pca = pca_.fit_transform(X) + else: + if isinstance(X, DaskArray): + if issparse(X._meta): + msg = "Dask sparse arrays do not support zero-centering (yet)" + raise TypeError(msg) from dask_ml.decomposition import TruncatedSVD - svd_solver = _handle_dask_ml_args(svd_solver, "TruncatedSVD") + svd_solver = _handle_dask_ml_args(svd_solver, TruncatedSVD) else: from sklearn.decomposition import TruncatedSVD - svd_solver = _handle_sklearn_args(svd_solver, "TruncatedSVD") + svd_solver = _handle_sklearn_args(svd_solver, TruncatedSVD) logg.debug( " without zero-centering: \n" - " the explained variance does not correspond to the exact statistical defintion\n" + " the explained variance does not correspond to the exact statistical definition\n" " the first component, e.g., might be heavily influenced by different means\n" " the following components often resemble the exact PCA very closely" ) @@ -314,42 +373,42 @@ def pca( n_components=n_comps, random_state=random_state, algorithm=svd_solver ) X_pca = pca_.fit_transform(X) - else: - msg = "This shouldn’t happen. Please open a bug report." - raise AssertionError(msg) if X_pca.dtype.descr != np.dtype(dtype).descr: X_pca = X_pca.astype(dtype) if data_is_AnnData: - adata.obsm["X_pca"] = X_pca + key_obsm, key_varm, key_uns = ( + ("X_pca", "PCs", "pca") if key_added is None else [key_added] * 3 + ) + adata.obsm[key_obsm] = X_pca if mask_var is not None: - adata.varm["PCs"] = np.zeros(shape=(adata.n_vars, n_comps)) - adata.varm["PCs"][mask_var] = pca_.components_.T + adata.varm[key_varm] = np.zeros(shape=(adata.n_vars, n_comps)) + adata.varm[key_varm][mask_var] = pca_.components_.T else: - adata.varm["PCs"] = pca_.components_.T - - uns_entry = { - "params": { - "zero_center": zero_center, - "use_highly_variable": mask_var_param == "highly_variable", - "mask_var": mask_var_param, - }, - "variance": pca_.explained_variance_, - "variance_ratio": pca_.explained_variance_ratio_, - } + adata.varm[key_varm] = pca_.components_.T + + params = dict( + zero_center=zero_center, + use_highly_variable=mask_var_param == "highly_variable", + mask_var=mask_var_param, + ) if layer is not None: - uns_entry["params"]["layer"] = layer - adata.uns["pca"] = uns_entry + params["layer"] = layer + adata.uns[key_uns] = dict( + params=params, + variance=pca_.explained_variance_, + variance_ratio=pca_.explained_variance_ratio_, + ) logg.info(" finished", time=logg_start) logg.debug( "and added\n" - " 'X_pca', the PCA coordinates (adata.obs)\n" - " 'PC1', 'PC2', ..., the loadings (adata.var)\n" - " 'pca_variance', the variance / eigenvalues (adata.uns)\n" - " 'pca_variance_ratio', the variance ratio (adata.uns)" + f" {key_obsm!r}, the PCA coordinates (adata.obs)\n" + f" {key_varm!r}, the loadings (adata.varm)\n" + f" 'pca_variance', the variance / eigenvalues (adata.uns[{key_uns!r}])\n" + f" 'pca_variance_ratio', the variance ratio (adata.uns[{key_uns!r}])" ) return adata if copy else None else: @@ -391,7 +450,7 @@ def _handle_mask_var( if use_highly_variable or ( use_highly_variable is None and mask_var is _empty - and "highly_variable" in adata.var.keys() + and "highly_variable" in adata.var.columns ): mask_var = "highly_variable" @@ -401,110 +460,84 @@ def _handle_mask_var( return mask_var, _check_mask(adata, mask_var, "var") -def _pca_with_sparse( - X: spmatrix, - n_pcs: int, +@overload +def _handle_dask_ml_args( + svd_solver: str | None, method: type[dmld.PCA | dmld.IncrementalPCA] +) -> SvdSolvPCADaskML: ... +@overload +def _handle_dask_ml_args( + svd_solver: str | None, method: type[dmld.TruncatedSVD] +) -> SvdSolvTruncatedSVDDaskML: ... +def _handle_dask_ml_args(svd_solver: str | None, method: MethodDaskML) -> SvdSolvDaskML: + import dask_ml.decomposition as dmld + + args: AbstractSet[SvdSolvDaskML] + default: SvdSolvDaskML + match method: + case dmld.PCA | dmld.IncrementalPCA: + args = get_literal_vals(SvdSolvPCADaskML) + default = "auto" + case dmld.TruncatedSVD: + args = get_literal_vals(SvdSolvTruncatedSVDDaskML) + default = "tsqr" + case _: + msg = f"Unknown {method=} in _handle_dask_ml_args" + raise ValueError(msg) + return _handle_x_args(svd_solver, method, args, default) + + +@overload +def _handle_sklearn_args( + svd_solver: str | None, method: type[skld.TruncatedSVD], *, sparse: None = None +) -> SvdSolvTruncatedSVDSklearn: ... +@overload +def _handle_sklearn_args( + svd_solver: str | None, method: type[skld.PCA], *, sparse: Literal[False] +) -> SvdSolvPCADenseSklearn: ... +@overload +def _handle_sklearn_args( + svd_solver: str | None, method: type[skld.PCA], *, sparse: Literal[True] +) -> SvdSolvPCASparseSklearn: ... +def _handle_sklearn_args( + svd_solver: str | None, method: MethodSklearn, *, sparse: bool | None = None +) -> SvdSolvSkearn: + import sklearn.decomposition as skld + + args: AbstractSet[SvdSolvSkearn] + default: SvdSolvSkearn + suffix = "" + match (method, sparse): + case (skld.TruncatedSVD, None): + args = get_literal_vals(SvdSolvTruncatedSVDSklearn) + default = "randomized" + case (skld.PCA, False): + args = get_literal_vals(SvdSolvPCADenseSklearn) + default = "arpack" + case (skld.PCA, True): + args = get_literal_vals(SvdSolvPCASparseSklearn) + default = "arpack" + suffix = " (with sparse input)" + case _: + msg = f"Unknown {method=} ({sparse=}) in _handle_sklearn_args" + raise ValueError(msg) + + return _handle_x_args(svd_solver, method, args, default, suffix=suffix) + + +def _handle_x_args( + svd_solver: str | None, + method: type, + args: Container[T], + default: T, *, - solver: str = "arpack", - mu: NDArray[np.floating] | None = None, - random_state: AnyRandom = None, -): - random_state = check_random_state(random_state) - np.random.set_state(random_state.get_state()) - random_init = np.random.rand(np.min(X.shape)) - X = check_array(X, accept_sparse=["csr", "csc"]) - - if mu is None: - mu = np.asarray(X.mean(0)).flatten()[None, :] - mdot = mu.dot - mmat = mdot - mhdot = mu.T.dot - mhmat = mu.T.dot - Xdot = X.dot - Xmat = Xdot - XHdot = X.T.conj().dot - XHmat = XHdot - ones = np.ones(X.shape[0])[None, :].dot - - def matvec(x): - return Xdot(x) - mdot(x) - - def matmat(x): - return Xmat(x) - mmat(x) - - def rmatvec(x): - return XHdot(x) - mhdot(ones(x)) - - def rmatmat(x): - return XHmat(x) - mhmat(ones(x)) - - XL = LinearOperator( - matvec=matvec, - dtype=X.dtype, - matmat=matmat, - shape=X.shape, - rmatvec=rmatvec, - rmatmat=rmatmat, - ) - - u, s, v = svds(XL, solver=solver, k=n_pcs, v0=random_init) - # u_based_decision was changed in https://github.com/scikit-learn/scikit-learn/pull/27491 - u, v = svd_flip( - u, v, u_based_decision=pkg_version("scikit-learn") < Version("1.5.0rc1") - ) - idx = np.argsort(-s) - v = v[idx, :] - - X_pca = (u * s)[:, idx] - ev = s[idx] ** 2 / (X.shape[0] - 1) - - total_var = _get_mean_var(X)[1].sum() - ev_ratio = ev / total_var - - output = { - "X_pca": X_pca, - "variance": ev, - "variance_ratio": ev_ratio, - "components": v, - } - return output - - -def _handle_dask_ml_args(svd_solver: str, method: str) -> str: - method2args = { - "PCA": {"auto", "full", "tsqr", "randomized"}, - "IncrementalPCA": {"auto", "full", "tsqr", "randomized"}, - "TruncatedSVD": {"tsqr", "randomized"}, - } - method2default = { - "PCA": "auto", - "IncrementalPCA": "auto", - "TruncatedSVD": "tsqr", - } - - return _handle_x_args("dask_ml", svd_solver, method, method2args, method2default) - - -def _handle_sklearn_args(svd_solver: str, method: str) -> str: - method2args = { - "PCA": {"auto", "full", "arpack", "randomized"}, - "TruncatedSVD": {"arpack", "randomized"}, - "PCA (with sparse input)": {"lobpcg", "arpack"}, - } - method2default = { - "PCA": "arpack", - "TruncatedSVD": "randomized", - "PCA (with sparse input)": "arpack", - } - - return _handle_x_args("sklearn", svd_solver, method, method2args, method2default) - - -def _handle_x_args(lib, svd_solver, method, method2args, method2default): - if svd_solver not in method2args[method]: - if svd_solver is not None: - warnings.warn( - f"Ignoring {svd_solver} and using {method2default[method]}, {lib}.decomposition.{method} only supports {method2args[method]}" - ) - svd_solver = method2default[method] - return svd_solver + suffix: str = "", +) -> T: + if svd_solver in args: + return svd_solver + if svd_solver is not None: + msg = ( + f"Ignoring {svd_solver=} and using {default}, " + f"{method.__module__}.{method.__qualname__}{suffix} only supports {args}." + ) + warnings.warn(msg) + return default diff --git a/src/scanpy/preprocessing/_pca/_compat.py b/src/scanpy/preprocessing/_pca/_compat.py new file mode 100644 index 0000000000..28eef2ba1a --- /dev/null +++ b/src/scanpy/preprocessing/_pca/_compat.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +from packaging.version import Version +from scipy.sparse.linalg import LinearOperator, svds +from sklearn.utils import check_array, check_random_state +from sklearn.utils.extmath import svd_flip + +from ..._compat import pkg_version +from .._utils import _get_mean_var + +if TYPE_CHECKING: + from typing import Literal + + from numpy.typing import NDArray + from scipy import sparse + from sklearn.decomposition import PCA + + from ..._compat import _LegacyRandom + + CSMatrix = sparse.csr_matrix | sparse.csc_matrix + + +def _pca_compat_sparse( + x: CSMatrix, + n_pcs: int, + *, + solver: Literal["arpack", "lobpcg"], + mu: NDArray[np.floating] | None = None, + random_state: _LegacyRandom = None, +) -> tuple[NDArray[np.floating], PCA]: + """Sparse PCA for scikit-learn <1.4""" + random_state = check_random_state(random_state) + np.random.set_state(random_state.get_state()) + random_init = np.random.rand(np.min(x.shape)) + x = check_array(x, accept_sparse=["csr", "csc"]) + + if mu is None: + mu = np.asarray(x.mean(0)).flatten()[None, :] + ones = np.ones(x.shape[0])[None, :].dot + + def mat_op(v: NDArray[np.floating]): + return (x @ v) - (mu @ v) + + def rmat_op(v: NDArray[np.floating]): + return (x.T.conj() @ v) - (mu.T @ ones(v)) + + linop = LinearOperator( + dtype=x.dtype, + shape=x.shape, + matvec=mat_op, + matmat=mat_op, + rmatvec=rmat_op, + rmatmat=rmat_op, + ) + + u, s, v = svds(linop, solver=solver, k=n_pcs, v0=random_init) + # u_based_decision was changed in https://github.com/scikit-learn/scikit-learn/pull/27491 + u, v = svd_flip( + u, v, u_based_decision=pkg_version("scikit-learn") < Version("1.5.0rc1") + ) + idx = np.argsort(-s) + v = v[idx, :] + + X_pca = (u * s)[:, idx] + ev = s[idx] ** 2 / (x.shape[0] - 1) + + total_var = _get_mean_var(x)[1].sum() + ev_ratio = ev / total_var + + from sklearn.decomposition import PCA + + pca = PCA(n_components=n_pcs, svd_solver=solver, random_state=random_state) + pca.explained_variance_ = ev + pca.explained_variance_ratio_ = ev_ratio + pca.components_ = v + return X_pca, pca diff --git a/src/scanpy/preprocessing/_pca/_dask_sparse.py b/src/scanpy/preprocessing/_pca/_dask_sparse.py new file mode 100644 index 0000000000..c2bff7ccca --- /dev/null +++ b/src/scanpy/preprocessing/_pca/_dask_sparse.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, cast, overload + +import numpy as np +import scipy.linalg +from numpy.typing import NDArray + +from scanpy._utils._doctests import doctest_needs + +from .._utils import _get_mean_var + +if TYPE_CHECKING: + from typing import Literal + + from numpy.typing import DTypeLike + from scipy import sparse + + from ..._compat import DaskArray + + CSMatrix = sparse.csr_matrix | sparse.csc_matrix + + +@dataclass +class PCASparseDask: + n_components: int | None = None + + @doctest_needs("dask") + def fit(self, x: DaskArray) -> PCASparseDaskFit: + """Fit the model on `x`. + + This method transforms `self` into a `PCASparseDaskFit` object and returns it. + + Examples + -------- + >>> import dask.array as da + >>> import scipy.sparse as sp + >>> x = ( + ... da.array(sp.random(100, 200, density=0.3, dtype="float32").toarray()) + ... .rechunk((10, -1)) + ... .map_blocks(sp.csr_matrix) + ... ) + >>> x + dask.array + >>> pca_fit = PCASparseDask().fit(x) + >>> assert isinstance(pca_fit, PCASparseDaskFit) + >>> pca_fit.transform(x) + dask.array + """ + if x._meta.format != "csr": + msg = ( + "Only dask arrays with CSR-meta format are supported. " + f"Got {x._meta.format} as meta." + ) + raise ValueError(msg) + if x.chunksize[1] != x.shape[1]: + msg = ( + "Only dask arrays with chunking along the first axis are supported. " + f"Got chunksize {x.chunksize} with shape {x.shape}. " + "Rechunking should be simple and cost nothing from AnnData's on-disk format when the on-disk layout has this chunking." + ) + raise ValueError(msg) + self.__class__ = PCASparseDaskFit + self = cast(PCASparseDaskFit, self) + + self.n_components_ = ( + min(x.shape) if self.n_components is None else self.n_components + ) + self.n_samples_ = x.shape[0] + self.n_features_in_ = x.shape[1] if x.ndim > 1 else 1 + self.dtype_ = x.dtype + covariance, self.mean_ = _cov_sparse_dask(x) + self.explained_variance_, self.components_ = scipy.linalg.eigh( + covariance, lower=False + ) + + # Arrange eigenvectors and eigenvalues in descending order + self.explained_variance_ = self.explained_variance_[::-1] + self.components_ = np.flip(self.components_, axis=1) + self.components_ = self.components_.T[: self.n_components_, :] + + self.explained_variance_ratio_ = self.explained_variance_ / np.sum( + self.explained_variance_ + ) + if self.n_components_ < min(self.n_samples_, self.n_features_in_): + self.noise_variance_ = self.explained_variance_[self.n_components_ :].mean() + else: + self.noise_variance_ = np.array([0.0]) + self.explained_variance_ = self.explained_variance_[: self.n_components_] + + self.explained_variance_ratio_ = self.explained_variance_ratio_[ + : self.n_components_ + ] + return self + + def fit_transform(self, x: DaskArray, y: DaskArray | None = None) -> DaskArray: + if y is None: + y = x + return self.fit(x).transform(y) + + +@dataclass +class PCASparseDaskFit(PCASparseDask): + n_components_: int = field(init=False) + n_samples_: int = field(init=False) + n_features_in_: int = field(init=False) + dtype_: np.dtype = field(init=False) + mean_: NDArray[np.floating] = field(init=False) + components_: NDArray[np.floating] = field(init=False) + explained_variance_: NDArray[np.floating] = field(init=False) + explained_variance_ratio_: NDArray[np.floating] = field(init=False) + noise_variance_: NDArray[np.floating] = field(init=False) + + def transform(self, x: DaskArray) -> DaskArray: + if TYPE_CHECKING: + # The type checker does not understand imports from dask.array + import dask.array.core as da + else: + import dask.array as da + + def transform_block( + x_part: CSMatrix, + mean_: NDArray[np.floating], + components_: NDArray[np.floating], + ): + pre_mean = mean_ @ components_.T + mean_impact = np.ones((x_part.shape[0], 1)) @ pre_mean.reshape(1, -1) + return (x_part @ components_.T) - mean_impact + + return da.map_blocks( + transform_block, + x, + mean_=self.mean_, + components_=self.components_, + chunks=(x.chunks[0], self.n_components_), + meta=np.zeros([0], dtype=x.dtype), + dtype=x.dtype, + ) + + +@overload +def _cov_sparse_dask( + x: DaskArray, *, return_gram: Literal[False] = False, dtype: DTypeLike | None = None +) -> tuple[NDArray[np.floating], NDArray[np.floating]]: ... +@overload +def _cov_sparse_dask( + x: DaskArray, *, return_gram: Literal[True], dtype: DTypeLike | None = None +) -> tuple[NDArray[np.floating], NDArray[np.floating], NDArray[np.floating]]: ... +def _cov_sparse_dask( + x: DaskArray, *, return_gram: bool = False, dtype: DTypeLike | None = None +) -> ( + tuple[NDArray[np.floating], NDArray[np.floating], NDArray[np.floating]] + | tuple[NDArray[np.floating], NDArray[np.floating]] +): + """\ + Computes the covariance matrix and row/col means of matrix `x`. + + Parameters + ---------- + + x + A sparse matrix + return_gram + If `True`, the gram matrix will be returned and a copy will be created + to store the results of the covariance, + while if `False`, the local gram matrix result will be overwritten. + (only used for unit testing at the moment) + dtype + The data type of the result (excluding the means) + + Returns + ------- + + :math:`\\cov(X, X)` + The covariance matrix of `x` in the form :math:`\\cov(X, X) = \\E(XX) - \\E(X)\\E(X)`. + :math:`\\gram(X, X)` + When return_gram is `True`, the gram matrix of `x` in the form :math:`\\frac{1}{n} X.T \\dot X`. + :math:`\\mean(X)` + The row means of `x`. + """ + if TYPE_CHECKING: + import dask.array.core as da + import dask.base as dask + else: + import dask + import dask.array as da + + if dtype is None: + dtype = np.float64 if np.issubdtype(x.dtype, np.integer) else x.dtype + else: + dtype = np.dtype(dtype) + + def gram_block(x_part: CSMatrix): + gram_matrix: CSMatrix = x_part.T @ x_part + return gram_matrix.toarray()[None, ...] # need new axis for summing + + gram_matrix_dask: DaskArray = da.map_blocks( + gram_block, + x, + new_axis=(1,), + chunks=((1,) * x.blocks.size, (x.shape[1],), (x.shape[1],)), + meta=np.array([], dtype=x.dtype), + dtype=x.dtype, + ).sum(axis=0) + mean_x_dask, _ = _get_mean_var(x) + gram_matrix, mean_x = cast( + tuple[NDArray, NDArray[np.float64]], + dask.compute(gram_matrix_dask, mean_x_dask), + ) + gram_matrix = gram_matrix.astype(dtype) + gram_matrix /= x.shape[0] + + cov_result = gram_matrix.copy() if return_gram else gram_matrix + cov_result -= mean_x[:, None] @ mean_x[None, :] + + if return_gram: + return cov_result, gram_matrix, mean_x + return cov_result, mean_x diff --git a/src/scanpy/preprocessing/_qc.py b/src/scanpy/preprocessing/_qc.py index f4e14054f9..5af8def042 100644 --- a/src/scanpy/preprocessing/_qc.py +++ b/src/scanpy/preprocessing/_qc.py @@ -1,15 +1,19 @@ from __future__ import annotations +from functools import singledispatch from typing import TYPE_CHECKING from warnings import warn import numba import numpy as np import pandas as pd -from scipy.sparse import csr_matrix, issparse, isspmatrix_coo, isspmatrix_csr -from sklearn.utils.sparsefuncs import mean_variance_axis +from scipy.sparse import csr_matrix, issparse, isspmatrix_coo, isspmatrix_csr, spmatrix -from .._utils import _doc_params +from scanpy.preprocessing._distributed import materialize_as_ndarray +from scanpy.preprocessing._utils import _get_mean_var + +from .._compat import DaskArray, njit +from .._utils import _doc_params, axis_nnz, axis_sum from ._docs import ( doc_adata_basic, doc_expr_reps, @@ -23,16 +27,16 @@ from collections.abc import Collection from anndata import AnnData - from scipy.sparse import spmatrix -def _choose_mtx_rep(adata, use_raw: bool = False, layer: str | None = None): +def _choose_mtx_rep(adata, *, use_raw: bool = False, layer: str | None = None): is_layer = layer is not None if use_raw and is_layer: - raise ValueError( + msg = ( "Cannot use expression from both layer and raw. You provided:" - f"'use_raw={use_raw}' and 'layer={layer}'" + f"{use_raw=!r} and {layer=!r}" ) + raise ValueError(msg) if is_layer: return adata.layers[layer] elif use_raw: @@ -98,21 +102,20 @@ def describe_obs( ) # Handle whether X is passed if X is None: - X = _choose_mtx_rep(adata, use_raw, layer) + X = _choose_mtx_rep(adata, use_raw=use_raw, layer=layer) if isspmatrix_coo(X): X = csr_matrix(X) # COO not subscriptable if issparse(X): X.eliminate_zeros() obs_metrics = pd.DataFrame(index=adata.obs_names) - if issparse(X): - obs_metrics[f"n_{var_type}_by_{expr_type}"] = X.getnnz(axis=1) - else: - obs_metrics[f"n_{var_type}_by_{expr_type}"] = np.count_nonzero(X, axis=1) + obs_metrics[f"n_{var_type}_by_{expr_type}"] = materialize_as_ndarray( + axis_nnz(X, axis=1) + ) if log1p: obs_metrics[f"log1p_n_{var_type}_by_{expr_type}"] = np.log1p( obs_metrics[f"n_{var_type}_by_{expr_type}"] ) - obs_metrics[f"total_{expr_type}"] = np.ravel(X.sum(axis=1)) + obs_metrics[f"total_{expr_type}"] = np.ravel(axis_sum(X, axis=1)) if log1p: obs_metrics[f"log1p_total_{expr_type}"] = np.log1p( obs_metrics[f"total_{expr_type}"] @@ -126,7 +129,7 @@ def describe_obs( ) for qc_var in qc_vars: obs_metrics[f"total_{expr_type}_{qc_var}"] = np.ravel( - X[:, adata.var[qc_var].values].sum(axis=1) + axis_sum(X[:, adata.var[qc_var].values], axis=1) ) if log1p: obs_metrics[f"log1p_total_{expr_type}_{qc_var}"] = np.log1p( @@ -141,6 +144,7 @@ def describe_obs( adata.obs[obs_metrics.columns] = obs_metrics else: return obs_metrics + return None @_doc_params( @@ -185,40 +189,31 @@ def describe_var( """ # Handle whether X is passed if X is None: - X = _choose_mtx_rep(adata, use_raw, layer) + X = _choose_mtx_rep(adata, use_raw=use_raw, layer=layer) if isspmatrix_coo(X): X = csr_matrix(X) # COO not subscriptable if issparse(X): X.eliminate_zeros() var_metrics = pd.DataFrame(index=adata.var_names) - if issparse(X): - # Current memory bottleneck for csr matrices: - var_metrics["n_cells_by_{expr_type}"] = X.getnnz(axis=0) - var_metrics["mean_{expr_type}"] = mean_variance_axis(X, axis=0)[0] - else: - var_metrics["n_cells_by_{expr_type}"] = np.count_nonzero(X, axis=0) - var_metrics["mean_{expr_type}"] = X.mean(axis=0) + var_metrics[f"n_cells_by_{expr_type}"], var_metrics[f"mean_{expr_type}"] = ( + materialize_as_ndarray((axis_nnz(X, axis=0), _get_mean_var(X, axis=0)[0])) + ) if log1p: - var_metrics["log1p_mean_{expr_type}"] = np.log1p( - var_metrics["mean_{expr_type}"] + var_metrics[f"log1p_mean_{expr_type}"] = np.log1p( + var_metrics[f"mean_{expr_type}"] ) - var_metrics["pct_dropout_by_{expr_type}"] = ( - 1 - var_metrics["n_cells_by_{expr_type}"] / X.shape[0] + var_metrics[f"pct_dropout_by_{expr_type}"] = ( + 1 - var_metrics[f"n_cells_by_{expr_type}"] / X.shape[0] ) * 100 - var_metrics["total_{expr_type}"] = np.ravel(X.sum(axis=0)) + var_metrics[f"total_{expr_type}"] = np.ravel(axis_sum(X, axis=0)) if log1p: - var_metrics["log1p_total_{expr_type}"] = np.log1p( - var_metrics["total_{expr_type}"] + var_metrics[f"log1p_total_{expr_type}"] = np.log1p( + var_metrics[f"total_{expr_type}"] ) - # Relabel - new_colnames = [] - for col in var_metrics.columns: - new_colnames.append(col.format(**locals())) - var_metrics.columns = new_colnames if inplace: adata.var[var_metrics.columns] = var_metrics - else: - return var_metrics + return None + return var_metrics @_doc_params( @@ -303,7 +298,7 @@ def calculate_qc_metrics( FutureWarning, ) # Pass X so I only have to do it once - X = _choose_mtx_rep(adata, use_raw, layer) + X = _choose_mtx_rep(adata, use_raw=use_raw, layer=layer) if isspmatrix_coo(X): X = csr_matrix(X) # COO not subscriptable if issparse(X): @@ -387,9 +382,19 @@ def top_proportions_sparse_csr(data, indptr, n): return values -def top_segment_proportions( - mtx: np.ndarray | spmatrix, ns: Collection[int] -) -> np.ndarray: +def check_ns(func): + def check_ns_inner(mtx: np.ndarray | spmatrix | DaskArray, ns: Collection[int]): + if not (max(ns) <= mtx.shape[1] and min(ns) > 0): + msg = "Positions outside range of features." + raise IndexError(msg) + return func(mtx, ns) + + return check_ns_inner + + +@singledispatch +@check_ns +def top_segment_proportions(mtx: np.ndarray, ns: Collection[int]) -> np.ndarray: """ Calculates total percentage of counts in top ns genes. @@ -402,20 +407,6 @@ def top_segment_proportions( 1-indexed, e.g. `ns=[50]` will calculate cumulative proportion up to the 50th most expressed gene. """ - # Pretty much just does dispatch - if not (max(ns) <= mtx.shape[1] and min(ns) > 0): - raise IndexError("Positions outside range of features.") - if issparse(mtx): - if not isspmatrix_csr(mtx): - mtx = csr_matrix(mtx) - return top_segment_proportions_sparse_csr(mtx.data, mtx.indptr, np.array(ns)) - else: - return top_segment_proportions_dense(mtx, ns) - - -def top_segment_proportions_dense( - mtx: np.ndarray | spmatrix, ns: Collection[int] -) -> np.ndarray: # Currently ns is considered to be 1 indexed ns = np.sort(ns) sums = mtx.sum(axis=1) @@ -432,7 +423,26 @@ def top_segment_proportions_dense( return values / sums[:, None] -@numba.njit(cache=True, parallel=True) +@top_segment_proportions.register(DaskArray) +@check_ns +def _(mtx: DaskArray, ns: Collection[int]) -> DaskArray: + if not isinstance(mtx._meta, csr_matrix | np.ndarray): + msg = f"DaskArray must have csr matrix or ndarray meta, got {mtx._meta}." + raise ValueError(msg) + return mtx.map_blocks( + lambda x: top_segment_proportions(x, ns), meta=np.array([]) + ).compute() + + +@top_segment_proportions.register(spmatrix) +@check_ns +def _(mtx: spmatrix, ns: Collection[int]) -> DaskArray: + if not isspmatrix_csr(mtx): + mtx = csr_matrix(mtx) + return top_segment_proportions_sparse_csr(mtx.data, mtx.indptr, np.array(ns)) + + +@njit def top_segment_proportions_sparse_csr(data, indptr, ns): # work around https://github.com/numba/numba/issues/5056 indptr = indptr.astype(np.int64) diff --git a/src/scanpy/preprocessing/_recipes.py b/src/scanpy/preprocessing/_recipes.py index 4579739939..4748d75e5c 100644 --- a/src/scanpy/preprocessing/_recipes.py +++ b/src/scanpy/preprocessing/_recipes.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: from anndata import AnnData - from .._utils import AnyRandom + from .._compat import _LegacyRandom @old_positionals( @@ -36,7 +36,7 @@ def recipe_weinreb17( cv_threshold: int = 2, n_pcs: int = 50, svd_solver="randomized", - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, copy: bool = False, ) -> AnnData | None: """\ @@ -59,7 +59,8 @@ def recipe_weinreb17( from ._deprecated import normalize_per_cell_weinreb16_deprecated, zscore_deprecated if issparse(adata.X): - raise ValueError("`recipe_weinreb16 does not support sparse matrices.") + msg = "`recipe_weinreb16 does not support sparse matrices." + raise ValueError(msg) if copy: adata = adata.copy() if log: diff --git a/src/scanpy/preprocessing/_scale.py b/src/scanpy/preprocessing/_scale.py index f6f4b4e586..ee15f977b9 100644 --- a/src/scanpy/preprocessing/_scale.py +++ b/src/scanpy/preprocessing/_scale.py @@ -11,7 +11,7 @@ from scipy.sparse import issparse, isspmatrix_csc, spmatrix from .. import logging as logg -from .._compat import DaskArray, old_positionals +from .._compat import DaskArray, njit, old_positionals from .._utils import ( _check_array_function_arguments, axis_mul_or_truediv, @@ -30,9 +30,12 @@ if TYPE_CHECKING: from numpy.typing import NDArray + from scipy import sparse as sp + CSMatrix = sp.csr_matrix | sp.csc_matrix -@numba.njit(cache=True, parallel=True) + +@njit def _scale_sparse_numba(indptr, indices, data, *, std, mask_obs, clip): for i in numba.prange(len(indptr) - 1): if mask_obs[i]: @@ -43,8 +46,10 @@ def _scale_sparse_numba(indptr, indices, data, *, std, mask_obs, clip): data[j] /= std[indices[j]] -@numba.njit(parallel=True, cache=True) -def clip_array(X: np.ndarray, max_value: float | None = 10, zero_center: bool = True): +@njit +def clip_array( + X: NDArray[np.floating], *, max_value: float, zero_center: bool +) -> NDArray[np.floating]: a_min, a_max = -max_value, max_value if X.ndim > 1: for r, c in numba.pndindex(X.shape): @@ -61,6 +66,14 @@ def clip_array(X: np.ndarray, max_value: float | None = 10, zero_center: bool = return X +def clip_set(x: CSMatrix, *, max_value: float, zero_center: bool = True) -> CSMatrix: + x = x.copy() + x[x > max_value] = max_value + if zero_center: + x[x < -max_value] = -max_value + return x + + @renamed_arg("X", "data", pos_0=True) @old_positionals("zero_center", "max_value", "copy", "layer", "obsm") @singledispatch @@ -120,13 +133,11 @@ def scale( """ _check_array_function_arguments(layer=layer, obsm=obsm) if layer is not None: - raise ValueError( - f"`layer` argument inappropriate for value of type {type(data)}" - ) + msg = f"`layer` argument inappropriate for value of type {type(data)}" + raise ValueError(msg) if obsm is not None: - raise ValueError( - f"`obsm` argument inappropriate for value of type {type(data)}" - ) + msg = f"`obsm` argument inappropriate for value of type {type(data)}" + raise ValueError(msg) return scale_array( data, zero_center=zero_center, max_value=max_value, copy=copy, mask_obs=mask_obs ) @@ -148,12 +159,11 @@ def scale_array( | tuple[ np.ndarray | DaskArray, NDArray[np.float64] | DaskArray, NDArray[np.float64] ] - | DaskArray ): if copy: X = X.copy() + mask_obs = _check_mask(X, mask_obs, "obs") if mask_obs is not None: - mask_obs = _check_mask(X, mask_obs, "obs") scale_rv = scale_array( X[mask_obs, :], zero_center=zero_center, @@ -172,7 +182,7 @@ def scale_array( if not zero_center and max_value is not None: logg.info( # Be careful of what? This should be more specific - "... be careful when using `max_value` " "without `zero_center`." + "... be careful when using `max_value` without `zero_center`." ) if np.issubdtype(X.dtype, np.integer): @@ -188,7 +198,8 @@ def scale_array( if zero_center: if isinstance(X, DaskArray) and issparse(X._meta): warnings.warn( - "zero-center being used with `DaskArray` sparse chunks. This can be bad if you have large chunks or intend to eventually read the whole data into memory.", + "zero-center being used with `DaskArray` sparse chunks. " + "This can be bad if you have large chunks or intend to eventually read the whole data into memory.", UserWarning, ) X -= mean @@ -204,23 +215,13 @@ def scale_array( # do the clipping if max_value is not None: logg.debug(f"... clipping at max_value {max_value}") - if isinstance(X, DaskArray) and issparse(X._meta): - - def clip_set(x): - x = x.copy() - x[x > max_value] = max_value - if zero_center: - x[x < -max_value] = -max_value - return x - - X = da.map_blocks(clip_set, X) + if isinstance(X, DaskArray): + clip = clip_set if issparse(X._meta) else clip_array + X = X.map_blocks(clip, max_value=max_value, zero_center=zero_center) + elif issparse(X): + X.data = clip_array(X.data, max_value=max_value, zero_center=False) else: - if isinstance(X, DaskArray): - X = X.map_blocks(clip_array, max_value, zero_center) - elif issparse(X): - X.data = clip_array(X.data, max_value=max_value, zero_center=False) - else: - X = clip_array(X, max_value=max_value, zero_center=zero_center) + X = clip_array(X, max_value=max_value, zero_center=zero_center) if return_mean_std: return X, mean, std else: diff --git a/src/scanpy/preprocessing/_scrublet/__init__.py b/src/scanpy/preprocessing/_scrublet/__init__.py index 976dafe89f..68b7f59526 100644 --- a/src/scanpy/preprocessing/_scrublet/__init__.py +++ b/src/scanpy/preprocessing/_scrublet/__init__.py @@ -15,7 +15,7 @@ from .core import Scrublet if TYPE_CHECKING: - from ..._utils import AnyRandom + from ..._compat import _LegacyRandom from ...neighbors import _Metric, _MetricFn @@ -58,7 +58,7 @@ def scrublet( threshold: float | None = None, verbose: bool = True, copy: bool = False, - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, ) -> AnnData | None: """\ Predict doublets using Scrublet :cite:p:`Wolock2019`. @@ -245,7 +245,7 @@ def _run_scrublet(ad_obs: AnnData, ad_sim: AnnData | None = None): return {"obs": ad_obs.obs, "uns": ad_obs.uns["scrublet"]} if batch_key is not None: - if batch_key not in adata.obs.keys(): + if batch_key not in adata.obs.columns: msg = ( "`batch_key` must be a column of .obs in the input AnnData object," f"but {batch_key!r} is not in {adata.obs.keys()!r}." @@ -309,7 +309,7 @@ def _scrublet_call_doublets( knn_dist_metric: _Metric | _MetricFn = "euclidean", get_doublet_neighbor_parents: bool = False, threshold: float | None = None, - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, verbose: bool = True, ) -> AnnData: """\ @@ -503,7 +503,7 @@ def scrublet_simulate_doublets( layer: str | None = None, sim_doublet_ratio: float = 2.0, synthetic_doublet_umi_subsampling: float = 1.0, - random_seed: AnyRandom = 0, + random_seed: _LegacyRandom = 0, ) -> AnnData: """\ Simulate doublets by adding the counts of random observed transcriptome pairs. diff --git a/src/scanpy/preprocessing/_scrublet/core.py b/src/scanpy/preprocessing/_scrublet/core.py index b608443de6..1236f42a7a 100644 --- a/src/scanpy/preprocessing/_scrublet/core.py +++ b/src/scanpy/preprocessing/_scrublet/core.py @@ -1,6 +1,5 @@ from __future__ import annotations -import sys from dataclasses import InitVar, dataclass, field from typing import TYPE_CHECKING, cast @@ -10,7 +9,7 @@ from scipy import sparse from ... import logging as logg -from ..._utils import get_random_state +from ..._utils import _get_legacy_random from ...neighbors import ( Neighbors, _get_indices_distances_from_sparse_matrix, @@ -22,19 +21,13 @@ from numpy.random import RandomState from numpy.typing import NDArray - from ..._utils import AnyRandom + from ..._compat import _LegacyRandom from ...neighbors import _Metric, _MetricFn __all__ = ["Scrublet"] -if sys.version_info > (3, 10): - kw_only = lambda yes: {"kw_only": yes} # noqa: E731 -else: - kw_only = lambda _: {} # noqa: E731 - - -@dataclass(**kw_only(True)) +@dataclass(kw_only=True) class Scrublet: """\ Initialize Scrublet object with counts matrix and doublet prediction parameters @@ -73,14 +66,14 @@ class Scrublet: # init fields counts_obs: InitVar[sparse.csr_matrix | sparse.csc_matrix | NDArray[np.integer]] = ( - field(**kw_only(False)) + field(kw_only=False) ) total_counts_obs: InitVar[NDArray[np.integer] | None] = None sim_doublet_ratio: float = 2.0 n_neighbors: InitVar[int | None] = None expected_doublet_rate: float = 0.1 stdev_doublet_rate: float = 0.02 - random_state: InitVar[AnyRandom] = 0 + random_state: InitVar[_LegacyRandom] = 0 # private fields @@ -181,7 +174,7 @@ def __post_init__( counts_obs: sparse.csr_matrix | sparse.csc_matrix | NDArray[np.integer], total_counts_obs: NDArray[np.integer] | None, n_neighbors: int | None, - random_state: AnyRandom, + random_state: _LegacyRandom, ) -> None: self._counts_obs = sparse.csc_matrix(counts_obs) self._total_counts_obs = ( @@ -194,7 +187,7 @@ def __post_init__( if n_neighbors is None else n_neighbors ) - self._random_state = get_random_state(random_state) + self._random_state = _get_legacy_random(random_state) def simulate_doublets( self, @@ -278,6 +271,7 @@ def set_manifold( def calculate_doublet_scores( self, + *, use_approx_neighbors: bool | None = None, distance_metric: _Metric | _MetricFn = "euclidean", get_doublet_neighbor_parents: bool = False, diff --git a/src/scanpy/preprocessing/_scrublet/pipeline.py b/src/scanpy/preprocessing/_scrublet/pipeline.py index 5f6c62838c..6e52a6650c 100644 --- a/src/scanpy/preprocessing/_scrublet/pipeline.py +++ b/src/scanpy/preprocessing/_scrublet/pipeline.py @@ -12,7 +12,7 @@ if TYPE_CHECKING: from typing import Literal - from ..._utils import AnyRandom + from ..._compat import _LegacyRandom from .core import Scrublet @@ -49,11 +49,12 @@ def truncated_svd( self: Scrublet, n_prin_comps: int = 30, *, - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, algorithm: Literal["arpack", "randomized"] = "arpack", ) -> None: if self._counts_sim_norm is None: - raise RuntimeError("_counts_sim_norm is not set") + msg = "_counts_sim_norm is not set" + raise RuntimeError(msg) from sklearn.decomposition import TruncatedSVD svd = TruncatedSVD( @@ -68,11 +69,12 @@ def pca( self: Scrublet, n_prin_comps: int = 50, *, - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, svd_solver: Literal["auto", "full", "arpack", "randomized"] = "arpack", ) -> None: if self._counts_sim_norm is None: - raise RuntimeError("_counts_sim_norm is not set") + msg = "_counts_sim_norm is not set" + raise RuntimeError(msg) from sklearn.decomposition import PCA X_obs = self._counts_obs_norm.toarray() diff --git a/src/scanpy/preprocessing/_scrublet/sparse_utils.py b/src/scanpy/preprocessing/_scrublet/sparse_utils.py index b4ff1a36b0..795559583c 100644 --- a/src/scanpy/preprocessing/_scrublet/sparse_utils.py +++ b/src/scanpy/preprocessing/_scrublet/sparse_utils.py @@ -7,17 +7,17 @@ from scanpy.preprocessing._utils import _get_mean_var -from ..._utils import get_random_state +from ..._utils import _get_legacy_random if TYPE_CHECKING: from numpy.typing import NDArray - from ..._utils import AnyRandom + from .._compat import _LegacyRandom def sparse_multiply( E: sparse.csr_matrix | sparse.csc_matrix | NDArray[np.float64], - a: float | int | NDArray[np.float64], + a: float | NDArray[np.float64], ) -> sparse.csr_matrix | sparse.csc_matrix: """multiply each row of E by a scalar""" @@ -47,10 +47,10 @@ def subsample_counts( *, rate: float, original_totals, - random_seed: AnyRandom = 0, + random_seed: _LegacyRandom = 0, ) -> tuple[sparse.csr_matrix | sparse.csc_matrix, NDArray[np.int64]]: if rate < 1: - random_seed = get_random_state(random_seed) + random_seed = _get_legacy_random(random_seed) E.data = random_seed.binomial(np.round(E.data).astype(int), rate) current_totals = np.asarray(E.sum(1)).squeeze() unsampled_orig_totals = original_totals - current_totals diff --git a/src/scanpy/preprocessing/_simple.py b/src/scanpy/preprocessing/_simple.py index 1e61b60ace..68a140d584 100644 --- a/src/scanpy/preprocessing/_simple.py +++ b/src/scanpy/preprocessing/_simple.py @@ -7,21 +7,23 @@ import warnings from functools import singledispatch -from typing import TYPE_CHECKING +from itertools import repeat +from typing import TYPE_CHECKING, TypeVar, overload import numba import numpy as np import scipy as sp from anndata import AnnData from pandas.api.types import CategoricalDtype -from scipy.sparse import csr_matrix, issparse, isspmatrix_csr, spmatrix +from scipy.sparse import csc_matrix, csr_matrix, issparse, isspmatrix_csr, spmatrix from sklearn.utils import check_array, sparsefuncs from .. import logging as logg -from .._compat import old_positionals +from .._compat import DaskArray, deprecated, njit, old_positionals from .._settings import settings as sett from .._utils import ( _check_array_function_arguments, + _resolve_axis, axis_sum, is_backed_type, raise_not_implemented_error_if_backed_type, @@ -29,27 +31,86 @@ sanitize_anndata, view_to_actual, ) -from ..get import _get_obs_rep, _set_obs_rep +from ..get import _check_mask, _get_obs_rep, _set_obs_rep from ._distributed import materialize_as_ndarray +from ._utils import _to_dense -# install dask if available try: import dask.array as da except ImportError: da = None -# backwards compat -from ._deprecated.highly_variable_genes import filter_genes_dispersion # noqa: F401 - if TYPE_CHECKING: from collections.abc import Collection, Iterable, Sequence from numbers import Number from typing import Literal + import pandas as pd from numpy.typing import NDArray - from .._compat import DaskArray - from .._utils import AnyRandom + from .._compat import _LegacyRandom + from .._utils import RNGLike, SeedLike + + +CSMatrix = csr_matrix | csc_matrix + +A = TypeVar("A", bound=np.ndarray | CSMatrix | DaskArray) + +def get_rows_to_keep_1(indptr, data): + lens = np.zeros(len(indptr) - 1, dtype=type(data[0])) + for i in range(len(lens)): + lens[i] = np.sum(data[indptr[i] : indptr[i + 1]]) + return lens + + +def get_rows_to_keep(indptr, dtype): + lens = indptr[1:] - indptr[:-1] + return lens + + +@njit() +def get_cols_to_keep(indices, data, colcount, nthr, Flag): + counts = np.zeros((nthr, colcount), dtype=type(data[0])) + for i in numba.prange(nthr): + start = i * indices.shape[0] // nthr + end = (i + 1) * indices.shape[0] // nthr + for j in range(start, end): + if data[j] != 0: # and indices[j]>> sc.pp.filter_cells(adata, min_genes=0) >>> adata.n_obs 640 - >>> adata.obs['n_genes'].min() + >>> int(adata.obs['n_genes'].min()) 1 >>> # filter manually >>> adata_copy = adata[adata.obs['n_genes'] >= 3] >>> adata_copy.n_obs 554 - >>> adata_copy.obs['n_genes'].min() + >>> int(adata_copy.obs['n_genes'].min()) 3 >>> # actually do some filtering >>> sc.pp.filter_cells(adata, min_genes=3) >>> adata.n_obs 554 - >>> adata.obs['n_genes'].min() + >>> int(adata.obs['n_genes'].min()) 3 """ if copy: @@ -142,10 +203,11 @@ def filter_cells( option is not None for option in [min_genes, min_counts, max_genes, max_counts] ) if n_given_options != 1: - raise ValueError( + msg = ( "Only provide one of the optional parameters `min_counts`, " "`min_genes`, `max_counts`, `max_genes` per call." ) + raise ValueError(msg) if isinstance(data, AnnData): raise_not_implemented_error_if_backed_type(data.X, "filter_cells") adata = data.copy() if copy else data @@ -169,11 +231,30 @@ def filter_cells( X = data # proceed with processing the data matrix min_number = min_counts if min_genes is None else min_genes max_number = max_counts if max_genes is None else max_genes - number_per_cell = axis_sum( - X if min_genes is None and max_genes is None else X > 0, axis=1 - ) + import time + + t0 = time.time() + if isinstance(X, sp.sparse._csr.csr_matrix): + if min_genes is None and max_genes is None: + number_per_cell = get_rows_to_keep_1(X.indptr, X.data) + else: + number_per_cell = get_rows_to_keep(X.indptr, type(X.data[0])) + + elif isinstance(X, sp.sparse._csc.csc_matrix): + nclos = X.shape[0] + nthr = numba.get_num_threads() + if min_genes is None and max_genes is None: + number_per_cell = get_cols_to_keep(X.indices, X.data, nclos, nthr, False) + else: + number_per_cell = get_cols_to_keep(X.indices, X.data, nclos, nthr, True) + + else: + number_per_cell = axis_sum( + X if min_genes is None and max_genes is None else X > 0, axis=1 + ) + if issparse(X): - number_per_cell = number_per_cell.A1 + number_per_cell = number_per_cell if min_number is not None: cell_subset = number_per_cell >= min_number if max_number is not None: @@ -257,10 +338,11 @@ def filter_genes( option is not None for option in [min_cells, min_counts, max_cells, max_counts] ) if n_given_options != 1: - raise ValueError( + msg = ( "Only provide one of the optional parameters `min_counts`, " "`min_cells`, `max_counts`, `max_cells` per call." ) + raise ValueError(msg) if isinstance(data, AnnData): raise_not_implemented_error_if_backed_type(data.X, "filter_genes") @@ -286,11 +368,36 @@ def filter_genes( X = data # proceed with processing the data matrix min_number = min_counts if min_cells is None else min_cells max_number = max_counts if max_cells is None else max_cells - number_per_gene = axis_sum( - X if min_cells is None and max_cells is None else X > 0, axis=0 - ) + import time + + t0 = time.time() + + if isinstance(X, sp.sparse._csr.csr_matrix): + ncols = X.shape[1] + nthr = numba.get_num_threads() + if min_cells is None and max_cells is None: + number_per_gene = get_cols_to_keep(X.indices, X.data, ncols, nthr, False) + else: + number_per_gene = get_cols_to_keep(X.indices, X.data, ncols, nthr, True) + + elif isinstance(X, sp.sparse._csc.csc_matrix): + if min_cells is None and max_cells is None: + number_per_gene = get_rows_to_keep_1( + X.indptr, X.data + ) + else: + number_per_gene = get_rows_to_keep(X.indptr, type(X.data[0])) + else: + number_per_gene = axis_sum( + X if min_cells is None and max_cells is None else X > 0, axis=0 + ) if issparse(X): - number_per_gene = number_per_gene.A1 + if isinstance(X, sp.sparse._csr.csr_matrix) or isinstance( + X, sp.sparse._csc.csc_matrix + ): + number_per_gene = number_per_gene + else: + number_per_gene = number_per_gene if min_number is not None: gene_subset = number_per_gene >= min_number if max_number is not None: @@ -373,12 +480,8 @@ def log1p_sparse(X: spmatrix, *, base: Number | None = None, copy: bool = False) @log1p.register(np.ndarray) def log1p_array(X: np.ndarray, *, base: Number | None = None, copy: bool = False): # Can force arrays to be np.ndarrays, but would be useful to not - # X = check_array(X, dtype=(np.float64, np.float32), ensure_2d=False, copy=copy) if copy: - if not np.issubdtype(X.dtype, np.floating): - X = X.astype(float) - else: - X = X.copy() + X = X.astype(float) if not np.issubdtype(X.dtype, np.floating) else X.copy() elif not (np.issubdtype(X.dtype, np.floating) or np.issubdtype(X.dtype, complex)): X = X.astype(float) np.log1p(X, out=X) @@ -406,13 +509,13 @@ def log1p_anndata( if chunked: if (layer is not None) or (obsm is not None): - raise NotImplementedError( + msg = ( "Currently cannot perform chunked operations on arrays not stored in X." ) + raise NotImplementedError(msg) if adata.isbacked and adata.file._filemode != "r+": - raise NotImplementedError( - "log1p is not implemented for backed AnnData with backed mode not r+" - ) + msg = "log1p is not implemented for backed AnnData with backed mode not r+" + raise NotImplementedError(msg) for chunk, start, end in adata.chunked_X(chunk_size): adata.X[start:end] = log1p(chunk, base=base, copy=False) else: @@ -420,8 +523,10 @@ def log1p_anndata( if is_backed_type(X): msg = f"log1p is not implemented for matrices of type {type(X)}" if layer is not None: - raise NotImplementedError(f"{msg} from layers") - raise NotImplementedError(f"{msg} without `chunked=True`") + msg = f"{msg} from layers" + raise NotImplementedError(msg) + msg = f"{msg} without `chunked=True`" + raise NotImplementedError(msg) X = log1p(X, copy=False, base=base) _set_obs_rep(adata, X, layer=layer, obsm=obsm) @@ -476,8 +581,19 @@ def sqrt( return X.sqrt() -def normalize_per_cell( # noqa: PLR0917 +@deprecated("Use sc.pp.normalize_total instead") +@old_positionals( + "counts_per_cell_after", + "counts_per_cell", + "key_n_counts", + "copy", + "layers", + "use_rep", + "min_counts", +) +def normalize_per_cell( data: AnnData | np.ndarray | spmatrix, + *, counts_per_cell_after: float | None = None, counts_per_cell: np.ndarray | None = None, key_n_counts: str = "n_counts", @@ -489,16 +605,16 @@ def normalize_per_cell( # noqa: PLR0917 """\ Normalize total counts per cell. - .. warning:: - .. deprecated:: 1.3.7 - Use :func:`~scanpy.pp.normalize_total` instead. - The new function is equivalent to the present - function, except that + .. deprecated:: 1.3.7 - * the new function doesn't filter cells based on `min_counts`, - use :func:`~scanpy.pp.filter_cells` if filtering is needed. - * some arguments were renamed - * `copy` is replaced by `inplace` + Use :func:`~scanpy.pp.normalize_total` instead. + The new function is equivalent to the present + function, except that + + * the new function doesn't filter cells based on `min_counts`, + use :func:`~scanpy.pp.filter_cells` if filtering is needed. + * some arguments were renamed + * `copy` is replaced by `inplace` Normalize each cell by total counts over all genes, so that every cell has the same total count after normalization. @@ -569,7 +685,11 @@ def normalize_per_cell( # noqa: PLR0917 adata.obs[key_n_counts] = counts_per_cell adata._inplace_subset_obs(cell_subset) counts_per_cell = counts_per_cell[cell_subset] - normalize_per_cell(adata.X, counts_per_cell_after, counts_per_cell) + normalize_per_cell( + adata.X, + counts_per_cell_after=counts_per_cell_after, + counts_per_cell=counts_per_cell, + ) layers = adata.layers.keys() if layers == "all" else layers if use_rep == "after": @@ -579,14 +699,15 @@ def normalize_per_cell( # noqa: PLR0917 elif use_rep is None: after = None else: - raise ValueError('use_rep should be "after", "X" or None') + msg = 'use_rep should be "after", "X" or None' + raise ValueError(msg) for layer in layers: _subset, counts = filter_cells(adata.layers[layer], min_counts=min_counts) temp = normalize_per_cell(adata.layers[layer], after, counts, copy=True) adata.layers[layer] = temp logg.info( - " finished ({time_passed}): normalized adata.X and added" + " finished ({time_passed}): normalized adata.X and added\n" f" {key_n_counts!r}, counts per cell before normalization (adata.obs)", time=start, ) @@ -595,7 +716,8 @@ def normalize_per_cell( # noqa: PLR0917 X = data.copy() if copy else data if counts_per_cell is None: if not copy: - raise ValueError("Can only be run with copy=True") + msg = "Can only be run with copy=True" + raise ValueError(msg) cell_subset, counts_per_cell = filter_cells(X, min_counts=min_counts) X = X[cell_subset] counts_per_cell = counts_per_cell[cell_subset] @@ -612,6 +734,34 @@ def normalize_per_cell( # noqa: PLR0917 return X if copy else None +DT = TypeVar("DT") + + +@njit +def get_resid( + data: np.ndarray, + regressor: np.ndarray, + coeff: np.ndarray, +) -> np.ndarray: + for i in numba.prange(data.shape[0]): + data[i] -= regressor[i] @ coeff + return data + + +def numpy_regress_out( + data: np.ndarray, + regressor: np.ndarray, +) -> np.ndarray: + """\ + Numba kernel for regress out unwanted sorces of variantion. + Finding coefficient using Linear regression (Linear Least Squares). + """ + inv_gram_matrix = np.linalg.inv(regressor.T @ regressor) + coeff = inv_gram_matrix @ (regressor.T @ data) + data = get_resid(data, regressor, coeff) + return data + + @old_positionals("layer", "n_jobs", "copy") def regress_out( adata: AnnData, @@ -649,6 +799,8 @@ def regress_out( `adata.X` | `adata.layers[layer]` : :class:`numpy.ndarray` | :class:`scipy.sparse._csr.csr_matrix` (dtype `float`) Corrected count data matrix. """ + from joblib import Parallel, delayed + start = logg.info(f"regressing out {keys}") adata = adata.copy() if copy else adata @@ -663,8 +815,7 @@ def regress_out( raise_not_implemented_error_if_backed_type(X, "regress_out") if issparse(X): - logg.info(" sparse input is densified and may " "lead to high memory use") - X = X.toarray() + logg.info(" sparse input is densified and may lead to high memory use") n_jobs = sett.n_jobs if n_jobs is None else n_jobs @@ -674,13 +825,16 @@ def regress_out( adata.obs[keys[0]].dtype, CategoricalDtype ): if len(keys) > 1: - raise ValueError( + msg = ( "If providing categorical variable, " "only a single one is allowed. For this one " "we regress on the mean for each category." ) + raise ValueError(msg) logg.debug("... regressing on per-gene means within categories") regressors = np.zeros(X.shape, dtype="float32") + X = _to_dense(X, order="F") if issparse(X) else X + # TODO figure out if we should use a numba kernel for this for category in adata.obs[keys[0]].cat.categories: mask = (category == adata.obs[keys[0]]).values for ix, x in enumerate(X.T): @@ -689,55 +843,68 @@ def regress_out( # regress on one or several ordinal variables else: # create data frame with selected keys (if given) - if keys: - regressors = adata.obs[keys] - else: - regressors = adata.obs.copy() + regressors = adata.obs[keys] if keys else adata.obs.copy() # add column of ones at index 0 (first column) regressors.insert(0, "ones", 1.0) + regressors = regressors.to_numpy() - len_chunk = np.ceil(min(1000, X.shape[1]) / n_jobs).astype(int) - n_chunks = np.ceil(X.shape[1] / len_chunk).astype(int) - - tasks = [] - # split the adata.X matrix by columns in chunks of size n_chunk - # (the last chunk could be of smaller size than the others) - chunk_list = np.array_split(X, n_chunks, axis=1) - if variable_is_categorical: - regressors_chunk = np.array_split(regressors, n_chunks, axis=1) - for idx, data_chunk in enumerate(chunk_list): - # each task is a tuple of a data_chunk eg. (adata.X[:,0:100]) and - # the regressors. This data will be passed to each of the jobs. - if variable_is_categorical: - regres = regressors_chunk[idx] - else: - regres = regressors - tasks.append(tuple((data_chunk, regres, variable_is_categorical))) + # if the regressors are not categorical and the matrix is not singular + # use the shortcut numpy_regress_out + if not variable_is_categorical and np.linalg.det(regressors.T @ regressors) != 0: + X = _to_dense(X, order="C") if issparse(X) else X + res = numpy_regress_out(X, regressors) - from joblib import Parallel, delayed + # for a categorical variable or if the above checks failed, + # we fall back to the GLM implemetation of regression. + else: + # split the adata.X matrix by columns in chunks of size n_chunk + # (the last chunk could be of smaller size than the others) + len_chunk = int(np.ceil(min(1000, X.shape[1]) / n_jobs)) + n_chunks = int(np.ceil(X.shape[1] / len_chunk)) + X = _to_dense(X, order="F") if issparse(X) else X + chunk_list = np.array_split(X, n_chunks, axis=1) + regressors_chunk = ( + np.array_split(regressors, n_chunks, axis=1) + if variable_is_categorical + else repeat(regressors) + ) - # TODO: figure out how to test that this doesn't oversubscribe resources - res = Parallel(n_jobs=n_jobs)(delayed(_regress_out_chunk)(task) for task in tasks) + # each task is passed a data chunk (e.g. `adata.X[:, 0:100]``) and the regressors. + # This data will be passed to each of the jobs. + # TODO: figure out how to test that this doesn't oversubscribe resources + res = Parallel(n_jobs=n_jobs)( + delayed(_regress_out_chunk)( + data_chunk, regres, variable_is_categorical=variable_is_categorical + ) + for data_chunk, regres in zip(chunk_list, regressors_chunk, strict=False) + ) + + # res is a list of vectors (each corresponding to a regressed gene column). + # The transpose is needed to get the matrix in the shape needed + res = np.vstack(res).T - # res is a list of vectors (each corresponding to a regressed gene column). - # The transpose is needed to get the matrix in the shape needed - _set_obs_rep(adata, np.vstack(res).T, layer=layer) + _set_obs_rep(adata, res, layer=layer) logg.info(" finished", time=start) return adata if copy else None -def _regress_out_chunk(data): - # data is a tuple containing the selected columns from adata.X - # and the regressors dataFrame - data_chunk = data[0] - regressors = data[1] - variable_is_categorical = data[2] - - responses_chunk_list = [] +def _regress_out_chunk( + data_chunk: NDArray[np.floating], + regressors: pd.DataFrame | NDArray[np.floating], + *, + variable_is_categorical: bool, +) -> NDArray[np.floating]: import statsmodels.api as sm - from statsmodels.tools.sm_exceptions import PerfectSeparationError + import statsmodels.tools.sm_exceptions as sme + Psw = ( + sme.PerfectSeparationWarning + if hasattr(sme, "PerfectSeparationWarning") + else None + ) + + responses_chunk_list = [] for col_index in range(data_chunk.shape[1]): # if all values are identical, the statsmodel.api.GLM throws an error; # but then no regression is necessary anyways... @@ -749,13 +916,18 @@ def _regress_out_chunk(data): regres = np.c_[np.ones(regressors.shape[0]), regressors[:, col_index]] else: regres = regressors + try: - result = sm.GLM( - data_chunk[:, col_index], regres, family=sm.families.Gaussian() - ).fit() - new_column = result.resid_response - except PerfectSeparationError: # this emulates R's behavior - logg.warning("Encountered PerfectSeparationError, setting to 0 as in R.") + with warnings.catch_warnings(): + # See issue #3260 - for statsmodels>=0.14.0 + if Psw: + warnings.simplefilter("error", Psw) + result = sm.GLM( + data_chunk[:, col_index], regres, family=sm.families.Gaussian() + ).fit() + new_column = result.resid_response + except (sme.PerfectSeparationError, *([Psw] if Psw else [])): + logg.warning("Encountered perfect separation, setting to 0 as in R.") new_column = np.zeros(data_chunk.shape[0]) responses_chunk_list.append(new_column) @@ -763,17 +935,55 @@ def _regress_out_chunk(data): return np.vstack(responses_chunk_list) -@old_positionals("n_obs", "random_state", "copy") -def subsample( - data: AnnData | np.ndarray | spmatrix, +@overload +def sample( + data: AnnData, fraction: float | None = None, *, - n_obs: int | None = None, - random_state: AnyRandom = 0, + n: int | None = None, + rng: RNGLike | SeedLike | None = 0, + copy: Literal[False] = False, + replace: bool = False, + axis: Literal["obs", 0, "var", 1] = "obs", + p: str | NDArray[np.bool_] | NDArray[np.floating] | None = None, +) -> None: ... +@overload +def sample( + data: AnnData, + fraction: float | None = None, + *, + n: int | None = None, + rng: RNGLike | SeedLike | None = None, + copy: Literal[True], + replace: bool = False, + axis: Literal["obs", 0, "var", 1] = "obs", + p: str | NDArray[np.bool_] | NDArray[np.floating] | None = None, +) -> AnnData: ... +@overload +def sample( + data: A, + fraction: float | None = None, + *, + n: int | None = None, + rng: RNGLike | SeedLike | None = None, + copy: bool = False, + replace: bool = False, + axis: Literal["obs", 0, "var", 1] = "obs", + p: str | NDArray[np.bool_] | NDArray[np.floating] | None = None, +) -> tuple[A, NDArray[np.int64]]: ... +def sample( + data: AnnData | np.ndarray | CSMatrix | DaskArray, + fraction: float | None = None, + *, + n: int | None = None, + rng: RNGLike | SeedLike | None = None, copy: bool = False, -) -> AnnData | tuple[np.ndarray | spmatrix, NDArray[np.int64]] | None: + replace: bool = False, + axis: Literal["obs", 0, "var", 1] = "obs", + p: str | NDArray[np.bool_] | NDArray[np.floating] | None = None, +) -> AnnData | None | tuple[np.ndarray | CSMatrix | DaskArray, NDArray[np.int64]]: """\ - Subsample to a fraction of the number of observations. + Sample observations or variables with or without replacement. Parameters ---------- @@ -781,49 +991,89 @@ def subsample( The (annotated) data matrix of shape `n_obs` × `n_vars`. Rows correspond to cells and columns to genes. fraction - Subsample to this `fraction` of the number of observations. - n_obs - Subsample to this number of observations. + Sample to this `fraction` of the number of observations or variables. + (All of them, even if there are `0`\\ s/`False`\\ s in `p`.) + This can be larger than 1.0, if `replace=True`. + See `axis` and `replace`. + n + Sample to this number of observations or variables. See `axis`. random_state Random seed to change subsampling. copy If an :class:`~anndata.AnnData` is passed, determines whether a copy is returned. + replace + If True, samples are drawn with replacement. + axis + Sample `obs`\\ ervations (axis 0) or `var`\\ iables (axis 1). + p + Drawing probabilities (floats) or mask (bools). + Either an `axis`-sized array, or the name of a column. + If `p` is an array of probabilities, it must sum to 1. Returns ------- - Returns `X[obs_indices], obs_indices` if data is array-like, otherwise - subsamples the passed :class:`~anndata.AnnData` (`copy == False`) or - returns a subsampled copy of it (`copy == True`). + If `isinstance(data, AnnData)` and `copy=False`, + this function returns `None`. Otherwise: + + `data[indices, :]` | `data[:, indices]` (depending on `axis`) + If `data` is array-like or `copy=True`, returns the subset. + `indices` : numpy.ndarray + If `data` is array-like, also returns the indices into the original. """ - np.random.seed(random_state) - old_n_obs = data.n_obs if isinstance(data, AnnData) else data.shape[0] - if n_obs is not None: - new_n_obs = n_obs - elif fraction is not None: - if fraction > 1 or fraction < 0: - raise ValueError(f"`fraction` needs to be within [0, 1], not {fraction}") - new_n_obs = int(fraction * old_n_obs) - logg.debug(f"... subsampled to {new_n_obs} data points") - else: - raise ValueError("Either pass `n_obs` or `fraction`.") - obs_indices = np.random.choice(old_n_obs, size=new_n_obs, replace=False) - if isinstance(data, AnnData): - if data.isbacked: - if copy: - return data[obs_indices].to_memory() - else: - raise NotImplementedError( - "Inplace subsampling is not implemented for backed objects." - ) + # parameter validation + if not copy and isinstance(data, AnnData) and data.isbacked: + msg = "Inplace sampling (`copy=False`) is not implemented for backed objects." + raise NotImplementedError(msg) + axis, axis_name = _resolve_axis(axis) + p = _check_mask(data, p, dim=axis_name, allow_probabilities=True) + if p is not None and p.dtype == bool: + p = p.astype(np.float64) / p.sum() + old_n = data.shape[axis] + match (fraction, n): + case (None, None): + msg = "Either `fraction` or `n` must be set." + raise TypeError(msg) + case (None, _): + pass + case (_, None): + if fraction < 0: + msg = f"`{fraction=}` needs to be nonnegative." + raise ValueError(msg) + if not replace and fraction > 1: + msg = f"If `replace=False`, `{fraction=}` needs to be within [0, 1]." + raise ValueError(msg) + n = int(fraction * old_n) + logg.debug(f"... sampled to {n} {axis_name}") + case _: + msg = "Providing both `fraction` and `n` is not allowed." + raise TypeError(msg) + del fraction + + # actually do subsampling + rng = np.random.default_rng(rng) + indices = rng.choice(old_n, size=n, replace=replace, p=p) + + # overload 1: inplace AnnData subset + if not copy and isinstance(data, AnnData): + if axis_name == "obs": + data._inplace_subset_obs(indices) else: - if copy: - return data[obs_indices].copy() - else: - data._inplace_subset_obs(obs_indices) - else: - X = data - return X[obs_indices], obs_indices + data._inplace_subset_var(indices) + return None + + subset = data[indices] if axis_name == "obs" else data[:, indices] + + # overload 2: copy AnnData subset + if copy and isinstance(data, AnnData): + assert isinstance(subset, AnnData) + return subset.to_memory() if data.isbacked else subset.copy() + + # overload 3: return array and indices + assert isinstance(subset, np.ndarray | CSMatrix | DaskArray), type(subset) + if copy: + subset = subset.copy() + return subset, indices @renamed_arg("target_counts", "counts_per_cell") @@ -832,7 +1082,7 @@ def downsample_counts( counts_per_cell: int | Collection[int] | None = None, total_counts: int | None = None, *, - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, replace: bool = False, copy: bool = False, ) -> AnnData | None: @@ -874,9 +1124,8 @@ def downsample_counts( total_counts_call = total_counts is not None counts_per_cell_call = counts_per_cell is not None if total_counts_call is counts_per_cell_call: - raise ValueError( - "Must specify exactly one of `total_counts` or `counts_per_cell`." - ) + msg = "Must specify exactly one of `total_counts` or `counts_per_cell`." + raise ValueError(msg) if copy: adata = adata.copy() if total_counts_call: @@ -896,11 +1145,12 @@ def _downsample_per_cell(X, counts_per_cell, random_state, replace): # np.random.choice needs int arguments in numba code: counts_per_cell = counts_per_cell.astype(np.int_, copy=False) if not isinstance(counts_per_cell, np.ndarray) or len(counts_per_cell) != n_obs: - raise ValueError( + msg = ( "If provided, 'counts_per_cell' must be either an integer, or " "coercible to an `np.ndarray` of length as number of observations" " by `np.asarray(counts_per_cell)`." ) + raise ValueError(msg) if issparse(X): original_type = type(X) if not isspmatrix_csr(X): @@ -956,15 +1206,19 @@ def _downsample_total_counts(X, total_counts, random_state, replace): X = original_type(X) else: v = X.reshape(np.multiply(*X.shape)) - _downsample_array(v, total_counts, random_state, replace=replace, inplace=True) + _downsample_array( + v, total_counts, random_state=random_state, replace=replace, inplace=True + ) return X -@numba.njit(cache=True) +# TODO: can/should this be parallelized? +@njit() # noqa: TID251 def _downsample_array( col: np.ndarray, target: int, - random_state: AnyRandom = 0, + *, + random_state: _LegacyRandom = 0, replace: bool = True, inplace: bool = False, ): @@ -1005,7 +1259,6 @@ def _pca_fallback(data, n_comps=2): # calculate eigenvectors & eigenvalues of the covariance matrix # use 'eigh' rather than 'eig' since C is symmetric, # the performance gain is substantial - # evals, evecs = np.linalg.eigh(C) evals, evecs = sp.sparse.linalg.eigsh(C, k=n_comps) # sort eigenvalues in decreasing order idcs = np.argsort(evals)[::-1] diff --git a/src/scanpy/preprocessing/_utils.py b/src/scanpy/preprocessing/_utils.py index 64adb036d9..3ca74734c0 100644 --- a/src/scanpy/preprocessing/_utils.py +++ b/src/scanpy/preprocessing/_utils.py @@ -8,6 +8,7 @@ from scipy import sparse from sklearn.random_projection import sample_without_replacement +from .._compat import njit from .._utils import axis_sum, elem_mul if TYPE_CHECKING: @@ -15,8 +16,8 @@ from numpy.typing import DTypeLike, NDArray - from .._compat import DaskArray - from .._utils import AnyRandom, _SupportedArray + from .._compat import DaskArray, _LegacyRandom + from .._utils import _SupportedArray @singledispatch @@ -40,7 +41,8 @@ def _get_mean_var( mean_sq = axis_mean(elem_mul(X, X), axis=axis, dtype=np.float64) var = mean_sq - mean**2 # enforce R convention (unbiased estimator) for variance - var *= X.shape[axis] / (X.shape[axis] - 1) + if X.shape[axis] != 1: + var *= X.shape[axis] / (X.shape[axis] - 1) return mean, var @@ -62,7 +64,8 @@ def sparse_mean_variance_axis(mtx: sparse.spmatrix, axis: int): ax_minor = 0 shape = mtx.shape[::-1] else: - raise ValueError("This function only works on sparse csr and csc matrices") + msg = "This function only works on sparse csr and csc matrices" + raise ValueError(msg) if axis == ax_minor: return sparse_mean_var_major_axis( mtx.data, @@ -82,7 +85,7 @@ def sparse_mean_variance_axis(mtx: sparse.spmatrix, axis: int): ) -@numba.njit(cache=True, parallel=True) +@njit def sparse_mean_var_minor_axis( data, indices, indptr, *, major_len, minor_len, n_threads ): @@ -115,7 +118,7 @@ def sparse_mean_var_minor_axis( return means, variances -@numba.njit(cache=True, parallel=True) +@njit def sparse_mean_var_major_axis(data, indptr, *, major_len, minor_len, n_threads): """ Computes mean and variance for a sparse array for the major axis. @@ -148,7 +151,7 @@ def sample_comb( dims: tuple[int, ...], nsamp: int, *, - random_state: AnyRandom = None, + random_state: _LegacyRandom = None, method: Literal[ "auto", "tracking_selection", "reservoir_sampling", "pool" ] = "auto", @@ -158,3 +161,46 @@ def sample_comb( np.prod(dims), nsamp, random_state=random_state, method=method ) return np.vstack(np.unravel_index(idx, dims)).T + + +def _to_dense( + X: sparse.spmatrix, + order: Literal["C", "F"] = "C", +) -> NDArray: + """\ + Numba kernel for np.toarray() function + """ + out = np.zeros(X.shape, dtype=X.dtype, order=order) + if X.format == "csr": + _to_dense_csr_numba(X.indptr, X.indices, X.data, out, X.shape) + elif X.format == "csc": + _to_dense_csc_numba(X.indptr, X.indices, X.data, out, X.shape) + else: + out = X.toarray(order=order) + return out + + +@njit +def _to_dense_csc_numba( + indptr: NDArray, + indices: NDArray, + data: NDArray, + X: NDArray, + shape: tuple[int, int], +) -> None: + for c in numba.prange(X.shape[1]): + for i in range(indptr[c], indptr[c + 1]): + X[indices[i], c] = data[i] + + +@njit +def _to_dense_csr_numba( + indptr: NDArray, + indices: NDArray, + data: NDArray, + X: NDArray, + shape: tuple[int, int], +) -> None: + for r in numba.prange(shape[0]): + for i in range(indptr[r], indptr[r + 1]): + X[r, indices[i]] = data[i] diff --git a/src/scanpy/queries/_queries.py b/src/scanpy/queries/_queries.py index 24a2482449..e992f937e3 100644 --- a/src/scanpy/queries/_queries.py +++ b/src/scanpy/queries/_queries.py @@ -1,6 +1,6 @@ from __future__ import annotations -import collections.abc as cabc +from collections.abc import Iterable from functools import singledispatch from types import MappingProxyType from typing import TYPE_CHECKING @@ -12,7 +12,7 @@ from ..get import rank_genes_groups_df if TYPE_CHECKING: - from collections.abc import Iterable, Mapping + from collections.abc import Mapping from typing import Any import pandas as pd @@ -60,16 +60,16 @@ def simple_query( """ if isinstance(attrs, str): attrs = [attrs] - elif isinstance(attrs, cabc.Iterable): + elif isinstance(attrs, Iterable): attrs = list(attrs) else: - raise TypeError(f"attrs must be of type list or str, was {type(attrs)}.") + msg = f"attrs must be of type list or str, was {type(attrs)}." + raise TypeError(msg) try: from pybiomart import Server except ImportError: - raise ImportError( - "This method requires the `pybiomart` module to be installed." - ) + msg = "This method requires the `pybiomart` module to be installed." + raise ImportError(msg) server = Server(host, use_cache=use_cache) dataset = server.marts["ENSEMBL_MART_ENSEMBL"].datasets[f"{org}_gene_ensembl"] res = dataset.query(attributes=attrs, filters=filters, use_attr_names=True) @@ -273,17 +273,17 @@ def enrich( try: from gprofiler import GProfiler except ImportError: - raise ImportError( - "This method requires the `gprofiler-official` module to be installed." - ) + msg = "This method requires the `gprofiler-official` module to be installed." + raise ImportError(msg) gprofiler = GProfiler(user_agent="scanpy", return_dataframe=True) gprofiler_kwargs = dict(gprofiler_kwargs) for k in ["organism"]: if gprofiler_kwargs.get(k) is not None: - raise ValueError( + msg = ( f"Argument `{k}` should be passed directly through `enrich`, " "not through `gprofiler_kwargs`" ) + raise ValueError(msg) return gprofiler.profile(container, organism=org, **gprofiler_kwargs) diff --git a/src/scanpy/readwrite.py b/src/scanpy/readwrite.py index 98b6438a90..c568519cd7 100644 --- a/src/scanpy/readwrite.py +++ b/src/scanpy/readwrite.py @@ -10,24 +10,38 @@ import h5py import numpy as np import pandas as pd -from anndata import ( - AnnData, - read_csv, - read_excel, - read_h5ad, - read_hdf, - read_loom, - read_mtx, - read_text, -) +from packaging.version import Version + +if Version(anndata.__version__) >= Version("0.11.0rc2"): + from anndata.io import ( + read_csv, + read_excel, + read_h5ad, + read_hdf, + read_loom, + read_mtx, + read_text, + ) +else: + from anndata import ( + read_csv, + read_excel, + read_h5ad, + read_hdf, + read_loom, + read_mtx, + read_text, + ) +from anndata import AnnData from matplotlib.image import imread from . import logging as logg -from ._compat import old_positionals +from ._compat import add_note, deprecated, old_positionals from ._settings import settings from ._utils import _empty if TYPE_CHECKING: + from datetime import datetime from typing import BinaryIO, Literal from ._utils import Empty @@ -118,7 +132,7 @@ def read( See the h5py :ref:`dataset_compression`. (Default: `settings.cache_compression`) kwargs - Parameters passed to :func:`~anndata.read_loom`. + Parameters passed to :func:`~anndata.io.read_loom`. Returns ------- @@ -142,13 +156,14 @@ def read( filekey = str(filename) filename = settings.writedir / (filekey + "." + settings.file_format_data) if not filename.exists(): - raise ValueError( + msg = ( f"Reading with filekey {filekey!r} failed, " f"the inferred filename {filename!r} does not exist. " "If you intended to provide a filename, either use a filename " f"ending on one of the available extensions {avail_exts} " "or pass the parameter `ext`." ) + raise ValueError(msg) return read_h5ad(filename, backed=backed) @@ -206,40 +221,46 @@ def read_10x_h5( adata = _read_v3_10x_h5(filename, start=start) if genome: if genome not in adata.var["genome"].values: - raise ValueError( - f"Could not find data corresponding to genome '{genome}' in '{filename}'. " - f'Available genomes are: {list(adata.var["genome"].unique())}.' + msg = ( + f"Could not find data corresponding to genome {genome!r} in {filename}. " + f"Available genomes are: {list(adata.var['genome'].unique())}." ) + raise ValueError(msg) adata = adata[:, adata.var["genome"] == genome] if gex_only: adata = adata[:, adata.var["feature_types"] == "Gene Expression"] if adata.is_view: adata = adata.copy() else: - adata = _read_legacy_10x_h5(filename, genome=genome, start=start) + adata = _read_legacy_10x_h5(Path(filename), genome=genome, start=start) return adata -def _read_legacy_10x_h5(filename, *, genome=None, start=None): +def _read_legacy_10x_h5( + path: Path, *, genome: str | None = None, start: datetime | None = None +): """ Read hdf5 file from Cell Ranger v2 or earlier versions. """ - with h5py.File(str(filename), "r") as f: + with h5py.File(str(path), "r") as f: try: children = list(f.keys()) if not genome: if len(children) > 1: - raise ValueError( - f"'{filename}' contains more than one genome. For legacy 10x h5 " - "files you must specify the genome if more than one is present. " + msg = ( + f"{path} contains more than one genome. " + "For legacy 10x h5 files you must specify the genome " + "if more than one is present. " f"Available genomes are: {children}" ) + raise ValueError(msg) genome = children[0] elif genome not in children: - raise ValueError( - f"Could not find genome '{genome}' in '{filename}'. " + msg = ( + f"Could not find genome {genome!r} in {path}. " f"Available genomes are: {children}" ) + raise ValueError(msg) dsets = {} _collect_datasets(dsets, f[genome]) @@ -270,7 +291,8 @@ def _read_legacy_10x_h5(filename, *, genome=None, start=None): logg.info("", time=start) return adata except KeyError: - raise Exception("File is missing one or more required datasets.") + msg = "File is missing one or more required datasets." + raise Exception(msg) def _collect_datasets(dsets: dict, group: h5py.Group): @@ -341,7 +363,8 @@ def _read_v3_10x_h5(filename, *, start=None): ] ) else: - raise ValueError("10x h5 has no features group") + msg = "10x h5 has no features group" + raise ValueError(msg) adata = AnnData( matrix, obs=obs_dict, @@ -350,9 +373,11 @@ def _read_v3_10x_h5(filename, *, start=None): logg.info("", time=start) return adata except KeyError: - raise Exception("File is missing one or more required datasets.") + msg = "File is missing one or more required datasets." + raise Exception(msg) +@deprecated("Use `squidpy.read.visium` instead.") def read_visium( path: Path | str, genome: str | None = None, @@ -365,6 +390,9 @@ def read_visium( """\ Read 10x-Genomics-formatted visum dataset. + .. deprecated:: 1.11.0 + Use :func:`squidpy.read.visium` instead. + In addition to reading regular 10x output, this looks for the `spatial` folder and loads images, coordinates and scale factors. @@ -451,11 +479,11 @@ def read_visium( if not f.exists(): if any(x in str(f) for x in ["hires_image", "lowres_image"]): logg.warning( - f"You seem to be missing an image file.\n" - f"Could not find '{f}'." + f"You seem to be missing an image file.\nCould not find {f}." ) else: - raise OSError(f"Could not find '{f}'") + msg = f"Could not find {f}" + raise OSError(msg) adata.uns["spatial"][library_id]["images"] = dict() for res in ["hires", "lowres"]: @@ -464,7 +492,8 @@ def read_visium( str(files[f"{res}_image"]) ) except Exception: - raise OSError(f"Could not find '{res}_image'") + msg = f"Could not find '{res}_image'" + raise OSError(msg) # read json scalefactors adata.uns["spatial"][library_id]["scalefactors"] = json.loads( @@ -606,7 +635,8 @@ def _read_10x_mtx( adata.var_names = genes[0].values adata.var["gene_symbols"] = genes[1].values else: - raise ValueError("`var_names` needs to be 'gene_symbols' or 'gene_ids'") + msg = "`var_names` needs to be 'gene_symbols' or 'gene_ids'" + raise ValueError(msg) if not is_legacy: adata.var["feature_types"] = genes[2].values barcodes = pd.read_csv(path / f"{prefix}barcodes.tsv{suffix}", header=None) @@ -650,11 +680,12 @@ def write( if ext is None: ext = ext_ elif ext != ext_: - raise ValueError( + msg = ( "It suffices to provide the file type by " "providing a proper extension to the filename." 'One of "txt", "csv", "h5" or "npz".' ) + raise ValueError(msg) else: key = filename ext = settings.file_format_data if ext is None else ext @@ -672,8 +703,9 @@ def write( # ------------------------------------------------------------------------------- +@old_positionals("as_header") def read_params( - filename: Path | str, asheader: bool = False + filename: Path | str, *, as_header: bool = False ) -> dict[str, int | float | bool | str | None]: """\ Read parameter dictionary from text file. @@ -701,13 +733,12 @@ def read_params( params = OrderedDict([]) for line in filename.open(): - if "=" in line: - if not asheader or line.startswith("#"): - line = line[1:] if line.startswith("#") else line - key, val = line.split("=") - key = key.strip() - val = val.strip() - params[key] = convert_string(val) + if "=" in line and (not as_header or line.startswith("#")): + line = line[1:] if line.startswith("#") else line + key, val = line.split("=") + key = key.strip() + val = val.strip() + params[key] = convert_string(val) return params @@ -750,9 +781,8 @@ def _read( **kwargs, ): if ext is not None and ext not in avail_exts: - raise ValueError( - "Please provide one of the available extensions.\n" f"{avail_exts}" - ) + msg = f"Please provide one of the available extensions.\n{avail_exts}" + raise ValueError(msg) else: ext = is_valid_filename(filename, return_ext=True) is_present = _check_datafile_present_and_download(filename, backup_url=backup_url) @@ -776,7 +806,8 @@ def _read( return read_h5ad(path_cache) if not is_present: - raise FileNotFoundError(f"Did not find file {filename}.") + msg = f"Did not find file {filename}." + raise FileNotFoundError(msg) logg.debug(f"reading {filename}") if not cache and not suppress_cache_warning: logg.hint( @@ -786,7 +817,8 @@ def _read( # do the actual reading if ext == "xlsx" or ext == "xls": if sheet is None: - raise ValueError("Provide `sheet` parameter when reading '.xlsx' files.") + msg = "Provide `sheet` parameter when reading '.xlsx' files." + raise ValueError(msg) else: adata = read_excel(filename, sheet) elif ext in {"mtx", "mtx.gz"}: @@ -800,7 +832,7 @@ def _read( elif ext in {"txt", "tab", "data", "tsv"}: if ext == "data": logg.hint( - "... assuming '.data' means tab or white-space " "separated text file", + "... assuming '.data' means tab or white-space separated text file" ) logg.hint("change this by passing `ext` to sc.read") adata = read_text(filename, delimiter, first_column_names) @@ -809,7 +841,8 @@ def _read( elif ext == "loom": adata = read_loom(filename=filename, **kwargs) else: - raise ValueError(f"Unknown extension {ext}.") + msg = f"Unknown extension {ext}." + raise ValueError(msg) if cache: logg.info( f"... writing an {settings.file_format_data} " @@ -980,15 +1013,11 @@ def _get_filename_from_key(key, ext=None) -> Path: def _download(url: str, path: Path): - try: - import ipywidgets # noqa: F401 - from tqdm.auto import tqdm - except ImportError: - from tqdm import tqdm - from urllib.error import URLError from urllib.request import Request, urlopen + from tqdm.auto import tqdm + blocksize = 1024 * 8 blocknum = 0 @@ -998,14 +1027,17 @@ def _download(url: str, path: Path): try: open_url = urlopen(req) except URLError: - logg.warning( - "Failed to open the url with default certificates, trying with certifi." - ) + msg = "Failed to open the url with default certificates." + try: + from certifi import where + except ImportError as e: + add_note(e, f"{msg} Please install `certifi` and try again.") + raise + else: + logg.warning(f"{msg} Trying to use certifi.") from ssl import create_default_context - from certifi import where - open_url = urlopen(req, context=create_default_context(cafile=where())) with open_url as resp: @@ -1053,7 +1085,7 @@ def _check_datafile_present_and_download(path, backup_url=None): return True -def is_valid_filename(filename: Path, return_ext=False): +def is_valid_filename(filename: Path, *, return_ext: bool = False): """Check whether the argument is a filename.""" ext = filename.suffixes @@ -1075,11 +1107,10 @@ def is_valid_filename(filename: Path, return_ext=False): return "mtx.gz" if return_ext else True elif not return_ext: return False - raise ValueError( - f"""\ + msg = f"""\ {filename!r} does not end on a valid extension. Please, provide one of the available extensions. {avail_exts} Text files with .gz and .bz2 extensions are also supported.\ """ - ) + raise ValueError(msg) diff --git a/src/scanpy/tools/_dendrogram.py b/src/scanpy/tools/_dendrogram.py index f60f0ae2e9..f33aca1ff7 100644 --- a/src/scanpy/tools/_dendrogram.py +++ b/src/scanpy/tools/_dendrogram.py @@ -60,8 +60,8 @@ def dendrogram( to compute a correlation matrix. The hierarchical clustering can be visualized using - :func:`scanpy.pl.dendrogram` or multiple other visualizations that can - include a dendrogram: :func:`~scanpy.pl.matrixplot`, + :func:`scanpy.pl.dendrogram` or multiple other visualizations + that can include a dendrogram: :func:`~scanpy.pl.matrixplot`, :func:`~scanpy.pl.heatmap`, :func:`~scanpy.pl.dotplot`, and :func:`~scanpy.pl.stacked_violin`. @@ -78,15 +78,15 @@ def dendrogram( {use_rep} var_names List of var_names to use for computing the hierarchical clustering. - If `var_names` is given, then `use_rep` and `n_pcs` is ignored. + If `var_names` is given, then `use_rep` and `n_pcs` are ignored. use_raw Only when `var_names` is not None. Use `raw` attribute of `adata` if present. cor_method - correlation method to use. + Correlation method to use. Options are 'pearson', 'kendall', and 'spearman' linkage_method - linkage method to use. See :func:`scipy.cluster.hierarchy.linkage` + Linkage method to use. See :func:`scipy.cluster.hierarchy.linkage` for more information. optimal_ordering Same as the optimal_ordering argument of :func:`scipy.cluster.hierarchy.linkage` @@ -124,15 +124,17 @@ def dendrogram( groupby = [groupby] for group in groupby: if group not in adata.obs_keys(): - raise ValueError( + msg = ( "groupby has to be a valid observation. " f"Given value: {group}, valid observations: {adata.obs_keys()}" ) + raise ValueError(msg) if not isinstance(adata.obs[group].dtype, CategoricalDtype): - raise ValueError( + msg = ( "groupby has to be a categorical observation. " f"Given value: {group}, Column type: {adata.obs[group].dtype}" ) + raise ValueError(msg) if var_names is None: rep_df = pd.DataFrame( @@ -188,7 +190,7 @@ def dendrogram( if inplace: if key_added is None: - key_added = f'dendrogram_{"_".join(groupby)}' + key_added = f"dendrogram_{'_'.join(groupby)}" logg.info(f"Storing dendrogram info using `.uns[{key_added!r}]`") adata.uns[key_added] = dat else: diff --git a/src/scanpy/tools/_diffmap.py b/src/scanpy/tools/_diffmap.py index dee643c39b..b69c2ef18f 100644 --- a/src/scanpy/tools/_diffmap.py +++ b/src/scanpy/tools/_diffmap.py @@ -8,7 +8,7 @@ if TYPE_CHECKING: from anndata import AnnData - from .._utils import AnyRandom + from .._compat import _LegacyRandom @old_positionals("neighbors_key", "random_state", "copy") @@ -17,15 +17,15 @@ def diffmap( n_comps: int = 15, *, neighbors_key: str | None = None, - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, copy: bool = False, ) -> AnnData | None: """\ Diffusion Maps :cite:p:`Coifman2005,Haghverdi2015,Wolf2018`. - Diffusion maps :cite:p:`Coifman2005` has been proposed for visualizing single-cell - data by :cite:t:`Haghverdi2015`. The tool uses the adapted Gaussian kernel suggested - by :cite:t:`Haghverdi2016` in the implementation of :cite:t:`Wolf2018`. + Diffusion maps :cite:p:`Coifman2005` have been proposed for visualizing single-cell + data by :cite:t:`Haghverdi2015`. This tool uses the adapted Gaussian kernel suggested + by :cite:t:`Haghverdi2016` with the implementation of :cite:t:`Wolf2018`. The width ("sigma") of the connectivity kernel is implicitly determined by the number of neighbors used to compute the single-cell graph in @@ -42,12 +42,12 @@ def diffmap( n_comps The number of dimensions of the representation. neighbors_key - If not specified, diffmap looks .uns['neighbors'] for neighbors settings - and .obsp['connectivities'], .obsp['distances'] for connectivities and - distances respectively (default storage places for pp.neighbors). - If specified, diffmap looks .uns[neighbors_key] for neighbors settings and - .obsp[.uns[neighbors_key]['connectivities_key']], - .obsp[.uns[neighbors_key]['distances_key']] for connectivities and distances + If not specified, diffmap looks in .uns['neighbors'] for neighbors settings + and .obsp['connectivities'] and .obsp['distances'] for connectivities and + distances, respectively (default storage places for pp.neighbors). + If specified, diffmap looks in .uns[neighbors_key] for neighbors settings and + .obsp[.uns[neighbors_key]['connectivities_key']] and + .obsp[.uns[neighbors_key]['distances_key']] for connectivities and distances, respectively. random_state A numpy random seed @@ -77,11 +77,11 @@ def diffmap( neighbors_key = "neighbors" if neighbors_key not in adata.uns: - raise ValueError( - "You need to run `pp.neighbors` first to compute a neighborhood graph." - ) + msg = "You need to run `pp.neighbors` first to compute a neighborhood graph." + raise ValueError(msg) if n_comps <= 2: - raise ValueError("Provide any value greater than 2 for `n_comps`. ") + msg = "Provide any value greater than 2 for `n_comps`. " + raise ValueError(msg) adata = adata.copy() if copy else adata _diffmap( adata, n_comps=n_comps, neighbors_key=neighbors_key, random_state=random_state diff --git a/src/scanpy/tools/_dpt.py b/src/scanpy/tools/_dpt.py index 8c5f7d857b..a9adc2a112 100644 --- a/src/scanpy/tools/_dpt.py +++ b/src/scanpy/tools/_dpt.py @@ -18,7 +18,7 @@ def _diffmap(adata, n_comps=15, neighbors_key=None, random_state=0): - start = logg.info(f"computing Diffusion Maps using n_comps={n_comps}(=n_dcs)") + start = logg.info(f"computing Diffusion Maps using {n_comps=}(=n_dcs)") dpt = DPT(adata, neighbors_key=neighbors_key) dpt.compute_transitions() dpt.compute_eigen(n_comps=n_comps, random_state=random_state) @@ -53,7 +53,7 @@ def dpt( :cite:p:`Haghverdi2016,Wolf2019`. Reconstruct the progression of a biological process from snapshot - data. `Diffusion Pseudotime` has been introduced by :cite:t:`Haghverdi2016` and + data. `Diffusion Pseudotime` was introduced by :cite:t:`Haghverdi2016` and implemented within Scanpy :cite:p:`Wolf2018`. Here, we use a further developed version, which is able to deal with disconnected graphs :cite:p:`Wolf2019` and can be run in a `hierarchical` mode by setting the parameter @@ -64,9 +64,9 @@ def dpt( adata.uns['iroot'] = np.flatnonzero(adata.obs['cell_types'] == 'Stem')[0] - This requires to run :func:`~scanpy.pp.neighbors`, first. In order to - reproduce the original implementation of DPT, use `method=='gauss'` in - this. Using the default `method=='umap'` only leads to minor quantitative + This requires running :func:`~scanpy.pp.neighbors`, first. In order to + reproduce the original implementation of DPT, use `method=='gauss'`. + Using the default `method=='umap'` only leads to minor quantitative differences, though. .. versionadded:: 1.1 @@ -96,12 +96,12 @@ def dpt( maximum correlation in Kendall tau criterion of :cite:t:`Haghverdi2016` to stabilize the splitting. neighbors_key - If not specified, dpt looks .uns['neighbors'] for neighbors settings - and .obsp['connectivities'], .obsp['distances'] for connectivities and - distances respectively (default storage places for pp.neighbors). - If specified, dpt looks .uns[neighbors_key] for neighbors settings and - .obsp[.uns[neighbors_key]['connectivities_key']], - .obsp[.uns[neighbors_key]['distances_key']] for connectivities and distances + If not specified, dpt looks in .uns['neighbors'] for neighbors settings + and .obsp['connectivities'] and .obsp['distances'] for connectivities and + distances, respectively (default storage places for pp.neighbors). + If specified, dpt looks in .uns[neighbors_key] for neighbors settings and + .obsp[.uns[neighbors_key]['connectivities_key']] and + .obsp[.uns[neighbors_key]['distances_key']] for connectivities and distances, respectively. copy Copy instance before computation and return a copy. @@ -129,7 +129,8 @@ def dpt( if neighbors_key is None: neighbors_key = "neighbors" if neighbors_key not in adata.uns: - raise ValueError("You need to run `pp.neighbors` and `tl.diffmap` first.") + msg = "You need to run `pp.neighbors` and `tl.diffmap` first." + raise ValueError(msg) if "iroot" not in adata.uns and "xroot" not in adata.var: logg.warning( "No root cell found. To compute pseudotime, pass the index or " @@ -137,7 +138,7 @@ def dpt( " adata.uns['iroot'] = root_cell_index\n" " adata.var['xroot'] = adata[root_cell_name, :].X" ) - if "X_diffmap" not in adata.obsm.keys(): + if "X_diffmap" not in adata.obsm: logg.warning( "Trying to run `tl.dpt` without prior call of `tl.diffmap`. " "Falling back to `tl.diffmap` with default parameters." @@ -152,7 +153,7 @@ def dpt( allow_kendall_tau_shift=allow_kendall_tau_shift, neighbors_key=neighbors_key, ) - start = logg.info(f"computing Diffusion Pseudotime using n_dcs={n_dcs}") + start = logg.info(f"computing Diffusion Pseudotime using {n_dcs=}") if n_branchings > 1: logg.info(" this uses a hierarchical implementation") if dpt.iroot is not None: @@ -262,7 +263,7 @@ def detect_branchings(self): """ logg.debug( f" detect {self.n_branchings} " - f'branching{"" if self.n_branchings == 1 else "s"}', + f"branching{'' if self.n_branchings == 1 else 's'}", ) # a segment is a subset of points of the data set (defined by the # indices of the points in the segment) @@ -799,9 +800,8 @@ def _detect_branching( elif self.flavor == "wolf17_bi" or self.flavor == "wolf17_bi_un": ssegs = self._detect_branching_single_wolf17_bi(Dseg, tips) else: - raise ValueError( - '`flavor` needs to be in {"haghverdi16", "wolf17_tri", "wolf17_bi"}.' - ) + msg = '`flavor` needs to be in {"haghverdi16", "wolf17_tri", "wolf17_bi"}.' + raise ValueError(msg) # make sure that each data point has a unique association with a segment masks = np.zeros((len(ssegs), Dseg.shape[0]), dtype=bool) for iseg, seg in enumerate(ssegs): @@ -1039,9 +1039,11 @@ def kendall_tau_split(self, a: np.ndarray, b: np.ndarray) -> int: Splitting index according to above description. """ if a.size != b.size: - raise ValueError("a and b need to have the same size") + msg = "a and b need to have the same size" + raise ValueError(msg) if a.ndim != b.ndim != 1: - raise ValueError("a and b need to be one-dimensional arrays") + msg = "a and b need to be one-dimensional arrays" + raise ValueError(msg) import scipy as sp min_length = 5 diff --git a/src/scanpy/tools/_draw_graph.py b/src/scanpy/tools/_draw_graph.py index 9d922fe49e..0727715671 100644 --- a/src/scanpy/tools/_draw_graph.py +++ b/src/scanpy/tools/_draw_graph.py @@ -1,24 +1,29 @@ from __future__ import annotations import random -from typing import TYPE_CHECKING, Literal, get_args +from importlib.util import find_spec +from typing import TYPE_CHECKING, Literal import numpy as np from .. import _utils from .. import logging as logg from .._compat import old_positionals -from .._utils import _choose_graph +from .._utils import _choose_graph, get_literal_vals from ._utils import get_init_pos_from_paga if TYPE_CHECKING: + from typing import LiteralString, TypeVar + from anndata import AnnData from scipy.sparse import spmatrix - from .._utils import AnyRandom + from .._compat import _LegacyRandom + + S = TypeVar("S", bound=LiteralString) + _Layout = Literal["fr", "drl", "kk", "grid_fr", "lgl", "rt", "rt_circular", "fa"] -_LAYOUTS = get_args(_Layout) @old_positionals( @@ -38,7 +43,7 @@ def draw_graph( *, init_pos: str | bool | None = None, root: int | None = None, - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, n_jobs: int | None = None, adjacency: spmatrix | None = None, key_added_ext: str | None = None, @@ -51,19 +56,19 @@ def draw_graph( Force-directed graph drawing :cite:p:`Islam2011,Jacomy2014,Chippada2018`. An alternative to tSNE that often preserves the topology of the data - better. This requires to run :func:`~scanpy.pp.neighbors`, first. + better. This requires running :func:`~scanpy.pp.neighbors`, first. - The default layout ('fa', `ForceAtlas2`, :cite:t:`Jacomy2014`) uses the package |fa2|_ - :cite:p:`Chippada2018`, which can be installed via `pip install fa2`. + The default layout ('fa', `ForceAtlas2`, :cite:t:`Jacomy2014`) uses the package |fa2-modified|_ + :cite:p:`Chippada2018`, which can be installed via `pip install fa2-modified`. `Force-directed graph drawing`_ describes a class of long-established algorithms for visualizing graphs. - It has been suggested for visualizing single-cell data by :cite:t:`Islam2011`. + It was suggested for visualizing single-cell data by :cite:t:`Islam2011`. Many other layouts as implemented in igraph :cite:p:`Csardi2006` are available. Similar approaches have been used by :cite:t:`Zunder2015` or :cite:t:`Weinreb2017`. - .. |fa2| replace:: `fa2` - .. _fa2: https://github.com/bhargavchippada/forceatlas2 + .. |fa2-modified| replace:: `fa2-modified` + .. _fa2-modified: https://github.com/AminAlam/fa2_modified .. _Force-directed graph drawing: https://en.wikipedia.org/wiki/Force-directed_graph_drawing Parameters @@ -93,9 +98,9 @@ def draw_graph( Use precomputed coordinates for initialization. If `False`/`None` (the default), initialize randomly. neighbors_key - If not specified, draw_graph looks .obsp['connectivities'] for connectivities + If not specified, draw_graph looks at .obsp['connectivities'] for connectivities (default storage place for pp.neighbors). - If specified, draw_graph looks + If specified, draw_graph looks at .obsp[.uns[neighbors_key]['connectivities_key']] for connectivities. obsp Use .obsp[obsp] as adjacency. You can't specify both @@ -118,13 +123,14 @@ def draw_graph( `draw_graph` parameters. """ start = logg.info(f"drawing single-cell graph using layout {layout!r}") - if layout not in _LAYOUTS: - raise ValueError(f"Provide a valid layout, one of {_LAYOUTS}.") + if layout not in (layouts := get_literal_vals(_Layout)): + msg = f"Provide a valid layout, one of {layouts}." + raise ValueError(msg) adata = adata.copy() if copy else adata if adjacency is None: adjacency = _choose_graph(adata, obsp, neighbors_key) # init coordinates - if init_pos in adata.obsm.keys(): + if init_pos in adata.obsm: init_coords = adata.obsm[init_pos] elif init_pos == "paga" or init_pos: init_coords = get_init_pos_from_paga( @@ -137,47 +143,10 @@ def draw_graph( else: np.random.seed(random_state) init_coords = np.random.random((adjacency.shape[0], 2)) - # see whether fa2 is installed - if layout == "fa": - try: - from fa2 import ForceAtlas2 - except ImportError: - logg.warning( - "Package 'fa2' is not installed, falling back to layout 'fr'." - "To use the faster and better ForceAtlas2 layout, " - "install package 'fa2' (`pip install fa2`)." - ) - layout = "fr" + layout = coerce_fa2_layout(layout) # actual drawing if layout == "fa": - forceatlas2 = ForceAtlas2( - # Behavior alternatives - outboundAttractionDistribution=False, # Dissuade hubs - linLogMode=False, # NOT IMPLEMENTED - adjustSizes=False, # Prevent overlap (NOT IMPLEMENTED) - edgeWeightInfluence=1.0, - # Performance - jitterTolerance=1.0, # Tolerance - barnesHutOptimize=True, - barnesHutTheta=1.2, - multiThreaded=False, # NOT IMPLEMENTED - # Tuning - scalingRatio=2.0, - strongGravityMode=False, - gravity=1.0, - # Log - verbose=False, - ) - if "maxiter" in kwds: - iterations = kwds["maxiter"] - elif "iterations" in kwds: - iterations = kwds["iterations"] - else: - iterations = 500 - positions = forceatlas2.forceatlas2( - adjacency, pos=init_coords, iterations=iterations - ) - positions = np.array(positions) + positions = np.array(fa2_positions(adjacency, init_coords, **kwds)) else: # igraph doesn't use numpy seed random.seed(random_state) @@ -202,3 +171,51 @@ def draw_graph( deep=f"added\n {key_added!r}, graph_drawing coordinates (adata.obsm)", ) return adata if copy else None + + +def fa2_positions( + adjacency: spmatrix | np.ndarray, init_coords: np.ndarray, **kwds +) -> list[tuple[float, float]]: + from fa2_modified import ForceAtlas2 + + forceatlas2 = ForceAtlas2( + # Behavior alternatives + outboundAttractionDistribution=False, # Dissuade hubs + linLogMode=False, # NOT IMPLEMENTED + adjustSizes=False, # Prevent overlap (NOT IMPLEMENTED) + edgeWeightInfluence=1.0, + # Performance + jitterTolerance=1.0, # Tolerance + barnesHutOptimize=True, + barnesHutTheta=1.2, + multiThreaded=False, # NOT IMPLEMENTED + # Tuning + scalingRatio=2.0, + strongGravityMode=False, + gravity=1.0, + # Log + verbose=False, + ) + if "maxiter" in kwds: + iterations = kwds["maxiter"] + elif "iterations" in kwds: + iterations = kwds["iterations"] + else: + iterations = 500 + return forceatlas2.forceatlas2(adjacency, pos=init_coords, iterations=iterations) + + +def coerce_fa2_layout(layout: S) -> S | Literal["fa", "fr"]: + # see whether fa2 is installed + if layout != "fa": + return layout + + if find_spec("fa2_modified") is None: + logg.warning( + "Package 'fa2-modified' is not installed, falling back to layout 'fr'." + "To use the faster and better ForceAtlas2 layout, " + "install package 'fa2-modified' (`pip install fa2-modified`)." + ) + return "fr" + + return "fa" diff --git a/src/scanpy/tools/_embedding_density.py b/src/scanpy/tools/_embedding_density.py index 5ae69361dc..b930cc0aeb 100644 --- a/src/scanpy/tools/_embedding_density.py +++ b/src/scanpy/tools/_embedding_density.py @@ -70,7 +70,7 @@ def embedding_density( The annotated data matrix. basis The embedding over which the density will be calculated. This embedded - representation should be found in `adata.obsm['X_[basis]']``. + representation is found in `adata.obsm['X_[basis]']``. groupby Key for categorical observation/cell annotation for which densities are calculated per category. @@ -130,10 +130,11 @@ def embedding_density( basis = "draw_graph_fa" if f"X_{basis}" not in adata.obsm_keys(): - raise ValueError( + msg = ( "Cannot find the embedded representation " f"`adata.obsm['X_{basis}']`. Compute the embedding first." ) + raise ValueError(msg) if components is None: components = "1,2" @@ -142,17 +143,20 @@ def embedding_density( components = np.array(components).astype(int) - 1 if len(components) != 2: - raise ValueError("Please specify exactly 2 components, or `None`.") + msg = "Please specify exactly 2 components, or `None`." + raise ValueError(msg) if basis == "diffmap": components += 1 if groupby is not None: if groupby not in adata.obs: - raise ValueError(f"Could not find {groupby!r} `.obs` column.") + msg = f"Could not find {groupby!r} `.obs` column." + raise ValueError(msg) if adata.obs[groupby].dtype.name != "category": - raise ValueError(f"{groupby!r} column does not contain categorical data") + msg = f"{groupby!r} column does not contain categorical data" + raise ValueError(msg) # Define new covariate name if key_added is not None: diff --git a/src/scanpy/tools/_ingest.py b/src/scanpy/tools/_ingest.py index 3d05937d81..256e1a97c6 100644 --- a/src/scanpy/tools/_ingest.py +++ b/src/scanpy/tools/_ingest.py @@ -91,10 +91,10 @@ def ingest( The method to map labels in `adata_ref.obs` to `adata.obs`. The only supported value is 'knn'. neighbors_key - If not specified, ingest looks adata_ref.uns['neighbors'] + If not specified, ingest looks at adata_ref.uns['neighbors'] for neighbors settings and adata_ref.obsp['distances'] for distances (default storage places for pp.neighbors). - If specified, ingest looks adata_ref.uns[neighbors_key] for + If specified, ingest looks at adata_ref.uns[neighbors_key] for neighbors settings and adata_ref.obsp[adata_ref.uns[neighbors_key]['distances_key']] for distances. inplace @@ -123,11 +123,12 @@ def ingest( # anndata version check anndata_version = pkg_version("anndata") if anndata_version < ANNDATA_MIN_VERSION: - raise ValueError( + msg = ( f"ingest only works correctly with anndata>={ANNDATA_MIN_VERSION} " f"(you have {anndata_version}) as prior to {ANNDATA_MIN_VERSION}, " "`AnnData.concatenate` did not concatenate `.obsm`." ) + raise ValueError(msg) start = logg.info("running ingest") obs = [obs] if isinstance(obs, str) else obs @@ -153,7 +154,7 @@ def ingest( ing.map_labels(col, labeling_method[i]) logg.info(" finished", time=start) - return ing.to_adata(inplace) + return ing.to_adata(inplace=inplace) def _rp_forest_generate( @@ -187,12 +188,13 @@ def __init__(self, dim, axis=0, vals=None): def __setitem__(self, key, value): if value.shape[self._axis] != self._dim: - raise ValueError( - f"Value passed for key '{key}' is of incorrect shape. " + msg = ( + f"Value passed for key {key!r} is of incorrect shape. " f"Value has shape {value.shape[self._axis]} " f"for dimension {self._axis} while " f"it should have {self._dim}." ) + raise ValueError(msg) self._data[key] = value def __getitem__(self, key): @@ -296,16 +298,12 @@ def _init_neighbors(self, adata, neighbors_key): self._use_rep = "X_pca" self._n_pcs = neighbors["params"]["n_pcs"] self._rep = adata.obsm["X_pca"][:, : self._n_pcs] - elif adata.n_vars > settings.N_PCS and "X_pca" in adata.obsm.keys(): + elif adata.n_vars > settings.N_PCS and "X_pca" in adata.obsm: self._use_rep = "X_pca" self._rep = adata.obsm["X_pca"][:, : settings.N_PCS] self._n_pcs = self._rep.shape[1] - if "metric_kwds" in neighbors["params"]: - self._metric_kwds = neighbors["params"]["metric_kwds"] - else: - self._metric_kwds = {} - + self._metric_kwds = neighbors["params"].get("metric_kwds", {}) self._metric = neighbors["params"]["metric"] self._neigh_random_state = neighbors["params"].get("random_state", 0) @@ -316,7 +314,7 @@ def _init_pca(self, adata): self._pca_use_hvg = adata.uns["pca"]["params"]["use_highly_variable"] mask = "highly_variable" - if self._pca_use_hvg and mask not in adata.var.keys(): + if self._pca_use_hvg and mask not in adata.var.columns: msg = f"Did not find `adata.var[{mask!r}']`." raise ValueError(msg) @@ -344,10 +342,11 @@ def __init__(self, adata: AnnData, neighbors_key: str | None = None): if neighbors_key in adata.uns: self._init_neighbors(adata, neighbors_key) else: - raise ValueError( + msg = ( f'There is no neighbors data in `adata.uns["{neighbors_key}"]`.\n' "Please run pp.neighbors." ) + raise ValueError(msg) if "X_umap" in adata.obsm: self._init_umap(adata) @@ -375,7 +374,7 @@ def _same_rep(self): return self._pca(self._n_pcs) if self._use_rep == "X": return adata.X - if self._use_rep in adata.obsm.keys(): + if self._use_rep in adata.obsm: return adata.obsm[self._use_rep] return adata.X @@ -397,10 +396,11 @@ def fit(self, adata_new): new_var_names = adata_new.var_names.str.upper() if not ref_var_names.equals(new_var_names): - raise ValueError( + msg = ( "Variables in the new adata are different " "from variables in the reference adata" ) + raise ValueError(msg) self._obs = pd.DataFrame(index=adata_new.obs.index) self._obsm = _DimDict(adata_new.n_obs, axis=0) @@ -444,9 +444,8 @@ def map_embedding(self, method): elif method == "pca": self._obsm["X_pca"] = self._pca() else: - raise NotImplementedError( - "Ingest supports only umap and pca embeddings for now." - ) + msg = "Ingest supports only umap and pca embeddings for now." + raise NotImplementedError(msg) def _knn_classify(self, labels): # ensure it's categorical @@ -465,9 +464,11 @@ def map_labels(self, labels, method): if method == "knn": self._obs[labels] = self._knn_classify(labels) else: - raise NotImplementedError("Ingest supports knn labeling for now.") + msg = "Ingest supports knn labeling for now." + raise NotImplementedError(msg) - def to_adata(self, inplace=False): + @old_positionals("inplace") + def to_adata(self, *, inplace: bool = False) -> AnnData | None: """\ Returns `adata_new` with mapped embeddings and labels. diff --git a/src/scanpy/tools/_leiden.py b/src/scanpy/tools/_leiden.py index 4514fd8978..cccd7f96ce 100644 --- a/src/scanpy/tools/_leiden.py +++ b/src/scanpy/tools/_leiden.py @@ -17,6 +17,8 @@ from anndata import AnnData from scipy import sparse + from .._compat import _LegacyRandom + try: from leidenalg.VertexPartition import MutableVertexPartition except ImportError: @@ -32,7 +34,7 @@ def leiden( resolution: float = 1, *, restrict_to: tuple[str, Sequence[str]] | None = None, - random_state: _utils.AnyRandom = 0, + random_state: _LegacyRandom = 0, key_added: str = "leiden", adjacency: sparse.spmatrix | None = None, directed: bool | None = None, @@ -50,9 +52,9 @@ def leiden( Cluster cells using the Leiden algorithm :cite:p:`Traag2019`, an improved version of the Louvain algorithm :cite:p:`Blondel2008`. - It has been proposed for single-cell analysis by :cite:t:`Levine2015`. + It was proposed for single-cell analysis by :cite:t:`Levine2015`. - This requires having ran :func:`~scanpy.pp.neighbors` or + This requires having run :func:`~scanpy.pp.neighbors` or :func:`~scanpy.external.pp.bbknn` first. Parameters @@ -90,9 +92,9 @@ def leiden( :func:`~leidenalg.find_partition`. neighbors_key Use neighbors connectivities as adjacency. - If not specified, leiden looks .obsp['connectivities'] for connectivities + If not specified, leiden looks at .obsp['connectivities'] for connectivities (default storage place for pp.neighbors). - If specified, leiden looks + If specified, leiden looks at .obsp[.uns[neighbors_key]['connectivities_key']] for connectivities. obsp Use .obsp[obsp] as adjacency. You can't specify both @@ -118,19 +120,18 @@ def leiden( and `n_iterations`. """ if flavor not in {"igraph", "leidenalg"}: - raise ValueError( - f"flavor must be either 'igraph' or 'leidenalg', but '{flavor}' was passed" + msg = ( + f"flavor must be either 'igraph' or 'leidenalg', but {flavor!r} was passed" ) + raise ValueError(msg) _utils.ensure_igraph() if flavor == "igraph": if directed: - raise ValueError( - "Cannot use igraph's leiden implemntation with a directed graph." - ) + msg = "Cannot use igraph’s leiden implementation with a directed graph." + raise ValueError(msg) if partition_type is not None: - raise ValueError( - "Do not pass in partition_type argument when using igraph." - ) + msg = "Do not pass in partition_type argument when using igraph." + raise ValueError(msg) else: try: import leidenalg @@ -138,9 +139,8 @@ def leiden( msg = 'In the future, the default backend for leiden will be igraph instead of leidenalg.\n\n To achieve the future defaults please pass: flavor="igraph" and n_iterations=2. directed must also be False to work with igraph\'s implementation.' _utils.warn_once(msg, FutureWarning, stacklevel=3) except ImportError: - raise ImportError( - "Please install the leiden algorithm: `conda install -c conda-forge leidenalg` or `pip3 install leidenalg`." - ) + msg = "Please install the leiden algorithm: `conda install -c conda-forge leidenalg` or `pip3 install leidenalg`." + raise ImportError(msg) clustering_args = dict(clustering_args) start = logg.info("running Leiden clustering") diff --git a/src/scanpy/tools/_louvain.py b/src/scanpy/tools/_louvain.py index e6259f035d..1800dbe08e 100644 --- a/src/scanpy/tools/_louvain.py +++ b/src/scanpy/tools/_louvain.py @@ -22,6 +22,8 @@ from anndata import AnnData from scipy.sparse import spmatrix + from .._compat import _LegacyRandom + try: from louvain.VertexPartition import MutableVertexPartition except ImportError: @@ -50,7 +52,7 @@ def louvain( adata: AnnData, resolution: float | None = None, *, - random_state: _utils.AnyRandom = 0, + random_state: _LegacyRandom = 0, restrict_to: tuple[str, Sequence[str]] | None = None, key_added: str = "louvain", adjacency: spmatrix | None = None, @@ -67,10 +69,10 @@ def louvain( Cluster cells into subgroups :cite:p:`Blondel2008,Levine2015,Traag2017`. Cluster cells using the Louvain algorithm :cite:p:`Blondel2008` in the implementation - of :cite:t:`Traag2017`. The Louvain algorithm has been proposed for single-cell + of :cite:t:`Traag2017`. The Louvain algorithm was proposed for single-cell analysis by :cite:t:`Levine2015`. - This requires having ran :func:`~scanpy.pp.neighbors` or + This requires having run :func:`~scanpy.pp.neighbors` or :func:`~scanpy.external.pp.bbknn` first, or explicitly passing a ``adjacency`` matrix. @@ -141,9 +143,8 @@ def louvain( partition_kwargs = dict(partition_kwargs) start = logg.info("running Louvain clustering") if (flavor != "vtraag") and (partition_type is not None): - raise ValueError( - "`partition_type` is only a valid argument " 'when `flavour` is "vtraag"' - ) + msg = '`partition_type` is only a valid argument when `flavour` is "vtraag"' + raise ValueError(msg) adata = adata.copy() if copy else adata if adjacency is None: adjacency = _choose_graph(adata, obsp, neighbors_key) @@ -163,10 +164,7 @@ def louvain( if not directed: logg.debug(" using the undirected graph") g = _utils.get_igraph_from_adjacency(adjacency, directed=directed) - if use_weights: - weights = np.array(g.es["weight"]).astype(np.float64) - else: - weights = None + weights = np.array(g.es["weight"]).astype(np.float64) if use_weights else None if flavor == "vtraag": import louvain @@ -240,7 +238,8 @@ def louvain( for k, v in partition.items(): groups[k] = v else: - raise ValueError('`flavor` needs to be "vtraag" or "igraph" or "taynaud".') + msg = '`flavor` needs to be "vtraag" or "igraph" or "taynaud".' + raise ValueError(msg) if restrict_to is not None: if key_added == "louvain": key_added += "_R" diff --git a/src/scanpy/tools/_marker_gene_overlap.py b/src/scanpy/tools/_marker_gene_overlap.py index 534f5c3b33..43408ff2c3 100644 --- a/src/scanpy/tools/_marker_gene_overlap.py +++ b/src/scanpy/tools/_marker_gene_overlap.py @@ -4,7 +4,7 @@ from __future__ import annotations -import collections.abc as cabc +from collections.abc import Set as AbstractSet from typing import TYPE_CHECKING import numpy as np @@ -30,10 +30,7 @@ def _calc_overlap_count(markers1: dict, markers2: dict): overlaps = np.zeros((len(markers1), len(markers2))) for j, marker_group in enumerate(markers1): - tmp = [ - len(markers2[i].intersection(markers1[marker_group])) - for i in markers2.keys() - ] + tmp = [len(markers2[i].intersection(markers1[marker_group])) for i in markers2] overlaps[j, :] = tmp return overlaps @@ -51,7 +48,7 @@ def _calc_overlap_coef(markers1: dict, markers2: dict): tmp = [ len(markers2[i].intersection(markers1[marker_group])) / max(min(len(markers2[i]), len(markers1[marker_group])), 1) - for i in markers2.keys() + for i in markers2 ] overlap_coef[j, :] = tmp @@ -70,7 +67,7 @@ def _calc_jaccard(markers1: dict, markers2: dict): tmp = [ len(markers2[i].intersection(markers1[marker_group])) / len(markers2[i].union(markers1[marker_group])) - for i in markers2.keys() + for i in markers2 ] jacc_results[j, :] = tmp @@ -91,7 +88,7 @@ def marker_gene_overlap( inplace: bool = False, ): """\ - Calculate an overlap score between data-deriven marker genes and + Calculate an overlap score between data-derived marker genes and provided markers Marker gene overlap scores can be quoted as overlap counts, overlap @@ -138,7 +135,7 @@ def marker_gene_overlap( Returns ------- - Returns :class:`pandas.DataFrame` if `inplace=True`, else returns an `AnnData` object where it sets the following field: + Returns :class:`pandas.DataFrame` if `inplace=False`, else returns an `AnnData` object where it sets the following field: `adata.uns[key_added]` : :class:`pandas.DataFrame` (dtype `float`) Marker gene overlap scores. Default for `key_added` is `'marker_gene_overlap'`. @@ -165,49 +162,56 @@ def marker_gene_overlap( """ # Test user inputs if inplace: - raise NotImplementedError( + msg = ( "Writing Pandas dataframes to h5ad is currently under development." "\nPlease use `inplace=False`." ) + raise NotImplementedError(msg) if key not in adata.uns: - raise ValueError( + msg = ( "Could not find marker gene data. " "Please run `sc.tl.rank_genes_groups()` first." ) + raise ValueError(msg) avail_methods = {"overlap_count", "overlap_coef", "jaccard", "enrich"} if method not in avail_methods: - raise ValueError(f"Method must be one of {avail_methods}.") + msg = f"Method must be one of {avail_methods}." + raise ValueError(msg) if normalize == "None": normalize = None avail_norm = {"reference", "data", None} if normalize not in avail_norm: - raise ValueError(f"Normalize must be one of {avail_norm}.") + msg = f"Normalize must be one of {avail_norm}." + raise ValueError(msg) if normalize is not None and method != "overlap_count": - raise ValueError("Can only normalize with method=`overlap_count`.") + msg = "Can only normalize with method=`overlap_count`." + raise ValueError(msg) - if not all(isinstance(val, cabc.Set) for val in reference_markers.values()): + if not all(isinstance(val, AbstractSet) for val in reference_markers.values()): try: reference_markers = { key: set(val) for key, val in reference_markers.items() } except Exception: - raise ValueError( + msg = ( "Please ensure that `reference_markers` contains " "sets or lists of markers as values." ) + raise ValueError(msg) if adj_pval_threshold is not None: if "pvals_adj" not in adata.uns[key]: - raise ValueError( + msg = ( "Could not find adjusted p-value data. " "Please run `sc.tl.rank_genes_groups()` with a " "method that outputs adjusted p-values." ) + raise ValueError(msg) if adj_pval_threshold < 0: logg.warning( diff --git a/src/scanpy/tools/_paga.py b/src/scanpy/tools/_paga.py index 98f0ac622b..b7f1e86e5d 100644 --- a/src/scanpy/tools/_paga.py +++ b/src/scanpy/tools/_paga.py @@ -107,21 +107,22 @@ def paga( """ check_neighbors = "neighbors" if neighbors_key is None else neighbors_key if check_neighbors not in adata.uns: - raise ValueError( - "You need to run `pp.neighbors` first to compute a neighborhood graph." - ) + msg = "You need to run `pp.neighbors` first to compute a neighborhood graph." + raise ValueError(msg) if groups is None: for k in ("leiden", "louvain"): if k in adata.obs.columns: groups = k break if groups is None: - raise ValueError( + msg = ( "You need to run `tl.leiden` or `tl.louvain` to compute " "community labels, or specify `groups='an_existing_key'`" ) + raise ValueError(msg) elif groups not in adata.obs.columns: - raise KeyError(f"`groups` key {groups!r} not found in `adata.obs`.") + msg = f"`groups` key {groups!r} not found in `adata.obs`." + raise KeyError(msg) adata = adata.copy() if copy else adata _utils.sanitize_anndata(adata) @@ -170,9 +171,8 @@ def compute_connectivities(self): elif self._model == "v1.0": return self._compute_connectivities_v1_0() else: - raise ValueError( - f"`model` {self._model} needs to be one of {_AVAIL_MODELS}." - ) + msg = f"`model` {self._model} needs to be one of {_AVAIL_MODELS}." + raise ValueError(msg) def _compute_connectivities_v1_2(self): import igraph @@ -196,10 +196,7 @@ def _compute_connectivities_v1_2(self): inter_es = inter_es.tocoo() for i, j, v in zip(inter_es.row, inter_es.col, inter_es.data): expected_random_null = (es[i] * ns[j] + es[j] * ns[i]) / (n - 1) - if expected_random_null != 0: - scaled_value = v / expected_random_null - else: - scaled_value = 1 + scaled_value = v / expected_random_null if expected_random_null != 0 else 1 if scaled_value > 1: scaled_value = 1 connectivities[i, j] = scaled_value @@ -229,10 +226,7 @@ def _compute_connectivities_v1_0(self): for i, j, v in zip(inter_es.row, inter_es.col, inter_es.data): # have n_neighbors**2 inside sqrt for backwards compat geom_mean_approx_knn = np.sqrt(n_neighbors_sq * ns[i] * ns[j]) - if geom_mean_approx_knn != 0: - scaled_value = v / geom_mean_approx_knn - else: - scaled_value = 1 + scaled_value = v / geom_mean_approx_knn if geom_mean_approx_knn != 0 else 1 connectivities[i, j] = scaled_value # set attributes self.ns = ns @@ -279,15 +273,17 @@ def compute_transitions(self): "The key 'velocyto_transitions' has been changed to 'velocity_graph'." ) else: - raise ValueError( + msg = ( "The passed AnnData needs to have an `uns` annotation " "with key 'velocity_graph' - a sparse matrix from RNA velocity." ) + raise ValueError(msg) if self._adata.uns[vkey].shape != (self._adata.n_obs, self._adata.n_obs): - raise ValueError( + msg = ( f"The passed 'velocity_graph' have shape {self._adata.uns[vkey].shape} " f"but shoud have shape {(self._adata.n_obs, self._adata.n_obs)}" ) + raise ValueError(msg) # restore this at some point # if 'expected_n_edges_random' not in self._adata.uns['paga']: # raise ValueError( diff --git a/src/scanpy/tools/_rank_genes_groups.py b/src/scanpy/tools/_rank_genes_groups.py index 1fc2727091..05e5738d99 100644 --- a/src/scanpy/tools/_rank_genes_groups.py +++ b/src/scanpy/tools/_rank_genes_groups.py @@ -2,8 +2,7 @@ from __future__ import annotations -from math import floor -from typing import TYPE_CHECKING, Literal, get_args +from typing import TYPE_CHECKING, Literal import numpy as np import pandas as pd @@ -14,6 +13,7 @@ from .._compat import old_positionals from .._utils import ( check_nonnegative_integers, + get_literal_vals, raise_not_implemented_error_if_backed_type, ) from ..get import _check_mask @@ -28,9 +28,11 @@ _CorrMethod = Literal["benjamini-hochberg", "bonferroni"] -# Used with get_args +# Used with get_literal_vals _Method = Literal["logreg", "t-test", "wilcoxon", "t-test_overestim_var"] +_CONST_MAX_SIZE = 10000000 + def _select_top_n(scores: NDArray, n_top: int): n_from = scores.shape[0] @@ -46,9 +48,7 @@ def _ranks( X: np.ndarray | sparse.csr_matrix | sparse.csc_matrix, mask_obs: NDArray[np.bool_] | None = None, mask_obs_rest: NDArray[np.bool_] | None = None, -): - CONST_MAX_SIZE = 10000000 - +) -> Generator[tuple[pd.DataFrame, int, int], None, None]: n_genes = X.shape[1] if issparse(X): @@ -70,7 +70,7 @@ def _ranks( get_chunk = lambda X, left, right: adapt(X[:, left:right]) # Calculate chunk frames - max_chunk = floor(CONST_MAX_SIZE / n_cells) + max_chunk = max(_CONST_MAX_SIZE // n_cells, 1) for left in range(0, n_genes, max_chunk): right = min(left + max_chunk, n_genes) @@ -80,7 +80,7 @@ def _ranks( yield ranks, left, right -def _tiecorrect(ranks): +def _tiecorrect(ranks: pd.DataFrame) -> np.float64: size = np.float64(ranks.shape[0]) if size < 2: return np.repeat(ranks.shape[1], 1.0) @@ -98,7 +98,7 @@ class _RankGenes: def __init__( self, adata: AnnData, - groups: list[str] | Literal["all"], + groups: Iterable[str] | Literal["all"], groupby: str, *, mask_var: NDArray[np.bool_] | None = None, @@ -123,15 +123,17 @@ def __init__( ) if len(invalid_groups_selected) > 0: - raise ValueError( - "Could not calculate statistics for groups {} since they only " - "contain one sample.".format(", ".join(invalid_groups_selected)) + msg = ( + f"Could not calculate statistics for groups {', '.join(invalid_groups_selected)} " + "since they only contain one sample." ) + raise ValueError(msg) adata_comp = adata if layer is not None: if use_raw: - raise ValueError("Cannot specify `layer` and have `use_raw=True`.") + msg = "Cannot specify `layer` and have `use_raw=True`." + raise ValueError(msg) X = adata_comp.layers[layer] else: if use_raw and adata.raw is not None: @@ -252,7 +254,8 @@ def t_test( # hack for overestimating the variance for small groups ns_rest = ns_group else: - raise ValueError("Method does not exist.") + msg = "Method does not exist." + raise ValueError(msg) # TODO: Come up with better solution. Mask unexpressed genes? # See https://github.com/scipy/scipy/issues/10269 @@ -275,7 +278,7 @@ def t_test( yield group_index, scores, pvals def wilcoxon( - self, tie_correct: bool + self, *, tie_correct: bool ) -> Generator[tuple[int, NDArray[np.floating], NDArray[np.floating]], None, None]: from scipy import stats @@ -287,10 +290,7 @@ def wilcoxon( # initialize space for z-scores scores = np.zeros(n_genes) # initialize space for tie correction coefficients - if tie_correct: - T = np.zeros(n_genes) - else: - T = 1 + T = np.zeros(n_genes) if tie_correct else 1 for group_index, mask_obs in enumerate(self.groups_masks_obs): if group_index == self.ireference: @@ -346,10 +346,7 @@ def wilcoxon( for group_index, mask_obs in enumerate(self.groups_masks_obs): n_active = np.count_nonzero(mask_obs) - if tie_correct: - T_i = T[group_index] - else: - T_i = 1 + T_i = T[group_index] if tie_correct else 1 std_dev = np.sqrt( T_i * n_active * (n_cells - n_active) * (n_cells + 1) / 12.0 @@ -374,7 +371,8 @@ def logreg( X = self.X[self.grouping_mask.values, :] if len(self.groups_order) == 1: - raise ValueError("Cannot perform logistic regression on a single cluster.") + msg = "Cannot perform logistic regression on a single cluster." + raise ValueError(msg) clf = LogisticRegression(**kwds) clf.fit(X, self.grouping.cat.codes) @@ -408,7 +406,7 @@ def compute_statistics( if method in {"t-test", "t-test_overestim_var"}: generate_test_results = self.t_test(method) elif method == "wilcoxon": - generate_test_results = self.wilcoxon(tie_correct) + generate_test_results = self.wilcoxon(tie_correct=tie_correct) elif method == "logreg": generate_test_results = self.logreg(**kwds) @@ -588,7 +586,7 @@ def rank_genes_groups( Notes ----- There are slight inconsistencies depending on whether sparse - or dense data are passed. See `here `__. + or dense data are passed. See `here `__. Examples -------- @@ -598,13 +596,13 @@ def rank_genes_groups( >>> # to visualize the results >>> sc.pl.rank_genes_groups(adata) """ - if mask_var is not None: - mask_var = _check_mask(adata, mask_var, "var") + mask_var = _check_mask(adata, mask_var, "var") if use_raw is None: use_raw = adata.raw is not None elif use_raw is True and adata.raw is None: - raise ValueError("Received `use_raw=True`, but `adata.raw` is empty.") + msg = "Received `use_raw=True`, but `adata.raw` is empty." + raise ValueError(msg) if method is None: method = "t-test" @@ -613,21 +611,23 @@ def rank_genes_groups( rankby_abs = not kwds.pop("only_positive") # backwards compat start = logg.info("ranking genes") - avail_methods = set(get_args(_Method)) - if method not in avail_methods: - raise ValueError(f"Method must be one of {avail_methods}.") + if method not in (avail_methods := get_literal_vals(_Method)): + msg = f"Method must be one of {avail_methods}." + raise ValueError(msg) avail_corr = {"benjamini-hochberg", "bonferroni"} if corr_method not in avail_corr: - raise ValueError(f"Correction method must be one of {avail_corr}.") + msg = f"Correction method must be one of {avail_corr}." + raise ValueError(msg) adata = adata.copy() if copy else adata _utils.sanitize_anndata(adata) # for clarity, rename variable if groups == "all": groups_order = "all" - elif isinstance(groups, (str, int)): - raise ValueError("Specify a sequence of groups") + elif isinstance(groups, str | int): + msg = "Specify a sequence of groups" + raise ValueError(msg) else: groups_order = list(groups) if isinstance(groups_order[0], int): @@ -636,9 +636,8 @@ def rank_genes_groups( groups_order += [reference] if reference != "rest" and reference not in adata.obs[groupby].cat.categories: cats = adata.obs[groupby].cat.categories.tolist() - raise ValueError( - f"reference = {reference} needs to be one of groupby = {cats}." - ) + msg = f"reference = {reference} needs to be one of groupby = {cats}." + raise ValueError(msg) if key_added is None: key_added = "rank_genes_groups" @@ -733,10 +732,7 @@ def rank_genes_groups( def _calc_frac(X): - if issparse(X): - n_nonzero = X.getnnz(axis=0) - else: - n_nonzero = np.count_nonzero(X, axis=0) + n_nonzero = X.getnnz(axis=0) if issparse(X) else np.count_nonzero(X, axis=0) return n_nonzero / X.shape[0] @@ -758,7 +754,7 @@ def filter_rank_genes_groups( use_raw: bool | None = None, key_added: str = "rank_genes_groups_filtered", min_in_group_fraction: float = 0.25, - min_fold_change: int | float = 1, + min_fold_change: float = 1, max_out_group_fraction: float = 0.5, compare_abs: bool = False, ) -> None: @@ -862,7 +858,7 @@ def filter_rank_genes_groups( if not use_logfolds or not use_fraction: sub_X = adata.raw[:, var_names].X if use_raw else adata[:, var_names].X - in_group = adata.obs[groupby] == cluster + in_group = (adata.obs[groupby] == cluster).to_numpy() X_in = sub_X[in_group] X_out = sub_X[~in_group] diff --git a/src/scanpy/tools/_score_genes.py b/src/scanpy/tools/_score_genes.py index a743abff37..7dff1a300c 100644 --- a/src/scanpy/tools/_score_genes.py +++ b/src/scanpy/tools/_score_genes.py @@ -15,14 +15,20 @@ from ..get import _get_obs_rep if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Callable, Generator, Sequence from typing import Literal from anndata import AnnData from numpy.typing import DTypeLike, NDArray from scipy.sparse import csc_matrix, csr_matrix - from .._utils import AnyRandom + from .._compat import _LegacyRandom + + try: + _StrIdx = pd.Index[str] + except TypeError: # Sphinx + _StrIdx = pd.Index + _GetSubset = Callable[[_StrIdx], np.ndarray | csr_matrix | csc_matrix] def _sparse_nanmean( @@ -32,7 +38,8 @@ def _sparse_nanmean( np.nanmean equivalent for sparse matrices """ if not issparse(X): - raise TypeError("X must be a sparse matrix") + msg = "X must be a sparse matrix" + raise TypeError(msg) # count the number of nan elements per row/column (dep. on axis) Z = X.copy() @@ -59,11 +66,12 @@ def score_genes( adata: AnnData, gene_list: Sequence[str] | pd.Index[str], *, + ctrl_as_ref: bool = True, ctrl_size: int = 50, gene_pool: Sequence[str] | pd.Index[str] | None = None, n_bins: int = 25, score_name: str = "score", - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, copy: bool = False, use_raw: bool | None = None, layer: str | None = None, @@ -71,8 +79,8 @@ def score_genes( """\ Score a set of genes :cite:p:`Satija2015`. - The score is the average expression of a set of genes subtracted with the - average expression of a reference set of genes. The reference set is + The score is the average expression of a set of genes after subtraction by + the average expression of a reference set of genes. The reference set is randomly sampled from the `gene_pool` for each binned expression value. This reproduces the approach in Seurat :cite:p:`Satija2015` and has been implemented @@ -84,6 +92,9 @@ def score_genes( The annotated data matrix. gene_list The list of gene names used for score calculation. + ctrl_as_ref + Allow the algorithm to use the control genes as reference. + Will be changed to `False` in scanpy 2.0. ctrl_size Number of reference genes to be sampled from each bin. If `len(gene_list)` is not too low, you can set `ctrl_size=len(gene_list)`. @@ -118,15 +129,74 @@ def score_genes( """ start = logg.info(f"computing score {score_name!r}") adata = adata.copy() if copy else adata - use_raw = _check_use_raw(adata, use_raw) + use_raw = _check_use_raw(adata, use_raw, layer=layer) if is_backed_type(adata.X) and not use_raw: - raise NotImplementedError( - f"score_genes is not implemented for matrices of type {type(adata.X)}" - ) + msg = f"score_genes is not implemented for matrices of type {type(adata.X)}" + raise NotImplementedError(msg) if random_state is not None: np.random.seed(random_state) + gene_list, gene_pool, get_subset = _check_score_genes_args( + adata, gene_list, gene_pool, use_raw=use_raw, layer=layer + ) + del use_raw, layer, random_state + + # Trying here to match the Seurat approach in scoring cells. + # Basically we need to compare genes against random genes in a matched + # interval of expression. + + control_genes = pd.Index([], dtype="string") + for r_genes in _score_genes_bins( + gene_list, + gene_pool, + ctrl_as_ref=ctrl_as_ref, + ctrl_size=ctrl_size, + n_bins=n_bins, + get_subset=get_subset, + ): + control_genes = control_genes.union(r_genes) + + if len(control_genes) == 0: + msg = "No control genes found in any cut." + if ctrl_as_ref: + msg += " Try setting `ctrl_as_ref=False`." + raise RuntimeError(msg) + + means_list, means_control = ( + _nan_means(get_subset(genes), axis=1, dtype="float64") + for genes in (gene_list, control_genes) + ) + score = means_list - means_control + + adata.obs[score_name] = pd.Series( + np.array(score).ravel(), index=adata.obs_names, dtype="float64" + ) + + logg.info( + " finished", + time=start, + deep=( + "added\n" + f" {score_name!r}, score of gene set (adata.obs).\n" + f" {len(control_genes)} total control genes are used." + ), + ) + return adata if copy else None + + +def _check_score_genes_args( + adata: AnnData, + gene_list: pd.Index[str] | Sequence[str], + gene_pool: pd.Index[str] | Sequence[str] | None, + *, + layer: str | None, + use_raw: bool, +) -> tuple[pd.Index[str], pd.Index[str], _GetSubset]: + """Restrict `gene_list` and `gene_pool` to present genes in `adata`. + + Also returns a function to get subset of `adata.X` based on a set of genes passed. + """ var_names = adata.raw.var_names if use_raw else adata.var_names gene_list = pd.Index([gene_list] if isinstance(gene_list, str) else gene_list) genes_to_ignore = gene_list.difference(var_names, sort=False) # first get missing @@ -134,18 +204,16 @@ def score_genes( if len(genes_to_ignore) > 0: logg.warning(f"genes are not in var_names and ignored: {genes_to_ignore}") if len(gene_list) == 0: - raise ValueError("No valid genes were passed for scoring.") + msg = "No valid genes were passed for scoring." + raise ValueError(msg) if gene_pool is None: - gene_pool = pd.Index(var_names, dtype="string") + gene_pool = var_names.astype("string") else: gene_pool = pd.Index(gene_pool, dtype="string").intersection(var_names) if len(gene_pool) == 0: - raise ValueError("No valid genes were passed for reference set.") - - # Trying here to match the Seurat approach in scoring cells. - # Basically we need to compare genes against random genes in a matched - # interval of expression. + msg = "No valid genes were passed for reference set." + raise ValueError(msg) def get_subset(genes: pd.Index[str]): x = _get_obs_rep(adata, use_raw=use_raw, layer=layer) @@ -154,6 +222,18 @@ def get_subset(genes: pd.Index[str]): idx = var_names.get_indexer(genes) return x[:, idx] + return gene_list, gene_pool, get_subset + + +def _score_genes_bins( + gene_list: pd.Index[str], + gene_pool: pd.Index[str], + *, + ctrl_as_ref: bool, + ctrl_size: int, + n_bins: int, + get_subset: _GetSubset, +) -> Generator[pd.Index[str], None, None]: # average expression of genes obs_avg = pd.Series(_nan_means(get_subset(gene_pool), axis=0), index=gene_pool) # Sometimes (and I don’t know how) missing data may be there, with NaNs for missing entries @@ -161,35 +241,22 @@ def get_subset(genes: pd.Index[str]): n_items = int(np.round(len(obs_avg) / (n_bins - 1))) obs_cut = obs_avg.rank(method="min") // n_items - control_genes = pd.Index([], dtype="string") + keep_ctrl_in_obs_cut = False if ctrl_as_ref else obs_cut.index.isin(gene_list) # now pick `ctrl_size` genes from every cut for cut in np.unique(obs_cut.loc[gene_list]): - r_genes: pd.Index[str] = obs_cut[obs_cut == cut].index + r_genes: pd.Index[str] = obs_cut[(obs_cut == cut) & ~keep_ctrl_in_obs_cut].index + if len(r_genes) == 0: + msg = ( + f"No control genes for {cut=}. You might want to increase " + f"gene_pool size (current size: {len(gene_pool)})" + ) + logg.warning(msg) if ctrl_size < len(r_genes): r_genes = r_genes.to_series().sample(ctrl_size).index - control_genes = control_genes.union(r_genes.difference(gene_list)) - - means_list, means_control = ( - _nan_means(get_subset(genes), axis=1, dtype="float64") - for genes in (gene_list, control_genes) - ) - score = means_list - means_control - - adata.obs[score_name] = pd.Series( - np.array(score).ravel(), index=adata.obs_names, dtype="float64" - ) - - logg.info( - " finished", - time=start, - deep=( - "added\n" - f" {score_name!r}, score of gene set (adata.obs).\n" - f" {len(control_genes)} total control genes are used." - ), - ) - return adata if copy else None + if ctrl_as_ref: # otherwise `r_genes` is already filtered + r_genes = r_genes.difference(gene_list) + yield r_genes def _nan_means( diff --git a/src/scanpy/tools/_sim.py b/src/scanpy/tools/_sim.py index bf562b8408..f6ea2fede8 100644 --- a/src/scanpy/tools/_sim.py +++ b/src/scanpy/tools/_sim.py @@ -62,7 +62,7 @@ def sim( Sample from a stochastic differential equation model built from literature-curated boolean gene regulatory networks, as suggested by - :cite:t:`Wittmann2009`. The Scanpy implementation is due to :cite:t:`Wolf2018`. + :cite:t:`Wittmann2009`. The Scanpy implementation can be found in :cite:t:`Wolf2018`. Parameters ---------- @@ -120,7 +120,7 @@ def add_args(p): "default": "", "metavar": "f", "type": str, - "help": "Specify a parameter file " '(default: "sim/${exkey}_params.txt")', + "help": 'Specify a parameter file (default: "sim/${exkey}_params.txt")', } } p = _utils.add_args(p, dadd_args) @@ -198,7 +198,7 @@ def sample_dynamic_data(**params): X[::step], dir=writedir, noiseObs=noiseObs, - append=(False if restart == 0 else True), + append=restart != 0, branching=branching, nrRealizations=nrRealizations, ) @@ -208,7 +208,7 @@ def sample_dynamic_data(**params): noiseDyn * np.random.randn(500, 3), dir=writedir, noiseObs=noiseObs, - append=(False if restart == 0 else True), + append=restart != 0, branching=branching, nrRealizations=nrRealizations, ) @@ -216,7 +216,7 @@ def sample_dynamic_data(**params): break logg.debug( f"mean nr of offdiagonal edges {nrOffEdges_list.mean()} " - f"compared to total nr {grnsim.dim * (grnsim.dim - 1) / 2.}" + f"compared to total nr {grnsim.dim * (grnsim.dim - 1) / 2.0}" ) # more complex models @@ -270,7 +270,7 @@ def sample_dynamic_data(**params): X[::step], dir=writedir, noiseObs=noiseObs, - append=(False if restart == 0 else True), + append=restart != 0, branching=branching, nrRealizations=nrRealizations, ) @@ -358,16 +358,14 @@ def write_data( for g in range(dim): if np.abs(Coupl[gp, g]) > 1e-10: f.write( - f"{names[gp]:10} " - f"{names[g]:10} " - f"{Coupl[gp, g]:10.3} \n" + f"{names[gp]:10} {names[g]:10} {Coupl[gp, g]:10.3} \n" ) # write simulated data # the binary mode option in the following line is a fix for python 3 # variable names if varNames: - header += f'{"it":>2} ' - for v in varNames.keys(): + header += f"{'it':>2} " + for v in varNames: header += f"{v:>7} " with (dir / f"sim_{id}.txt").open("ab" if append else "wb") as f: np.savetxt( @@ -429,15 +427,17 @@ def __init__( self.verbosity = verbosity # checks if initType not in ["branch", "random"]: - raise RuntimeError("initType must be either: branch, random") - if model not in self.availModels.keys(): + msg = "initType must be either: branch, random" + raise RuntimeError(msg) + if model not in self.availModels: message = "model not among predefined models \n" # noqa: F841 # TODO FIX # read from file from .. import sim_models model = Path(sim_models.__file__).parent / f"{model}.txt" if not model.is_file(): - raise RuntimeError(f"Model file {model} does not exist") + msg = f"Model file {model} does not exist" + raise RuntimeError(msg) self.model = model # set the coupling matrix, and with that the adjacency matrix self.set_coupl(Coupl=Coupl) @@ -461,7 +461,8 @@ def sim_model(self, tmax, X0, noiseDyn=0, restart=0): elif self.modelType == "var": Xdiff = self.Xdiff_var(X[t - 1]) else: - raise ValueError(f"Unknown modelType {self.modelType!r}") + msg = f"Unknown modelType {self.modelType!r}" + raise ValueError(msg) X[t] = X[t - 1] + Xdiff # add dynamic noise X[t] += noiseDyn * np.random.randn(self.dim) @@ -501,7 +502,7 @@ def Xdiff_hill(self, Xt): ) if verbosity > 0: Xdiff_syn_tuple_str += ( - f'{"a" if v else "i"}' + f"{'a' if v else 'i'}" f"({self.pas[child][iv]}, {threshold:.2})" ) Xdiff_syn += Xdiff_syn_tuple @@ -605,12 +606,12 @@ def set_coupl(self, Coupl=None): or via sampling. """ self.varNames = {str(i): i for i in range(self.dim)} - if self.model not in self.availModels.keys() and Coupl is None: + if self.model not in self.availModels and Coupl is None: self.read_model() elif "var" in self.model.name: # vector auto regressive process self.Coupl = Coupl - self.boolRules = {s: "" for s in self.varNames.keys()} + self.boolRules = {s: "" for s in self.varNames} names = list(self.varNames.keys()) for gp in range(self.dim): pas = [] @@ -819,7 +820,7 @@ def parents_from_boolRule(self, rule): pa_old = [] pa_delete = [] for pa in rule_pa: - if pa not in self.varNames.keys(): + if pa not in self.varNames: settings.m(0, "list of available variables:") settings.m(0, list(self.varNames.keys())) message = ( @@ -842,24 +843,23 @@ def parents_from_boolRule(self, rule): def build_boolCoeff(self): """Compute coefficients for tuple space.""" # coefficients for hill functions from boolean update rules - self.boolCoeff = {s: [] for s in self.varNames.keys()} + self.boolCoeff = {s: [] for s in self.varNames} # parents - self.pas = {s: [] for s in self.varNames.keys()} + self.pas = {s: [] for s in self.varNames} # - for key in self.boolRules.keys(): - rule = self.boolRules[key] + for key, rule in self.boolRules.items(): self.pas[key] = self.parents_from_boolRule(rule) pasIndices = [self.varNames[pa] for pa in self.pas[key]] # check whether there are coupling matrix entries for each parent for g in range(self.dim): if g in pasIndices: if np.abs(self.Coupl[self.varNames[key], g]) < 1e-10: - raise ValueError(f"specify coupling value for {key} <- {g}") + msg = f"specify coupling value for {key} <- {g}" + raise ValueError(msg) else: if np.abs(self.Coupl[self.varNames[key], g]) > 1e-10: - raise ValueError( - "there should be no coupling value for " f"{key} <- {g}" - ) + msg = f"there should be no coupling value for {key} <- {g}" + raise ValueError(msg) if self.verbosity > 1: settings.m(0, "..." + key) settings.m(0, rule) @@ -958,7 +958,7 @@ def _check_branching( check = False if check: Xsamples.append(X) - logg.debug(f'realization {restart}: {"" if check else "no"} new branch') + logg.debug(f"realization {restart}: {'' if check else 'no'} new branch") return check, Xsamples @@ -1048,9 +1048,8 @@ def sample_coupling_matrix( check = True break if not check: - raise ValueError( - "did not find graph without cycles after" f"{max_trial} trials" - ) + msg = f"did not find graph without cycles after {max_trial} trials" + raise ValueError(msg) return Coupl, Adj, Adj_signed, n_edges @@ -1150,11 +1149,10 @@ def sim_givenAdj(self, Adj: np.ndarray, model="line"): # if there is more than a child with a single parent # order these children (there are two in three dim) # by distance to the source/parent - if nrchildren_par[1] > 1: - if Adj[children_sorted[0], parents[0]] == 0: - help = children_sorted[0] - children_sorted[0] = children_sorted[1] - children_sorted[1] = help + if nrchildren_par[1] > 1 and Adj[children_sorted[0], parents[0]] == 0: + help = children_sorted[0] + children_sorted[0] = children_sorted[1] + children_sorted[1] = help for gp in children_sorted: for g in range(dim): diff --git a/src/scanpy/tools/_top_genes.py b/src/scanpy/tools/_top_genes.py index 00dc764d82..3b4e709b5f 100644 --- a/src/scanpy/tools/_top_genes.py +++ b/src/scanpy/tools/_top_genes.py @@ -38,7 +38,7 @@ def correlation_matrix( """\ Calculate correlation matrix. - Calculate a correlation matrix for genes strored in sample annotation + Calculate a correlation matrix for genes stored in sample annotation using :func:`~scanpy.tl.rank_genes_groups`. Parameters @@ -73,7 +73,7 @@ def correlation_matrix( spearman Spearman rank correlation annotation_key - Allows to define the name of the anndata entry where results are stored. + Allows defining the name of the anndata entry where results are stored. """ # TODO: At the moment, only works for int identifiers @@ -181,10 +181,7 @@ def ROC_AUC_analysis( y_true = mask for i, j in enumerate(name_list): vec = adata[:, [j]].X - if issparse(vec): - y_score = vec.todense() - else: - y_score = vec + y_score = vec.todense() if issparse(vec) else vec ( fpr[name_list[i]], diff --git a/src/scanpy/tools/_tsne.py b/src/scanpy/tools/_tsne.py index 1e3ade92e6..62fa8b9d57 100644 --- a/src/scanpy/tools/_tsne.py +++ b/src/scanpy/tools/_tsne.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from anndata import AnnData - from .._utils import AnyRandom + from .._compat import _LegacyRandom @old_positionals( @@ -34,19 +34,20 @@ def tsne( n_pcs: int | None = None, *, use_rep: str | None = None, - perplexity: float | int = 30, - early_exaggeration: float | int = 12, - learning_rate: float | int = 1000, - random_state: AnyRandom = 0, + perplexity: float = 30, + metric: str = "euclidean", + early_exaggeration: float = 12, + learning_rate: float = 1000, + random_state: _LegacyRandom = 0, use_fast_tsne: bool = False, n_jobs: int | None = None, + key_added: str | None = None, copy: bool = False, - metric: str = "euclidean", ) -> AnnData | None: """\ t-SNE :cite:p:`vanDerMaaten2008,Amir2013,Pedregosa2011`. - t-distributed stochastic neighborhood embedding (tSNE, :cite:t:`vanDerMaaten2008`) has been + t-distributed stochastic neighborhood embedding (tSNE, :cite:t:`vanDerMaaten2008`) was proposed for visualizating single-cell data by :cite:t:`Amir2013`. Here, by default, we use the implementation of *scikit-learn* :cite:p:`Pedregosa2011`. You can achieve a huge speedup and better convergence if you install Multicore-tSNE_ @@ -88,6 +89,13 @@ def tsne( n_jobs Number of jobs for parallel computation. `None` means using :attr:`scanpy._settings.ScanpyConfig.n_jobs`. + key_added + If not specified, the embedding is stored as + :attr:`~anndata.AnnData.obsm`\\ `['X_tsne']` and the the parameters in + :attr:`~anndata.AnnData.uns`\\ `['tsne']`. + If specified, the embedding is stored as + :attr:`~anndata.AnnData.obsm`\\ ``[key_added]`` and the the parameters in + :attr:`~anndata.AnnData.uns`\\ ``[key_added]``. copy Return a copy instead of writing to `adata`. @@ -95,9 +103,9 @@ def tsne( ------- Returns `None` if `copy=False`, else returns an `AnnData` object. Sets the following fields: - `adata.obsm['X_tsne']` : :class:`numpy.ndarray` (dtype `float`) + `adata.obsm['X_tsne' | key_added]` : :class:`numpy.ndarray` (dtype `float`) tSNE coordinates of data. - `adata.uns['tsne']` : :class:`dict` + `adata.uns['tsne' | key_added]` : :class:`dict` tSNE parameters. """ @@ -165,26 +173,26 @@ def tsne( X_tsne = tsne.fit_transform(X) # update AnnData instance - adata.obsm["X_tsne"] = X_tsne # annotate samples with tSNE coordinates - adata.uns["tsne"] = { - "params": { - k: v - for k, v in { - "perplexity": perplexity, - "early_exaggeration": early_exaggeration, - "learning_rate": learning_rate, - "n_jobs": n_jobs, - "metric": metric, - "use_rep": use_rep, - }.items() - if v is not None - } - } + params = dict( + perplexity=perplexity, + early_exaggeration=early_exaggeration, + learning_rate=learning_rate, + n_jobs=n_jobs, + metric=metric, + use_rep=use_rep, + ) + key_uns, key_obsm = ("tsne", "X_tsne") if key_added is None else [key_added] * 2 + adata.obsm[key_obsm] = X_tsne # annotate samples with tSNE coordinates + adata.uns[key_uns] = dict(params={k: v for k, v in params.items() if v is not None}) logg.info( " finished", time=start, - deep="added\n 'X_tsne', tSNE coordinates (adata.obsm)", + deep=( + f"added\n" + f" {key_obsm!r}, tSNE coordinates (adata.obsm)\n" + f" {key_uns!r}, tSNE parameters (adata.uns)" + ), ) return adata if copy else None diff --git a/src/scanpy/tools/_umap.py b/src/scanpy/tools/_umap.py index 8f8c555ee5..926e6d3d4f 100644 --- a/src/scanpy/tools/_umap.py +++ b/src/scanpy/tools/_umap.py @@ -17,7 +17,7 @@ from anndata import AnnData - from .._utils import AnyRandom + from .._compat import _LegacyRandom _InitPos = Literal["paga", "spectral", "random"] @@ -49,12 +49,13 @@ def umap( gamma: float = 1.0, negative_sample_rate: int = 5, init_pos: _InitPos | np.ndarray | None = "spectral", - random_state: AnyRandom = 0, + random_state: _LegacyRandom = 0, a: float | None = None, b: float | None = None, - copy: bool = False, method: Literal["umap", "rapids"] = "umap", - neighbors_key: str | None = None, + key_added: str | None = None, + neighbors_key: str = "neighbors", + copy: bool = False, ) -> AnnData | None: """\ Embed the neighborhood graph using UMAP :cite:p:`McInnes2018`. @@ -122,8 +123,6 @@ def umap( More specific parameters controlling the embedding. If `None` these values are set automatically as determined by `min_dist` and `spread`. - copy - Return a copy instead of writing to adata. method Chosen implementation. @@ -134,32 +133,40 @@ def umap( .. deprecated:: 1.10.0 Use :func:`rapids_singlecell.tl.umap` instead. + key_added + If not specified, the embedding is stored as + :attr:`~anndata.AnnData.obsm`\\ `['X_umap']` and the the parameters in + :attr:`~anndata.AnnData.uns`\\ `['umap']`. + If specified, the embedding is stored as + :attr:`~anndata.AnnData.obsm`\\ ``[key_added]`` and the the parameters in + :attr:`~anndata.AnnData.uns`\\ ``[key_added]``. neighbors_key - If not specified, umap looks .uns['neighbors'] for neighbors settings - and .obsp['connectivities'] for connectivities - (default storage places for pp.neighbors). - If specified, umap looks .uns[neighbors_key] for neighbors settings and - .obsp[.uns[neighbors_key]['connectivities_key']] for connectivities. + Umap looks in + :attr:`~anndata.AnnData.uns`\\ ``[neighbors_key]`` for neighbors settings and + :attr:`~anndata.AnnData.obsp`\\ ``[.uns[neighbors_key]['connectivities_key']]`` for connectivities. + copy + Return a copy instead of writing to adata. Returns ------- Returns `None` if `copy=False`, else returns an `AnnData` object. Sets the following fields: - `adata.obsm['X_umap']` : :class:`numpy.ndarray` (dtype `float`) + `adata.obsm['X_umap' | key_added]` : :class:`numpy.ndarray` (dtype `float`) UMAP coordinates of data. - `adata.uns['umap']` : :class:`dict` + `adata.uns['umap' | key_added]` : :class:`dict` UMAP parameters. """ adata = adata.copy() if copy else adata - if neighbors_key is None: - neighbors_key = "neighbors" + key_obsm, key_uns = ("X_umap", "umap") if key_added is None else [key_added] * 2 + if neighbors_key is None: # backwards compat + neighbors_key = "neighbors" if neighbors_key not in adata.uns: - raise ValueError( - f"Did not find .uns[{neighbors_key!r}]. Run `sc.pp.neighbors` first." - ) + msg = f"Did not find .uns[{neighbors_key!r}]. Run `sc.pp.neighbors` first." + raise ValueError(msg) + start = logg.info("computing UMAP") neighbors = NeighborsView(adata, neighbors_key) @@ -178,11 +185,8 @@ def umap( if a is None or b is None: a, b = find_ab_params(spread, min_dist) - else: - a = a - b = b - adata.uns["umap"] = {"params": {"a": a, "b": b}} - if isinstance(init_pos, str) and init_pos in adata.obsm.keys(): + adata.uns[key_uns] = dict(params=dict(a=a, b=b)) + if isinstance(init_pos, str) and init_pos in adata.obsm: init_coords = adata.obsm[init_pos] elif isinstance(init_pos, str) and init_pos == "paga": init_coords = get_init_pos_from_paga( @@ -194,7 +198,7 @@ def umap( init_coords = check_array(init_coords, dtype=np.float32, accept_sparse=False) if random_state != 0: - adata.uns["umap"]["params"]["random_state"] = random_state + adata.uns[key_uns]["params"]["random_state"] = random_state random_state = check_random_state(random_state) neigh_params = neighbors["params"] @@ -236,10 +240,11 @@ def umap( warnings.warn(msg, FutureWarning) metric = neigh_params.get("metric", "euclidean") if metric != "euclidean": - raise ValueError( + msg = ( f"`sc.pp.neighbors` was called with `metric` {metric!r}, " "but umap `method` 'rapids' only supports the 'euclidean' metric." ) + raise ValueError(msg) from cuml import UMAP n_neighbors = neighbors["params"]["n_neighbors"] @@ -262,10 +267,14 @@ def umap( random_state=random_state, ) X_umap = umap.fit_transform(X_contiguous) - adata.obsm["X_umap"] = X_umap # annotate samples with UMAP coordinates + adata.obsm[key_obsm] = X_umap # annotate samples with UMAP coordinates logg.info( " finished", time=start, - deep=("added\n" " 'X_umap', UMAP coordinates (adata.obsm)"), + deep=( + "added\n" + f" {key_obsm!r}, UMAP coordinates (adata.obsm)\n" + f" {key_uns!r}, UMAP parameters (adata.uns)" + ), ) return adata if copy else None diff --git a/src/scanpy/tools/_utils.py b/src/scanpy/tools/_utils.py index b3ffa7d324..4d24b5e276 100644 --- a/src/scanpy/tools/_utils.py +++ b/src/scanpy/tools/_utils.py @@ -30,11 +30,10 @@ def _choose_representation( use_rep = "X" if use_rep is None: if adata.n_vars > settings.N_PCS: - if "X_pca" in adata.obsm.keys(): + if "X_pca" in adata.obsm: if n_pcs is not None and n_pcs > adata.obsm["X_pca"].shape[1]: - raise ValueError( - "`X_pca` does not have enough PCs. Rerun `sc.pp.pca` with adjusted `n_comps`." - ) + msg = "`X_pca` does not have enough PCs. Rerun `sc.pp.pca` with adjusted `n_comps`." + raise ValueError(msg) X = adata.obsm["X_pca"][:, :n_pcs] logg.info(f" using 'X_pca' with n_pcs = {X.shape[1]}") else: @@ -50,23 +49,25 @@ def _choose_representation( logg.info(" using data matrix X directly") X = adata.X else: - if use_rep in adata.obsm.keys() and n_pcs is not None: + if use_rep in adata.obsm and n_pcs is not None: if n_pcs > adata.obsm[use_rep].shape[1]: - raise ValueError( + msg = ( f"{use_rep} does not have enough Dimensions. Provide a " "Representation with equal or more dimensions than" "`n_pcs` or lower `n_pcs` " ) + raise ValueError(msg) X = adata.obsm[use_rep][:, :n_pcs] - elif use_rep in adata.obsm.keys() and n_pcs is None: + elif use_rep in adata.obsm and n_pcs is None: X = adata.obsm[use_rep] elif use_rep == "X": X = adata.X else: - raise ValueError( + msg = ( f"Did not find {use_rep} in `.obsm.keys()`. " "You need to compute it first." ) + raise ValueError(msg) settings.verbosity = verbosity # resetting verbosity return X @@ -86,7 +87,7 @@ def preprocess_with_pca(adata, n_pcs: int | None = None, random_state=0): logg.info(" using data matrix X directly (no PCA)") return adata.X elif n_pcs is None and "X_pca" in adata.obsm_keys(): - logg.info(f' using \'X_pca\' with n_pcs = {adata.obsm["X_pca"].shape[1]}') + logg.info(f" using 'X_pca' with n_pcs = {adata.obsm['X_pca'].shape[1]}") return adata.obsm["X_pca"] elif "X_pca" in adata.obsm_keys() and adata.obsm["X_pca"].shape[1] >= n_pcs: logg.info(f" using 'X_pca' with n_pcs = {n_pcs}") @@ -128,5 +129,6 @@ def get_init_pos_from_paga( else: init_pos[subset] = group_pos else: - raise ValueError("Plot PAGA first, so that adata.uns['paga']" "with key 'pos'.") + msg = "Plot PAGA first, so that adata.uns['paga'] with key 'pos'." + raise ValueError(msg) return init_pos diff --git a/src/scanpy/tools/_utils_clustering.py b/src/scanpy/tools/_utils_clustering.py index 47f652fbdf..3c771e5d74 100644 --- a/src/scanpy/tools/_utils_clustering.py +++ b/src/scanpy/tools/_utils_clustering.py @@ -37,12 +37,12 @@ def restrict_adjacency( adjacency: spmatrix, ) -> tuple[spmatrix, NDArray[np.bool_]]: if not isinstance(restrict_categories[0], str): - raise ValueError( - "You need to use strings to label categories, " "e.g. '1' instead of 1." - ) + msg = "You need to use strings to label categories, e.g. '1' instead of 1." + raise ValueError(msg) for c in restrict_categories: if c not in adata.obs[restrict_key].cat.categories: - raise ValueError(f"'{c}' is not a valid category for '{restrict_key}'") + msg = f"{c!r} is not a valid category for {restrict_key!r}" + raise ValueError(msg) restrict_indices = adata.obs[restrict_key].isin(restrict_categories).values adjacency = adjacency[restrict_indices, :] adjacency = adjacency[:, restrict_indices] diff --git a/src/testing/scanpy/_helpers/__init__.py b/src/testing/scanpy/_helpers/__init__.py index b85faec3f3..3cff738132 100644 --- a/src/testing/scanpy/_helpers/__init__.py +++ b/src/testing/scanpy/_helpers/__init__.py @@ -5,13 +5,22 @@ from __future__ import annotations import warnings +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +from importlib.util import find_spec from itertools import permutations +from typing import TYPE_CHECKING import numpy as np from anndata.tests.helpers import asarray, assert_equal import scanpy as sc +if TYPE_CHECKING: + from collections.abc import MutableSequence + + from scanpy._compat import DaskArray + # TODO: Report more context on the fields being compared on error # TODO: Allow specifying paths to ignore on comparison @@ -124,13 +133,50 @@ def _check_check_values_warnings(function, adata, expected_warning, kwargs={}): # Delayed imports for case where we aren't using dask -def as_dense_dask_array(*args, **kwargs): +def as_dense_dask_array(*args, **kwargs) -> DaskArray: from anndata.tests.helpers import as_dense_dask_array return as_dense_dask_array(*args, **kwargs) -def as_sparse_dask_array(*args, **kwargs): +def as_sparse_dask_array(*args, **kwargs) -> DaskArray: from anndata.tests.helpers import as_sparse_dask_array return as_sparse_dask_array(*args, **kwargs) + + +@dataclass(init=False) +class MultiContext(AbstractContextManager): + contexts: MutableSequence[AbstractContextManager] + + def __init__(self, *contexts: AbstractContextManager): + self.contexts = list(contexts) + + def __enter__(self): + for ctx in self.contexts: + ctx.__enter__() + + def __exit__(self, exc_type, exc_value, traceback): + for ctx in reversed(self.contexts): + ctx.__exit__(exc_type, exc_value, traceback) + + +@contextmanager +def maybe_dask_process_context(): + """ + Running numba with dask's threaded scheduler causes crashes, + so we need to switch to single-threaded (or processes, which is slower) + scheduler for tests that use numba. + """ + if not find_spec("dask"): + yield + return + + import dask.config + + prev_scheduler = dask.config.get("scheduler", "threads") + dask.config.set(scheduler="single-threaded") + try: + yield + finally: + dask.config.set(scheduler=prev_scheduler) diff --git a/src/testing/scanpy/_helpers/data.py b/src/testing/scanpy/_helpers/data.py index 8b3fd491f1..d98f4e36c0 100644 --- a/src/testing/scanpy/_helpers/data.py +++ b/src/testing/scanpy/_helpers/data.py @@ -6,16 +6,7 @@ from __future__ import annotations import warnings - -try: - from functools import cache -except ImportError: # Python < 3.9 - from functools import lru_cache - - def cache(func): - return lru_cache(maxsize=None)(func) - - +from functools import cache from typing import TYPE_CHECKING import scanpy as sc diff --git a/src/testing/scanpy/_pytest/__init__.py b/src/testing/scanpy/_pytest/__init__.py index 0403be07d8..e365a90495 100644 --- a/src/testing/scanpy/_pytest/__init__.py +++ b/src/testing/scanpy/_pytest/__init__.py @@ -19,6 +19,7 @@ @pytest.fixture(autouse=True) def _global_test_context( request: pytest.FixtureRequest, + cache: pytest.Cache, tmp_path_factory: pytest.TempPathFactory, ) -> Generator[None, None, None]: """Switch to agg backend, reset settings, and close all figures at teardown.""" @@ -33,7 +34,11 @@ def _global_test_context( sc.settings.logfile = sys.stderr sc.settings.verbosity = "hint" sc.settings.autoshow = True - sc.settings.datasetdir = tmp_path_factory.mktemp("scanpy_data") + # create directory for debug data + cache.mkdir("debug") + # reuse data files between test runs (unless overwritten in the test) + sc.settings.datasetdir = cache.mkdir("scanpy-data") + # create new writedir for each test run sc.settings.writedir = tmp_path_factory.mktemp("scanpy_write") if isinstance(request.node, pytest.DoctestItem): @@ -70,8 +75,7 @@ def pytest_addoption(parser: pytest.Parser) -> None: action="store_true", default=False, help=( - "Run tests that retrieve stuff from the internet. " - "This increases test time." + "Run tests that retrieve stuff from the internet. This increases test time." ), ) @@ -99,12 +103,11 @@ def _modify_doctests(request: pytest.FixtureRequest) -> None: func = _import_name(request.node.name) needs_mod: str | None - if needs_mod := getattr(func, "_doctest_needs", None): - needs_marker = needs[needs_mod] - if needs_marker.mark.args[0]: - pytest.skip(reason=needs_marker.mark.kwargs["reason"]) skip_reason: str | None - if skip_reason := getattr(func, "_doctest_skip_reason", None): + if ( + (needs_mod := getattr(func, "_doctest_needs", None)) + and (skip_reason := needs[needs_mod].skip_reason) + ) or (skip_reason := getattr(func, "_doctest_skip_reason", None)): pytest.skip(reason=skip_reason) if getattr(func, "_doctest_internet", False) and not request.config.getoption( "--internet-tests" @@ -127,6 +130,6 @@ def pytest_itemcollected(item: pytest.Item) -> None: ) -assert ( - "scanpy" not in sys.modules -), "scanpy is already imported, this will mess up test coverage" +assert "scanpy" not in sys.modules, ( + "scanpy is already imported, this will mess up test coverage" +) diff --git a/src/testing/scanpy/_pytest/fixtures/__init__.py b/src/testing/scanpy/_pytest/fixtures/__init__.py index 16c3f8a4df..7578473786 100644 --- a/src/testing/scanpy/_pytest/fixtures/__init__.py +++ b/src/testing/scanpy/_pytest/fixtures/__init__.py @@ -12,10 +12,10 @@ import pytest from .data import ( - _pbmc3ks_parametrized_session, backed_adata, pbmc3k_parametrized, pbmc3k_parametrized_small, + pbmc3ks_parametrized_session, ) if TYPE_CHECKING: @@ -25,7 +25,7 @@ __all__ = [ "float_dtype", "_doctest_env", - "_pbmc3ks_parametrized_session", + "pbmc3ks_parametrized_session", "pbmc3k_parametrized", "pbmc3k_parametrized_small", "backed_adata", @@ -37,9 +37,8 @@ def float_dtype(request): return request.param -@pytest.fixture() +@pytest.fixture def _doctest_env(cache: pytest.Cache, tmp_path: Path) -> Generator[None, None, None]: - from scanpy import settings from scanpy._compat import chdir showwarning_orig = warnings.showwarning @@ -61,8 +60,6 @@ def showwarning(message, category, filename, lineno, file=None, line=None): # n ] + [("ignore", None, Warning, None, 0)] warnings.showwarning = showwarning - old_dd, settings.datasetdir = settings.datasetdir, cache.mkdir("scanpy-data") with chdir(tmp_path): yield warnings.showwarning = showwarning_orig - settings.datasetdir = old_dd diff --git a/src/testing/scanpy/_pytest/fixtures/data.py b/src/testing/scanpy/_pytest/fixtures/data.py index 3d907c9fec..4d44d8239b 100644 --- a/src/testing/scanpy/_pytest/fixtures/data.py +++ b/src/testing/scanpy/_pytest/fixtures/data.py @@ -16,7 +16,11 @@ from anndata._core.sparse_dataset import ( BaseCompressedSparseDataset as SparseDataset, ) - from anndata.experimental import sparse_dataset + + if Version(anndata_version) >= Version("0.11.0rc2"): + from anndata.io import sparse_dataset + else: + from anndata.experimental import sparse_dataset def make_sparse(x): return sparse_dataset(x) @@ -40,7 +44,7 @@ def make_sparse(x): ), ids=lambda x: f"{x[0].__name__}-{x[1]}", ) -def _pbmc3ks_parametrized_session(request) -> dict[bool, AnnData]: +def pbmc3ks_parametrized_session(request) -> dict[bool, AnnData]: from ..._helpers.data import pbmc3k sparsity_func, dtype = request.param @@ -51,13 +55,13 @@ def _pbmc3ks_parametrized_session(request) -> dict[bool, AnnData]: @pytest.fixture -def pbmc3k_parametrized(_pbmc3ks_parametrized_session) -> Callable[[], AnnData]: - return _pbmc3ks_parametrized_session[False].copy +def pbmc3k_parametrized(pbmc3ks_parametrized_session) -> Callable[[], AnnData]: + return pbmc3ks_parametrized_session[False].copy @pytest.fixture -def pbmc3k_parametrized_small(_pbmc3ks_parametrized_session) -> Callable[[], AnnData]: - return _pbmc3ks_parametrized_session[True].copy +def pbmc3k_parametrized_small(pbmc3ks_parametrized_session) -> Callable[[], AnnData]: + return pbmc3ks_parametrized_session[True].copy @pytest.fixture( diff --git a/src/testing/scanpy/_pytest/marks.py b/src/testing/scanpy/_pytest/marks.py index 5695009a40..22b32269d2 100644 --- a/src/testing/scanpy/_pytest/marks.py +++ b/src/testing/scanpy/_pytest/marks.py @@ -1,15 +1,31 @@ from __future__ import annotations -import sys from enum import Enum, auto from importlib.util import find_spec +from typing import TYPE_CHECKING import pytest +from packaging.version import Version +if TYPE_CHECKING: + from collections.abc import Callable -def _next_val(name: str, start: int, count: int, last_values: list[str]) -> str: - """Distribution name for matching modules""" - return name.replace("_", "-") + +SKIP_EXTRA: dict[str, Callable[[], str | None]] = {} + + +def _skip_if_skmisc_too_old() -> str | None: + import numpy as np + import skmisc + + if Version(skmisc.__version__) <= Version("0.3.1") and Version( + np.__version__ + ) >= Version("2"): + return "scikit-misc≤0.3.1 requires numpy<2" + return None + + +SKIP_EXTRA["skmisc"] = _skip_if_skmisc_too_old class QuietMarkDecorator(pytest.MarkDecorator): @@ -25,11 +41,15 @@ class needs(QuietMarkDecorator, Enum): :func:`pytest.importorskip` skips tests after they started running. """ - # _generate_next_value_ needs to come before members, also it’s finnicky: - # https://github.com/python/mypy/issues/7591#issuecomment-652800625 - _generate_next_value_ = ( - staticmethod(_next_val) if sys.version_info >= (3, 10) else _next_val - ) + # _generate_next_value_ needs to come before members + @staticmethod + def _generate_next_value_( + name: str, start: int, count: int, last_values: list[str] + ) -> str: + """Distribution name for matching modules""" + return name.replace("_", "-") + + mod: str dask = auto() dask_ml = auto() @@ -59,8 +79,18 @@ class needs(QuietMarkDecorator, Enum): wishbone = "wishbone-dev" def __init__(self, mod: str) -> None: - reason = f"needs module `{self._name_}`" - if self._name_.casefold() != mod.casefold().replace("-", "_"): - reason = f"{reason} (`pip install {mod}`)" - dec = pytest.mark.skipif(not find_spec(self._name_), reason=reason) + self.mod = mod + reason = self.skip_reason + dec = pytest.mark.skipif(bool(reason), reason=reason or "") super().__init__(dec.mark) + + @property + def skip_reason(self) -> str | None: + if find_spec(self._name_): + if skip_extra := SKIP_EXTRA.get(self._name_): + return skip_extra() + return None + reason = f"needs module `{self._name_}`" + if self._name_.casefold() != self.mod.casefold().replace("-", "_"): + reason = f"{reason} (`pip install {self.mod}`)" + return reason diff --git a/src/testing/scanpy/_pytest/params.py b/src/testing/scanpy/_pytest/params.py index af80d1709d..f405e33d5e 100644 --- a/src/testing/scanpy/_pytest/params.py +++ b/src/testing/scanpy/_pytest/params.py @@ -15,19 +15,22 @@ from .._pytest.marks import needs if TYPE_CHECKING: - from collections.abc import Iterable - from typing import Literal + from collections.abc import Callable, Iterable + from typing import Any, Literal from _pytest.mark.structures import ParameterSet def param_with( at: ParameterSet, + transform: Callable[..., Iterable[Any]] = lambda x: (x,), *, marks: Iterable[pytest.Mark | pytest.MarkDecorator] = (), id: str | None = None, ) -> ParameterSet: - return pytest.param(*at.values, marks=[*at.marks, *marks], id=id or at.id) + return pytest.param( + *transform(*at.values), marks=[*at.marks, *marks], id=id or at.id + ) MAP_ARRAY_TYPES: dict[ diff --git a/tests/_data/regress_test_small.npy b/tests/_data/regress_test_small.npy new file mode 100644 index 0000000000..5a590fb35f Binary files /dev/null and b/tests/_data/regress_test_small.npy differ diff --git a/tests/_data/visium_data/1.0.0/spatial/scalefactors_json.json b/tests/_data/visium_data/1.0.0/spatial/scalefactors_json.json index 9f47f51518..5479b589c0 100644 --- a/tests/_data/visium_data/1.0.0/spatial/scalefactors_json.json +++ b/tests/_data/visium_data/1.0.0/spatial/scalefactors_json.json @@ -1 +1,6 @@ -{"spot_diameter_fullres": 89.42751063343188, "tissue_hires_scalef": 0.150015, "fiducial_diameter_fullres": 144.45982486939, "tissue_lowres_scalef": 0.045004502} \ No newline at end of file +{ + "spot_diameter_fullres": 89.42751063343188, + "tissue_hires_scalef": 0.150015, + "fiducial_diameter_fullres": 144.45982486939, + "tissue_lowres_scalef": 0.045004502 +} diff --git a/tests/_images/dotplot/expected.png b/tests/_images/dotplot/expected.png index ea54ae3447..9c4b822369 100644 Binary files a/tests/_images/dotplot/expected.png and b/tests/_images/dotplot/expected.png differ diff --git a/tests/_images/dotplot_dict/expected.png b/tests/_images/dotplot_dict/expected.png index ec1df6560c..d805ea94db 100644 Binary files a/tests/_images/dotplot_dict/expected.png and b/tests/_images/dotplot_dict/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.black_tup-na_in_legend.False-legend.on_bottom-groups.3]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.black_tup-na_in_legend.False-legend.on_bottom-groups.3]/expected.png new file mode 100644 index 0000000000..11588cba72 Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.black_tup-na_in_legend.False-legend.on_bottom-groups.3]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.black_tup-na_in_legend.False-legend.on_bottom-groups.all]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.black_tup-na_in_legend.False-legend.on_bottom-groups.all]/expected.png new file mode 100644 index 0000000000..bf12e84957 Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.black_tup-na_in_legend.False-legend.on_bottom-groups.all]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.black_tup-na_in_legend.True-legend.on_bottom-groups.3]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.black_tup-na_in_legend.True-legend.on_bottom-groups.3]/expected.png new file mode 100644 index 0000000000..b500ad0300 Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.black_tup-na_in_legend.True-legend.on_bottom-groups.3]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.black_tup-na_in_legend.True-legend.on_bottom-groups.all]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.black_tup-na_in_legend.True-legend.on_bottom-groups.all]/expected.png new file mode 100644 index 0000000000..b4f6750350 Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.black_tup-na_in_legend.True-legend.on_bottom-groups.all]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.default-na_in_legend.False-legend.on_bottom-groups.3]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.default-na_in_legend.False-legend.on_bottom-groups.3]/expected.png new file mode 100644 index 0000000000..248aab23bf Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.default-na_in_legend.False-legend.on_bottom-groups.3]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.default-na_in_legend.False-legend.on_bottom-groups.all]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.default-na_in_legend.False-legend.on_bottom-groups.all]/expected.png new file mode 100644 index 0000000000..61b80cb288 Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.default-na_in_legend.False-legend.on_bottom-groups.all]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.default-na_in_legend.True-legend.on_bottom-groups.3]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.default-na_in_legend.True-legend.on_bottom-groups.3]/expected.png new file mode 100644 index 0000000000..4b943bcd27 Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.default-na_in_legend.True-legend.on_bottom-groups.3]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.default-na_in_legend.True-legend.on_bottom-groups.all]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.default-na_in_legend.True-legend.on_bottom-groups.all]/expected.png new file mode 100644 index 0000000000..ce64b59054 Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[pca-na_color.default-na_in_legend.True-legend.on_bottom-groups.all]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.black_tup-na_in_legend.False-legend.on_bottom-groups.3]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.black_tup-na_in_legend.False-legend.on_bottom-groups.3]/expected.png new file mode 100644 index 0000000000..dc31533948 Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.black_tup-na_in_legend.False-legend.on_bottom-groups.3]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.black_tup-na_in_legend.False-legend.on_bottom-groups.all]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.black_tup-na_in_legend.False-legend.on_bottom-groups.all]/expected.png new file mode 100644 index 0000000000..3e0cf8c43f Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.black_tup-na_in_legend.False-legend.on_bottom-groups.all]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.black_tup-na_in_legend.True-legend.on_bottom-groups.3]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.black_tup-na_in_legend.True-legend.on_bottom-groups.3]/expected.png new file mode 100644 index 0000000000..75485fcf1e Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.black_tup-na_in_legend.True-legend.on_bottom-groups.3]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.black_tup-na_in_legend.True-legend.on_bottom-groups.all]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.black_tup-na_in_legend.True-legend.on_bottom-groups.all]/expected.png new file mode 100644 index 0000000000..26af76bb5a Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.black_tup-na_in_legend.True-legend.on_bottom-groups.all]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.default-na_in_legend.False-legend.on_bottom-groups.3]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.default-na_in_legend.False-legend.on_bottom-groups.3]/expected.png new file mode 100644 index 0000000000..87520dce9f Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.default-na_in_legend.False-legend.on_bottom-groups.3]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.default-na_in_legend.False-legend.on_bottom-groups.all]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.default-na_in_legend.False-legend.on_bottom-groups.all]/expected.png new file mode 100644 index 0000000000..636b77dad7 Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.default-na_in_legend.False-legend.on_bottom-groups.all]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.default-na_in_legend.True-legend.on_bottom-groups.3]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.default-na_in_legend.True-legend.on_bottom-groups.3]/expected.png new file mode 100644 index 0000000000..ebe3016322 Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.default-na_in_legend.True-legend.on_bottom-groups.3]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.default-na_in_legend.True-legend.on_bottom-groups.all]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.default-na_in_legend.True-legend.on_bottom-groups.all]/expected.png new file mode 100644 index 0000000000..aa9555a667 Binary files /dev/null and b/tests/_images/embedding-missing-values/test_missing_values_categorical[spatial-na_color.default-na_in_legend.True-legend.on_bottom-groups.all]/expected.png differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.default]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.default]/expected.png deleted file mode 100644 index 4a15aff087..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.default]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.norm]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.norm]/expected.png deleted file mode 100644 index 79701cba60..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.norm]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.numbers]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.numbers]/expected.png deleted file mode 100644 index 3f7ab6acdf..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.numbers]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.percentile]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.percentile]/expected.png deleted file mode 100644 index 523c5493e6..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.percentile]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.vcenter]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.vcenter]/expected.png deleted file mode 100644 index 07837c079a..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_data-vbounds.vcenter]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.default]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.default]/expected.png deleted file mode 100644 index 4a15aff087..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.default]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.norm]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.norm]/expected.png deleted file mode 100644 index 79701cba60..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.norm]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.numbers]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.numbers]/expected.png deleted file mode 100644 index 3f7ab6acdf..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.numbers]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.percentile]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.percentile]/expected.png deleted file mode 100644 index 523c5493e6..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.percentile]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.vcenter]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.vcenter]/expected.png deleted file mode 100644 index 07837c079a..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.on_right-vbounds.vcenter]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.off-vbounds.default]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-vbounds.default]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.off-vbounds.default]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-vbounds.default]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.off-vbounds.norm]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-vbounds.norm]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.off-vbounds.norm]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-vbounds.norm]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.off-vbounds.numbers]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-vbounds.numbers]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.off-vbounds.numbers]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-vbounds.numbers]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.off-vbounds.percentile]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-vbounds.percentile]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.off-vbounds.percentile]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-vbounds.percentile]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.off-vbounds.vcenter]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-vbounds.vcenter]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-legend.off-vbounds.vcenter]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.black_tup-vbounds.vcenter]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.default]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.default]/expected.png deleted file mode 100644 index e740fcca0b..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.default]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.norm]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.norm]/expected.png deleted file mode 100644 index 8de704a908..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.norm]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.numbers]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.numbers]/expected.png deleted file mode 100644 index a8881f76ca..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.numbers]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.percentile]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.percentile]/expected.png deleted file mode 100644 index 9c2785f8c1..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.percentile]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.vcenter]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.vcenter]/expected.png deleted file mode 100644 index 4705e7ff2e..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_data-vbounds.vcenter]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.default]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.default]/expected.png deleted file mode 100644 index e740fcca0b..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.default]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.norm]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.norm]/expected.png deleted file mode 100644 index 8de704a908..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.norm]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.numbers]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.numbers]/expected.png deleted file mode 100644 index a8881f76ca..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.numbers]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.percentile]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.percentile]/expected.png deleted file mode 100644 index 9c2785f8c1..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.percentile]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.vcenter]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.vcenter]/expected.png deleted file mode 100644 index 4705e7ff2e..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.on_right-vbounds.vcenter]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.off-vbounds.default]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-vbounds.default]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.off-vbounds.default]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-vbounds.default]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.off-vbounds.norm]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-vbounds.norm]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.off-vbounds.norm]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-vbounds.norm]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.off-vbounds.numbers]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-vbounds.numbers]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.off-vbounds.numbers]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-vbounds.numbers]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.off-vbounds.percentile]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-vbounds.percentile]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.off-vbounds.percentile]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-vbounds.percentile]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.off-vbounds.vcenter]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-vbounds.vcenter]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-legend.off-vbounds.vcenter]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[pca-na_color.default-vbounds.vcenter]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.default]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.default]/expected.png deleted file mode 100644 index 7cf84b81c9..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.default]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.norm]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.norm]/expected.png deleted file mode 100644 index aad366ae7c..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.norm]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.numbers]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.numbers]/expected.png deleted file mode 100644 index 9f09b085b4..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.numbers]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.percentile]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.percentile]/expected.png deleted file mode 100644 index fdef79fbfc..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.percentile]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.vcenter]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.vcenter]/expected.png deleted file mode 100644 index a18b111bf4..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_data-vbounds.vcenter]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.default]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.default]/expected.png deleted file mode 100644 index 7cf84b81c9..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.default]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.norm]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.norm]/expected.png deleted file mode 100644 index aad366ae7c..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.norm]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.numbers]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.numbers]/expected.png deleted file mode 100644 index 9f09b085b4..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.numbers]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.percentile]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.percentile]/expected.png deleted file mode 100644 index fdef79fbfc..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.percentile]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.vcenter]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.vcenter]/expected.png deleted file mode 100644 index a18b111bf4..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.on_right-vbounds.vcenter]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.off-vbounds.default]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-vbounds.default]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.off-vbounds.default]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-vbounds.default]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.off-vbounds.norm]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-vbounds.norm]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.off-vbounds.norm]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-vbounds.norm]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.off-vbounds.numbers]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-vbounds.numbers]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.off-vbounds.numbers]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-vbounds.numbers]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.off-vbounds.percentile]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-vbounds.percentile]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.off-vbounds.percentile]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-vbounds.percentile]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.off-vbounds.vcenter]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-vbounds.vcenter]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-legend.off-vbounds.vcenter]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.black_tup-vbounds.vcenter]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.default]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.default]/expected.png deleted file mode 100644 index d308779f92..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.default]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.norm]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.norm]/expected.png deleted file mode 100644 index 3696ba3d28..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.norm]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.numbers]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.numbers]/expected.png deleted file mode 100644 index 96352358d1..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.numbers]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.percentile]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.percentile]/expected.png deleted file mode 100644 index 96384f44bf..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.percentile]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.vcenter]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.vcenter]/expected.png deleted file mode 100644 index c97398776d..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_data-vbounds.vcenter]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.default]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.default]/expected.png deleted file mode 100644 index d308779f92..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.default]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.norm]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.norm]/expected.png deleted file mode 100644 index 3696ba3d28..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.norm]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.numbers]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.numbers]/expected.png deleted file mode 100644 index 96352358d1..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.numbers]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.percentile]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.percentile]/expected.png deleted file mode 100644 index 96384f44bf..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.percentile]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.vcenter]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.vcenter]/expected.png deleted file mode 100644 index c97398776d..0000000000 Binary files a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.on_right-vbounds.vcenter]/expected.png and /dev/null differ diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.off-vbounds.default]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-vbounds.default]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.off-vbounds.default]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-vbounds.default]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.off-vbounds.norm]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-vbounds.norm]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.off-vbounds.norm]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-vbounds.norm]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.off-vbounds.numbers]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-vbounds.numbers]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.off-vbounds.numbers]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-vbounds.numbers]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.off-vbounds.percentile]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-vbounds.percentile]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.off-vbounds.percentile]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-vbounds.percentile]/expected.png diff --git a/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.off-vbounds.vcenter]/expected.png b/tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-vbounds.vcenter]/expected.png similarity index 100% rename from tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-legend.off-vbounds.vcenter]/expected.png rename to tests/_images/embedding-missing-values/test_missing_values_continuous[spatial-na_color.default-vbounds.vcenter]/expected.png diff --git a/tests/_images/highest_expr_genes/expected.png b/tests/_images/highest_expr_genes/expected.png new file mode 100644 index 0000000000..1df4b95ff2 Binary files /dev/null and b/tests/_images/highest_expr_genes/expected.png differ diff --git a/tests/_images/matrixplot/expected.png b/tests/_images/matrixplot/expected.png index 216626af46..cb2421ba02 100644 Binary files a/tests/_images/matrixplot/expected.png and b/tests/_images/matrixplot/expected.png differ diff --git a/tests/_images/matrixplot2/expected.png b/tests/_images/matrixplot2/expected.png index b4cdc8dc02..58990e2526 100644 Binary files a/tests/_images/matrixplot2/expected.png and b/tests/_images/matrixplot2/expected.png differ diff --git a/tests/_images/matrixplot_std_scale_group/expected.png b/tests/_images/matrixplot_std_scale_group/expected.png index 6d1970b3bb..05fe80da7d 100644 Binary files a/tests/_images/matrixplot_std_scale_group/expected.png and b/tests/_images/matrixplot_std_scale_group/expected.png differ diff --git a/tests/_images/scatter_HES_percent_mito_n_genes_bulk_labels/expected.png b/tests/_images/scatter_HES_percent_mito_n_genes_bulk_labels/expected.png new file mode 100644 index 0000000000..2609a53567 Binary files /dev/null and b/tests/_images/scatter_HES_percent_mito_n_genes_bulk_labels/expected.png differ diff --git a/tests/_images/stacked_violin/expected.png b/tests/_images/stacked_violin/expected.png index 8130c0e8d8..1cadbe8ad7 100644 Binary files a/tests/_images/stacked_violin/expected.png and b/tests/_images/stacked_violin/expected.png differ diff --git a/tests/_images/stacked_violin_no_cat_obs/expected.png b/tests/_images/stacked_violin_no_cat_obs/expected.png index 08c6a1fc82..750a346b0d 100644 Binary files a/tests/_images/stacked_violin_no_cat_obs/expected.png and b/tests/_images/stacked_violin_no_cat_obs/expected.png differ diff --git a/tests/_images/stacked_violin_std_scale_group/expected.png b/tests/_images/stacked_violin_std_scale_group/expected.png index 963cab263a..af4bf8948b 100644 Binary files a/tests/_images/stacked_violin_std_scale_group/expected.png and b/tests/_images/stacked_violin_std_scale_group/expected.png differ diff --git a/tests/_images/stacked_violin_std_scale_var_dict/expected.png b/tests/_images/stacked_violin_std_scale_var_dict/expected.png index a648f2ff8a..de1c02455e 100644 Binary files a/tests/_images/stacked_violin_std_scale_var_dict/expected.png and b/tests/_images/stacked_violin_std_scale_var_dict/expected.png differ diff --git a/tests/_images/stacked_violin_swap_axes_pbmc68k_reduced/expected.png b/tests/_images/stacked_violin_swap_axes_pbmc68k_reduced/expected.png new file mode 100644 index 0000000000..ab80e8e713 Binary files /dev/null and b/tests/_images/stacked_violin_swap_axes_pbmc68k_reduced/expected.png differ diff --git a/tests/conftest.py b/tests/conftest.py index 52bc61168a..2d7f8e7aad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ import sys from pathlib import Path from textwrap import dedent -from typing import TYPE_CHECKING, TypedDict, Union, cast +from typing import TYPE_CHECKING, TypedDict, cast import pytest @@ -88,7 +88,7 @@ def fmt_descr(descr): return f"{descr} ({basename})" if basename else descr result = cast( - Union[CompareResult, None], + CompareResult | None, compare_images(str(expected), str(actual), tol=tol, in_decorator=True), ) if result is None: @@ -133,7 +133,8 @@ def save_and_compare(*path_parts: Path | os.PathLike, tol: int): plt.savefig(actual_pth, dpi=40) plt.close() if not expected_pth.is_file(): - raise OSError(f"No expected output found at {expected_pth}.") + msg = f"No expected output found at {expected_pth}." + raise OSError(msg) check_same_image(expected_pth, actual_pth, tol=tol) return save_and_compare diff --git a/tests/external/test_hashsolo.py b/tests/external/test_hashsolo.py index a6f0a79971..d338d1b6ec 100644 --- a/tests/external/test_hashsolo.py +++ b/tests/external/test_hashsolo.py @@ -1,6 +1,7 @@ from __future__ import annotations import numpy as np +import pandas as pd from anndata import AnnData import scanpy.external as sce @@ -27,9 +28,11 @@ def test_cell_demultiplexing(): sce.pp.hashsolo(test_data, test_data.obs.columns) doublets = ["Doublet"] * 10 - classes = list( - np.repeat(np.arange(10), 98).reshape(98, 10, order="F").ravel().astype(str) - ) + classes = np.repeat(np.arange(10), 98).reshape(98, 10, order="F").ravel().tolist() negatives = ["Negative"] * 10 - classification = doublets + classes + negatives - assert test_data.obs["Classification"].astype(str).tolist() == classification + expected = pd.array(doublets + classes + negatives, dtype="string") + classification = test_data.obs["Classification"].array.astype("string") + # This is a bit flaky, so allow some mismatches: + if (expected != classification).sum() > 3: + # Compare lists for better error message + assert classification.tolist() == expected.tolist() diff --git a/tests/external/test_scanorama_integrate.py b/tests/external/test_scanorama_integrate.py index 19a53a4d27..baa2007fc0 100644 --- a/tests/external/test_scanorama_integrate.py +++ b/tests/external/test_scanorama_integrate.py @@ -1,18 +1,11 @@ from __future__ import annotations -import sys - -import pytest - import scanpy as sc import scanpy.external as sce from testing.scanpy._helpers.data import pbmc68k_reduced from testing.scanpy._pytest.marks import needs -pytestmark = [ - needs.scanorama, - pytest.mark.skipif(sys.version_info < (3, 10), reason="annoy is unstable on 3.9"), -] +pytestmark = [needs.scanorama] def test_scanorama_integrate(): diff --git a/tests/external/test_wishbone.py b/tests/external/test_wishbone.py index 7fadef63c6..db649a5b9d 100644 --- a/tests/external/test_wishbone.py +++ b/tests/external/test_wishbone.py @@ -22,6 +22,6 @@ def test_run_wishbone(): components=[2, 3], num_waypoints=150, ) - assert all( - [k in adata.obs for k in ["trajectory_wishbone", "branch_wishbone"]] - ), "Run Wishbone Error!" + assert all([k in adata.obs for k in ["trajectory_wishbone", "branch_wishbone"]]), ( + "Run Wishbone Error!" + ) diff --git a/tests/notebooks/test_paga_paul15_subsampled.py b/tests/notebooks/test_paga_paul15_subsampled.py index 6d4ee886ba..5d8c17d336 100644 --- a/tests/notebooks/test_paga_paul15_subsampled.py +++ b/tests/notebooks/test_paga_paul15_subsampled.py @@ -129,7 +129,7 @@ def test_paga_paul15_subsampled(image_comparer, plt): left_margin=0.15, n_avg=50, annotations=["distance"], - show_yticks=True if ipath == 0 else False, + show_yticks=ipath == 0, show_colorbar=False, color_map="Greys", color_maps_annotations={"distance": "viridis"}, @@ -138,6 +138,6 @@ def test_paga_paul15_subsampled(image_comparer, plt): show=False, ) # add a test for this at some point - # data.to_csv('./write/paga_path_{}.csv'.format(descr)) + # data.to_csv(f"./write/paga_path_{descr}.csv") save_and_compare_images("paga_path") diff --git a/tests/test_aggregated.py b/tests/test_aggregated.py index 0cc064e74e..5bd87e231d 100644 --- a/tests/test_aggregated.py +++ b/tests/test_aggregated.py @@ -8,12 +8,18 @@ from scipy import sparse import scanpy as sc -from scanpy._utils import _resolve_axis +from scanpy._utils import _resolve_axis, get_literal_vals +from scanpy.get._aggregated import AggType from testing.scanpy._helpers import assert_equal from testing.scanpy._helpers.data import pbmc3k_processed from testing.scanpy._pytest.params import ARRAY_TYPES_MEM +@pytest.fixture(params=get_literal_vals(AggType)) +def metric(request: pytest.FixtureRequest) -> AggType: + return request.param + + @pytest.fixture def df_base(): ax_base = ["A", "B"] @@ -88,7 +94,6 @@ def test_mask(axis): @pytest.mark.parametrize("array_type", ARRAY_TYPES_MEM) -@pytest.mark.parametrize("metric", ["sum", "mean", "var", "count_nonzero"]) def test_aggregate_vs_pandas(metric, array_type): adata = pbmc3k_processed().raw.to_adata() adata = adata[ @@ -135,7 +140,6 @@ def test_aggregate_vs_pandas(metric, array_type): @pytest.mark.parametrize("array_type", ARRAY_TYPES_MEM) -@pytest.mark.parametrize("metric", ["sum", "mean", "var", "count_nonzero"]) def test_aggregate_axis(array_type, metric): adata = pbmc3k_processed().raw.to_adata() adata = adata[ @@ -222,7 +226,8 @@ def test_aggregate_axis_specification(axis_name): { "a": ["a", "a", "b", "b"], "b": ["c", "d", "d", "d"], - } + }, + index=["a_c", "a_d", "b_d1", "b_d2"], ), ["a", "b"], ["count_nonzero"], # , "sum", "mean"], @@ -253,7 +258,8 @@ def test_aggregate_axis_specification(axis_name): { "a": ["a", "a", "b", "b"], "b": ["c", "d", "d", "d"], - } + }, + index=["a_c", "a_d", "b_d1", "b_d2"], ), ["a", "b"], ["sum", "mean", "count_nonzero"], @@ -284,7 +290,8 @@ def test_aggregate_axis_specification(axis_name): { "a": ["a", "a", "b", "b"], "b": ["c", "d", "d", "d"], - } + }, + index=["a_c", "a_d", "b_d1", "b_d2"], ), ["a", "b"], ["mean"], @@ -381,7 +388,6 @@ def test_combine_categories(label_cols, cols, expected): @pytest.mark.parametrize("array_type", ARRAY_TYPES_MEM) -@pytest.mark.parametrize("metric", ["sum", "mean", "var", "count_nonzero"]) def test_aggregate_arraytype(array_type, metric): adata = pbmc3k_processed().raw.to_adata() adata = adata[ diff --git a/tests/test_backed.py b/tests/test_backed.py index 787edf9c21..bfa1d79592 100644 --- a/tests/test_backed.py +++ b/tests/test_backed.py @@ -91,8 +91,8 @@ def test_log1p_backed_errors(backed_adata): def test_scatter_backed(backed_adata): sc.pp.pca(backed_adata, chunked=True) - sc.pl.scatter(backed_adata, color="0", basis="pca") + sc.pl.scatter(backed_adata, color="0", basis="pca", show=False) def test_dotplot_backed(backed_adata): - sc.pl.dotplot(backed_adata, ["0", "1", "2", "3"], groupby="cat") + sc.pl.dotplot(backed_adata, ["0", "1", "2", "3"], groupby="cat", show=False) diff --git a/tests/test_binary.py b/tests/test_binary.py index 6b70c824f3..2cf1aa1bee 100644 --- a/tests/test_binary.py +++ b/tests/test_binary.py @@ -2,6 +2,7 @@ import os import re +from contextlib import nullcontext from pathlib import Path from subprocess import PIPE from typing import TYPE_CHECKING @@ -19,7 +20,7 @@ @pytest.fixture -def set_path(monkeypatch: MonkeyPatch) -> None: +def _set_path(monkeypatch: MonkeyPatch) -> None: monkeypatch.setenv("PATH", str(HERE / "_scripts"), prepend=os.pathsep) @@ -31,15 +32,18 @@ def test_builtin_settings(capsys: CaptureFixture): @pytest.mark.parametrize("args", [[], ["-h"]]) def test_help_displayed(args: list[str], capsys: CaptureFixture): - try: # -h raises it, no args doesn’t. Maybe not ideal but meh. + # -h raises it, no args doesn’t. Maybe not ideal but meh. + ctx = pytest.raises(SystemExit) if args else nullcontext() + with ctx as se: main(args) - except SystemExit as se: - assert se.code == 0 + if se is not None: + assert se.value.code == 0 captured = capsys.readouterr() assert captured.out.startswith("usage: ") -def test_help_output(set_path: None, capsys: CaptureFixture): +@pytest.mark.usefixtures("_set_path") +def test_help_output(capsys: CaptureFixture): with pytest.raises(SystemExit, match="^0$"): main(["-h"]) captured = capsys.readouterr() @@ -50,7 +54,8 @@ def test_help_output(set_path: None, capsys: CaptureFixture): ) -def test_external(set_path: None): +@pytest.mark.usefixtures("_set_path") +def test_external(): # We need to capture the output manually, since subprocesses don’t write to sys.stderr cmdline = ["testbin", "-t", "--testarg", "testpos"] cmd = main(cmdline, stdout=PIPE, encoding="utf-8", check=True) diff --git a/tests/test_clustering.py b/tests/test_clustering.py index 1972842a05..260513ff45 100644 --- a/tests/test_clustering.py +++ b/tests/test_clustering.py @@ -75,13 +75,13 @@ def test_leiden_random_state(adata_neighbors, flavor): @needs.igraph def test_leiden_igraph_directed(adata_neighbors): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"Cannot use igraph’s leiden.*directed"): sc.tl.leiden(adata_neighbors, flavor="igraph", directed=True) @needs.igraph def test_leiden_wrong_flavor(adata_neighbors): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"flavor must be.*'igraph'.*'leidenalg'.*but"): sc.tl.leiden(adata_neighbors, flavor="foo") @@ -90,7 +90,7 @@ def test_leiden_wrong_flavor(adata_neighbors): def test_leiden_igraph_partition_type(adata_neighbors): import leidenalg - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"Do not pass in partition_type"): sc.tl.leiden( adata_neighbors, flavor="igraph", @@ -147,7 +147,7 @@ def test_leiden_objective_function(adata_neighbors): @needs.igraph @pytest.mark.parametrize( - "clustering,key", + ("clustering", "key"), [ pytest.param(sc.tl.louvain, "louvain", marks=needs.louvain), pytest.param(sc.tl.leiden, "leiden", marks=needs.leidenalg), @@ -215,7 +215,7 @@ def test_partition_type(adata_neighbors): @pytest.mark.parametrize( - "clustering,default_key,default_res,custom_resolutions", + ("clustering", "default_key", "default_res", "custom_resolutions"), [ pytest.param(sc.tl.leiden, "leiden", 0.8, [0.9, 1.1], marks=needs.leidenalg), pytest.param(sc.tl.louvain, "louvain", 0.8, [0.9, 1.1], marks=needs.louvain), diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 7e69821ac9..5e0fc1e125 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -24,6 +24,15 @@ from anndata import AnnData +@pytest.fixture(autouse=True) +def _tmp_dataset_dir(tmp_path: Path) -> None: + """Make sure that datasets are downloaded during the test run. + + The default test environment stores them in a cached location. + """ + sc.settings.datasetdir = tmp_path / "scanpy_data" + + @pytest.mark.internet def test_burczynski06(): with pytest.warns(UserWarning, match=r"Variable names are not unique"): @@ -102,6 +111,7 @@ def test_pbmc68k_reduced(): sc.datasets.pbmc68k_reduced() +@pytest.mark.filterwarnings("ignore:Use `squidpy.*` instead:FutureWarning") @pytest.mark.internet def test_visium_datasets(): """Tests that reading/ downloading works and is does not have global effects.""" @@ -112,6 +122,7 @@ def test_visium_datasets(): assert_adata_equal(hheart, hheart_again) +@pytest.mark.filterwarnings("ignore:Use `squidpy.*` instead:FutureWarning") @pytest.mark.internet def test_visium_datasets_dir_change(tmp_path: Path): """Test that changing the dataset dir doesn't break reading.""" @@ -123,6 +134,7 @@ def test_visium_datasets_dir_change(tmp_path: Path): assert_adata_equal(mbrain, mbrain_again) +@pytest.mark.filterwarnings("ignore:Use `squidpy.*` instead:FutureWarning") @pytest.mark.internet def test_visium_datasets_images(): """Test that image download works and is does not have global effects.""" @@ -173,7 +185,7 @@ def test_download_failure(): *DS_MARKS[ds], ], ) - for ds in set(sc.datasets.__all__) - DS_DYNAMIC + for ds in sorted(set(sc.datasets.__all__) - DS_DYNAMIC) ], ) def test_doc_shape(ds_name): diff --git a/tests/test_dendrogram.py b/tests/test_dendrogram.py index 18b952eff2..44a08fcf67 100644 --- a/tests/test_dendrogram.py +++ b/tests/test_dendrogram.py @@ -18,7 +18,7 @@ def test_dendrogram_key_added(groupby, key_added): adata = pbmc68k_reduced() sc.tl.dendrogram(adata, groupby=groupby, key_added=key_added, use_rep="X_pca") if isinstance(groupby, list): - dendrogram_key = f'dendrogram_{"_".join(groupby)}' + dendrogram_key = f"dendrogram_{'_'.join(groupby)}" else: dendrogram_key = f"dendrogram_{groupby}" diff --git a/tests/test_embedding.py b/tests/test_embedding.py index 86fbd7e656..692157a084 100644 --- a/tests/test_embedding.py +++ b/tests/test_embedding.py @@ -9,36 +9,56 @@ from testing.scanpy._pytest.marks import needs -def test_tsne(): - pbmc = pbmc68k_reduced() +@pytest.mark.parametrize( + ("key_added", "key_obsm", "key_uns"), + [ + pytest.param(None, "X_tsne", "tsne", id="None"), + pytest.param("custom_key", "custom_key", "custom_key", id="custom_key"), + ], +) +def test_tsne(key_added: str | None, key_obsm: str, key_uns: str): + pbmc = pbmc68k_reduced()[:200].copy() euclidean1 = sc.tl.tsne(pbmc, metric="euclidean", copy=True) with pytest.warns(UserWarning, match="In previous versions of scanpy"): - euclidean2 = sc.tl.tsne(pbmc, metric="euclidean", n_jobs=2, copy=True) + euclidean2 = sc.tl.tsne( + pbmc, metric="euclidean", n_jobs=2, key_added=key_added, copy=True + ) cosine = sc.tl.tsne(pbmc, metric="cosine", copy=True) # Reproducibility - np.testing.assert_equal(euclidean1.obsm["X_tsne"], euclidean2.obsm["X_tsne"]) + np.testing.assert_equal(euclidean1.obsm["X_tsne"], euclidean2.obsm[key_obsm]) # Metric has some effect assert not np.array_equal(euclidean1.obsm["X_tsne"], cosine.obsm["X_tsne"]) # Params are recorded assert euclidean1.uns["tsne"]["params"]["n_jobs"] == 1 - assert euclidean2.uns["tsne"]["params"]["n_jobs"] == 2 + assert euclidean2.uns[key_uns]["params"]["n_jobs"] == 2 assert cosine.uns["tsne"]["params"]["n_jobs"] == 1 assert euclidean1.uns["tsne"]["params"]["metric"] == "euclidean" - assert euclidean2.uns["tsne"]["params"]["metric"] == "euclidean" + assert euclidean2.uns[key_uns]["params"]["metric"] == "euclidean" assert cosine.uns["tsne"]["params"]["metric"] == "cosine" -def test_umap_init_dtype(): - pbmc = pbmc68k_reduced()[:100, :].copy() - sc.tl.umap(pbmc, init_pos=pbmc.obsm["X_pca"][:, :2].astype(np.float32)) - embed1 = pbmc.obsm["X_umap"].copy() - sc.tl.umap(pbmc, init_pos=pbmc.obsm["X_pca"][:, :2].astype(np.float64)) - embed2 = pbmc.obsm["X_umap"].copy() - assert_array_almost_equal(embed1, embed2) - assert_array_almost_equal(embed1, embed2) +@pytest.mark.parametrize( + ("key_added", "key_obsm", "key_uns"), + [ + pytest.param(None, "X_umap", "umap", id="None"), + pytest.param("custom_key", "custom_key", "custom_key", id="custom_key"), + ], +) +def test_umap_init_dtype(key_added: str | None, key_obsm: str, key_uns: str): + pbmc1 = pbmc68k_reduced()[:100, :].copy() + pbmc2 = pbmc1.copy() + for pbmc, dtype, k in [(pbmc1, np.float32, None), (pbmc2, np.float64, key_added)]: + sc.tl.umap(pbmc, init_pos=pbmc.obsm["X_pca"][:, :2].astype(dtype), key_added=k) + + # check that embeddings are close for different dtypes + assert_array_almost_equal(pbmc1.obsm["X_umap"], pbmc2.obsm[key_obsm]) + + # check that params are recorded + assert pbmc1.uns["umap"]["params"]["a"] == pbmc2.uns[key_uns]["params"]["a"] + assert pbmc1.uns["umap"]["params"]["b"] == pbmc2.uns[key_uns]["params"]["b"] @pytest.mark.parametrize( diff --git a/tests/test_embedding_plots.py b/tests/test_embedding_plots.py deleted file mode 100644 index abc156a574..0000000000 --- a/tests/test_embedding_plots.py +++ /dev/null @@ -1,550 +0,0 @@ -from __future__ import annotations - -from functools import partial -from pathlib import Path - -import matplotlib as mpl -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -import pytest -import seaborn as sns -from matplotlib.colors import Normalize -from matplotlib.testing.compare import compare_images - -import scanpy as sc -from testing.scanpy._helpers.data import pbmc3k_processed - -HERE: Path = Path(__file__).parent -ROOT = HERE / "_images" - -MISSING_VALUES_ROOT = ROOT / "embedding-missing-values" - - -def check_images(pth1, pth2, *, tol): - result = compare_images(pth1, pth2, tol=tol) - assert result is None, result - - -@pytest.fixture(scope="module") -def adata(): - """A bit cute.""" - from matplotlib.image import imread - from sklearn.cluster import DBSCAN - from sklearn.datasets import make_blobs - - empty_pixel = np.array([1.0, 1.0, 1.0, 0]).reshape(1, 1, -1) - image = imread(HERE.parent / "docs/_static/img/Scanpy_Logo_RGB.png") - x, y = np.where(np.logical_and.reduce(~np.equal(image, empty_pixel), axis=2)) - - # Just using to calculate the hex coords - hexes = plt.hexbin(x, y, gridsize=(44, 100)) - counts = hexes.get_array() - pixels = hexes.get_offsets()[counts != 0] - plt.close() - - labels = DBSCAN(eps=20, min_samples=2).fit(pixels).labels_ - order = np.argsort(labels) - adata = sc.AnnData( - make_blobs( - pd.Series(labels[order]).value_counts().values, - n_features=20, - shuffle=False, - random_state=42, - )[0], - obs={"label": pd.Categorical(labels[order].astype(str))}, - obsm={"spatial": pixels[order, ::-1]}, - uns={ - "spatial": { - "scanpy_img": { - "images": {"hires": image}, - "scalefactors": { - "tissue_hires_scalef": 1, - "spot_diameter_fullres": 10, - }, - } - } - }, - ) - sc.pp.pca(adata) - - # Adding some missing values - adata.obs["label_missing"] = adata.obs["label"].copy() - adata.obs["label_missing"][::2] = np.nan - - adata.obs["1_missing"] = adata.obs_vector("1") - adata.obs.loc[ - adata.obsm["spatial"][:, 0] < adata.obsm["spatial"][:, 0].mean(), "1_missing" - ] = np.nan - - return adata - - -@pytest.fixture -def fixture_request(request): - """Returns a Request object. - - Allows you to access names of parameterized tests from within a test. - """ - return request - - -@pytest.fixture( - params=[(0, 0, 0, 1), None], - ids=["na_color.black_tup", "na_color.default"], -) -def na_color(request): - return request.param - - -@pytest.fixture(params=[True, False], ids=["na_in_legend.True", "na_in_legend.False"]) -def na_in_legend(request): - return request.param - - -@pytest.fixture( - params=[partial(sc.pl.pca, show=False), partial(sc.pl.spatial, show=False)], - ids=["pca", "spatial"], -) -def plotfunc(request): - return request.param - - -@pytest.fixture( - params=["on data", "right margin", None], - ids=["legend.on_data", "legend.on_right", "legend.off"], -) -def legend_loc(request): - return request.param - - -@pytest.fixture( - params=[lambda x: list(x.cat.categories[:3]), lambda x: []], - ids=["groups.3", "groups.all"], -) -def groupsfunc(request): - return request.param - - -@pytest.fixture( - params=[ - pytest.param( - {"vmin": None, "vmax": None, "vcenter": None, "norm": None}, - id="vbounds.default", - ), - pytest.param( - {"vmin": 0, "vmax": 5, "vcenter": None, "norm": None}, id="vbounds.numbers" - ), - pytest.param( - {"vmin": "p15", "vmax": "p90", "vcenter": None, "norm": None}, - id="vbounds.percentile", - ), - pytest.param( - {"vmin": 0, "vmax": "p99", "vcenter": 0.1, "norm": None}, - id="vbounds.vcenter", - ), - pytest.param( - {"vmin": None, "vmax": None, "vcenter": None, "norm": Normalize(0, 5)}, - id="vbounds.norm", - ), - ] -) -def vbounds(request): - return request.param - - -def test_missing_values_categorical( - *, - fixture_request, - image_comparer, - adata, - plotfunc, - na_color, - na_in_legend, - legend_loc, - groupsfunc, -): - save_and_compare_images = partial(image_comparer, MISSING_VALUES_ROOT, tol=15) - - base_name = fixture_request.node.name - - # Passing through a dict so it's easier to use default values - kwargs = {} - kwargs["legend_loc"] = legend_loc - kwargs["groups"] = groupsfunc(adata.obs["label"]) - if na_color is not None: - kwargs["na_color"] = na_color - kwargs["na_in_legend"] = na_in_legend - - plotfunc(adata, color=["label", "label_missing"], **kwargs) - - save_and_compare_images(base_name) - - -def test_missing_values_continuous( - *, fixture_request, image_comparer, adata, plotfunc, na_color, legend_loc, vbounds -): - save_and_compare_images = partial(image_comparer, MISSING_VALUES_ROOT, tol=15) - - base_name = fixture_request.node.name - - # Passing through a dict so it's easier to use default values - kwargs = {} - kwargs.update(vbounds) - kwargs["legend_loc"] = legend_loc - if na_color is not None: - kwargs["na_color"] = na_color - - plotfunc(adata, color=["1", "1_missing"], **kwargs) - - save_and_compare_images(base_name) - - -def test_enumerated_palettes(fixture_request, adata, tmpdir, plotfunc): - tmpdir = Path(tmpdir) - base_name = fixture_request.node.name - - categories = adata.obs["label"].cat.categories - colors_rgb = dict(zip(categories, sns.color_palette(n_colors=12))) - - dict_pth = tmpdir / f"rgbdict_{base_name}.png" - list_pth = tmpdir / f"rgblist_{base_name}.png" - - # making a copy so colors aren't saved - plotfunc(adata.copy(), color="label", palette=colors_rgb) - plt.savefig(dict_pth, dpi=40) - plt.close() - plotfunc(adata.copy(), color="label", palette=[colors_rgb[c] for c in categories]) - plt.savefig(list_pth, dpi=40) - plt.close() - - check_images(dict_pth, list_pth, tol=15) - - -def test_dimension_broadcasting(adata, tmpdir, check_same_image): - tmpdir = Path(tmpdir) - - with pytest.raises(ValueError): - sc.pl.pca( - adata, color=["label", "1_missing"], dimensions=[(0, 1), (1, 2), (2, 3)] - ) - - dims_pth = tmpdir / "broadcast_dims.png" - color_pth = tmpdir / "broadcast_colors.png" - - sc.pl.pca(adata, color=["label", "label", "label"], dimensions=(2, 3), show=False) - plt.savefig(dims_pth, dpi=40) - plt.close() - sc.pl.pca(adata, color="label", dimensions=[(2, 3), (2, 3), (2, 3)], show=False) - plt.savefig(color_pth, dpi=40) - plt.close() - - check_same_image(dims_pth, color_pth, tol=5) - - -def test_marker_broadcasting(adata, tmpdir, check_same_image): - tmpdir = Path(tmpdir) - - with pytest.raises(ValueError): - sc.pl.pca(adata, color=["label", "1_missing"], marker=[".", "^", "x"]) - - dims_pth = tmpdir / "broadcast_markers.png" - color_pth = tmpdir / "broadcast_colors_for_markers.png" - - sc.pl.pca(adata, color=["label", "label", "label"], marker="^", show=False) - plt.savefig(dims_pth, dpi=40) - plt.close() - sc.pl.pca(adata, color="label", marker=["^", "^", "^"], show=False) - plt.savefig(color_pth, dpi=40) - plt.close() - - check_same_image(dims_pth, color_pth, tol=5) - - -def test_dimensions_same_as_components(adata, tmpdir, check_same_image): - tmpdir = Path(tmpdir) - adata = adata.copy() - adata.obs["mean"] = np.ravel(adata.X.mean(axis=1)) - - comp_pth = tmpdir / "components_plot.png" - dims_pth = tmpdir / "dimension_plot.png" - - # TODO: Deprecate components kwarg - # with pytest.warns(FutureWarning, match=r"components .* deprecated"): - sc.pl.pca( - adata, - color=["mean", "label"], - components=["1,2", "2,3"], - show=False, - ) - plt.savefig(comp_pth, dpi=40) - plt.close() - - sc.pl.pca( - adata, - color=["mean", "mean", "label", "label"], - dimensions=[(0, 1), (1, 2), (0, 1), (1, 2)], - show=False, - ) - plt.savefig(dims_pth, dpi=40) - plt.close() - - check_same_image(dims_pth, comp_pth, tol=5) - - -def test_embedding_colorbar_location(image_comparer): - save_and_compare_images = partial(image_comparer, ROOT, tol=15) - - adata = pbmc3k_processed().raw.to_adata() - - sc.pl.pca(adata, color="LDHB", colorbar_loc=None) - - save_and_compare_images("no_colorbar") - - -# Spatial specific - - -def test_visium_circles(image_comparer): # standard visium data - save_and_compare_images = partial(image_comparer, ROOT, tol=15) - - adata = sc.read_visium(HERE / "_data" / "visium_data" / "1.0.0") - adata.obs = adata.obs.astype({"array_row": "str"}) - - sc.pl.spatial( - adata, - color="array_row", - groups=["24", "33"], - crop_coord=(100, 400, 400, 100), - alpha=0.5, - size=1.3, - show=False, - ) - - save_and_compare_images("spatial_visium") - - -def test_visium_default(image_comparer): # default values - from packaging.version import parse as parse_version - - if parse_version(mpl.__version__) < parse_version("3.7.0"): - pytest.xfail("Matplotlib 3.7.0+ required for this test") - - save_and_compare_images = partial(image_comparer, ROOT, tol=5) - - adata = sc.read_visium(HERE / "_data" / "visium_data" / "1.0.0") - adata.obs = adata.obs.astype({"array_row": "str"}) - - # Points default to transparent if an image is included - sc.pl.spatial(adata, show=False) - - save_and_compare_images("spatial_visium_default") - - -def test_visium_empty_img_key(image_comparer): # visium coordinates but image empty - save_and_compare_images = partial(image_comparer, ROOT, tol=15) - - adata = sc.read_visium(HERE / "_data" / "visium_data" / "1.0.0") - adata.obs = adata.obs.astype({"array_row": "str"}) - - sc.pl.spatial(adata, img_key=None, color="array_row", show=False) - - save_and_compare_images("spatial_visium_empty_image") - - sc.pl.embedding(adata, basis="spatial", color="array_row", show=False) - save_and_compare_images("spatial_visium_embedding") - - -def test_spatial_general(image_comparer): # general coordinates - save_and_compare_images = partial(image_comparer, ROOT, tol=15) - - adata = sc.read_visium(HERE / "_data" / "visium_data" / "1.0.0") - adata.obs = adata.obs.astype({"array_row": "str"}) - spatial_metadata = adata.uns.pop( - "spatial" - ) # spatial data don't have imgs, so remove entry from uns - # Required argument for now - spot_size = list(spatial_metadata.values())[0]["scalefactors"][ - "spot_diameter_fullres" - ] - - sc.pl.spatial(adata, show=False, spot_size=spot_size) - save_and_compare_images("spatial_general_nocol") - - # category - sc.pl.spatial(adata, show=False, spot_size=spot_size, color="array_row") - save_and_compare_images("spatial_general_cat") - - # continuous - sc.pl.spatial(adata, show=False, spot_size=spot_size, color="array_col") - save_and_compare_images("spatial_general_cont") - - -def test_spatial_external_img(image_comparer): # external image - save_and_compare_images = partial(image_comparer, ROOT, tol=15) - - adata = sc.read_visium(HERE / "_data" / "visium_data" / "1.0.0") - adata.obs = adata.obs.astype({"array_row": "str"}) - - img = adata.uns["spatial"]["custom"]["images"]["hires"] - scalef = adata.uns["spatial"]["custom"]["scalefactors"]["tissue_hires_scalef"] - sc.pl.spatial( - adata, - color="array_row", - scale_factor=scalef, - img=img, - basis="spatial", - show=False, - ) - save_and_compare_images("spatial_external_img") - - -@pytest.fixture(scope="module") -def equivalent_spatial_plotters(adata): - no_spatial = adata.copy() - del no_spatial.uns["spatial"] - - img_key = "hires" - library_id = list(adata.uns["spatial"])[0] - spatial_data = adata.uns["spatial"][library_id] - img = spatial_data["images"][img_key] - scale_factor = spatial_data["scalefactors"][f"tissue_{img_key}_scalef"] - spot_size = spatial_data["scalefactors"]["spot_diameter_fullres"] - - orig_plotter = partial(sc.pl.spatial, adata, color="1", show=False) - removed_plotter = partial( - sc.pl.spatial, - no_spatial, - color="1", - img=img, - scale_factor=scale_factor, - spot_size=spot_size, - show=False, - ) - - return (orig_plotter, removed_plotter) - - -@pytest.fixture(scope="module") -def equivalent_spatial_plotters_no_img(equivalent_spatial_plotters): - orig, removed = equivalent_spatial_plotters - return (partial(orig, img_key=None), partial(removed, img=None, scale_factor=None)) - - -@pytest.fixture( - params=[ - pytest.param({"crop_coord": (50, 200, 0, 500)}, id="crop"), - pytest.param({"size": 0.5}, id="size:.5"), - pytest.param({"size": 2}, id="size:2"), - pytest.param({"spot_size": 5}, id="spotsize"), - pytest.param({"bw": True}, id="bw"), - # Shape of the image for particular fixture, should not be hardcoded like this - pytest.param({"img": np.ones((774, 1755, 4)), "scale_factor": 1.0}, id="img"), - pytest.param( - {"na_color": (0, 0, 0, 0), "color": "1_missing"}, id="na_color.transparent" - ), - pytest.param( - {"na_color": "lightgray", "color": "1_missing"}, id="na_color.lightgray" - ), - ] -) -def spatial_kwargs(request): - return request.param - - -def test_manual_equivalency(equivalent_spatial_plotters, tmpdir, spatial_kwargs): - """ - Tests that manually passing values to sc.pl.spatial is similar to automatic extraction. - """ - orig, removed = equivalent_spatial_plotters - - TESTDIR = Path(tmpdir) - orig_pth = TESTDIR / "orig.png" - removed_pth = TESTDIR / "removed.png" - - orig(**spatial_kwargs) - plt.savefig(orig_pth, dpi=40) - plt.close() - removed(**spatial_kwargs) - plt.savefig(removed_pth, dpi=40) - plt.close() - - check_images(orig_pth, removed_pth, tol=1) - - -def test_manual_equivalency_no_img( - equivalent_spatial_plotters_no_img, tmpdir, spatial_kwargs -): - if "bw" in spatial_kwargs: - # Has no meaning when there is no image - pytest.skip() - orig, removed = equivalent_spatial_plotters_no_img - - TESTDIR = Path(tmpdir) - orig_pth = TESTDIR / "orig.png" - removed_pth = TESTDIR / "removed.png" - - orig(**spatial_kwargs) - plt.savefig(orig_pth, dpi=40) - plt.close() - removed(**spatial_kwargs) - plt.savefig(removed_pth, dpi=40) - plt.close() - - check_images(orig_pth, removed_pth, tol=1) - - -def test_white_background_vs_no_img(adata, tmpdir, spatial_kwargs): - if {"bw", "img", "img_key", "na_color"}.intersection(spatial_kwargs): - # These arguments don't make sense for this check - pytest.skip() - - white_background = np.ones_like( - adata.uns["spatial"]["scanpy_img"]["images"]["hires"] - ) - TESTDIR = Path(tmpdir) - white_pth = TESTDIR / "white_background.png" - noimg_pth = TESTDIR / "no_img.png" - - sc.pl.spatial( - adata, - color="2", - img=white_background, - scale_factor=1.0, - show=False, - **spatial_kwargs, - ) - plt.savefig(white_pth) - sc.pl.spatial(adata, color="2", img_key=None, show=False, **spatial_kwargs) - plt.savefig(noimg_pth) - - check_images(white_pth, noimg_pth, tol=1) - - -def test_spatial_na_color(adata, tmpdir): - """ - Check that na_color defaults to transparent when an image is present, light gray when not. - """ - white_background = np.ones_like( - adata.uns["spatial"]["scanpy_img"]["images"]["hires"] - ) - TESTDIR = Path(tmpdir) - lightgray_pth = TESTDIR / "lightgray.png" - transparent_pth = TESTDIR / "transparent.png" - noimg_pth = TESTDIR / "noimg.png" - whiteimg_pth = TESTDIR / "whiteimg.png" - - def plot(pth, **kwargs): - sc.pl.spatial(adata, color="1_missing", show=False, **kwargs) - plt.savefig(pth, dpi=40) - plt.close() - - plot(lightgray_pth, na_color="lightgray", img_key=None) - plot(transparent_pth, na_color=(0.0, 0.0, 0.0, 0.0), img_key=None) - plot(noimg_pth, img_key=None) - plot(whiteimg_pth, img=white_background, scale_factor=1.0) - - check_images(lightgray_pth, noimg_pth, tol=1) - check_images(transparent_pth, whiteimg_pth, tol=1) - with pytest.raises(AssertionError): - check_images(lightgray_pth, transparent_pth, tol=1) diff --git a/tests/test_filter_rank_genes_groups.py b/tests/test_filter_rank_genes_groups.py index 26851bb102..a64ac983f3 100644 --- a/tests/test_filter_rank_genes_groups.py +++ b/tests/test_filter_rank_genes_groups.py @@ -1,159 +1,96 @@ from __future__ import annotations import numpy as np +import pytest from scanpy.tools import filter_rank_genes_groups, rank_genes_groups from testing.scanpy._helpers.data import pbmc68k_reduced -names_no_reference = np.array( +NAMES_NO_REF = [ + ["CD3D", "ITM2A", "CD3D", "CCL5", "CD7", "nan", "CD79A", "nan", "NKG7", "LYZ"], + ["CD3E", "CD3D", "nan", "NKG7", "CD3D", "AIF1", "CD79B", "nan", "GNLY", "CST3"], + ["IL32", "RPL39", "nan", "CST7", "nan", "nan", "nan", "SNHG7", "CD7", "nan"], + ["nan", "SRSF7", "IL32", "GZMA", "nan", "LST1", "IGJ", "nan", "CTSW", "nan"], + ["nan", "nan", "CD2", "CTSW", "CD8B", "TYROBP", "ISG20", "SNHG8", "GZMB", "nan"], +] + +NAMES_REF = [ + ["CD3D", "ITM2A", "CD3D", "nan", "CD3D", "nan", "CD79A", "nan", "CD7"], + ["nan", "nan", "nan", "CD3D", "nan", "AIF1", "nan", "nan", "NKG7"], + ["nan", "nan", "nan", "NKG7", "nan", "FCGR3A", "ISG20", "SNHG7", "CTSW"], + ["nan", "CD3D", "nan", "CCL5", "CD7", "nan", "CD79B", "nan", "GNLY"], + ["CD3E", "IL32", "nan", "IL32", "CD27", "FCER1G", "nan", "nan", "nan"], +] + +NAMES_NO_REF_COMPARE_ABS = [ [ - ["CD3D", "ITM2A", "CD3D", "CCL5", "CD7", "nan", "CD79A", "nan", "NKG7", "LYZ"], - ["CD3E", "CD3D", "nan", "NKG7", "CD3D", "AIF1", "CD79B", "nan", "GNLY", "CST3"], - ["IL32", "RPL39", "nan", "CST7", "nan", "nan", "nan", "SNHG7", "CD7", "nan"], - ["nan", "SRSF7", "IL32", "GZMA", "nan", "LST1", "IGJ", "nan", "CTSW", "nan"], - [ - "nan", - "nan", - "CD2", - "CTSW", - "CD8B", - "TYROBP", - "ISG20", - "SNHG8", - "GZMB", - "nan", - ], - ] -) - -names_reference = np.array( + *("CD3D", "ITM2A", "HLA-DRB1", "CCL5", "HLA-DPA1"), + *("nan", "CD79A", "nan", "NKG7", "LYZ"), + ], [ - ["CD3D", "ITM2A", "CD3D", "nan", "CD3D", "nan", "CD79A", "nan", "CD7"], - ["nan", "nan", "nan", "CD3D", "nan", "AIF1", "nan", "nan", "NKG7"], - ["nan", "nan", "nan", "NKG7", "nan", "FCGR3A", "ISG20", "SNHG7", "CTSW"], - ["nan", "CD3D", "nan", "CCL5", "CD7", "nan", "CD79B", "nan", "GNLY"], - ["CD3E", "IL32", "nan", "IL32", "CD27", "FCER1G", "nan", "nan", "nan"], - ] -) - -names_compare_abs = np.array( + *("HLA-DPA1", "nan", "CD3D", "NKG7", "HLA-DRB1"), + *("AIF1", "CD79B", "nan", "GNLY", "CST3"), + ], [ - [ - "CD3D", - "ITM2A", - "HLA-DRB1", - "CCL5", - "HLA-DPA1", - "nan", - "CD79A", - "nan", - "NKG7", - "LYZ", - ], - [ - "HLA-DPA1", - "nan", - "CD3D", - "NKG7", - "HLA-DRB1", - "AIF1", - "CD79B", - "nan", - "GNLY", - "CST3", - ], - [ - "nan", - "PSAP", - "CD74", - "CST7", - "CD74", - "PSAP", - "FCER1G", - "SNHG7", - "CD7", - "HLA-DRA", - ], - [ - "IL32", - "nan", - "HLA-DRB5", - "GZMA", - "HLA-DRB5", - "LST1", - "nan", - "nan", - "CTSW", - "HLA-DRB1", - ], - [ - "nan", - "FCER1G", - "HLA-DPB1", - "CTSW", - "HLA-DPB1", - "TYROBP", - "TYROBP", - "S100A10", - "GZMB", - "HLA-DPA1", - ], - ] -) - - -def test_filter_rank_genes_groups(): - adata = pbmc68k_reduced() - - # fix filter defaults - args = { - "adata": adata, - "key_added": "rank_genes_groups_filtered", - "min_in_group_fraction": 0.25, - "min_fold_change": 1, - "max_out_group_fraction": 0.5, - } - - rank_genes_groups( - adata, "bulk_labels", reference="Dendritic", method="wilcoxon", n_genes=5 - ) - filter_rank_genes_groups(**args) - - assert np.array_equal( - names_reference, - np.array(adata.uns["rank_genes_groups_filtered"]["names"].tolist()), - ) + *("nan", "PSAP", "CD74", "CST7", "CD74"), + *("PSAP", "FCER1G", "SNHG7", "CD7", "HLA-DRA"), + ], + [ + *("IL32", "nan", "HLA-DRB5", "GZMA", "HLA-DRB5"), + *("LST1", "nan", "nan", "CTSW", "HLA-DRB1"), + ], + [ + *("nan", "FCER1G", "HLA-DPB1", "CTSW", "HLA-DPB1"), + *("TYROBP", "TYROBP", "S100A10", "GZMB", "HLA-DPA1"), + ], +] - rank_genes_groups(adata, "bulk_labels", method="wilcoxon", n_genes=5) - filter_rank_genes_groups(**args) - assert np.array_equal( - names_no_reference, - np.array(adata.uns["rank_genes_groups_filtered"]["names"].tolist()), - ) +EXPECTED = { + ("Dendritic", False): np.array(NAMES_REF), + ("rest", False): np.array(NAMES_NO_REF), + ("rest", True): np.array(NAMES_NO_REF_COMPARE_ABS), +} - rank_genes_groups(adata, "bulk_labels", method="wilcoxon", pts=True, n_genes=5) - filter_rank_genes_groups(**args) - assert np.array_equal( - names_no_reference, - np.array(adata.uns["rank_genes_groups_filtered"]["names"].tolist()), - ) +@pytest.mark.parametrize( + ("reference", "pts", "abs"), + [ + pytest.param("Dendritic", False, False, id="ref-no_pts-no_abs"), + pytest.param("rest", False, False, id="rest-no_pts-no_abs"), + pytest.param("rest", True, False, id="rest-pts-no_abs"), + pytest.param("rest", True, True, id="rest-pts-abs"), + ], +) +def test_filter_rank_genes_groups(reference, pts, abs): + adata = pbmc68k_reduced() - # test compare_abs rank_genes_groups( - adata, "bulk_labels", method="wilcoxon", pts=True, rankby_abs=True, n_genes=5 - ) - - filter_rank_genes_groups( adata, - compare_abs=True, - min_in_group_fraction=-1, - max_out_group_fraction=1, - min_fold_change=3.1, + "bulk_labels", + reference=reference, + pts=pts, + method="wilcoxon", + rankby_abs=abs, + n_genes=5, ) + if abs: + filter_rank_genes_groups( + adata, + compare_abs=True, + min_in_group_fraction=-1, + max_out_group_fraction=1, + min_fold_change=3.1, + ) + else: + filter_rank_genes_groups( + adata, + min_in_group_fraction=0.25, + min_fold_change=1, + max_out_group_fraction=0.5, + ) assert np.array_equal( - names_compare_abs, + EXPECTED[reference, abs], np.array(adata.uns["rank_genes_groups_filtered"]["names"].tolist()), ) diff --git a/tests/test_get.py b/tests/test_get.py index a171220243..05cb1b6a9d 100644 --- a/tests/test_get.py +++ b/tests/test_get.py @@ -24,7 +24,7 @@ def transpose_adata(adata: AnnData, *, expect_duplicates: bool = False) -> AnnDa TRANSPOSE_PARAMS = pytest.mark.parametrize( - "dim,transform,func", + ("dim", "transform", "func"), [ ("obs", lambda x, expect_duplicates=False: x, sc.get.obs_df), ("var", transpose_adata, sc.get.var_df), @@ -327,7 +327,7 @@ def test_non_unique_cols_value_error(): index=[f"gene_{i}" for i in range(N)], ), ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"adata\.obs contains duplicated columns"): sc.get.obs_df(adata, ["repeated_col"]) @@ -337,7 +337,7 @@ def test_non_unique_var_index_value_error(): obs=pd.DataFrame(index=["cell-0", "cell-1"]), var=pd.DataFrame(index=["gene-0", "gene-0", "gene-1"]), ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"adata\.var_names contains duplicated items"): sc.get.obs_df(adata, ["gene-0"]) @@ -456,7 +456,7 @@ def shared_key_adata(request): r"'var_id'.* adata\.obs .* adata\.raw\.var\['gene_symbols'\]", ) else: - assert False + pytest.fail("add branch for new kind") def test_shared_key_errors(shared_key_adata): diff --git a/tests/test_highly_variable_genes.py b/tests/test_highly_variable_genes.py index a002a0ceb7..528a86ea99 100644 --- a/tests/test_highly_variable_genes.py +++ b/tests/test_highly_variable_genes.py @@ -13,13 +13,15 @@ from scipy import sparse import scanpy as sc +from scanpy.preprocessing._utils import _get_mean_var from testing.scanpy._helpers import _check_check_values_warnings from testing.scanpy._helpers.data import pbmc3k, pbmc68k_reduced from testing.scanpy._pytest.marks import needs from testing.scanpy._pytest.params import ARRAY_TYPES if TYPE_CHECKING: - from typing import Callable, Literal + from collections.abc import Callable + from typing import Literal FILE = Path(__file__).parent / Path("_scripts/seurat_hvg.csv") FILE_V3 = Path(__file__).parent / Path("_scripts/seurat_hvg_v3.csv.gz") @@ -126,11 +128,32 @@ def test_keep_layer(base, flavor): elif flavor == "cell_ranger": sc.pp.highly_variable_genes(adata, flavor=flavor) else: - assert False + pytest.fail(f"Unknown {flavor=}") assert np.allclose(X_orig.toarray(), adata.X.toarray()) +@pytest.mark.parametrize( + "flavor", + [ + "seurat", + pytest.param( + "cell_ranger", + marks=pytest.mark.xfail(reason="can’t deal with duplicate bin edges"), + ), + ], +) +def test_no_filter_genes(flavor): + """Test that even with columns containing all-zeros in the data, n_top_genes is respected.""" + adata = sc.datasets.pbmc3k() + means, _ = _get_mean_var(adata.X) + assert (means == 0).any() + sc.pp.normalize_total(adata, target_sum=10000) + sc.pp.log1p(adata) + sc.pp.highly_variable_genes(adata, flavor=flavor, n_top_genes=10000) + assert adata.var["highly_variable"].sum() == 10000 + + def _check_pearson_hvg_columns(output_df: pd.DataFrame, n_top_genes: int): assert pd.api.types.is_float_dtype(output_df["residual_variances"].dtype) @@ -233,7 +256,7 @@ def test_pearson_residuals_general( "residual_variances", "highly_variable_rank", ]: - assert key in output_df.keys() + assert key in output_df.columns # check consistency with normalization method if subset: @@ -302,7 +325,7 @@ def test_pearson_residuals_batch(pbmc3k_parametrized_small, subset, n_top_genes) "highly_variable_nbatches", "highly_variable_intersection", ]: - assert key in output_df.keys() + assert key in output_df.columns # general checks on ranks, hvg flag and residual variance _check_pearson_hvg_columns(output_df, n_top_genes) @@ -352,8 +375,8 @@ def test_compare_to_upstream( # noqa: PLR0917 array_type: Callable, ): if func == "fgd" and flavor == "cell_ranger": - msg = "The deprecated filter_genes_dispersion behaves differently with cell_ranger" - request.node.add_marker(pytest.mark.xfail(reason=msg)) + reason = "The deprecated filter_genes_dispersion behaves differently with cell_ranger" + request.applymarker(pytest.mark.xfail(reason=reason)) hvg_info = pd.read_csv(ref_path) pbmc = pbmc68k_reduced() @@ -534,6 +557,14 @@ def test_batches(): assert np.all(np.isin(colnames, hvg1.columns)) +def test_degenerate_batches(): + adata = AnnData( + X=np.random.randn(10, 100), + obs=dict(batch=pd.Categorical([*([1] * 4), *([2] * 5), 3])), + ) + sc.pp.highly_variable_genes(adata, batch_key="batch") + + @needs.skmisc def test_seurat_v3_mean_var_output_with_batchkey(): pbmc = pbmc3k() @@ -598,7 +629,8 @@ def test_subset_inplace_consistency(flavor, array_type, batch_key): pass else: - raise ValueError(f"Unknown flavor {flavor}") + msg = f"Unknown flavor {flavor}" + raise ValueError(msg) n_genes = adata.shape[1] diff --git a/tests/test_logging.py b/tests/test_logging.py index 3f8a3ee97d..81b4acbf38 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -142,6 +142,8 @@ def test_call_outputs(func): """ output_io = StringIO() with redirect_stdout(output_io): - func() + out = func() + if out is not None: + print(out) output = output_io.getvalue() assert output != "" diff --git a/tests/test_neighbors_common.py b/tests/test_neighbors_common.py index b7c63eddc6..2ae59b2b5a 100644 --- a/tests/test_neighbors_common.py +++ b/tests/test_neighbors_common.py @@ -57,6 +57,7 @@ def mk_knn_matrix( @pytest.mark.parametrize("style", ["basic", "rapids", "sklearn"]) @pytest.mark.parametrize("duplicates", [True, False], ids=["duplicates", "unique"]) def test_ind_dist_shortcut_manual( + *, n_neighbors: int | None, style: Literal["basic", "rapids", "sklearn"], duplicates: bool, diff --git a/tests/test_neighbors_key_added.py b/tests/test_neighbors_key_added.py index 4f30cee047..1da87f8ba9 100644 --- a/tests/test_neighbors_key_added.py +++ b/tests/test_neighbors_key_added.py @@ -33,7 +33,8 @@ def test_neighbors_key_added(adata): def test_neighbors_pca_keys_added_without_previous_pca_run(adata): - assert "pca" not in adata.uns and "X_pca" not in adata.obsm + assert "pca" not in adata.uns + assert "X_pca" not in adata.obsm with pytest.warns( UserWarning, match=r".*Falling back to preprocessing with `sc.pp.pca` and default params", diff --git a/tests/test_normalization.py b/tests/test_normalization.py index 922d55a40a..9cf20c0b52 100644 --- a/tests/test_normalization.py +++ b/tests/test_normalization.py @@ -198,12 +198,12 @@ def _check_pearson_pca_fields(ad, n_cells, n_comps): "Missing `.uns` keys. Expected `['pearson_residuals_normalization', 'pca']`, " f"but only {list(ad.uns.keys())} were found" ) - assert ( - "X_pca" in ad.obsm - ), f"Missing `obsm` key `'X_pca'`, only {list(ad.obsm.keys())} were found" - assert ( - "PCs" in ad.varm - ), f"Missing `varm` key `'PCs'`, only {list(ad.varm.keys())} were found" + assert "X_pca" in ad.obsm, ( + f"Missing `obsm` key `'X_pca'`, only {list(ad.obsm.keys())} were found" + ) + assert "PCs" in ad.varm, ( + f"Missing `varm` key `'PCs'`, only {list(ad.varm.keys())} were found" + ) assert ad.obsm["X_pca"].shape == ( n_cells, n_comps, @@ -236,10 +236,10 @@ def test_normalize_pearson_residuals_pca( n_cells, n_genes = adata.shape n_unmasked = n_genes - 5 adata.var["test_mask"] = np.r_[ - np.repeat(True, n_unmasked), np.repeat(False, n_genes - n_unmasked) + np.repeat(True, n_unmasked), np.repeat(False, n_genes - n_unmasked) # noqa: FBT003 ] n_var_copy = locals()[n_var_copy_name] - assert isinstance(n_var_copy, (int, np.integer)) + assert isinstance(n_var_copy, int | np.integer) if do_hvg: sc.experimental.pp.highly_variable_genes( diff --git a/tests/test_package_structure.py b/tests/test_package_structure.py index 19a6836e65..3541c561a5 100644 --- a/tests/test_package_structure.py +++ b/tests/test_package_structure.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib import os from collections import defaultdict from inspect import Parameter, signature @@ -69,6 +70,13 @@ def test_descend_classes_and_funcs(): assert {p.values[0] for p in api_functions} == funcs +@pytest.mark.filterwarnings("error::FutureWarning:.*Import anndata.*") +def test_import_future_anndata_import_warning(): + import scanpy + + importlib.reload(scanpy) + + @pytest.mark.parametrize(("f", "qualname"), api_functions) def test_function_headers(f, qualname): filename = getsourcefile(f) @@ -130,6 +138,7 @@ class ExpectedSig(TypedDict): copy_sigs["sc.pp.filter_cells"] = None # unclear `inplace` situation copy_sigs["sc.pp.filter_genes"] = None # unclear `inplace` situation copy_sigs["sc.pp.subsample"] = None # returns indices along matrix +copy_sigs["sc.pp.sample"] = None # returns indices along matrix # partial exceptions: “data” instead of “adata” copy_sigs["sc.pp.log1p"]["first_name"] = "data" copy_sigs["sc.pp.normalize_per_cell"]["first_name"] = "data" diff --git a/tests/test_paga.py b/tests/test_paga.py index 2f13d862b4..d8de573fee 100644 --- a/tests/test_paga.py +++ b/tests/test_paga.py @@ -19,7 +19,7 @@ @pytest.fixture(scope="module") -def _pbmc_session(): +def pbmc_session(): pbmc = pbmc68k_reduced() sc.tl.paga(pbmc, groups="bulk_labels") pbmc.obs["cool_feature"] = pbmc[:, "CST3"].X.squeeze().copy() @@ -28,12 +28,12 @@ def _pbmc_session(): @pytest.fixture -def pbmc(_pbmc_session): - return _pbmc_session.copy() +def pbmc(pbmc_session): + return pbmc_session.copy() @pytest.mark.parametrize( - "test_id,func", + ("test_id", "func"), [ ("", sc.pl.paga), ("continuous", partial(sc.pl.paga, color="CST3")), diff --git a/tests/test_pca.py b/tests/test_pca.py index 90ec4648b4..0130b6ac35 100644 --- a/tests/test_pca.py +++ b/tests/test_pca.py @@ -1,32 +1,38 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING +from contextlib import nullcontext +from functools import wraps +from typing import TYPE_CHECKING, Literal import anndata as ad import numpy as np import pytest from anndata import AnnData -from anndata.tests.helpers import ( - asarray, - assert_equal, -) +from anndata.tests import helpers +from anndata.tests.helpers import assert_equal from packaging.version import Version from scipy import sparse from scipy.sparse import issparse import scanpy as sc -from testing.scanpy._helpers import as_dense_dask_array, as_sparse_dask_array +from scanpy._compat import DaskArray, pkg_version +from scanpy._utils import get_literal_vals +from scanpy.preprocessing._pca import SvdSolver as SvdSolverSupported +from scanpy.preprocessing._pca._dask_sparse import _cov_sparse_dask +from testing.scanpy import _helpers from testing.scanpy._helpers.data import pbmc3k_normalized from testing.scanpy._pytest.marks import needs -from testing.scanpy._pytest.params import ( - ARRAY_TYPES, - ARRAY_TYPES_SPARSE_DASK_UNSUPPORTED, - param_with, -) +from testing.scanpy._pytest.params import ARRAY_TYPES as ARRAY_TYPES_ALL +from testing.scanpy._pytest.params import param_with if TYPE_CHECKING: - from typing import Literal + from collections.abc import Callable, Generator + + from anndata.typing import ArrayDataStructureType + + ArrayType = Callable[[np.ndarray], ArrayDataStructureType] + A_list = np.array( [ @@ -62,144 +68,229 @@ ) -# If one uses dask for PCA it will always require dask-ml -@pytest.fixture( - params=[ - param_with(at, marks=[needs.dask_ml]) if "dask" in at.id else at - for at in ARRAY_TYPES_SPARSE_DASK_UNSUPPORTED - ] -) -def array_type(request: pytest.FixtureRequest): - return request.param +if pkg_version("anndata") < Version("0.9"): + def to_memory(self: AnnData, *, copy: bool = False) -> AnnData: + """Compatibility version of AnnData.to_memory() that works with old AnnData versions""" + adata = self + if adata.isbacked: + adata = adata.to_memory() + return adata.copy() if copy else adata +else: + to_memory = AnnData.to_memory -@pytest.fixture(params=[None, "valid", "invalid"]) -def svd_solver_type(request: pytest.FixtureRequest): - return request.param + +def _chunked_1d( + f: Callable[[np.ndarray], DaskArray], +) -> Callable[[np.ndarray], DaskArray]: + @wraps(f) + def wrapper(a: np.ndarray) -> DaskArray: + da = f(a) + return da.rechunk((da.chunksize[0], -1)) + + return wrapper + + +DASK_CONVERTERS = { + f: _chunked_1d(f) + for f in (_helpers.as_dense_dask_array, _helpers.as_sparse_dask_array) +} -@pytest.fixture(params=[True, False], ids=["zero_center", "no_zero_center"]) -def zero_center(request: pytest.FixtureRequest): +def maybe_convert_array_to_dask(array_type): + # If one uses dask for PCA it will always require dask-ml. + # dask-ml can’t do 2D-chunked arrays, so rechunk them. + if as_dask_array := DASK_CONVERTERS.get(array_type): + return (as_dask_array,) + + # When not using dask, just return the array type + assert "dask" not in array_type.__name__, "add more branches or refactor" + return (array_type,) + + +ARRAY_TYPES = [ + param_with( + at, + maybe_convert_array_to_dask, + marks=[needs.dask_ml] if at.id == "dask_array_dense" else [], + ) + for at in ARRAY_TYPES_ALL +] + + +@pytest.fixture(params=ARRAY_TYPES) +def array_type(request: pytest.FixtureRequest) -> ArrayType: return request.param -@pytest.fixture -def pca_params( - array_type, svd_solver_type: Literal[None, "valid", "invalid"], zero_center -): - all_svd_solvers = {"auto", "full", "arpack", "randomized", "tsqr", "lobpcg"} - - expected_warning = None - svd_solver = None - if svd_solver_type is not None: - # TODO: are these right for sparse? - if array_type in {as_dense_dask_array, as_sparse_dask_array}: - svd_solver = ( - {"auto", "full", "tsqr", "randomized"} - if zero_center - else {"tsqr", "randomized"} - ) - elif array_type in {sparse.csr_matrix, sparse.csc_matrix}: - svd_solver = ( - {"lobpcg", "arpack"} if zero_center else {"arpack", "randomized"} - ) - elif array_type is asarray: - svd_solver = ( - {"auto", "full", "arpack", "randomized"} - if zero_center - else {"arpack", "randomized"} - ) - else: - assert False, f"Unknown array type {array_type}" - if svd_solver_type == "invalid": - svd_solver = all_svd_solvers - svd_solver - expected_warning = "Ignoring" - - svd_solver = np.random.choice(list(svd_solver)) - # explicit check for special case - if ( - svd_solver == "randomized" - and zero_center - and array_type in [sparse.csr_matrix, sparse.csc_matrix] - ): - expected_warning = "not work with sparse input" +SVDSolverDeprecated = Literal["lobpcg"] +SVDSolver = SvdSolverSupported | SVDSolverDeprecated - return (svd_solver, expected_warning) +SKLEARN_ADDITIONAL: frozenset[SvdSolverSupported] = frozenset( + {"covariance_eigh"} if pkg_version("scikit-learn") >= Version("1.5") else () +) + + +def gen_pca_params( + *, + array_type: ArrayType, + svd_solver_type: Literal[None, "valid", "invalid"], + zero_center: bool, +) -> Generator[tuple[SVDSolver | None, str | None, str | None], None, None]: + if array_type is DASK_CONVERTERS[_helpers.as_sparse_dask_array] and not zero_center: + xfail_reason = "Sparse-in-dask with zero_center=False not implemented yet" + yield None, None, xfail_reason + return + if svd_solver_type is None: + yield None, None, None + return + + all_svd_solvers = get_literal_vals(SVDSolver) + svd_solvers: set[SVDSolver] + match array_type, zero_center: + case (dc, True) if dc is DASK_CONVERTERS[_helpers.as_dense_dask_array]: + svd_solvers = {"auto", "full", "tsqr", "randomized"} + case (dc, False) if dc is DASK_CONVERTERS[_helpers.as_dense_dask_array]: + svd_solvers = {"tsqr", "randomized"} + case (dc, True) if dc is DASK_CONVERTERS[_helpers.as_sparse_dask_array]: + svd_solvers = {"covariance_eigh"} + case ((sparse.csr_matrix | sparse.csc_matrix), True): + svd_solvers = {"arpack"} | SKLEARN_ADDITIONAL + case ((sparse.csr_matrix | sparse.csc_matrix), False): + svd_solvers = {"arpack", "randomized"} + case (helpers.asarray, True): + svd_solvers = {"auto", "full", "arpack", "randomized"} | SKLEARN_ADDITIONAL + case (helpers.asarray, False): + svd_solvers = {"arpack", "randomized"} + case _: + pytest.fail(f"Unknown {array_type=} ({zero_center=})") + + if svd_solver_type == "invalid": + svd_solvers = all_svd_solvers - svd_solvers + warn_pat_expected = r"Ignoring svd_solver" + elif svd_solver_type == "valid": + warn_pat_expected = None + else: + pytest.fail(f"Unknown {svd_solver_type=}") + + # sorted to prevent https://github.com/pytest-dev/pytest-xdist/issues/432 + for svd_solver in sorted(svd_solvers): + # explicit check for special case + if ( + array_type in {sparse.csr_matrix, sparse.csc_matrix} + and zero_center + and svd_solver == "lobpcg" + ): + pat = r"legacy code" + else: + pat = warn_pat_expected + yield (svd_solver, pat, None) -def test_pca_warnings(array_type, zero_center, pca_params): - svd_solver, expected_warning = pca_params +@pytest.mark.parametrize( + ("array_type", "zero_center", "svd_solver", "warn_pat_expected"), + [ + pytest.param( + array_type.values[0], + zero_center, + svd_solver, + warn_pat_expected, + marks=( + array_type.marks + if xfail_reason is None + else [pytest.mark.xfail(reason=xfail_reason)] + ), + id=( + f"{array_type.id}-{'zero_center' if zero_center else 'no_zero_center'}-" + f"{svd_solver or svd_solver_type}-{'xfail' if xfail_reason else warn_pat_expected}" + ), + ) + for array_type in ARRAY_TYPES + for zero_center in [True, False] + for svd_solver_type in [None, "valid", "invalid"] + for svd_solver, warn_pat_expected, xfail_reason in gen_pca_params( + array_type=array_type.values[0], + zero_center=zero_center, + svd_solver_type=svd_solver_type, + ) + ], +) +def test_pca_warnings( + *, + array_type: ArrayType, + zero_center: bool, + svd_solver: SVDSolver, + warn_pat_expected: str | None, +): A = array_type(A_list).astype("float32") adata = AnnData(A) - if expected_warning is not None: - with pytest.warns(UserWarning, match=expected_warning): - sc.pp.pca(adata, svd_solver=svd_solver, zero_center=zero_center) - return - - try: - with warnings.catch_warnings(): - warnings.simplefilter("error") + if warn_pat_expected is not None: + with pytest.warns((UserWarning, FutureWarning), match=warn_pat_expected): warnings.filterwarnings( - "ignore", - "pkg_resources is deprecated as an API", - DeprecationWarning, + "ignore", r".*Using a dense eigensolver instead of LOBPCG", UserWarning ) sc.pp.pca(adata, svd_solver=svd_solver, zero_center=zero_center) - except UserWarning: - # TODO: Fix this case, maybe by increasing test data size. - # https://github.com/scverse/scanpy/issues/2744 - if svd_solver == "lobpcg": - pytest.xfail(reason="lobpcg doesn’t work with this small test data") - raise - + return -# This warning test is out of the fixture because it is a special case in the logic of the function -def test_pca_warnings_sparse(): - for array_type in (sparse.csr_matrix, sparse.csc_matrix): - A = array_type(A_list).astype("float32") - adata = AnnData(A) - with pytest.warns(UserWarning, match="not work with sparse input"): - sc.pp.pca(adata, svd_solver="randomized", zero_center=True) + warnings.simplefilter("error") + warnings.filterwarnings( + "ignore", "pkg_resources is deprecated as an API", DeprecationWarning + ) + sc.pp.pca(adata, svd_solver=svd_solver, zero_center=zero_center) def test_pca_transform(array_type): - A = array_type(A_list).astype("float32") + adata = AnnData(array_type(A_list).astype("float32")) A_pca_abs = np.abs(A_pca) - A_svd_abs = np.abs(A_svd) - - adata = AnnData(A) - with warnings.catch_warnings(record=True) as record: - sc.pp.pca(adata, n_comps=4, zero_center=True, dtype="float64") - assert len(record) == 0, record + warnings.filterwarnings("error") + sc.pp.pca(adata, n_comps=4, zero_center=True, dtype="float64") + adata = to_memory(adata) assert np.linalg.norm(A_pca_abs[:, :4] - np.abs(adata.obsm["X_pca"])) < 2e-05 - with warnings.catch_warnings(record=True) as record: + +def test_pca_transform_randomized(array_type): + adata = AnnData(array_type(A_list).astype("float32")) + A_pca_abs = np.abs(A_pca) + + warnings.filterwarnings("error") + if isinstance(adata.X, DaskArray) and issparse(adata.X._meta): + patterns = ( + r"Ignoring random_state=14 when using a sparse dask array", + r"Ignoring svd_solver='randomized' when using a sparse dask array", + ) + ctx = _helpers.MultiContext( + *(pytest.warns(UserWarning, match=pattern) for pattern in patterns) + ) + elif sparse.issparse(adata.X): + ctx = pytest.warns(UserWarning, match=r"Ignoring.*'randomized") + else: + ctx = nullcontext() + + with ctx: sc.pp.pca( adata, - n_comps=5, + n_comps=4, zero_center=True, svd_solver="randomized", dtype="float64", random_state=14, ) - if sparse.issparse(A): - assert any( - isinstance(r.message, UserWarning) - and "svd_solver 'randomized' does not work with sparse input" - in str(r.message) - for r in record - ) - else: - assert len(record) == 0 - assert np.linalg.norm(A_pca_abs - np.abs(adata.obsm["X_pca"])) < 2e-05 + assert np.linalg.norm(A_pca_abs[:, :4] - np.abs(adata.obsm["X_pca"])) < 2e-05 - with warnings.catch_warnings(record=True) as record: - sc.pp.pca(adata, n_comps=4, zero_center=False, dtype="float64", random_state=14) - assert len(record) == 0 + +def test_pca_transform_no_zero_center(request: pytest.FixtureRequest, array_type): + adata = AnnData(array_type(A_list).astype("float32")) + A_svd_abs = np.abs(A_svd) + if isinstance(adata.X, DaskArray) and issparse(adata.X._meta): + reason = "TruncatedSVD is not supported for sparse Dask yet" + request.applymarker(pytest.mark.xfail(reason=reason)) + + warnings.filterwarnings("error") + sc.pp.pca(adata, n_comps=4, zero_center=False, dtype="float64", random_state=14) assert np.linalg.norm(A_svd_abs[:, :4] - np.abs(adata.obsm["X_pca"])) < 2e-05 @@ -217,45 +308,66 @@ def test_pca_shapes(): sc.pp.pca(adata) assert adata.obsm["X_pca"].shape == (20, 19) - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=r"n_components=100 must be between 1 and.*20 with svd_solver='arpack'", + ): sc.pp.pca(adata, n_comps=100) -def test_pca_sparse(): +@pytest.mark.parametrize( + ("key_added", "keys_expected"), + [ + pytest.param(None, ("X_pca", "PCs", "pca"), id="None"), + pytest.param("custom_key", ("custom_key",) * 3, id="custom_key"), + ], +) +def test_pca_sparse(key_added: str | None, keys_expected: tuple[str, str, str]): """ Tests that implicitly centered pca on sparse arrays returns equivalent results to explicit centering on dense arrays. """ - pbmc = pbmc3k_normalized() + pbmc = pbmc3k_normalized()[:200].copy() pbmc_dense = pbmc.copy() pbmc_dense.X = pbmc_dense.X.toarray() implicit = sc.pp.pca(pbmc, dtype=np.float64, copy=True) - explicit = sc.pp.pca(pbmc_dense, dtype=np.float64, copy=True) + explicit = sc.pp.pca(pbmc_dense, dtype=np.float64, key_added=key_added, copy=True) + + key_obsm, key_varm, key_uns = keys_expected np.testing.assert_allclose( - implicit.uns["pca"]["variance"], explicit.uns["pca"]["variance"] + implicit.uns["pca"]["variance"], explicit.uns[key_uns]["variance"] ) np.testing.assert_allclose( - implicit.uns["pca"]["variance_ratio"], explicit.uns["pca"]["variance_ratio"] + implicit.uns["pca"]["variance_ratio"], explicit.uns[key_uns]["variance_ratio"] ) - np.testing.assert_allclose(implicit.obsm["X_pca"], explicit.obsm["X_pca"]) - np.testing.assert_allclose(implicit.varm["PCs"], explicit.varm["PCs"]) + np.testing.assert_allclose(implicit.obsm["X_pca"], explicit.obsm[key_obsm]) + np.testing.assert_allclose(implicit.varm["PCs"], explicit.varm[key_varm]) def test_pca_reproducible(array_type): pbmc = pbmc3k_normalized() pbmc.X = array_type(pbmc.X) - a = sc.pp.pca(pbmc, copy=True, dtype=np.float64, random_state=42) - b = sc.pp.pca(pbmc, copy=True, dtype=np.float64, random_state=42) - c = sc.pp.pca(pbmc, copy=True, dtype=np.float64, random_state=0) + with ( + pytest.warns(UserWarning, match=r"Ignoring random_state.*sparse dask array") + if isinstance(pbmc.X, DaskArray) and issparse(pbmc.X._meta) + else nullcontext() + ): + a = sc.pp.pca(pbmc, copy=True, dtype=np.float64, random_state=42) + b = sc.pp.pca(pbmc, copy=True, dtype=np.float64, random_state=42) + c = sc.pp.pca(pbmc, copy=True, dtype=np.float64, random_state=0) assert_equal(a, b) + # Test that changing random seed changes result # Does not show up reliably with 32 bit computation - assert not np.array_equal(a.obsm["X_pca"], c.obsm["X_pca"]) + # sparse-in-dask doesn’t use a random seed, so it also doesn’t work there. + if not (isinstance(pbmc.X, DaskArray) and issparse(pbmc.X._meta)): + a, c = map(to_memory, [a, c]) + assert not np.array_equal(a.obsm["X_pca"], c.obsm["X_pca"]) def test_pca_chunked(): @@ -302,9 +414,9 @@ def test_pca_n_pcs(): ) -# We use all ARRAY_TYPES here since this error should be raised before +# We use all possible array types here since this error should be raised before # PCA can realize that it got a Dask array -@pytest.mark.parametrize("array_type", ARRAY_TYPES) +@pytest.mark.parametrize("array_type", ARRAY_TYPES_ALL) def test_mask_highly_var_error(array_type): """Check if use_highly_variable=True throws an error if the annotation is missing.""" adata = AnnData(array_type(A_list).astype("float32")) @@ -344,25 +456,26 @@ def test_mask_var_argument_equivalence(float_dtype, array_type): adata_w_mask.var["mask"] = mask_var sc.pp.pca(adata_w_mask, mask_var="mask", dtype=float_dtype) + adata, adata_w_mask = map(to_memory, [adata, adata_w_mask]) assert np.allclose( adata.X.toarray() if issparse(adata.X) else adata.X, adata_w_mask.X.toarray() if issparse(adata_w_mask.X) else adata_w_mask.X, ) -def test_mask(array_type, request): - if array_type is as_dense_dask_array: - pytest.xfail("TODO: Dask arrays are not supported") +def test_mask(request: pytest.FixtureRequest, array_type): + if array_type in DASK_CONVERTERS.values(): + reason = "TODO: Dask arrays are not supported" + request.applymarker(pytest.mark.xfail(reason=reason)) adata = sc.datasets.blobs(n_variables=10, n_centers=3, n_observations=100) adata.X = array_type(adata.X) if isinstance(adata.X, np.ndarray) and Version(ad.__version__) < Version("0.9"): - request.node.add_marker( - pytest.mark.xfail( - reason="TODO: Previous version of anndata would return an F ordered array for one" - " case here, which suprisingly considerably changes the results of PCA. " - ) + reason = ( + "TODO: Previous version of anndata would return an F ordered array for one" + " case here, which surprisingly considerably changes the results of PCA." ) + request.applymarker(pytest.mark.xfail(reason=reason)) mask_var = np.random.choice([True, False], adata.shape[1]) adata_masked = adata[:, mask_var].copy() @@ -379,13 +492,10 @@ def test_mask(array_type, request): ) -def test_mask_order_warning(request): +def test_mask_order_warning(request: pytest.FixtureRequest): if Version(ad.__version__) >= Version("0.9"): - request.node.add_marker( - pytest.mark.xfail( - reason="Not expected to warn in later versions of anndata" - ) - ) + reason = "Not expected to warn in later versions of anndata" + request.applymarker(pytest.mark.xfail(reason=reason)) adata = ad.AnnData(X=np.random.randn(50, 5)) mask = np.array([True, False, True, False, True]) @@ -413,8 +523,11 @@ def test_mask_defaults(array_type, float_dtype): with_var = sc.pp.pca(adata, copy=True, dtype=float_dtype) assert without_var.uns["pca"]["params"]["mask_var"] is None assert with_var.uns["pca"]["params"]["mask_var"] == "highly_variable" + without_var, with_var = map(to_memory, [without_var, with_var]) assert not np.array_equal(without_var.obsm["X_pca"], with_var.obsm["X_pca"]) + with_no_mask = sc.pp.pca(adata, mask_var=None, copy=True, dtype=float_dtype) + with_no_mask = to_memory(with_no_mask) assert np.array_equal(without_var.obsm["X_pca"], with_no_mask.obsm["X_pca"]) @@ -442,3 +555,77 @@ def test_pca_layer(): ) np.testing.assert_equal(X_adata.obsm["X_pca"], layer_adata.obsm["X_pca"]) np.testing.assert_equal(X_adata.varm["PCs"], layer_adata.varm["PCs"]) + + +# Skipping these tests during min-deps testing shouldn't be an issue because the sparse-in-dask feature is not available on anndata<0.10 anyway +needs_anndata_dask = pytest.mark.skipif( + pkg_version("anndata") < Version("0.10"), + reason="Old AnnData doesn’t have dask test helpers", +) + + +@needs.dask +@needs_anndata_dask +@pytest.mark.parametrize( + "other_array_type", + [lambda x: x.toarray(), DASK_CONVERTERS[_helpers.as_sparse_dask_array]], + ids=["dense-mem", "sparse-dask"], +) +def test_covariance_eigh_impls(other_array_type): + warnings.filterwarnings("error") + + adata_sparse_mem = pbmc3k_normalized()[:200, :100].copy() + adata_other = adata_sparse_mem.copy() + adata_other.X = other_array_type(adata_other.X) + + sc.pp.pca(adata_sparse_mem, svd_solver="covariance_eigh") + sc.pp.pca(adata_other, svd_solver="covariance_eigh") + + to_memory(adata_other) + np.testing.assert_allclose( + np.abs(adata_sparse_mem.obsm["X_pca"]), np.abs(adata_other.obsm["X_pca"]) + ) + + +@needs.dask +@needs_anndata_dask +@pytest.mark.parametrize( + ("msg_re", "op"), + [ + ( + r"Only dask arrays with CSR-meta", + lambda a: a.map_blocks( + sparse.csc_matrix, meta=sparse.csc_matrix(np.array([])) + ), + ), + (r"Only dask arrays with chunking", lambda a: a.rechunk((a.shape[0], 100))), + ], + ids=["as-csc", "bad-chunking"], +) +def test_sparse_dask_input_errors(msg_re: str, op: Callable[[DaskArray], DaskArray]): + adata_sparse = pbmc3k_normalized() + adata_sparse.X = op(DASK_CONVERTERS[_helpers.as_sparse_dask_array](adata_sparse.X)) + + with pytest.raises(ValueError, match=msg_re): + sc.pp.pca(adata_sparse, svd_solver="covariance_eigh") + + +@needs.dask +@needs_anndata_dask +@pytest.mark.parametrize( + ("dtype", "dtype_arg", "rtol"), + [ + pytest.param(np.float32, None, 1e-5, id="float32"), + pytest.param(np.float32, np.float64, None, id="float32-float64"), + pytest.param(np.float64, None, None, id="float64"), + pytest.param(np.int64, None, None, id="int64"), + ], +) +def test_cov_sparse_dask(dtype, dtype_arg, rtol): + x_arr = A_list.astype(dtype) + x = DASK_CONVERTERS[_helpers.as_sparse_dask_array](x_arr) + cov, gram, mean = _cov_sparse_dask(x, return_gram=True, dtype=dtype_arg) + np.testing.assert_allclose(mean, np.mean(x_arr, axis=0)) + np.testing.assert_allclose(gram, (x_arr.T @ x_arr) / x.shape[0]) + tol_args = dict(rtol=rtol) if rtol is not None else {} + np.testing.assert_allclose(cov, np.cov(x_arr, rowvar=False, bias=True), **tol_args) diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 96b590268b..f135a68aa4 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -36,6 +36,23 @@ # If test images need to be updated, simply copy actual.png to expected.png. +@pytest.mark.parametrize("col", [None, "symb"]) +@pytest.mark.parametrize("layer", [None, "layer_name"]) +def test_highest_expr_genes(image_comparer, col, layer): + save_and_compare_images = partial(image_comparer, ROOT, tol=5) + + adata = pbmc3k() + if layer is not None: + adata.layers[layer] = adata.X + del adata.X + # check that only existing categories are shown + adata.var["symb"] = adata.var_names.astype("category") + + sc.pl.highest_expr_genes(adata, 20, gene_symbols=col, layer=layer, show=False) + + save_and_compare_images("highest_expr_genes") + + @needs.leidenalg def test_heatmap(image_comparer): save_and_compare_images = partial(image_comparer, ROOT, tol=15) @@ -156,7 +173,7 @@ def test_heatmap(image_comparer): reason="https://github.com/mwaskom/seaborn/issues/1953", ) @pytest.mark.parametrize( - "obs_keys,name", + ("obs_keys", "name"), [(None, "clustermap"), ("cell_type", "clustermap_withcolor")], ) def test_clustermap(image_comparer, obs_keys, name): @@ -167,9 +184,9 @@ def test_clustermap(image_comparer, obs_keys, name): save_and_compare_images(name) -@pytest.mark.parametrize( - "id,fn", - [ +params_dotplot_matrixplot_stacked_violin = [ + pytest.param(id, fn, id=id) + for id, fn in [ ( "dotplot", partial( @@ -317,10 +334,13 @@ def test_clustermap(image_comparer, obs_keys, name): figsize=(8, 2.5), ), ), - ], -) + ] +] + + +@pytest.mark.parametrize(("id", "fn"), params_dotplot_matrixplot_stacked_violin) def test_dotplot_matrixplot_stacked_violin(image_comparer, id, fn): - save_and_compare_images = partial(image_comparer, ROOT, tol=15) + save_and_compare_images = partial(image_comparer, ROOT, tol=5) adata = krumsiek11() adata.obs["numeric_column"] = adata.X[:, 0] @@ -367,6 +387,17 @@ def test_dotplot_obj(image_comparer): save_and_compare_images("dotplot_std_scale_var") +def test_dotplot_style_no_reset(): + pbmc = pbmc68k_reduced() + plot = sc.pl.dotplot(pbmc, "CD79A", "bulk_labels", return_fig=True) + assert isinstance(plot, sc.pl.DotPlot) + assert plot.cmap == sc.pl.DotPlot.DEFAULT_COLORMAP + plot.style(cmap="winter") + assert plot.cmap == "winter" + plot.style(color_on="square") + assert plot.cmap == "winter", "style() should not reset unspecified parameters" + + def test_dotplot_add_totals(image_comparer): save_and_compare_images = partial(image_comparer, ROOT, tol=5) @@ -424,6 +455,30 @@ def test_stacked_violin_obj(image_comparer, plt): save_and_compare_images("stacked_violin_return_fig") +# checking for https://github.com/scverse/scanpy/issues/3152 +def test_stacked_violin_swap_axes_match(image_comparer): + save_and_compare_images = partial(image_comparer, ROOT, tol=10) + pbmc = pbmc68k_reduced() + sc.tl.rank_genes_groups( + pbmc, + "bulk_labels", + method="wilcoxon", + tie_correct=True, + pts=True, + key_added="wilcoxon", + ) + swapped_ax = sc.pl.rank_genes_groups_stacked_violin( + pbmc, + n_genes=2, + key="wilcoxon", + groupby="bulk_labels", + swap_axes=True, + return_fig=True, + ) + swapped_ax.show() + save_and_compare_images("stacked_violin_swap_axes_pbmc68k_reduced") + + def test_tracksplot(image_comparer): save_and_compare_images = partial(image_comparer, ROOT, tol=15) @@ -561,221 +616,220 @@ def test_correlation(image_comparer): save_and_compare_images("correlation") -@pytest.mark.parametrize( - "name,fn", - [ - ( - "ranked_genes_sharey", - partial( - sc.pl.rank_genes_groups, n_genes=12, n_panels_per_row=3, show=False - ), - ), - ( - "ranked_genes", - partial( - sc.pl.rank_genes_groups, - n_genes=12, - n_panels_per_row=3, - sharey=False, - show=False, - ), - ), - ( - "ranked_genes_heatmap", - partial( - sc.pl.rank_genes_groups_heatmap, n_genes=4, cmap="YlGnBu", show=False - ), - ), - ( - "ranked_genes_heatmap_swap_axes", - partial( - sc.pl.rank_genes_groups_heatmap, - n_genes=20, - swap_axes=True, - use_raw=False, - show_gene_labels=False, - show=False, - vmin=-3, - vmax=3, - cmap="bwr", - ), +_RANK_GENES_GROUPS_PARAMS = [ + ( + "sharey", + partial(sc.pl.rank_genes_groups, n_genes=12, n_panels_per_row=3, show=False), + ), + ( + "basic", + partial( + sc.pl.rank_genes_groups, + n_genes=12, + n_panels_per_row=3, + sharey=False, + show=False, ), - ( - "ranked_genes_heatmap_swap_axes_vcenter", - partial( - sc.pl.rank_genes_groups_heatmap, - n_genes=20, - swap_axes=True, - use_raw=False, - show_gene_labels=False, - show=False, - vmin=-3, - vcenter=1, - vmax=3, - cmap="RdBu_r", - ), + ), + ( + "heatmap", + partial(sc.pl.rank_genes_groups_heatmap, n_genes=4, cmap="YlGnBu", show=False), + ), + ( + "heatmap_swap_axes", + partial( + sc.pl.rank_genes_groups_heatmap, + n_genes=20, + swap_axes=True, + use_raw=False, + show_gene_labels=False, + show=False, + vmin=-3, + vmax=3, + cmap="bwr", ), - ( - "ranked_genes_stacked_violin", - partial( - sc.pl.rank_genes_groups_stacked_violin, - n_genes=3, - show=False, - groups=["3", "0", "5"], - ), + ), + ( + "heatmap_swap_axes_vcenter", + partial( + sc.pl.rank_genes_groups_heatmap, + n_genes=20, + swap_axes=True, + use_raw=False, + show_gene_labels=False, + show=False, + vmin=-3, + vcenter=1, + vmax=3, + cmap="RdBu_r", ), - ( - "ranked_genes_dotplot", - partial(sc.pl.rank_genes_groups_dotplot, n_genes=4, show=False), + ), + ( + "stacked_violin", + partial( + sc.pl.rank_genes_groups_stacked_violin, + n_genes=3, + show=False, + groups=["3", "0", "5"], ), - ( - "ranked_genes_dotplot_gene_names", - partial( - sc.pl.rank_genes_groups_dotplot, - var_names={ - "T-cell": ["CD3D", "CD3E", "IL32"], - "B-cell": ["CD79A", "CD79B", "MS4A1"], - "myeloid": ["CST3", "LYZ"], - }, - values_to_plot="logfoldchanges", - cmap="bwr", - vmin=-3, - vmax=3, - show=False, - ), + ), + ( + "dotplot", + partial(sc.pl.rank_genes_groups_dotplot, n_genes=4, show=False), + ), + ( + "dotplot_gene_names", + partial( + sc.pl.rank_genes_groups_dotplot, + var_names={ + "T-cell": ["CD3D", "CD3E", "IL32"], + "B-cell": ["CD79A", "CD79B", "MS4A1"], + "myeloid": ["CST3", "LYZ"], + }, + values_to_plot="logfoldchanges", + cmap="bwr", + vmin=-3, + vmax=3, + show=False, ), - ( - "ranked_genes_dotplot_logfoldchange", - partial( - sc.pl.rank_genes_groups_dotplot, - n_genes=4, - values_to_plot="logfoldchanges", - vmin=-5, - vmax=5, - min_logfoldchange=3, - cmap="RdBu_r", - swap_axes=True, - title="log fold changes swap_axes", - show=False, - ), + ), + ( + "dotplot_logfoldchange", + partial( + sc.pl.rank_genes_groups_dotplot, + n_genes=4, + values_to_plot="logfoldchanges", + vmin=-5, + vmax=5, + min_logfoldchange=3, + cmap="RdBu_r", + swap_axes=True, + title="log fold changes swap_axes", + show=False, ), - ( - "ranked_genes_dotplot_logfoldchange_vcenter", - partial( - sc.pl.rank_genes_groups_dotplot, - n_genes=4, - values_to_plot="logfoldchanges", - vmin=-5, - vcenter=1, - vmax=5, - min_logfoldchange=3, - cmap="RdBu_r", - swap_axes=True, - title="log fold changes swap_axes", - show=False, - ), + ), + ( + "dotplot_logfoldchange_vcenter", + partial( + sc.pl.rank_genes_groups_dotplot, + n_genes=4, + values_to_plot="logfoldchanges", + vmin=-5, + vcenter=1, + vmax=5, + min_logfoldchange=3, + cmap="RdBu_r", + swap_axes=True, + title="log fold changes swap_axes", + show=False, ), - ( - "ranked_genes_matrixplot", - partial( - sc.pl.rank_genes_groups_matrixplot, - n_genes=5, - show=False, - title="matrixplot", - gene_symbols="symbol", - use_raw=False, - ), + ), + ( + "matrixplot", + partial( + sc.pl.rank_genes_groups_matrixplot, + n_genes=5, + show=False, + title="matrixplot", + gene_symbols="symbol", + use_raw=False, ), - ( - "ranked_genes_matrixplot_gene_names_symbol", - partial( - sc.pl.rank_genes_groups_matrixplot, - var_names={ - "T-cell": ["CD3D__", "CD3E__", "IL32__"], - "B-cell": ["CD79A__", "CD79B__", "MS4A1__"], - "myeloid": ["CST3__", "LYZ__"], - }, - values_to_plot="logfoldchanges", - cmap="bwr", - vmin=-3, - vmax=3, - gene_symbols="symbol", - use_raw=False, - show=False, - ), + ), + ( + "matrixplot_gene_names_symbol", + partial( + sc.pl.rank_genes_groups_matrixplot, + var_names={ + "T-cell": ["CD3D__", "CD3E__", "IL32__"], + "B-cell": ["CD79A__", "CD79B__", "MS4A1__"], + "myeloid": ["CST3__", "LYZ__"], + }, + values_to_plot="logfoldchanges", + cmap="bwr", + vmin=-3, + vmax=3, + gene_symbols="symbol", + use_raw=False, + show=False, ), - ( - "ranked_genes_matrixplot_n_genes_negative", - partial( - sc.pl.rank_genes_groups_matrixplot, - n_genes=-5, - show=False, - title="matrixplot n_genes=-5", - ), + ), + ( + "matrixplot_n_genes_negative", + partial( + sc.pl.rank_genes_groups_matrixplot, + n_genes=-5, + show=False, + title="matrixplot n_genes=-5", ), - ( - "ranked_genes_matrixplot_swap_axes", - partial( - sc.pl.rank_genes_groups_matrixplot, - n_genes=5, - show=False, - swap_axes=True, - values_to_plot="logfoldchanges", - vmin=-6, - vmax=6, - cmap="bwr", - title="log fold changes swap_axes", - ), + ), + ( + "matrixplot_swap_axes", + partial( + sc.pl.rank_genes_groups_matrixplot, + n_genes=5, + show=False, + swap_axes=True, + values_to_plot="logfoldchanges", + vmin=-6, + vmax=6, + cmap="bwr", + title="log fold changes swap_axes", ), - ( - "ranked_genes_matrixplot_swap_axes_vcenter", - partial( - sc.pl.rank_genes_groups_matrixplot, - n_genes=5, - show=False, - swap_axes=True, - values_to_plot="logfoldchanges", - vmin=-6, - vcenter=1, - vmax=6, - cmap="bwr", - title="log fold changes swap_axes", - ), + ), + ( + "matrixplot_swap_axes_vcenter", + partial( + sc.pl.rank_genes_groups_matrixplot, + n_genes=5, + show=False, + swap_axes=True, + values_to_plot="logfoldchanges", + vmin=-6, + vcenter=1, + vmax=6, + cmap="bwr", + title="log fold changes swap_axes", ), - ( - "ranked_genes_tracksplot", - partial( - sc.pl.rank_genes_groups_tracksplot, - n_genes=3, - show=False, - groups=["3", "2", "1"], - ), + ), + ( + "tracksplot", + partial( + sc.pl.rank_genes_groups_tracksplot, + n_genes=3, + show=False, + groups=["3", "2", "1"], ), - ( - "ranked_genes_violin", - partial( - sc.pl.rank_genes_groups_violin, - groups="0", - n_genes=5, - use_raw=True, - jitter=False, - strip=False, - show=False, - ), + ), + ( + "violin", + partial( + sc.pl.rank_genes_groups_violin, + groups="0", + n_genes=5, + use_raw=True, + jitter=False, + strip=False, + show=False, ), - ( - "ranked_genes_violin_not_raw", - partial( - sc.pl.rank_genes_groups_violin, - groups="0", - n_genes=5, - use_raw=False, - jitter=False, - strip=False, - show=False, - ), + ), + ( + "violin_not_raw", + partial( + sc.pl.rank_genes_groups_violin, + groups="0", + n_genes=5, + use_raw=False, + jitter=False, + strip=False, + show=False, ), - ], + ), +] + + +@pytest.mark.parametrize( + ("name", "fn"), + [pytest.param(name, fn, id=name) for name, fn in _RANK_GENES_GROUPS_PARAMS], ) def test_rank_genes_groups(image_comparer, name, fn): save_and_compare_images = partial(image_comparer, ROOT, tol=15) @@ -788,12 +842,13 @@ def test_rank_genes_groups(image_comparer, name, fn): with plt.rc_context({"axes.grid": True, "figure.figsize": (4, 4)}): fn(pbmc) - save_and_compare_images(name) + key = "ranked_genes" if name == "basic" else f"ranked_genes_{name}" + save_and_compare_images(key) plt.close() @pytest.fixture(scope="session") -def _gene_symbols_adatas(): +def gene_symbols_adatas_session() -> tuple[AnnData, AnnData]: """Create two anndata objects which are equivalent except for var_names Both have ensembl ids and hgnc symbols as columns in var. The first has ensembl @@ -826,21 +881,21 @@ def _gene_symbols_adatas(): @pytest.fixture -def gene_symbols_adatas(_gene_symbols_adatas): - a, b = _gene_symbols_adatas +def gene_symbols_adatas(gene_symbols_adatas_session) -> tuple[AnnData, AnnData]: + a, b = gene_symbols_adatas_session return a.copy(), b.copy() @pytest.mark.parametrize( "func", - ( + [ sc.pl.rank_genes_groups_dotplot, sc.pl.rank_genes_groups_heatmap, sc.pl.rank_genes_groups_matrixplot, sc.pl.rank_genes_groups_stacked_violin, sc.pl.rank_genes_groups_tracksplot, # TODO: add other rank_genes_groups plots here once they work - ), + ], ) def test_plot_rank_genes_groups_gene_symbols( gene_symbols_adatas, func, tmp_path, check_same_image @@ -876,14 +931,14 @@ def test_plot_rank_genes_groups_gene_symbols( @pytest.mark.parametrize( "func", - ( + [ sc.pl.rank_genes_groups_dotplot, sc.pl.rank_genes_groups_heatmap, sc.pl.rank_genes_groups_matrixplot, sc.pl.rank_genes_groups_stacked_violin, sc.pl.rank_genes_groups_tracksplot, # TODO: add other rank_genes_groups plots here once they work - ), + ], ) def test_rank_genes_groups_plots_n_genes_vs_var_names(tmp_path, func, check_same_image): """\ @@ -933,7 +988,7 @@ def wrapped(pth, **kwargs): @pytest.mark.parametrize( - "id,fn", + ("id", "fn"), [ ("heatmap", sc.pl.heatmap), ("dotplot", sc.pl.dotplot), @@ -955,8 +1010,8 @@ def test_genes_symbols(image_comparer, id, fn): save_and_compare_images(f"{id}_gene_symbols") -@pytest.fixture(scope="module") -def _pbmc_scatterplots_session(): +@pytest.fixture(scope="session") +def pbmc_scatterplots_session() -> AnnData: # Wrapped in another fixture to avoid mutation pbmc = pbmc68k_reduced() pbmc.obs["mask"] = pbmc.obs["louvain"].isin(["0", "1", "3"]) @@ -970,12 +1025,12 @@ def _pbmc_scatterplots_session(): @pytest.fixture -def pbmc_scatterplots(_pbmc_scatterplots_session): - return _pbmc_scatterplots_session.copy() +def pbmc_scatterplots(pbmc_scatterplots_session) -> AnnData: + return pbmc_scatterplots_session.copy() @pytest.mark.parametrize( - "id,fn", + ("id", "fn"), [ ("pca", partial(sc.pl.pca, color="bulk_labels")), ( @@ -1295,11 +1350,13 @@ def test_binary_scatter(image_comparer): def test_scatter_specify_layer_and_raw(): pbmc = pbmc68k_reduced() pbmc.layers["layer"] = pbmc.raw.X.copy() - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"Cannot use both a layer and.*raw"): sc.pl.umap(pbmc, color="HES4", use_raw=True, layer="layer") -@pytest.mark.parametrize("color", ["n_genes", "bulk_labels"]) +@pytest.mark.parametrize( + "color", ["n_genes", "bulk_labels", ["n_genes", "bulk_labels"]] +) def test_scatter_no_basis_per_obs(image_comparer, color): """Test scatterplot of per-obs points with no basis""" @@ -1315,7 +1372,8 @@ def test_scatter_no_basis_per_obs(image_comparer, color): # palette only applies to categorical, i.e. color=='bulk_labels' palette="Set2", ) - save_and_compare_images(f"scatter_HES_percent_mito_{color}") + color_str = color if isinstance(color, str) else "_".join(color) + save_and_compare_images(f"scatter_HES_percent_mito_{color_str}") def test_scatter_no_basis_per_var(image_comparer): @@ -1335,33 +1393,23 @@ def pbmc_filtered() -> Callable[[], AnnData]: return pbmc.copy -def test_scatter_no_basis_raw(check_same_image, pbmc_filtered, tmpdir): - adata = pbmc_filtered() - +@pytest.mark.parametrize("use_raw", [True, None]) +def test_scatter_no_basis_raw(check_same_image, pbmc_filtered, tmp_path, use_raw): """Test scatterplots of raw layer with no basis.""" - path1 = tmpdir / "scatter_EGFL7_F12_FAM185A_rawNone.png" - path2 = tmpdir / "scatter_EGFL7_F12_FAM185A_rawTrue.png" - path3 = tmpdir / "scatter_EGFL7_F12_FAM185A_rawToAdata.png" + adata = pbmc_filtered() - sc.pl.scatter(adata, x="EGFL7", y="F12", color="FAM185A", use_raw=None) - plt.savefig(path1) - plt.close() + sc.pl.scatter(adata.raw.to_adata(), x="EGFL7", y="F12", color="FAM185A") + plt.savefig(path1 := tmp_path / "scatter-raw-to-adata.png") - # is equivalent to: - sc.pl.scatter(adata, x="EGFL7", y="F12", color="FAM185A", use_raw=True) - plt.savefig(path2) + sc.pl.scatter(adata, x="EGFL7", y="F12", color="FAM185A", use_raw=use_raw) + plt.savefig(path2 := tmp_path / f"scatter-{use_raw=}.png") plt.close() - # and also to: - sc.pl.scatter(adata.raw.to_adata(), x="EGFL7", y="F12", color="FAM185A") - plt.savefig(path3) - check_same_image(path1, path2, tol=15) - check_same_image(path1, path3, tol=15) @pytest.mark.parametrize( - "x,y,color,use_raw", + ("x", "y", "color", "use_raw"), [ # test that plotting fails with a ValueError if trying to plot # var_names only found in raw and use_raw is False @@ -1380,7 +1428,9 @@ def test_scatter_no_basis_value_error(pbmc_filtered, x, y, color, use_raw): raise a `ValueError`. This test checks that this happens as expected. """ - with pytest.raises(ValueError): + with pytest.raises( + ValueError, match=r"inputs must all come from either `\.obs` or `\.var`" + ): sc.pl.scatter(pbmc_filtered(), x=x, y=y, color=color, use_raw=use_raw) @@ -1406,11 +1456,10 @@ def test_rankings(image_comparer): # TODO: Make more generic -def test_scatter_rep(tmpdir): +def test_scatter_rep(tmp_path): """ Test to make sure I can predict when scatter reps should be the same """ - TESTDIR = Path(tmpdir) rep_args = { "raw": {"use_raw": True}, "layer": {"layer": "layer", "use_raw": False}, @@ -1425,7 +1474,7 @@ def test_scatter_rep(tmpdir): columns=["rep", "gene", "result"], ) states["outpth"] = [ - TESTDIR / f"{state.gene}_{state.rep}_{state.result}.png" + tmp_path / f"{state.gene}_{state.rep}_{state.result}.png" for state in states.itertuples() ] pattern = np.array(list(chain.from_iterable(repeat(i, 5) for i in range(3)))) @@ -1550,11 +1599,13 @@ def test_color_cycler(caplog): colors = sns.color_palette("deep") cyl = sns.rcmod.cycler("color", sns.color_palette("deep")) - with caplog.at_level(logging.WARNING): - with plt.rc_context({"axes.prop_cycle": cyl, "patch.facecolor": colors[0]}): - sc.pl.umap(pbmc, color="phase") - plt.show() - plt.close() + with ( + caplog.at_level(logging.WARNING), + plt.rc_context({"axes.prop_cycle": cyl, "patch.facecolor": colors[0]}), + ): + sc.pl.umap(pbmc, color="phase") + plt.show() + plt.close() assert caplog.text == "" @@ -1577,14 +1628,14 @@ def test_repeated_colors_w_missing_value(): @pytest.mark.parametrize( "plot", - ( + [ sc.pl.rank_genes_groups_dotplot, sc.pl.rank_genes_groups_heatmap, sc.pl.rank_genes_groups_matrixplot, sc.pl.rank_genes_groups_stacked_violin, sc.pl.rank_genes_groups_tracksplot, # TODO: add other rank_genes_groups plots here once they work - ), + ], ) def test_filter_rank_genes_groups_plots(tmp_path, plot, check_same_image): N_GENES = 4 diff --git a/tests/test_plotting_embedded/conftest.py b/tests/test_plotting_embedded/conftest.py new file mode 100644 index 0000000000..d9e8ff8581 --- /dev/null +++ b/tests/test_plotting_embedded/conftest.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import pytest + +import scanpy as sc + +HERE: Path = Path(__file__).parent + + +@pytest.fixture(scope="module") +def adata(): + """A bit cute.""" + from matplotlib.image import imread + from sklearn.cluster import DBSCAN + from sklearn.datasets import make_blobs + + empty_pixel = np.array([1.0, 1.0, 1.0, 0]).reshape(1, 1, -1) + image = imread(HERE.parent.parent / "docs/_static/img/Scanpy_Logo_RGB.png") + x, y = np.where(np.logical_and.reduce(~np.equal(image, empty_pixel), axis=2)) + + # Just using to calculate the hex coords + hexes = plt.hexbin(x, y, gridsize=(44, 100)) + counts = hexes.get_array() + pixels = hexes.get_offsets()[counts != 0] + plt.close() + + labels = DBSCAN(eps=20, min_samples=2).fit(pixels).labels_ + order = np.argsort(labels) + adata = sc.AnnData( + make_blobs( + pd.Series(labels[order]).value_counts().values, + n_features=20, + shuffle=False, + random_state=42, + )[0], + obs={"label": pd.Categorical(labels[order].astype(str))}, + obsm={"spatial": pixels[order, ::-1]}, + uns={ + "spatial": { + "scanpy_img": { + "images": {"hires": image}, + "scalefactors": { + "tissue_hires_scalef": 1, + "spot_diameter_fullres": 10, + }, + } + } + }, + ) + sc.pp.pca(adata) + + # Adding some missing values + adata.obs["label_missing"] = adata.obs["label"].copy() + adata.obs.loc[::2, "label_missing"] = np.nan + + adata.obs["1_missing"] = adata.obs_vector("1") + adata.obs.loc[ + adata.obsm["spatial"][:, 0] < adata.obsm["spatial"][:, 0].mean(), "1_missing" + ] = np.nan + + return adata diff --git a/tests/test_plotting_embedded/test_embeddings.py b/tests/test_plotting_embedded/test_embeddings.py new file mode 100644 index 0000000000..c5dc8d3e53 --- /dev/null +++ b/tests/test_plotting_embedded/test_embeddings.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +from functools import partial, wraps +from pathlib import Path +from typing import TYPE_CHECKING + +import matplotlib.pyplot as plt +import numpy as np +import pytest +import seaborn as sns +from matplotlib.colors import Normalize +from matplotlib.testing.compare import compare_images + +import scanpy as sc +from testing.scanpy._helpers.data import pbmc3k_processed + +if TYPE_CHECKING: + from scanpy.plotting._utils import _LegendLoc + + +HERE: Path = Path(__file__).parent +ROOT = HERE.parent / "_images" + +MISSING_VALUES_ROOT = ROOT / "embedding-missing-values" + + +def check_images(pth1: Path, pth2: Path, *, tol: int) -> None: + result = compare_images(str(pth1), str(pth2), tol=tol) + assert result is None, result + + +@pytest.fixture( + params=[(0, 0, 0, 1), None], + ids=["na_color.black_tup", "na_color.default"], +) +def na_color(request): + return request.param + + +@pytest.fixture(params=[True, False], ids=["na_in_legend.True", "na_in_legend.False"]) +def na_in_legend(request): + return request.param + + +@pytest.fixture(params=[sc.pl.pca, sc.pl.spatial]) +def plotfunc(request): + if request.param is sc.pl.spatial: + + @wraps(request.param) + def f(adata, **kwargs): + with pytest.warns(FutureWarning, match=r"Use `squidpy.*` instead"): + return sc.pl.spatial(adata, **kwargs) + + else: + f = request.param + return partial(f, show=False) + + +@pytest.fixture( + params=["on data", "right margin", "lower center", None], + ids=["legend.on_data", "legend.on_right", "legend.on_bottom", "legend.off"], +) +def legend_loc(request) -> _LegendLoc | None: + return request.param + + +@pytest.fixture( + params=[lambda x: list(x.cat.categories[:3]), lambda x: []], + ids=["groups.3", "groups.all"], +) +def groupsfunc(request): + return request.param + + +@pytest.fixture( + params=[ + pytest.param( + {"vmin": None, "vmax": None, "vcenter": None, "norm": None}, + id="vbounds.default", + ), + pytest.param( + {"vmin": 0, "vmax": 5, "vcenter": None, "norm": None}, id="vbounds.numbers" + ), + pytest.param( + {"vmin": "p15", "vmax": "p90", "vcenter": None, "norm": None}, + id="vbounds.percentile", + ), + pytest.param( + {"vmin": 0, "vmax": "p99", "vcenter": 0.1, "norm": None}, + id="vbounds.vcenter", + ), + pytest.param( + {"vmin": None, "vmax": None, "vcenter": None, "norm": Normalize(0, 5)}, + id="vbounds.norm", + ), + ] +) +def vbounds(request): + return request.param + + +def test_missing_values_categorical( + *, + request: pytest.FixtureRequest, + image_comparer, + adata, + plotfunc, + na_color, + na_in_legend, + legend_loc, + groupsfunc, +): + save_and_compare_images = partial(image_comparer, MISSING_VALUES_ROOT, tol=15) + + base_name = request.node.name + + # Passing through a dict so it's easier to use default values + kwargs = {} + kwargs["legend_loc"] = legend_loc + kwargs["groups"] = groupsfunc(adata.obs["label"]) + if na_color is not None: + kwargs["na_color"] = na_color + kwargs["na_in_legend"] = na_in_legend + + plotfunc(adata, color=["label", "label_missing"], **kwargs) + + save_and_compare_images(base_name) + + +def test_missing_values_continuous( + *, + request: pytest.FixtureRequest, + image_comparer, + adata, + plotfunc, + na_color, + vbounds, +): + save_and_compare_images = partial(image_comparer, MISSING_VALUES_ROOT, tol=15) + + base_name = request.node.name + + # Passing through a dict so it's easier to use default values + kwargs = {} + kwargs.update(vbounds) + if na_color is not None: + kwargs["na_color"] = na_color + + plotfunc(adata, color=["1", "1_missing"], **kwargs) + + save_and_compare_images(base_name) + + +def test_enumerated_palettes(request, adata, tmp_path, plotfunc): + base_name = request.node.name + + categories = adata.obs["label"].cat.categories + colors_rgb = dict(zip(categories, sns.color_palette(n_colors=12))) + + dict_pth = tmp_path / f"rgbdict_{base_name}.png" + list_pth = tmp_path / f"rgblist_{base_name}.png" + + # making a copy so colors aren't saved + plotfunc(adata.copy(), color="label", palette=colors_rgb) + plt.savefig(dict_pth, dpi=40) + plt.close() + plotfunc(adata.copy(), color="label", palette=[colors_rgb[c] for c in categories]) + plt.savefig(list_pth, dpi=40) + plt.close() + + check_images(dict_pth, list_pth, tol=15) + + +def test_dimension_broadcasting(adata, tmp_path, check_same_image): + with pytest.raises( + ValueError, + match=r"Could not broadcast together arguments with shapes: \[2, 3, 1\]", + ): + sc.pl.pca( + adata, color=["label", "1_missing"], dimensions=[(0, 1), (1, 2), (2, 3)] + ) + + dims_pth = tmp_path / "broadcast_dims.png" + color_pth = tmp_path / "broadcast_colors.png" + + sc.pl.pca(adata, color=["label", "label", "label"], dimensions=(2, 3), show=False) + plt.savefig(dims_pth, dpi=40) + plt.close() + sc.pl.pca(adata, color="label", dimensions=[(2, 3), (2, 3), (2, 3)], show=False) + plt.savefig(color_pth, dpi=40) + plt.close() + + check_same_image(dims_pth, color_pth, tol=5) + + +def test_marker_broadcasting(adata, tmp_path, check_same_image): + with pytest.raises( + ValueError, + match=r"Could not broadcast together arguments with shapes: \[2, 1, 3\]", + ): + sc.pl.pca(adata, color=["label", "1_missing"], marker=[".", "^", "x"]) + + dims_pth = tmp_path / "broadcast_markers.png" + color_pth = tmp_path / "broadcast_colors_for_markers.png" + + sc.pl.pca(adata, color=["label", "label", "label"], marker="^", show=False) + plt.savefig(dims_pth, dpi=40) + plt.close() + sc.pl.pca(adata, color="label", marker=["^", "^", "^"], show=False) + plt.savefig(color_pth, dpi=40) + plt.close() + + check_same_image(dims_pth, color_pth, tol=5) + + +def test_dimensions_same_as_components(adata, tmp_path, check_same_image): + adata = adata.copy() + adata.obs["mean"] = np.ravel(adata.X.mean(axis=1)) + + comp_pth = tmp_path / "components_plot.png" + dims_pth = tmp_path / "dimension_plot.png" + + # TODO: Deprecate components kwarg + # with pytest.warns(FutureWarning, match=r"components .* deprecated"): + sc.pl.pca( + adata, + color=["mean", "label"], + components=["1,2", "2,3"], + show=False, + ) + plt.savefig(comp_pth, dpi=40) + plt.close() + + sc.pl.pca( + adata, + color=["mean", "mean", "label", "label"], + dimensions=[(0, 1), (1, 2), (0, 1), (1, 2)], + show=False, + ) + plt.savefig(dims_pth, dpi=40) + plt.close() + + check_same_image(dims_pth, comp_pth, tol=5) + + +def test_embedding_colorbar_location(image_comparer): + save_and_compare_images = partial(image_comparer, ROOT, tol=15) + + adata = pbmc3k_processed().raw.to_adata() + + sc.pl.pca(adata, color="LDHB", colorbar_loc=None) + + save_and_compare_images("no_colorbar") diff --git a/tests/test_plotting_embedded/test_spatial.py b/tests/test_plotting_embedded/test_spatial.py new file mode 100644 index 0000000000..873db68794 --- /dev/null +++ b/tests/test_plotting_embedded/test_spatial.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +from functools import partial +from pathlib import Path + +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +import pytest +from matplotlib.testing.compare import compare_images + +import scanpy as sc + +HERE: Path = Path(__file__).parent +ROOT = HERE.parent / "_images" +DATA_DIR = HERE.parent / "_data" + + +pytestmark = [ + pytest.mark.filterwarnings("ignore:Use `squidpy.*` instead:FutureWarning") +] + + +def check_images(pth1: Path, pth2: Path, *, tol: int) -> None: + result = compare_images(str(pth1), str(pth2), tol=tol) + assert result is None, result + + +def test_visium_circles(image_comparer): # standard visium data + save_and_compare_images = partial(image_comparer, ROOT, tol=15) + + adata = sc.read_visium(DATA_DIR / "visium_data" / "1.0.0") + adata.obs = adata.obs.astype({"array_row": "str"}) + + sc.pl.spatial( + adata, + color="array_row", + groups=["24", "33"], + crop_coord=(100, 400, 400, 100), + alpha=0.5, + size=1.3, + show=False, + ) + + save_and_compare_images("spatial_visium") + + +def test_visium_default(image_comparer): # default values + from packaging.version import parse as parse_version + + if parse_version(mpl.__version__) < parse_version("3.7.0"): + pytest.xfail("Matplotlib 3.7.0+ required for this test") + + save_and_compare_images = partial(image_comparer, ROOT, tol=5) + + adata = sc.read_visium(DATA_DIR / "visium_data" / "1.0.0") + adata.obs = adata.obs.astype({"array_row": "str"}) + + # Points default to transparent if an image is included + sc.pl.spatial(adata, show=False) + + save_and_compare_images("spatial_visium_default") + + +def test_visium_empty_img_key(image_comparer): # visium coordinates but image empty + save_and_compare_images = partial(image_comparer, ROOT, tol=15) + + adata = sc.read_visium(DATA_DIR / "visium_data" / "1.0.0") + adata.obs = adata.obs.astype({"array_row": "str"}) + + sc.pl.spatial(adata, img_key=None, color="array_row", show=False) + + save_and_compare_images("spatial_visium_empty_image") + + sc.pl.embedding(adata, basis="spatial", color="array_row", show=False) + save_and_compare_images("spatial_visium_embedding") + + +def test_spatial_general(image_comparer): # general coordinates + save_and_compare_images = partial(image_comparer, ROOT, tol=15) + + adata = sc.read_visium(DATA_DIR / "visium_data" / "1.0.0") + adata.obs = adata.obs.astype({"array_row": "str"}) + spatial_metadata = adata.uns.pop( + "spatial" + ) # spatial data don't have imgs, so remove entry from uns + # Required argument for now + spot_size = list(spatial_metadata.values())[0]["scalefactors"][ + "spot_diameter_fullres" + ] + + sc.pl.spatial(adata, show=False, spot_size=spot_size) + save_and_compare_images("spatial_general_nocol") + + # category + sc.pl.spatial(adata, show=False, spot_size=spot_size, color="array_row") + save_and_compare_images("spatial_general_cat") + + # continuous + sc.pl.spatial(adata, show=False, spot_size=spot_size, color="array_col") + save_and_compare_images("spatial_general_cont") + + +def test_spatial_external_img(image_comparer): # external image + save_and_compare_images = partial(image_comparer, ROOT, tol=15) + + adata = sc.read_visium(DATA_DIR / "visium_data" / "1.0.0") + adata.obs = adata.obs.astype({"array_row": "str"}) + + img = adata.uns["spatial"]["custom"]["images"]["hires"] + scalef = adata.uns["spatial"]["custom"]["scalefactors"]["tissue_hires_scalef"] + sc.pl.spatial( + adata, + color="array_row", + scale_factor=scalef, + img=img, + basis="spatial", + show=False, + ) + save_and_compare_images("spatial_external_img") + + +@pytest.fixture(scope="module") +def equivalent_spatial_plotters(adata): + no_spatial = adata.copy() + del no_spatial.uns["spatial"] + + img_key = "hires" + library_id = list(adata.uns["spatial"])[0] + spatial_data = adata.uns["spatial"][library_id] + img = spatial_data["images"][img_key] + scale_factor = spatial_data["scalefactors"][f"tissue_{img_key}_scalef"] + spot_size = spatial_data["scalefactors"]["spot_diameter_fullres"] + + orig_plotter = partial(sc.pl.spatial, adata, color="1", show=False) + removed_plotter = partial( + sc.pl.spatial, + no_spatial, + color="1", + img=img, + scale_factor=scale_factor, + spot_size=spot_size, + show=False, + ) + + return (orig_plotter, removed_plotter) + + +@pytest.fixture(scope="module") +def equivalent_spatial_plotters_no_img(equivalent_spatial_plotters): + orig, removed = equivalent_spatial_plotters + return (partial(orig, img_key=None), partial(removed, img=None, scale_factor=None)) + + +@pytest.fixture( + params=[ + pytest.param({"crop_coord": (50, 200, 0, 500)}, id="crop"), + pytest.param({"size": 0.5}, id="size:.5"), + pytest.param({"size": 2}, id="size:2"), + pytest.param({"spot_size": 5}, id="spotsize"), + pytest.param({"bw": True}, id="bw"), + # Shape of the image for particular fixture, should not be hardcoded like this + pytest.param({"img": np.ones((774, 1755, 4)), "scale_factor": 1.0}, id="img"), + pytest.param( + {"na_color": (0, 0, 0, 0), "color": "1_missing"}, id="na_color.transparent" + ), + pytest.param( + {"na_color": "lightgray", "color": "1_missing"}, id="na_color.lightgray" + ), + ] +) +def spatial_kwargs(request): + return request.param + + +def test_manual_equivalency(equivalent_spatial_plotters, tmp_path, spatial_kwargs): + """ + Tests that manually passing values to sc.pl.spatial is similar to automatic extraction. + """ + orig, removed = equivalent_spatial_plotters + + orig_pth = tmp_path / "orig.png" + removed_pth = tmp_path / "removed.png" + + orig(**spatial_kwargs) + plt.savefig(orig_pth, dpi=40) + plt.close() + removed(**spatial_kwargs) + plt.savefig(removed_pth, dpi=40) + plt.close() + + check_images(orig_pth, removed_pth, tol=1) + + +def test_manual_equivalency_no_img( + equivalent_spatial_plotters_no_img, tmp_path, spatial_kwargs +): + if "bw" in spatial_kwargs: + # Has no meaning when there is no image + pytest.skip() + orig, removed = equivalent_spatial_plotters_no_img + + orig_pth = tmp_path / "orig.png" + removed_pth = tmp_path / "removed.png" + + orig(**spatial_kwargs) + plt.savefig(orig_pth, dpi=40) + plt.close() + removed(**spatial_kwargs) + plt.savefig(removed_pth, dpi=40) + plt.close() + + check_images(orig_pth, removed_pth, tol=1) + + +def test_white_background_vs_no_img(adata, tmp_path, spatial_kwargs): + if {"bw", "img", "img_key", "na_color"}.intersection(spatial_kwargs): + # These arguments don't make sense for this check + pytest.skip() + + white_background = np.ones_like( + adata.uns["spatial"]["scanpy_img"]["images"]["hires"] + ) + white_pth = tmp_path / "white_background.png" + noimg_pth = tmp_path / "no_img.png" + + sc.pl.spatial( + adata, + color="2", + img=white_background, + scale_factor=1.0, + show=False, + **spatial_kwargs, + ) + plt.savefig(white_pth) + sc.pl.spatial(adata, color="2", img_key=None, show=False, **spatial_kwargs) + plt.savefig(noimg_pth) + + check_images(white_pth, noimg_pth, tol=1) + + +def test_spatial_na_color(adata, tmp_path): + """ + Check that na_color defaults to transparent when an image is present, light gray when not. + """ + white_background = np.ones_like( + adata.uns["spatial"]["scanpy_img"]["images"]["hires"] + ) + lightgray_pth = tmp_path / "lightgray.png" + transparent_pth = tmp_path / "transparent.png" + noimg_pth = tmp_path / "noimg.png" + whiteimg_pth = tmp_path / "whiteimg.png" + + def plot(pth, **kwargs): + sc.pl.spatial(adata, color="1_missing", show=False, **kwargs) + plt.savefig(pth, dpi=40) + plt.close() + + plot(lightgray_pth, na_color="lightgray", img_key=None) + plot(transparent_pth, na_color=(0.0, 0.0, 0.0, 0.0), img_key=None) + plot(noimg_pth, img_key=None) + plot(whiteimg_pth, img=white_background, scale_factor=1.0) + + check_images(lightgray_pth, noimg_pth, tol=1) + check_images(transparent_pth, whiteimg_pth, tol=1) + with pytest.raises(AssertionError): + check_images(lightgray_pth, transparent_pth, tol=1) diff --git a/tests/test_plotting_utils.py b/tests/test_plotting_utils.py index 6b53cd5b50..2090ffde38 100644 --- a/tests/test_plotting_utils.py +++ b/tests/test_plotting_utils.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import cast +from string import ascii_lowercase, ascii_uppercase +from typing import TYPE_CHECKING, cast import numpy as np import pytest @@ -8,8 +9,13 @@ from matplotlib import colormaps from matplotlib.colors import ListedColormap +from scanpy.plotting._anndata import _check_if_annotations from scanpy.plotting._utils import _validate_palette +if TYPE_CHECKING: + from typing import Any, Literal + + viridis = cast(ListedColormap, colormaps["viridis"]) @@ -27,3 +33,28 @@ def test_validate_palette_no_mod(palette, typ): adata = AnnData(uns=dict(test_colors=palette)) _validate_palette(adata, "test") assert palette is adata.uns["test_colors"], "Palette should not be modified" + + +@pytest.mark.parametrize( + ("axis_name", "args", "expected"), + [ + pytest.param("obs", {}, True, id="valid-nothing"), + pytest.param("obs", dict(x="B", colors=["obs_a"]), True, id="valid-basic"), + pytest.param("var", dict(colors=["A", "C", "obs_a"]), False, id="invalid-axis"), + pytest.param("obs", dict(x="A"), True, id="valid-raw"), + pytest.param("obs", dict(x="A", use_raw=False), False, id="invalid-noraw"), + pytest.param("obs", dict(colors=[(0, 0, 0), "red"]), True, id="valid-color"), + ], +) +def test_check_all_in_axis( + *, axis_name: Literal["obs", "var"], args: dict[str, Any], expected: bool +): + raw = AnnData( + np.random.randn(10, 20), + dict(obs_a=range(10), obs_names=list(ascii_lowercase[:10])), + dict(var_a=range(20), var_names=list(ascii_uppercase[:20])), + ) + adata = raw[:, 1:].copy() + adata.raw = raw + + assert _check_if_annotations(adata, axis_name, **args) is expected diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index 9504b14217..6282c5ccf4 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -1,6 +1,10 @@ from __future__ import annotations +import warnings +from importlib.util import find_spec from itertools import product +from pathlib import Path +from typing import TYPE_CHECKING import numpy as np import pandas as pd @@ -9,17 +13,30 @@ from anndata.tests.helpers import asarray, assert_equal from numpy.testing import assert_allclose from scipy import sparse as sp -from scipy.sparse import issparse +from scipy.sparse import coo_matrix, csc_matrix, csr_matrix, issparse import scanpy as sc from testing.scanpy._helpers import ( anndata_v0_8_constructor_compat, check_rep_mutation, check_rep_results, + maybe_dask_process_context, ) from testing.scanpy._helpers.data import pbmc3k, pbmc68k_reduced from testing.scanpy._pytest.params import ARRAY_TYPES +if TYPE_CHECKING: + from collections.abc import Callable + from typing import Any, Literal + + from numpy.typing import NDArray + + CSMatrix = sp.csc_matrix | sp.csr_matrix + + +HERE = Path(__file__).parent +DATA_PATH = HERE / "_data" + def test_log1p(tmp_path): A = np.random.rand(200, 10).astype(np.float32) @@ -129,34 +146,159 @@ def test_normalize_per_cell(): assert adata.X.sum(axis=1).tolist() == adata_sparse.X.sum(axis=1).A1.tolist() -def test_subsample(): - adata = AnnData(np.ones((200, 10))) - sc.pp.subsample(adata, n_obs=40) - assert adata.n_obs == 40 - sc.pp.subsample(adata, fraction=0.1) - assert adata.n_obs == 4 +def _random_probs(n: int, frac_zero: float) -> NDArray[np.float64]: + """ + Generate a random probability distribution of `n` values between 0 and 1. + """ + probs = np.random.randint(0, 10000, n).astype(np.float64) + probs[probs < np.quantile(probs, frac_zero)] = 0 + probs /= probs.sum() + np.testing.assert_almost_equal(probs.sum(), 1) + return probs + +@pytest.mark.parametrize("array_type", ARRAY_TYPES) +@pytest.mark.parametrize("which", ["copy", "inplace", "array"]) +@pytest.mark.parametrize( + ("axis", "f_or_n", "replace"), + [ + pytest.param(0, 40, False, id="obs-40-no_replace"), + pytest.param(0, 0.1, False, id="obs-0.1-no_replace"), + pytest.param(0, 201, True, id="obs-201-replace"), + pytest.param(0, 1, True, id="obs-1-replace"), + pytest.param(1, 10, False, id="var-10-no_replace"), + pytest.param(1, 11, True, id="var-11-replace"), + pytest.param(1, 2.0, True, id="var-2.0-replace"), + ], +) +@pytest.mark.parametrize( + "ps", + [ + dict(obs=None, var=None), + dict(obs=np.tile([True, False], 100), var=np.tile([True, False], 5)), + dict(obs=_random_probs(200, 0.3), var=_random_probs(10, 0.7)), + ], + ids=["all", "mask", "p"], +) +def test_sample( + *, + request: pytest.FixtureRequest, + array_type: Callable[[np.ndarray], np.ndarray | CSMatrix], + which: Literal["copy", "inplace", "array"], + axis: Literal[0, 1], + f_or_n: float | int, # noqa: PYI041 + replace: bool, + ps: dict[Literal["obs", "var"], NDArray[np.bool_] | None], +): + adata = AnnData(array_type(np.ones((200, 10)))) + p = ps["obs" if axis == 0 else "var"] + expected = int(adata.shape[axis] * f_or_n) if isinstance(f_or_n, float) else f_or_n + if p is not None and not replace and expected > (n_possible := (p != 0).sum()): + request.applymarker(pytest.xfail(f"Can’t draw {expected} out of {n_possible}")) + + # ignoring this warning declaratively is a pain so do it here + if find_spec("dask"): + import dask.array as da + + warnings.filterwarnings("ignore", category=da.PerformanceWarning) + # can’t guarantee that duplicates are drawn when `replace=True`, + # so we just ignore the warning instead using `with pytest.warns(...)` + warnings.filterwarnings( + "ignore" if replace else "error", r".*names are not unique", UserWarning + ) + rv = sc.pp.sample( + adata.X if which == "array" else adata, + f_or_n if isinstance(f_or_n, float) else None, + n=f_or_n if isinstance(f_or_n, int) else None, + replace=replace, + axis=axis, + # `copy` only effects AnnData inputs + copy=dict(copy=True, inplace=False, array=False)[which], + p=p, + ) -def test_subsample_copy(): + match which: + case "copy": + subset = rv + assert rv is not adata + assert adata.shape == (200, 10) + case "inplace": + subset = adata + assert rv is None + case "array": + subset, indices = rv + assert len(indices) == expected + assert adata.shape == (200, 10) + case _: + pytest.fail(f"Unknown `{which=}`") + + assert subset.shape == ((expected, 10) if axis == 0 else (200, expected)) + + +@pytest.mark.parametrize( + ("args", "exc", "pattern"), + [ + pytest.param( + dict(), TypeError, r"Either `fraction` or `n` must be set", id="empty" + ), + pytest.param( + dict(n=10, fraction=0.2), + TypeError, + r"Providing both `fraction` and `n` is not allowed", + id="both", + ), + pytest.param( + dict(fraction=2), + ValueError, + r"If `replace=False`, `fraction=2` needs to be", + id="frac>1", + ), + pytest.param( + dict(fraction=-0.3), + ValueError, + r"`fraction=-0\.3` needs to be nonnegative", + id="frac<0", + ), + pytest.param( + dict(n=3, p=np.ones(200, dtype=np.int32)), + ValueError, + r"mask/probabilities array must be boolean or floating point", + id="type(p)", + ), + ], +) +def test_sample_error(args: dict[str, Any], exc: type[Exception], pattern: str): adata = AnnData(np.ones((200, 10))) - assert sc.pp.subsample(adata, n_obs=40, copy=True).shape == (40, 10) - assert sc.pp.subsample(adata, fraction=0.1, copy=True).shape == (20, 10) + with pytest.raises(exc, match=pattern): + sc.pp.sample(adata, **args) -def test_subsample_copy_backed(tmp_path): - A = np.random.rand(200, 10).astype(np.float32) - adata_m = AnnData(A.copy()) - adata_d = AnnData(A.copy()) - filename = tmp_path / "test.h5ad" - adata_d.filename = filename - # This should not throw an error - assert sc.pp.subsample(adata_d, n_obs=40, copy=True).shape == (40, 10) +def test_sample_backwards_compat(): + expected = np.array( + [26, 86, 2, 55, 75, 93, 16, 73, 54, 95, 53, 92, 78, 13, 7, 30, 22, 24, 33, 8] + ) + legacy_result, indices = sc.pp.subsample(np.arange(100), n_obs=20) + assert np.array_equal(indices, legacy_result), "arange choices should match indices" + assert np.array_equal(legacy_result, expected) + + +def test_sample_copy_backed(tmp_path): + adata_m = AnnData(np.random.rand(200, 10).astype(np.float32)) + adata_d = adata_m.copy() + adata_d.filename = tmp_path / "test.h5ad" + + assert sc.pp.sample(adata_d, n=40, copy=True).shape == (40, 10) np.testing.assert_array_equal( - sc.pp.subsample(adata_m, n_obs=40, copy=True).X, - sc.pp.subsample(adata_d, n_obs=40, copy=True).X, + sc.pp.sample(adata_m, n=40, copy=True, rng=0).X, + sc.pp.sample(adata_d, n=40, copy=True, rng=0).X, ) + + +def test_sample_copy_backed_error(tmp_path): + adata_d = AnnData(np.random.rand(200, 10).astype(np.float32)) + adata_d.filename = tmp_path / "test.h5ad" with pytest.raises(NotImplementedError): - sc.pp.subsample(adata_d, n_obs=40, copy=False) + sc.pp.sample(adata_d, n=40, copy=False) @pytest.mark.parametrize("array_type", ARRAY_TYPES) @@ -168,7 +310,8 @@ def test_scale_matrix_types(array_type, zero_center, max_value): adata_casted = adata.copy() adata_casted.X = array_type(adata_casted.raw.X) sc.pp.scale(adata, zero_center=zero_center, max_value=max_value) - sc.pp.scale(adata_casted, zero_center=zero_center, max_value=max_value) + with maybe_dask_process_context(): + sc.pp.scale(adata_casted, zero_center=zero_center, max_value=max_value) X = adata_casted.X if "dask" in array_type.__name__: X = X.compute() @@ -321,6 +464,16 @@ def test_regress_out_constants(): assert_equal(adata, adata_copy) +def test_regress_out_reproducible(): + adata = pbmc68k_reduced() + adata = adata.raw.to_adata()[:200, :200].copy() + sc.pp.regress_out(adata, keys=["n_counts", "percent_mito"]) + # This file was generated from the original implementation in version 1.10.3 + # Now we compare new implementation with the old one + tester = np.load(DATA_PATH / "regress_test_small.npy") + np.testing.assert_allclose(adata.X, tester) + + def test_regress_out_constants_equivalent(): # Tests that constant values don't change results # (since support for constant values is implemented by us) @@ -356,11 +509,11 @@ def test_downsample_counts_per_cell(count_matrix_format, replace, dtype): X = np.random.randint(0, 100, (1000, 100)) * np.random.binomial(1, 0.3, (1000, 100)) X = X.astype(dtype) adata = anndata_v0_8_constructor_compat(X=count_matrix_format(X).astype(dtype)) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"Must specify exactly one"): sc.pp.downsample_counts( adata, counts_per_cell=TARGET, total_counts=TARGET, replace=replace ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"Must specify exactly one"): sc.pp.downsample_counts(adata, replace=replace) initial_totals = np.ravel(adata.X.sum(axis=1)) adata = sc.pp.downsample_counts( @@ -389,7 +542,7 @@ def test_downsample_counts_per_cell_multiple_targets( X = X.astype(dtype) adata = anndata_v0_8_constructor_compat(X=count_matrix_format(X).astype(dtype)) initial_totals = np.ravel(adata.X.sum(axis=1)) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"counts_per_cell.*length as number of obs"): sc.pp.downsample_counts(adata, counts_per_cell=[40, 10], replace=replace) adata = sc.pp.downsample_counts( adata, counts_per_cell=TARGETS, replace=replace, copy=True @@ -448,12 +601,12 @@ def test_recipe_weinreb(): @pytest.mark.parametrize("array_type", ARRAY_TYPES) @pytest.mark.parametrize( - "max_cells,max_counts,min_cells,min_counts", + ("max_cells", "max_counts", "min_cells", "min_counts"), [ - [100, None, None, None], - [None, 100, None, None], - [None, None, 20, None], - [None, None, None, 20], + (100, None, None, None), + (None, 100, None, None), + (None, None, 20, None), + (None, None, None, 20), ], ) def test_filter_genes(array_type, max_cells, max_counts, min_cells, min_counts): @@ -487,12 +640,12 @@ def test_filter_genes(array_type, max_cells, max_counts, min_cells, min_counts): @pytest.mark.parametrize("array_type", ARRAY_TYPES) @pytest.mark.parametrize( - "max_genes,max_counts,min_genes,min_counts", + ("max_genes", "max_counts", "min_genes", "min_counts"), [ - [100, None, None, None], - [None, 100, None, None], - [None, None, 20, None], - [None, None, None, 20], + (100, None, None, None), + (None, 100, None, None), + (None, None, 20, None), + (None, None, None, 20), ], ) def test_filter_cells(array_type, max_genes, max_counts, min_genes, min_counts): @@ -522,3 +675,14 @@ def test_filter_cells(array_type, max_genes, max_counts, min_genes, min_counts): if issparse(adata.X): adata.X = adata.X.todense() assert_allclose(X, adata.X, rtol=1e-5, atol=1e-5) + + +@pytest.mark.parametrize("array_type", [csr_matrix, csc_matrix, coo_matrix]) +@pytest.mark.parametrize("order", ["C", "F"]) +def test_todense(array_type, order): + x_org = np.array([[0, 1, 2], [3, 0, 4]]) + x_sparse = array_type(x_org) + x_dense = sc.pp._utils._to_dense(x_sparse, order=order) + np.testing.assert_array_equal(x_dense, x_org) + assert x_dense.flags["C_CONTIGUOUS"] == (order == "C") + assert x_dense.flags["F_CONTIGUOUS"] == (order == "F") diff --git a/tests/test_preprocessing_distributed.py b/tests/test_preprocessing_distributed.py index 374c6c4b31..afb120b982 100644 --- a/tests/test_preprocessing_distributed.py +++ b/tests/test_preprocessing_distributed.py @@ -31,7 +31,7 @@ pytestmark = [needs.zarr] -@pytest.fixture() +@pytest.fixture @filter_oldformatwarning def adata() -> AnnData: a = read_zarr(input_file) @@ -40,13 +40,13 @@ def adata() -> AnnData: return a -@filter_oldformatwarning @pytest.fixture( params=[ pytest.param("direct", marks=[needs.zappy]), pytest.param("dask", marks=[needs.dask, pytest.mark.anndata_dask_support]), ] ) +@filter_oldformatwarning def adata_dist(request: pytest.FixtureRequest) -> AnnData: # regular anndata except for X, which we replace on the next line a = read_zarr(input_file) @@ -75,12 +75,13 @@ def test_log1p(adata: AnnData, adata_dist: AnnData): npt.assert_allclose(result, adata.X) +@pytest.mark.filterwarnings("ignore:Use sc.pp.normalize_total instead:FutureWarning") def test_normalize_per_cell( request: pytest.FixtureRequest, adata: AnnData, adata_dist: AnnData ): if isinstance(adata_dist.X, DaskArray): - msg = "normalize_per_cell deprecated and broken for Dask" - request.node.add_marker(pytest.mark.xfail(reason=msg)) + reason = "normalize_per_cell deprecated and broken for Dask" + request.applymarker(pytest.mark.xfail(reason=reason)) normalize_per_cell(adata_dist) assert isinstance(adata_dist.X, DIST_TYPES) result = materialize_as_ndarray(adata_dist.X) @@ -156,7 +157,7 @@ def test_write_zarr(adata: AnnData, adata_dist: AnnData): elif adata_dist.uns["dist-mode"] == "direct": adata_dist.X.to_zarr(temp_store.dir_path("X"), chunks) else: - assert False, "add branch for new dist-mode" + pytest.fail("add branch for new dist-mode") # read back as zarr directly and check it is the same as adata.X adata_log1p = read_zarr(temp_store) diff --git a/tests/test_qc_metrics.py b/tests/test_qc_metrics.py index 83971fa2ce..7ca6534b7c 100644 --- a/tests/test_qc_metrics.py +++ b/tests/test_qc_metrics.py @@ -4,19 +4,25 @@ import pandas as pd import pytest from anndata import AnnData +from anndata.tests.helpers import assert_equal from scipy import sparse import scanpy as sc +from scanpy._compat import DaskArray +from scanpy._utils import axis_sum from scanpy.preprocessing._qc import ( describe_obs, describe_var, top_proportions, top_segment_proportions, ) +from testing.scanpy._helpers import as_sparse_dask_array, maybe_dask_process_context +from testing.scanpy._pytest.marks import needs +from testing.scanpy._pytest.params import ARRAY_TYPES @pytest.fixture -def anndata(): +def adata() -> AnnData: a = np.random.binomial(100, 0.005, (1000, 1000)) adata = AnnData( sparse.csr_matrix(a), @@ -26,6 +32,22 @@ def anndata(): return adata +def prepare_adata(adata: AnnData) -> AnnData: + if isinstance(adata.X, DaskArray): + adata.X = adata.X.rechunk((100, -1)) + adata.var["mito"] = np.concatenate( + (np.ones(100, dtype=bool), np.zeros(900, dtype=bool)) + ) + adata.var["negative"] = False + return adata + + +@pytest.fixture(params=ARRAY_TYPES) +def adata_prepared(request: pytest.FixtureRequest, adata: AnnData) -> AnnData: + adata.X = request.param(adata.X) + return prepare_adata(adata) + + @pytest.mark.parametrize( "a", [np.ones((100, 100)), sparse.csr_matrix(np.ones((100, 100)))], @@ -67,58 +89,101 @@ def test_top_segments(cls): # While many of these are trivial, # they’re also just making sure the metrics are there -def test_qc_metrics(): - adata = AnnData(X=sparse.csr_matrix(np.random.binomial(100, 0.005, (1000, 1000)))) - adata.var["mito"] = np.concatenate( - (np.ones(100, dtype=bool), np.zeros(900, dtype=bool)) +def test_qc_metrics(adata_prepared: AnnData): + with maybe_dask_process_context(): + sc.pp.calculate_qc_metrics( + adata_prepared, qc_vars=["mito", "negative"], inplace=True + ) + X = ( + adata_prepared.X.compute() + if isinstance(adata_prepared.X, DaskArray) + else adata_prepared.X ) - adata.var["negative"] = False - sc.pp.calculate_qc_metrics(adata, qc_vars=["mito", "negative"], inplace=True) - assert (adata.obs["n_genes_by_counts"] < adata.shape[1]).all() + max_X = X.max(axis=0) + if isinstance(max_X, sparse.spmatrix): + max_X = max_X.toarray() + elif isinstance(max_X, DaskArray): + max_X = max_X.compute() + assert (adata_prepared.obs["n_genes_by_counts"] < adata_prepared.shape[1]).all() + assert ( + adata_prepared.obs["n_genes_by_counts"] + >= adata_prepared.obs["log1p_n_genes_by_counts"] + ).all() + assert ( + adata_prepared.obs["total_counts"] + == np.ravel(axis_sum(adata_prepared.X, axis=1)) + ).all() assert ( - adata.obs["n_genes_by_counts"] >= adata.obs["log1p_n_genes_by_counts"] + adata_prepared.obs["total_counts"] >= adata_prepared.obs["log1p_total_counts"] ).all() - assert (adata.obs["total_counts"] == np.ravel(adata.X.sum(axis=1))).all() - assert (adata.obs["total_counts"] >= adata.obs["log1p_total_counts"]).all() assert ( - adata.obs["total_counts_mito"] >= adata.obs["log1p_total_counts_mito"] + adata_prepared.obs["total_counts_mito"] + >= adata_prepared.obs["log1p_total_counts_mito"] ).all() - assert (adata.obs["total_counts_negative"] == 0).all() + assert (adata_prepared.obs["total_counts_negative"] == 0).all() assert ( - adata.obs["pct_counts_in_top_50_genes"] - <= adata.obs["pct_counts_in_top_100_genes"] + adata_prepared.obs["pct_counts_in_top_50_genes"] + <= adata_prepared.obs["pct_counts_in_top_100_genes"] ).all() - for col in filter(lambda x: "negative" not in x, adata.obs.columns): - assert (adata.obs[col] >= 0).all() # Values should be positive or zero - assert (adata.obs[col] != 0).any().all() # Nothing should be all zeros + for col in filter(lambda x: "negative" not in x, adata_prepared.obs.columns): + assert (adata_prepared.obs[col] >= 0).all() # Values should be positive or zero + assert (adata_prepared.obs[col] != 0).any().all() # Nothing should be all zeros if col.startswith("pct_counts_in_top"): - assert (adata.obs[col] <= 100).all() - assert (adata.obs[col] >= 0).all() - for col in adata.var.columns: - assert (adata.var[col] >= 0).all() - assert (adata.var["mean_counts"] < np.ravel(adata.X.max(axis=0).todense())).all() - assert (adata.var["mean_counts"] >= adata.var["log1p_mean_counts"]).all() - assert (adata.var["total_counts"] >= adata.var["log1p_total_counts"]).all() - # Should return the same thing if run again - old_obs, old_var = adata.obs.copy(), adata.var.copy() - sc.pp.calculate_qc_metrics(adata, qc_vars=["mito", "negative"], inplace=True) - assert set(adata.obs.columns) == set(old_obs.columns) - assert set(adata.var.columns) == set(old_var.columns) - for col in adata.obs: - assert np.allclose(adata.obs[col], old_obs[col]) - for col in adata.var: - assert np.allclose(adata.var[col], old_var[col]) - # with log1p=False - adata = AnnData(X=sparse.csr_matrix(np.random.binomial(100, 0.005, (1000, 1000)))) - adata.var["mito"] = np.concatenate( - (np.ones(100, dtype=bool), np.zeros(900, dtype=bool)) - ) - adata.var["negative"] = False + assert (adata_prepared.obs[col] <= 100).all() + assert (adata_prepared.obs[col] >= 0).all() + for col in adata_prepared.var.columns: + assert (adata_prepared.var[col] >= 0).all() + assert (adata_prepared.var["mean_counts"] < np.ravel(max_X)).all() + assert ( + adata_prepared.var["mean_counts"] >= adata_prepared.var["log1p_mean_counts"] + ).all() + assert ( + adata_prepared.var["total_counts"] >= adata_prepared.var["log1p_total_counts"] + ).all() + + +def test_qc_metrics_idempotent(adata_prepared: AnnData): + with maybe_dask_process_context(): + sc.pp.calculate_qc_metrics( + adata_prepared, qc_vars=["mito", "negative"], inplace=True + ) + old_obs, old_var = adata_prepared.obs.copy(), adata_prepared.var.copy() + sc.pp.calculate_qc_metrics( + adata_prepared, qc_vars=["mito", "negative"], inplace=True + ) + assert set(adata_prepared.obs.columns) == set(old_obs.columns) + assert set(adata_prepared.var.columns) == set(old_var.columns) + for col in adata_prepared.obs: + assert np.allclose(adata_prepared.obs[col], old_obs[col]) + for col in adata_prepared.var: + assert np.allclose(adata_prepared.var[col], old_var[col]) + + +def test_qc_metrics_no_log1p(adata_prepared: AnnData): + with maybe_dask_process_context(): + sc.pp.calculate_qc_metrics( + adata_prepared, qc_vars=["mito", "negative"], log1p=False, inplace=True + ) + assert not np.any(adata_prepared.obs.columns.str.startswith("log1p_")) + assert not np.any(adata_prepared.var.columns.str.startswith("log1p_")) + + +@needs.dask +@pytest.mark.anndata_dask_support +@pytest.mark.parametrize("log1p", [True, False], ids=["log1p", "no_log1p"]) +def test_dask_against_in_memory(adata, log1p): + adata_as_dask = adata.copy() + adata_as_dask.X = as_sparse_dask_array(adata.X) + adata = prepare_adata(adata) + adata_as_dask = prepare_adata(adata_as_dask) + with maybe_dask_process_context(): + sc.pp.calculate_qc_metrics( + adata_as_dask, qc_vars=["mito", "negative"], log1p=log1p, inplace=True + ) sc.pp.calculate_qc_metrics( - adata, qc_vars=["mito", "negative"], log1p=False, inplace=True + adata, qc_vars=["mito", "negative"], log1p=log1p, inplace=True ) - assert not np.any(adata.obs.columns.str.startswith("log1p_")) - assert not np.any(adata.var.columns.str.startswith("log1p_")) + assert_equal(adata, adata_as_dask) def adata_mito(): @@ -166,8 +231,8 @@ def test_qc_metrics_percentage(): # In response to #421 sc.pp.calculate_qc_metrics(adata_dense, percent_top=[20, 30, 1001]) -def test_layer_raw(anndata): - adata = anndata.copy() +def test_layer_raw(adata: AnnData): + adata = adata.copy() adata.raw = adata.copy() adata.layers["counts"] = adata.X.copy() obs_orig, var_orig = sc.pp.calculate_qc_metrics(adata) @@ -180,8 +245,8 @@ def test_layer_raw(anndata): assert np.allclose(var_orig, var_raw) -def test_inner_methods(anndata): - adata = anndata.copy() +def test_inner_methods(adata: AnnData): + adata = adata.copy() full_inplace = adata.copy() partial_inplace = adata.copy() obs_orig, var_orig = sc.pp.calculate_qc_metrics(adata) diff --git a/tests/test_rank_genes_groups.py b/tests/test_rank_genes_groups.py index e81d1c1112..b938fd2ca3 100644 --- a/tests/test_rank_genes_groups.py +++ b/tests/test_rank_genes_groups.py @@ -59,14 +59,12 @@ def get_example_data(array_type: Callable[[np.ndarray], Any]) -> AnnData: return adata -def get_true_scores() -> ( - tuple[ - NDArray[np.object_], - NDArray[np.object_], - NDArray[np.floating], - NDArray[np.floating], - ] -): +def get_true_scores() -> tuple[ + NDArray[np.object_], + NDArray[np.object_], + NDArray[np.floating], + NDArray[np.floating], +]: with (DATA_PATH / "objs_t_test.pkl").open("rb") as f: true_scores_t_test, true_names_t_test = pickle.load(f) with (DATA_PATH / "objs_wilcoxon.pkl").open("rb") as f: @@ -307,6 +305,13 @@ def test_wilcoxon_tie_correction(reference): np.testing.assert_allclose(test_obj.stats[groups[0]]["pvals"], pvals) +def test_wilcoxon_huge_data(monkeypatch): + max_size = 300 + adata = pbmc68k_reduced() + monkeypatch.setattr(sc.tl._rank_genes_groups, "_CONST_MAX_SIZE", max_size) + rank_genes_groups(adata, groupby="bulk_labels", method="wilcoxon") + + @pytest.mark.parametrize( ("n_genes_add", "n_genes_out_add"), [pytest.param(0, 0, id="equal"), pytest.param(2, 1, id="more")], @@ -320,7 +325,7 @@ def test_mask_n_genes(n_genes_add, n_genes_out_add): pbmc = pbmc68k_reduced() mask_var = np.zeros(pbmc.shape[1]).astype(bool) - mask_var[:6].fill(True) + mask_var[:6].fill(True) # noqa: FBT003 no_genes = sum(mask_var) - 1 rank_genes_groups( diff --git a/tests/test_rank_genes_groups_logreg.py b/tests/test_rank_genes_groups_logreg.py index 3cc294487e..618de375f7 100644 --- a/tests/test_rank_genes_groups_logreg.py +++ b/tests/test_rank_genes_groups_logreg.py @@ -40,7 +40,7 @@ def test_rank_genes_groups_with_renamed_categories_use_rep(): assert adata.uns["rank_genes_groups"]["names"][0].tolist() == ("1", "3", "0") sc.tl.rank_genes_groups(adata, "blobs", method="logreg") - assert not adata.uns["rank_genes_groups"]["names"][0].tolist() == ("3", "1", "0") + assert adata.uns["rank_genes_groups"]["names"][0].tolist() != ("3", "1", "0") def test_rank_genes_groups_with_unsorted_groups(): diff --git a/tests/test_read_10x.py b/tests/test_read_10x.py index 1fe7d20315..301a156bec 100644 --- a/tests/test_read_10x.py +++ b/tests/test_read_10x.py @@ -23,7 +23,7 @@ def assert_anndata_equal(a1, a2): @pytest.mark.parametrize( - ["mtx_path", "h5_path"], + ("mtx_path", "h5_path"), [ pytest.param( ROOT / "1.2.0" / "filtered_gene_bc_matrices" / "hg19_chr21", @@ -109,7 +109,7 @@ def test_error_10x_h5_legacy(tmp_path): with h5py.File(onepth, "r") as one, h5py.File(twopth, "w") as two: one.copy("hg19_chr21", two) one.copy("hg19_chr21", two, name="hg19_chr21_copy") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"contains more than one genome"): sc.read_10x_h5(twopth) sc.read_10x_h5(twopth, genome="hg19_chr21_copy") @@ -140,9 +140,10 @@ def visium_pth(request, tmp_path) -> Path: (orig.parent / "tissue_positions.csv").write_text(csv) return visium2_pth else: - assert False + pytest.fail("add branch for new visium version") +@pytest.mark.filterwarnings("ignore:Use `squidpy.*` instead:FutureWarning") def test_read_visium_counts(visium_pth): """Test checking that read_visium reads the right genome""" spec_genome_v3 = sc.read_visium(visium_pth, genome="GRCh38") diff --git a/tests/test_scaling.py b/tests/test_scaling.py index b169f7a51e..0ad62bbc7d 100644 --- a/tests/test_scaling.py +++ b/tests/test_scaling.py @@ -114,13 +114,13 @@ def test_scale(*, typ, dtype, mask_obs, X, X_centered, X_scaled): def test_mask_string(): - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"Cannot refer to mask.* without.*anndata"): sc.pp.scale(np.array(X_original), mask_obs="mask") adata = AnnData(np.array(X_for_mask, dtype="float32")) adata.obs["some cells"] = np.array((0, 0, 1, 1, 1, 0, 0), dtype=bool) sc.pp.scale(adata, mask_obs="some cells") assert np.array_equal(adata.X, X_centered_for_mask) - assert "mean of some cells" in adata.var.keys() + assert "mean of some cells" in adata.var.columns @pytest.mark.parametrize("zero_center", [True, False]) diff --git a/tests/test_score_genes.py b/tests/test_score_genes.py index 4db68ed53d..4ac1b62224 100644 --- a/tests/test_score_genes.py +++ b/tests/test_score_genes.py @@ -1,6 +1,7 @@ from __future__ import annotations import pickle +from contextlib import nullcontext from pathlib import Path from typing import TYPE_CHECKING @@ -18,7 +19,8 @@ from numpy.typing import NDArray -HERE = Path(__file__).parent / "_data" +HERE = Path(__file__).parent +DATA_PATH = HERE / "_data" def _create_random_gene_names(n_genes, name_length) -> NDArray[np.str_]: @@ -71,7 +73,7 @@ def test_score_with_reference(): sc.pp.scale(adata) sc.tl.score_genes(adata, gene_list=adata.var_names[:100], score_name="Test") - with (HERE / "score_genes_reference_paul2015.pkl").open("rb") as file: + with (DATA_PATH / "score_genes_reference_paul2015.pkl").open("rb") as file: reference = pickle.load(file) # np.testing.assert_allclose(reference, adata.obs["Test"].to_numpy()) np.testing.assert_array_equal(reference, adata.obs["Test"].to_numpy()) @@ -220,7 +222,7 @@ def test_missing_genes(): # These genes have a different length of name non_extant_genes = _create_random_gene_names(n_genes=3, name_length=7) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=r"No valid genes were passed for scoring"): sc.tl.score_genes(adata, non_extant_genes) @@ -250,6 +252,7 @@ def test_layer(): sc.tl.score_genes(adata, gene_set, score_name="X_score") # score layer (`del` makes sure it actually uses the layer) adata.layers["test"] = adata.X.copy() + adata.raw = adata del adata.X sc.tl.score_genes(adata, gene_set, score_name="test_score", layer="test") @@ -262,3 +265,28 @@ def test_invalid_gene_pool(gene_pool): with pytest.raises(ValueError, match="reference set"): sc.tl.score_genes(adata, adata.var_names[:3], gene_pool=gene_pool) + + +def test_no_control_gene(): + np.random.seed(0) + adata = _create_adata(100, 1, p_zero=0, p_nan=0) + + with pytest.raises(RuntimeError, match="No control genes found"): + sc.tl.score_genes(adata, adata.var_names[:1], ctrl_size=1) + + +@pytest.mark.parametrize( + "ctrl_as_ref", [True, False], ids=["ctrl_as_ref", "no_ctrl_as_ref"] +) +def test_gene_list_is_control(*, ctrl_as_ref: bool): + np.random.seed(0) + adata = sc.datasets.blobs(n_variables=10, n_observations=100, n_centers=20) + adata.var_names = "g" + adata.var_names + with ( + pytest.raises(RuntimeError, match=r"No control genes found in any cut") + if ctrl_as_ref + else nullcontext() + ): + sc.tl.score_genes( + adata, gene_list="g3", ctrl_size=1, n_bins=5, ctrl_as_ref=ctrl_as_ref + ) diff --git a/tests/test_scrublet.py b/tests/test_scrublet.py index b3c7172384..246ffa4027 100644 --- a/tests/test_scrublet.py +++ b/tests/test_scrublet.py @@ -117,7 +117,7 @@ def _create_sim_from_parents(adata: AnnData, parents: np.ndarray) -> AnnData: ) -def test_scrublet_data(): +def test_scrublet_data(cache: pytest.Cache): """ Test that Scrublet processing is arranged correctly. @@ -156,18 +156,36 @@ def test_scrublet_data(): random_state=random_state, ) - # Require that the doublet scores are the same whether simulation is via - # the main function or manually provided - assert_allclose( - adata_scrublet_manual_sim.obs["doublet_score"], - adata_scrublet_auto_sim.obs["doublet_score"], - atol=1e-15, - rtol=1e-15, - ) + try: + # Require that the doublet scores are the same whether simulation is via + # the main function or manually provided + assert_allclose( + adata_scrublet_manual_sim.obs["doublet_score"], + adata_scrublet_auto_sim.obs["doublet_score"], + atol=1e-15, + rtol=1e-15, + ) + except AssertionError: + import zarr + + # try debugging https://github.com/scverse/scanpy/issues/3068 + cache_path = cache.mkdir("debug") + store_manual = zarr.ZipStore(cache_path / "scrublet-manual.zip", mode="w") + store_auto = zarr.ZipStore(cache_path / "scrublet-auto.zip", mode="w") + z_manual = zarr.zeros( + adata_scrublet_manual_sim.shape[0], chunks=10, store=store_manual + ) + z_auto = zarr.zeros( + adata_scrublet_auto_sim.shape[0], chunks=10, store=store_auto + ) + z_manual[...] = adata_scrublet_manual_sim.obs["doublet_score"].values + z_auto[...] = adata_scrublet_auto_sim.obs["doublet_score"].values + + raise @pytest.fixture(scope="module") -def _scrub_small_sess() -> AnnData: +def scrub_small_sess() -> AnnData: # Reduce size of input for faster test adata = pbmc200() sc.pp.filter_genes(adata, min_counts=100) @@ -176,9 +194,9 @@ def _scrub_small_sess() -> AnnData: return adata -@pytest.fixture() -def scrub_small(_scrub_small_sess: AnnData): - return _scrub_small_sess.copy() +@pytest.fixture +def scrub_small(scrub_small_sess: AnnData): + return scrub_small_sess.copy() test_params = { diff --git a/tests/test_utils.py b/tests/test_utils.py index 3bec055995..81369a6938 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,13 +2,15 @@ from operator import mul, truediv from types import ModuleType +from typing import TYPE_CHECKING import numpy as np import pytest from anndata.tests.helpers import asarray +from packaging.version import Version from scipy.sparse import csr_matrix, issparse -from scanpy._compat import DaskArray +from scanpy._compat import DaskArray, _legacy_numpy_gen, pkg_version from scanpy._utils import ( axis_mul_or_truediv, axis_sum, @@ -25,6 +27,9 @@ ARRAY_TYPES_SPARSE_DASK_UNSUPPORTED, ) +if TYPE_CHECKING: + from typing import Any + def test_descend_classes_and_funcs(): # create module hierarchy @@ -225,11 +230,15 @@ def test_is_constant(array_type): ], ) @pytest.mark.parametrize("block_type", [np.array, csr_matrix]) -def test_is_constant_dask(axis, expected, block_type): +def test_is_constant_dask(request: pytest.FixtureRequest, axis, expected, block_type): import dask.array as da - if (axis is None) and block_type is csr_matrix: - pytest.skip("Dask has weak support for scipy sparse matrices") + if block_type is csr_matrix and ( + axis is None or pkg_version("dask") < Version("2023.2.0") + ): + reason = "Dask has weak support for scipy sparse matrices" + # This test is flaky for old dask versions, but when `axis=None` it reliably fails + request.applymarker(pytest.mark.xfail(reason=reason, strict=axis is None)) x_data = [ [0, 0, 1, 1], @@ -242,3 +251,39 @@ def test_is_constant_dask(axis, expected, block_type): x = da.from_array(np.array(x_data), chunks=2).map_blocks(block_type) result = is_constant(x, axis=axis).compute() np.testing.assert_array_equal(expected, result) + + +@pytest.mark.parametrize("seed", [0, 1, 1256712675]) +@pytest.mark.parametrize("pass_seed", [True, False], ids=["pass_seed", "set_seed"]) +@pytest.mark.parametrize("func", ["choice"]) +def test_legacy_numpy_gen(*, seed: int, pass_seed: bool, func: str): + np.random.seed(seed) + state_before = np.random.get_state(legacy=False) + + arrs: dict[bool, np.ndarray] = {} + states_after: dict[bool, dict[str, Any]] = {} + for direct in [True, False]: + if not pass_seed: + np.random.seed(seed) + arrs[direct] = _mk_random(func, direct=direct, seed=seed if pass_seed else None) + states_after[direct] = np.random.get_state(legacy=False) + + np.testing.assert_array_equal(arrs[True], arrs[False]) + np.testing.assert_equal( + *states_after.values(), err_msg="both should affect global state the same" + ) + # they should affect the global state + with pytest.raises(AssertionError): + np.testing.assert_equal(states_after[True], state_before) + + +def _mk_random(func: str, *, direct: bool, seed: int | None) -> np.ndarray: + if direct and seed is not None: + np.random.seed(seed) + gen = np.random if direct else _legacy_numpy_gen(seed) + match func: + case "choice": + arr = np.arange(1000) + return gen.choice(arr, size=(100, 100)) + case _: + pytest.fail(f"Unknown {func=}")