diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..cd674b3d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,103 @@ +# PyARPES Specific +resources/ +scripts/ + +# Git +.git +.gitignore + +# Docker +docker-compose.yml +.docker + +# Byte-compiled / optimized / DLL files +__pycache__/ +*/__pycache__/ +*/*/__pycache__/ +*/*/*/__pycache__/ +*.py[cod] +*/*.py[cod] +*/*/*.py[cod] +*/*/*/*.py[cod] + +# C extensions +*.so + +# Test related +.pytest_cache/ + +# Conda related +conda/ + +# Documentation related +docs/ + +# Distribution / packaging +node_modules/ +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +*.lock + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Virtual environment +.env/ +.venv/ +venv/ + +# PyCharm +.idea + +# Python mode for VIM +.ropeproject +*/.ropeproject +*/*/.ropeproject +*/*/*/.ropeproject + +# Vim swap files +*.swp +*/*.swp +*/*/*.swp +*/*/*/*.swp \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3664f5cf..448d0002 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Sensitive +containers/fairmat-original + # Editors *.code-workspace .vscode/ diff --git a/README.rst b/README.rst index cbabec19..a2bc8d4a 100644 --- a/README.rst +++ b/README.rst @@ -55,7 +55,34 @@ Pip installation :: - pip install arpes + pip install arpes[all] + # optionally + # pip install arpes[standard] + # pip install arpes[core] + # ... see the feature bundles section below for more details + +Feature bundles and extra requirements +-------------------------------------- + +Not all users need all PyARPES features. You can specify which sets of feautures +you want using brackets after the `arpes` package name in order to specify +what functionality you need. Available options are + +1. `slim` +2. `standard` +3. `all` +4. `legacy_ui` +5. `ml` + +If you do not specify an extra requirements group, the installation will defualt to use +`slim`. `legacy_ui` provides support for in Jupyter interactive analysis tools via Bokeh. +These have been superseded by tools written with Qt. `ml` provides support for funtionality +requiring `scikit-learn` and its derivatives. Notably, this includes the decomposition functionality +for data exploration like `arpes.decomposition.pca_along` and `arpes.widgets.pca_explorer`. + +You should use `pip install arpes[standard]` unless you know that what you want is only +provided by `slim` or `all`. `legacy_ui` and `ml` are included in `all`. + Conda installation @@ -68,7 +95,7 @@ A minimal install looks like :: conda config --append channels conda-forge - conda install -c arpes -c conda-forge arpes + conda install -c arpes -c conda-forge arpes[all] Local installation from source diff --git a/arpes/analysis/decomposition.py b/arpes/analysis/decomposition.py index f9e79b65..0fca7198 100644 --- a/arpes/analysis/decomposition.py +++ b/arpes/analysis/decomposition.py @@ -1,5 +1,6 @@ """Provides array decomposition approaches like principal component analysis for xarray types.""" from functools import wraps +from arpes.feature_gate import Gates, gate from arpes.provenance import provenance from arpes.typing import DataType @@ -15,6 +16,7 @@ ) +@gate(Gates.ML) def decomposition_along( data: DataType, axes: List[str], decomposition_cls, correlation=False, **kwargs ) -> Tuple[DataType, Any]: @@ -103,6 +105,7 @@ def decomposition_along( return into, decomp +@gate(Gates.ML) @wraps(decomposition_along) def pca_along(*args, **kwargs): """Specializes `decomposition_along` with `sklearn.decomposition.PCA`.""" @@ -111,6 +114,7 @@ def pca_along(*args, **kwargs): return decomposition_along(*args, **kwargs, decomposition_cls=PCA) +@gate(Gates.ML) @wraps(decomposition_along) def factor_analysis_along(*args, **kwargs): """Specializes `decomposition_along` with `sklearn.decomposition.FactorAnalysis`.""" @@ -119,6 +123,7 @@ def factor_analysis_along(*args, **kwargs): return decomposition_along(*args, **kwargs, decomposition_cls=FactorAnalysis) +@gate(Gates.ML) @wraps(decomposition_along) def ica_along(*args, **kwargs): """Specializes `decomposition_along` with `sklearn.decomposition.FastICA`.""" @@ -127,6 +132,7 @@ def ica_along(*args, **kwargs): return decomposition_along(*args, **kwargs, decomposition_cls=FastICA) +@gate(Gates.ML) @wraps(decomposition_along) def nmf_along(*args, **kwargs): """Specializes `decomposition_along` with `sklearn.decomposition.NMF`.""" diff --git a/arpes/analysis/pocket.py b/arpes/analysis/pocket.py index 8c7b2fa1..de8abfac 100644 --- a/arpes/analysis/pocket.py +++ b/arpes/analysis/pocket.py @@ -1,6 +1,5 @@ """Contains electron/hole pocket analysis routines.""" import numpy as np -from sklearn.decomposition import PCA import xarray as xr from arpes.fits.fit_models import AffineBackgroundModel, LorentzianModel @@ -8,6 +7,7 @@ from arpes.typing import DataType from arpes.utilities import normalize_to_spectrum from arpes.utilities.conversion import slice_along_path +from arpes.feature_gate import gate, Gates from typing import List, Tuple @@ -19,6 +19,7 @@ ) +@gate(Gates.ML) def pocket_parameters(data: DataType, kf_method=None, sel=None, method_kwargs=None, **kwargs): """Estimates pocket center, anisotropy, principal vectors, and extent in either angle or k-space. @@ -35,6 +36,8 @@ def pocket_parameters(data: DataType, kf_method=None, sel=None, method_kwargs=No Returns: Extracted asymmetry parameters. """ + from sklearn.decomposition import PCA + slices, _ = curves_along_pocket(data, **kwargs) # slices, angles = if kf_method is None: diff --git a/arpes/analysis/self_energy.py b/arpes/analysis/self_energy.py index 5f25ff01..a0978a44 100644 --- a/arpes/analysis/self_energy.py +++ b/arpes/analysis/self_energy.py @@ -1,4 +1,5 @@ """Contains self-energy analysis routines.""" +from arpes.feature_gate import Gates, gate import xarray as xr import lmfit as lf import numpy as np @@ -71,6 +72,9 @@ def local_fermi_velocity(bare_band: xr.DataArray): return raw_velocity * METERS_PER_SECOND_PER_EV_ANGSTROM +gate(Gates.ML) + + def estimate_bare_band(dispersion: xr.DataArray, bare_band_specification: Optional[str] = None): """Estimates the bare band from a fitted dispersion. diff --git a/arpes/deep_learning/transforms.py b/arpes/deep_learning/transforms.py index 5894671e..3e510903 100644 --- a/arpes/deep_learning/transforms.py +++ b/arpes/deep_learning/transforms.py @@ -9,15 +9,19 @@ class Identity: """Represents a reversible identity transform.""" def encodes(self, x): + """Identity transform.""" return x def __call__(self, x): + """Passthrough for `self.encodes`.""" return x def decodes(self, x): + """The inverse of the idenity transform is also the identity.""" return x def __repr__(self): + """Just include the class name since there are no parameters here.""" return "Identity()" diff --git a/arpes/endstations/__init__.py b/arpes/endstations/__init__.py index 8ff9bae0..8c46c090 100644 --- a/arpes/endstations/__init__.py +++ b/arpes/endstations/__init__.py @@ -105,6 +105,7 @@ class EndstationBase: trace: Trace def __init__(self): + """Setup a trace object so that it can be used throughout the load steps.""" self.trace = Trace(silent=True) @classmethod diff --git a/arpes/feature_gate.py b/arpes/feature_gate.py new file mode 100644 index 00000000..02f0df54 --- /dev/null +++ b/arpes/feature_gate.py @@ -0,0 +1,163 @@ +"""Implements feature gates so that we can support optional dependencies better. + +The way this works is more or less by testing if functionality is available +at runtime by attempting an import. Sometimes, we may also perform more checks. + +Then, we provide a decorator which provides helpful error messages +if a feature gate is not passed. + +These gates are also used in import time in the `.all` modules in order to +control what gets imported when a user requests `arpes.all`. +""" + +import importlib +import enum +import functools +import warnings + +from dataclasses import dataclass, field +from typing import List, Optional + +__all__ = ["gate", "Gates", "failing_feature_gates"] + + +class Gates(str, enum.Enum): + """Defines which gates we will check. + + These more or less correspond onto extra requirements groups in + setup.py but in principle can be used for any functionality which + requires a certain hardware or software configuration. + """ + + LegacyUI = "legacy_ui" + ML = "ml" + Igor = "igor" + Qt = "qt" + + +@dataclass +class Gate: + @property + def message(self) -> str: + raise NotImplementedError() + + def check(self) -> bool: + raise NotImplementedError + + @staticmethod + def can_import_module(module_name) -> bool: + try: + importlib.import_module(module_name) + return True + except ImportError: + return False + + +@dataclass +class ImportModuleGate(Gate): + module_name: str + module_install_name: str + + _already_checked_gate: bool = False + _gate_did_pass: bool = False + + @property + def message(self) -> str: + return f"pip install {self.module_install_name}" + + def check(self) -> bool: + if not self._already_checked_gate: + self._already_checked_gate = True + self._gate_did_pass = self.can_import_module(self.module_name) + + return self._gate_did_pass + + +@dataclass +class ExtrasGate(Gate): + name: str + module_names: List[str] = field(default_factory=list) + module_install_names: List[str] = field(default_factory=list) + + _already_checked_gate: bool = False + _gate_did_pass: bool = False + + def __post_init__(self): + assert len(self.module_install_names) == len(self.module_names) + + @property + def message(self) -> str: + return ( + f"pip install arpes[{self.name}] OR pip install {' '.join(self.module_install_names)}" + ) + + def check(self) -> bool: + if not self._already_checked_gate: + self._already_checked_gate = True + self._gate_did_pass = all(self.can_import_module(name) for name in self.module_names) + + return self._gate_did_pass + + +ALL_GATES = { + Gates.LegacyUI: [ + ExtrasGate("legacy_ui", ["bokeh"], ["bokeh"]), + ], + Gates.ML: [ + ExtrasGate( + "ml", ["skimage", "sklearn", "cvxpy"], ["scikit-image", "scikit-learn", "cvxpy"] + ), + ], + Gates.Qt: [ + ExtrasGate("qt", ["pyqtgraph"], ["pyqtgraph"]), + ], + Gates.Igor: [ + ImportModuleGate("igor", "https://github.com/chstan/igorpy/tarball/712a4c4#egg=igor"), + ], +} + +FAILED_GATE_MESSAGE = """ +You need to install some packages before using this PyARPES functionality: + +{messages} +""" + + +def failing_feature_gates(gate_name) -> Optional[List[Gate]]: + """Determines whether a given feature should be turned on. + + This assessment is made according to whether appropriate modules + are installed and available. If not, we provide detailed instructions in order to + instruct the user. + """ + failing_gates = [] + for element in ALL_GATES[gate_name]: + if not element.check(): + failing_gates.append(element) + + if failing_gates: + warnings.warn( + FAILED_GATE_MESSAGE.format( + messages=" - " + "\n - ".join([g.message for g in failing_gates]) + ) + ) + + return failing_gates + + +def gate(gate_name: Gates): + """Runs a feature gate to determine whether we can support optional functionality.""" + + def decorate_inner(fn): + @functools.wraps(fn) + def wrapped_fn(*args, **kwargs): + if failing_feature_gates(gate_name): + raise RuntimeError( + "Cannot run function due to missing features. Please read instructions above." + ) + + return fn(*args, **kwargs) + + return wrapped_fn + + return decorate_inner diff --git a/arpes/io.py b/arpes/io.py index f766a534..1645f195 100644 --- a/arpes/io.py +++ b/arpes/io.py @@ -79,13 +79,20 @@ def load_data( return load_scan(desc, **kwargs) -DATA_EXAMPLES = { - "cut": ("ALG-MC", "cut.fits"), - "map": ("example_data", "fermi_surface.nc"), - "photon_energy": ("example_data", "photon_energy.nc"), - "nano_xps": ("example_data", "nano_xps.nc"), - "temperature_dependence": ("example_data", "temperature_dependence.nc"), -} +@dataclass +class DataExampleReference: + name: str + location: str = "example_data" + + +DATA_EXAMPLES = [ + DataExampleReference(name="cut.fits", location="ALG-MC"), + DataExampleReference(name="fermi_surface.nc"), + DataExampleReference(name="photon_energy.nc"), + DataExampleReference(name="nano_xps.nc"), + DataExampleReference(name="temperature_dependence.nc"), +] +DATA_EXAMPLES = {Path(example.name).stem: example for example in DATA_EXAMPLES} def load_example_data(example_name="cut") -> xr.Dataset: @@ -94,10 +101,11 @@ def load_example_data(example_name="cut") -> xr.Dataset: warnings.warn( f"Could not find requested example_name: {example_name}. Please provide one of {list(DATA_EXAMPLES.keys())}" ) + raise ValueError(f"No requested example {example_name}") - location, example = DATA_EXAMPLES[example_name] - file = Path(__file__).parent / "example_data" / example - return load_data(file=file, location=location) + example = DATA_EXAMPLES[example_name] + file = Path(__file__).parent / "example_data" / example.name + return load_data(file=file, location=example.location) @dataclass @@ -108,7 +116,7 @@ def cut(self) -> xr.DataArray: @property def map(self) -> xr.DataArray: - return load_example_data("map") + return load_example_data("fermi_surface") @property def photon_energy(self) -> xr.DataArray: diff --git a/arpes/plotting/all.py b/arpes/plotting/all.py index 8f617cda..098e4603 100644 --- a/arpes/plotting/all.py +++ b/arpes/plotting/all.py @@ -1,4 +1,5 @@ """Import many useful standard tools.""" +from arpes.feature_gate import Gates, failing_feature_gates from .annotations import * from .bands import * @@ -20,15 +21,19 @@ # Note, we lift Bokeh imports into definitions in case people don't want to install Bokeh # and also because of an undesirable interaction between pytest and Bokeh due to Bokeh's use # of jinja2. -from .interactive import * -from .band_tool import * -from .comparison_tool import * -from .curvature_tool import * -from .fit_inspection_tool import * -from .mask_tool import * -from .path_tool import * -from .dyn_tool import * -from .qt_tool import qt_tool -from .qt_ktool import ktool +if not failing_feature_gates(Gates.LegacyUI): + from .interactive import * + from .band_tool import * + from .comparison_tool import * + from .curvature_tool import * + from .fit_inspection_tool import * + from .mask_tool import * + from .path_tool import * + from .dyn_tool import * + +if not failing_feature_gates(Gates.Qt): + from .qt_tool import qt_tool + from .qt_ktool import ktool + from .fit_tool import * from .utils import savefig, remove_colorbars, fancy_labels diff --git a/arpes/plotting/band_tool.py b/arpes/plotting/band_tool.py index 5f3b8e41..0df82e5b 100644 --- a/arpes/plotting/band_tool.py +++ b/arpes/plotting/band_tool.py @@ -1,8 +1,8 @@ """An interactive band selection tool used to initialize curve fits.""" -import numpy as np -from bokeh import events +import numpy as np import xarray as xr + from arpes.analysis.band_analysis import fit_patterned_bands from arpes.exceptions import AnalysisError from arpes.models import band @@ -35,6 +35,7 @@ def tool_handler(self, doc): from bokeh.models.mappers import LinearColorMapper from bokeh.models import widgets from bokeh.plotting import figure + from bokeh import events if len(self.arr.shape) != 2: raise AnalysisError("Cannot use the band tool on non image-like spectra") diff --git a/arpes/plotting/dynamic_tool.py b/arpes/plotting/dynamic_tool.py index 0b43851a..5ad7fe2a 100644 --- a/arpes/plotting/dynamic_tool.py +++ b/arpes/plotting/dynamic_tool.py @@ -19,8 +19,6 @@ __all__ = ("make_dynamic",) -qt_info.setup_pyqtgraph() - class DynamicToolWindow(SimpleWindow): HELP_DIALOG_CLS = BasicHelpDialog @@ -155,6 +153,7 @@ def set_data(self, data: DataType): def make_dynamic(fn, data): """Starts a tool which makes any analysis function dynamic.""" + qt_info.setup_pyqtgraph() tool = DynamicTool(fn) tool.set_data(data) tool.start() diff --git a/arpes/widgets.py b/arpes/widgets.py index c0e42392..ec3db6c4 100644 --- a/arpes/widgets.py +++ b/arpes/widgets.py @@ -77,7 +77,7 @@ class SelectFromCollection: (i.e., `offsets`). """ - def __init__(self, ax, collection, alpha_other=0.3, on_select=None): + def __init__(self, ax, collection, alpha_other=0.03, on_select=None): self.canvas = ax.figure.canvas self.collection = collection self.alpha_other = alpha_other @@ -503,7 +503,7 @@ def set_axes(component_x, component_y): ax_components.clear() context["selected_components"] = [component_x, component_y] for_scatter, size = compute_for_scatter() - pts = ax_components.scatter(for_scatter.values[0], for_scatter.values[1], s=size) + pts = ax_components.scatter(for_scatter.values[0], for_scatter.values[1], s=size, alpha=0.2) if context["selector"] is not None: context["selector"].disconnect() diff --git a/arpes/xarray_extensions.py b/arpes/xarray_extensions.py index 5b5fb0df..938c28e0 100644 --- a/arpes/xarray_extensions.py +++ b/arpes/xarray_extensions.py @@ -38,6 +38,7 @@ """ import pandas as pd +from arpes.feature_gate import Gates, gate import lmfit import arpes import contextlib @@ -904,6 +905,8 @@ def inner_potential(self) -> float: return 10 + gate(Gates.ML) + def find_spectrum_energy_edges(self, indices=False): energy_marginal = self._obj.sum([d for d in self._obj.dims if d not in ["eV"]]) @@ -925,6 +928,8 @@ def find_spectrum_energy_edges(self, indices=False): delta = self._obj.G.stride(generic_dim_names=False) return edges * delta["eV"] + self._obj.coords["eV"].values[0] + gate(Gates.ML) + def find_spectrum_angular_edges_full(self, indices=False): # as a first pass, we need to find the bottom of the spectrum, we will use this # to select the active region and then to rebin into course steps in energy from 0 @@ -1064,6 +1069,8 @@ def mean_other(self, dim_or_dims, keep_attrs=False): [d for d in self._obj.dims if d not in dim_or_dims], keep_attrs=keep_attrs ) + gate(Gates.ML) + def find_spectrum_angular_edges(self, indices=False): angular_dim = "pixel" if "pixel" in self._obj.dims else "phi" energy_edge = self.find_spectrum_energy_edges() @@ -1775,12 +1782,14 @@ def plot(self, *args, rasterized=True, **kwargs): with plt.rc_context(rc={"text.usetex": False}): self._obj.plot(*args, **kwargs) + @gate(Gates.Qt) def show(self, detached=False, **kwargs): """Opens the Qt based image tool.""" import arpes.plotting.qt_tool arpes.plotting.qt_tool.qt_tool(self._obj, detached=detached, **kwargs) + @gate(Gates.LegacyUI) def show_d2(self, **kwargs): """Opens the Bokeh based second derivative image tool.""" from arpes.plotting.all import CurvatureTool @@ -1788,6 +1797,7 @@ def show_d2(self, **kwargs): curve_tool = CurvatureTool(**kwargs) return curve_tool.make_tool(self._obj) + @gate(Gates.LegacyUI) def show_band_tool(self, **kwargs): """Opens the Bokeh based band placement tool.""" from arpes.plotting.all import BandTool @@ -2475,6 +2485,7 @@ def __init__(self, xarray_obj: DataType): def eval(self, *args, **kwargs): return self._obj.results.G.map(lambda x: x.eval(*args, **kwargs)) + @gate(Gates.Qt) def show(self, detached=False): from arpes.plotting.fit_tool import fit_tool @@ -2621,6 +2632,7 @@ def param_as_dataset(self, param_name: str) -> xr.Dataset: } ) + @gate(Gates.LegacyUI) def show(self, detached: bool = False): """Opens a Bokeh based interactive fit inspection tool.""" from arpes.plotting.fit_tool import fit_tool diff --git a/building-from-source.md b/building-from-source.md index 1c21cb53..76f6d995 100644 --- a/building-from-source.md +++ b/building-from-source.md @@ -8,7 +8,7 @@ git clone https://gitlab.com/lanzara-group/python-arpes cd python-arpes tools/run-after-git-clone -pip install -e . +pip install -e .[standard] ``` Next you can test with diff --git a/containers/README.md b/containers/README.md new file mode 100644 index 00000000..6c181519 --- /dev/null +++ b/containers/README.md @@ -0,0 +1,5 @@ +# Containerized versions of PyARPES + +Most users should install according to the directions in the documentation. This means using a source install (git followed by editable install with pip) or should install directly from a package manager. It is useful for CI/CD and in cases where installations are to be set up automatically to use containers, however. + +If you want to add a container format, put relevant documentation about the build process and any build metadata into a subfolder here. \ No newline at end of file diff --git a/containers/fairmat-slim/Dockerfile b/containers/fairmat-slim/Dockerfile new file mode 100644 index 00000000..dd149723 --- /dev/null +++ b/containers/fairmat-slim/Dockerfile @@ -0,0 +1,51 @@ +# NOTE, the Docker build context should be set to the repo root +# not this folder! This means you should build using +# +# # (from project root) +# yarn build-container +# +# # (or manually) +# docker build -f containers/fairmat-slim/Dockerfile . +# +# or a similar invocation. + +FROM condaforge/mambaforge AS build +LABEL build_version="0.1.0" +LABEL description="Provides a minimal disk space containerized installation of PyARPES." + +# Multi stage environment build which serves two purposes: +# Stage 0 installs PyQt5 with pip instead of with conda. This needs to happen +# first so that we get version 5.15 which is not available on conda. If we install +# the correct version with pip after the conda environment builds, then +# pip clobbers the PyQt5 installation conda chose and `conda-pack` refuses to run. +COPY containers/fairmat-slim/environment-fairmat-slim-stage0.yml . +RUN mamba env create -f environment-fairmat-slim-stage0.yml + +# Stage 1 installs the rest of the dependencies. We specifically request +# matplotlib-base to prevent Qt installation. +COPY containers/fairmat-slim/environment-fairmat-slim-stage1.yml . +RUN mamba env update --prefix /opt/conda/envs/arpes -f environment-fairmat-slim-stage1.yml + +# Stage 2 installs PyARPES with all dependencies already installed. +# This can probably be pushed after the `conda-pack` step, +# which would make rebuilds very fast. We should consider this. +COPY containers/fairmat-slim/environment-fairmat-slim-stage2.yml . +COPY . ./ +RUN mamba env update --prefix /opt/conda/envs/arpes -f environment-fairmat-slim-stage2.yml + +RUN mamba install -c conda-forge conda-pack + +RUN conda-pack -n arpes -o /tmp/env.tar && \ + mkdir /venv && cd /venv && tar xf /tmp/env.tar && \ + rm /tmp/env.tar + +RUN /venv/bin/conda-unpack + +FROM debian:buster AS runtime + +COPY --from=build /venv /venv + +SHELL ["bin/bash", "-c"] + +# just verify there are no major surprises +ENTRYPOINT source /venv/bin/activate && python -c "from arpes.all import *" \ No newline at end of file diff --git a/containers/fairmat-slim/environment-fairmat-slim-stage0.yml b/containers/fairmat-slim/environment-fairmat-slim-stage0.yml new file mode 100644 index 00000000..808a06f6 --- /dev/null +++ b/containers/fairmat-slim/environment-fairmat-slim-stage0.yml @@ -0,0 +1,12 @@ +name: arpes +channels: + - conda-forge +dependencies: + - python=3.8 + + - nomkl + - pip + + # pip + - pip: + - PyQt5==5.15 diff --git a/containers/fairmat-slim/environment-fairmat-slim-stage1.yml b/containers/fairmat-slim/environment-fairmat-slim-stage1.yml new file mode 100644 index 00000000..88758891 --- /dev/null +++ b/containers/fairmat-slim/environment-fairmat-slim-stage1.yml @@ -0,0 +1,32 @@ +channels: + - conda-forge +dependencies: + - python=3.8 + - nomkl + + - astropy + - xarray>=0.16.1 + - h5py>=3.2.1 + + - pint + - pandas + - numpy>=1.20.0,<2.0.0 + - scipy>=1.6.0,<2.0.0 + - lmfit>=1.0.0,<2.0.0 + - netCDF4>=1.5.0,<2.0.0 + - numba>=0.53.0,<1.0.0 + - pyqtgraph>=0.12.0,<0.13.0 + - ase>=3.17.0,<3.22.0 + + # plotting + - colorcet + - matplotlib-base>=3.0.3 + - ipywidgets>=7.0.1,<8.0.0 + + # Misc deps + - packaging + - colorama + - imageio + - tqdm + - rx + - dill \ No newline at end of file diff --git a/containers/fairmat-slim/environment-fairmat-slim-stage2.yml b/containers/fairmat-slim/environment-fairmat-slim-stage2.yml new file mode 100644 index 00000000..843b4f06 --- /dev/null +++ b/containers/fairmat-slim/environment-fairmat-slim-stage2.yml @@ -0,0 +1,8 @@ +channels: + - conda-forge +dependencies: + - python=3.8 + + - pip + - pip: + - .[slim] diff --git a/docs/source/conf.py b/docs/source/conf.py index 51d21771..3fdd471a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -74,6 +74,7 @@ def autodoc_skip_member(app, what, name, obj, skip, options): def setup(app): + """Add the autodoc skip member hook, and any other module level config.""" app.connect("autodoc-skip-member", autodoc_skip_member) diff --git a/docs/source/dev-guide.rst b/docs/source/dev-guide.rst index f4b429f9..2942f031 100644 --- a/docs/source/dev-guide.rst +++ b/docs/source/dev-guide.rst @@ -22,7 +22,7 @@ or git clone https://github.com/chstan/arpes 3. Install PyARPES into your conda environment/virtualenv with - ``pip install -e .`` + ``pip install -e .[all]`` Tests ~~~~~ diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 70087b38..5cd6d2af 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -46,7 +46,7 @@ available either from the main repository at .. code:: bash - pip install -e . + pip install -e .[all] 5. *Recommended:* Configure IPython kernel according to the **Barebones Kernel Installation** below @@ -64,7 +64,7 @@ recommend .. code:: bash conda config --append channels conda-forge - conda install -c arpes arpes + conda install -c arpes arpes[all] Additional Suggested Steps ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/environment-readthedocs.yml b/environment-readthedocs.yml index 6a10f18d..75d771e3 100644 --- a/environment-readthedocs.yml +++ b/environment-readthedocs.yml @@ -5,6 +5,7 @@ channels: dependencies: - python=3.8 + - nomkl - astropy - xarray>=0.16.1 - h5py>=3.2.1 @@ -46,4 +47,4 @@ dependencies: - sphinx_rtd_theme - nbsphinx - sphinx_copybutton - - -e . + - -e .[core] diff --git a/environment.yml b/environment.yml index 4a307b02..cb0faee3 100644 --- a/environment.yml +++ b/environment.yml @@ -39,4 +39,4 @@ dependencies: - rx - dill - ase>=3.17.0,<3.22.0 - - -e . + - -e .[all] diff --git a/package.json b/package.json index 9938997b..c885e9e1 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,9 @@ "check-docstyle": "pydocstyle --config pyproject.toml", "build-docs": "python scripts/build_docs.py", "build-pypi": "python -m build --sdist --wheel .", - "build-conda": "yarn build-pypi && conda-build purge-all && conda-build ./conda -c anaconda -c conda-forge --output-folder conda-dist --numpy 1.20" + "build-conda": "yarn build-pypi && conda-build purge-all && conda-build ./conda -c anaconda -c conda-forge --output-folder conda-dist --numpy 1.20", + "build-container": "docker build -t pyarpes -f containers/fairmat-slim/Dockerfile .", + "run-container": "docker run -d pyarpes" }, "devDependencies": { "@arkweid/lefthook": "^0.7.6" diff --git a/pypi-readme.rst b/pypi-readme.rst index cbabec19..865233a0 100644 --- a/pypi-readme.rst +++ b/pypi-readme.rst @@ -55,7 +55,33 @@ Pip installation :: - pip install arpes + pip install arpes[all] + # optionally + # pip install arpes[standard] + # pip install arpes[core] + # ... see the feature bundles section below for more details + +Feature bundles and extra requirements +-------------------------------------- + +Not all users need all PyARPES features. You can specify which sets of feautures +you want using brackets after the `arpes` package name in order to specify +what functionality you need. Available options are + +1. `slim` +2. `standard` +3. `all` +4. `legacy_ui` +5. `ml` + +If you do not specify an extra requirements group, the installation will defualt to use +`slim`. `legacy_ui` provides support for in Jupyter interactive analysis tools via Bokeh. +These have been superseded by tools written with Qt. `ml` provides support for funtionality +requiring `scikit-learn` and its derivatives. Notably, this includes the decomposition functionality +for data exploration like `arpes.decomposition.pca_along` and `arpes.widgets.pca_explorer`. + +You should use `pip install arpes[standard]` unless you know that what you want is only +provided by `slim` or `all`. `legacy_ui` and `ml` are included in `all`. Conda installation @@ -68,7 +94,7 @@ A minimal install looks like :: conda config --append channels conda-forge - conda install -c arpes -c conda-forge arpes + conda install -c arpes -c conda-forge arpes[all] Local installation from source diff --git a/scripts/audit_packages.py b/scripts/audit_packages.py new file mode 100644 index 00000000..ddaa49eb --- /dev/null +++ b/scripts/audit_packages.py @@ -0,0 +1,60 @@ +"""A utility script to show how heavy the dependencies are for a conda environment.""" +import json +import os + +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Any + + +@dataclass +class Package: + """Handles parsing information from conda-meta to get package names and sizes.""" + + name: str + size_bytes: int + json: Dict[Any, Any] + + @property + def size_mb(self): + """Package size in megabytes.""" + return self.size_bytes // (1024 * 1024) + + @property + def size_kb(self): + """Package size in kilobytes.""" + return self.size_bytes // 1024 + + @classmethod + def conda_installed(cls): + """Finds and parse all available packages.""" + prefix = Path(os.getenv("CONDA_PREFIX")) + meta_files = list((prefix / "conda-meta").glob("*.json")) + + packages = [] + for package_meta in meta_files: + with open(package_meta, "r") as f: + packages.append(Package.from_json(json.load(f))) + + return packages + + @classmethod + def from_json(cls, json_data): + """Deserializes a Package instance from the JSON record.""" + return cls( + name=json_data["name"], + size_bytes=json_data["size"], + json=json_data, + ) + + +if __name__ == "__main__": + packages = Package.conda_installed() + packages_by_size = sorted(packages, key=lambda p: p.size_bytes, reverse=True) + + print(f"Total size: {sum(p.size_bytes for p in packages) // (1024 * 1024)} (MB)") + print("\nName" + " " * 20 + "Size (kb)") + print("-" * 33) + + for p in packages_by_size: + print(f"{p.name}{' ' * (24 - len(p.name))}{p.size_kb}") diff --git a/scripts/build_docs.py b/scripts/build_docs.py index 914a1368..7d2d4e66 100644 --- a/scripts/build_docs.py +++ b/scripts/build_docs.py @@ -15,19 +15,22 @@ @dataclass class BuildStep: + """Provides a platform-agnostic representation of a build step in a multistep build.""" + name: str = "Unnamed build step" @property def root(self) -> Path: + """The root of the currently building project.""" return (Path(__file__).parent / "..").absolute() @staticmethod - def is_windows(): + def is_windows() -> bool: + """Whether the platform we are building for is a variant of Windows.""" return sys.platform == "win32" def __call__(self, *args, **kwargs): - """Runs either call_windows or call_unix accordingy.""" - + """Runs either call_windows or call_unix accordingly.""" print(f"Running: {self.name}") if self.is_windows(): self.call_windows(*args, **kwargs) @@ -35,18 +38,27 @@ def __call__(self, *args, **kwargs): self.call_unix(*args, **kwargs) def call_windows(self, *args, **kwargs): + """Windows specific build variant.""" raise NotImplementedError def call_unix(self, *args, **kwargs): + """Unix (non-Windows) specific build variant.""" raise NotImplementedError @dataclass class Make(BuildStep): + """Runs make to build PyARPES documentation. + + This can be parameterized with a build variant, such as + to run `make clean` or `make html`. + """ + name: str = "Removing old build files" make_step: str = "" def call_windows(self): + """Run make.bat which is the Windows flavored build script.""" batch_script = str(self.root / "docs" / "make.bat") generated_path = (self.root / "docs" / "source" / "generated").resolve().absolute() @@ -56,18 +68,23 @@ def call_windows(self): subprocess.run(f"{batch_script} {self.make_step}", shell=True) def call_unix(self): + """Use make and the Makefile to build documentation.""" docs_root = str(self.root / "docs") subprocess.run(f"cd {docs_root} && make {self.make_step}", shell=True) @dataclass class MakeClean(Make): + """Run `make clean`.""" + name: str = "Run Sphinx Build (make clean)" make_step: str = "clean" @dataclass class MakeHtml(Make): + """Run `make html`.""" + name: str = "Run Sphinx Build (make html)" make_step: str = "html" diff --git a/setup.py b/setup.py index 74cb295d..481166dc 100755 --- a/setup.py +++ b/setup.py @@ -26,8 +26,6 @@ "astropy", "xarray>=0.16.1", "h5py>=3.2.1", - "pyqtgraph>=0.12.0,<0.13.0", - "PyQt5==5.15", "netCDF4>=1.5.0,<2.0.0", "colorcet", "pint", @@ -35,21 +33,25 @@ "numpy>=1.20.0,<2.0.0", "scipy>=1.6.0,<2.0.0", "lmfit>=1.0.0,<2.0.0", - "scikit-learn", # plotting "matplotlib>=3.0.3", - "bokeh>=2.0.0,<3.0.0", "ipywidgets>=7.0.1,<8.0.0", # Misc deps "packaging", "colorama", "imageio", - "titlecase", "tqdm", "rx", "dill", - "ase>=3.17.0,<3.22.0", "numba>=0.53.0,<1.0.0", + "ase>=3.17.0,<3.22.0", + ], + "qt": [ + "pyqtgraph>=0.12.0,<0.13.0", + "PyQt5==5.15", + ], + "legacy_ui": [ + "bokeh>=2.0.0,<3.0.0", ], "igor": ["igor==0.3.1"], "ml": [ @@ -58,9 +60,31 @@ "cvxpy", "libgcc", ], + "slim": [], # nomkl should be installed via conda + "all": [], + "standard": [], +} + +extra_groups = { + # base builds, from light to heavy installations + "slim": ["core"], + "standard": ["qt", "core"], + "all": ["ml", "legacy_ui", "qt", "core"], + # other feature sets + "legacy_ui": ["core"], + "ml": ["core"], } -requirements = [y for k, v in DEPENDENCY_GROUPS.items() for y in v if k not in {"igor", "ml"}] + +def compile_dependencies_for_group(group_name): + """Joins all dependencies required for an extras group.""" + all_groups = [group_name] + extra_groups[group_name] + return sum([DEPENDENCY_GROUPS[gname] for gname in all_groups], start=[]) + + +EXTRAS = { + group_name: compile_dependencies_for_group(group_name) for group_name in extra_groups.keys() +} DEV_DEPENDENCIES = { "jupyter": [ @@ -150,7 +174,8 @@ def run(self): dependency_links=[ "https://github.com/chstan/igorpy/tarball/712a4c4#egg=igor-0.3.1", ], - install_requires=requirements, + install_requires=DEPENDENCY_GROUPS["core"], + extras_require=EXTRAS, include_package_data=True, license="GPLv3", classifiers=[