Prepare CORNETO 1.0.0 RC6

corneto/backend/_picos_backend.py

@@ -2,9 +2,10 @@
 from typing import Any, List, Optional, Set, Tuple, Union
 
 import numpy as np
+from scipy.sparse import csr_matrix
 
 from corneto._constants import Direction, Solver, VarType
-from corneto._settings import _numpy_array
+from corneto._decorators import _delegate
 from corneto.backend._base import (
     Backend,
     CExpression,
@@ -20,9 +21,13 @@
 
 
 def _get_shape(a: Any):
+    if hasattr(a, "_corneto_shape"):
+        return a._corneto_shape
     if hasattr(a, "_csymbol_shape"):
         return a._csymbol_shape
     if hasattr(a, "_expr"):
+        if hasattr(a._expr, "_corneto_shape"):
+            return a._expr._corneto_shape
         return a._expr._csymbol_shape if hasattr(a._expr, "_csymbol_shape") else a._expr.shape
     if hasattr(a, "shape"):
         return a.shape
@@ -32,6 +37,8 @@ def _get_shape(a: Any):
 def _infer_shape(c: Any):
     shape = _get_shape(c)
     if len(shape) == 2 and (shape[0] == 1 or shape[1] == 1):
+        if hasattr(c, "_corneto_shape") or (hasattr(c, "_expr") and hasattr(c._expr, "_corneto_shape")):
+            return shape
         # This is the problematic case, as any transformation
         # using PICOS will result in a 2D array.
         if hasattr(c, "_proxy_symbols"):
@@ -58,14 +65,77 @@ def shape(self) -> Tuple[int, ...]:
     def _create_proxy_expr(self, expr: Any, symbols: Optional[Set["CSymbol"]] = None) -> "PicosExpression":
         return PicosExpression(expr, symbols)
 
+    @property
+    def T(self):
+        result = self._expr.T
+        result._corneto_shape = tuple(result.shape)
+        return self._create(result, {})
+
     def _elementwise_mul(self, other: Any) -> Any:
         return self._expr ^ other
 
     def _norm(self, p: int = 2) -> CExpression:
         return pc.Norm(self._expr, p=p)
 
     def _sum(self, axis: Optional[int] = None) -> Any:
-        return pc.sum(self._expr, axis=axis)
+        shape = _infer_shape(self)
+        if len(shape) > 2:
+            return pc.sum(self._expr, axis=axis)
+
+        if len(shape) == 0:
+            return self._expr
+
+        if len(shape) == 1:
+            if axis not in (None, 0, -1):
+                return pc.sum(self._expr, axis=axis)
+            rows, columns = shape[0], 1
+        else:
+            rows, columns = shape
+            if axis is not None and axis not in (0, 1, -1, -2):
+                return pc.sum(self._expr, axis=axis)
+
+        if axis is None or len(shape) == 1:
+            size = rows * columns
+            operator = csr_matrix(
+                (
+                    np.ones(size, dtype=float),
+                    (np.zeros(size, dtype=int), np.arange(size, dtype=int)),
+                ),
+                shape=(1, size),
+            )
+            flattened = self._expr.reshaped((size, 1))
+            result = pc.Constant(_get_unique_name("sum"), value=operator) * flattened
+            output_shape = (1,) if len(shape) == 1 else (1, 1)
+            result._corneto_shape = output_shape
+            return result
+
+        if axis in (0, -2):
+            operator = csr_matrix(
+                (
+                    np.ones(rows, dtype=float),
+                    (np.zeros(rows, dtype=int), np.arange(rows, dtype=int)),
+                ),
+                shape=(1, rows),
+            )
+            result = pc.Constant(_get_unique_name("sum"), value=operator) * self._expr
+            result._corneto_shape = (1, columns)
+            return result
+
+        operator = csr_matrix(
+            (
+                np.ones(columns, dtype=float),
+                (np.arange(columns, dtype=int), np.zeros(columns, dtype=int)),
+            ),
+            shape=(columns, 1),
+        )
+        result = self._expr * pc.Constant(_get_unique_name("sum"), value=operator)
+        result._corneto_shape = (rows, 1)
+        return result
+
+    @_delegate(override=True)
+    def sum(self, axis: Optional[int] = None) -> CExpression:
+        """Reduce with sparse affine operators instead of PICOS dense sums."""
+        return self._sum(axis=axis)
 
     def _max(self, axis: Optional[int] = None) -> Any:
         raise NotImplementedError()
@@ -100,7 +170,9 @@ def _vstack(self, other: Any) -> Any:
         return a // b
 
     def _reshape(self, shape: Tuple[int, ...]) -> Any:
-        return self._expr.reshaped(shape)
+        result = self._expr.reshaped(shape)
+        result._corneto_shape = tuple(shape)
+        return result
 
     @property
     def value(self) -> np.ndarray:
@@ -153,7 +225,10 @@ def value(self, value: Any) -> None:
 
 class PicosBackend(Backend):
     def __init__(self, default_solver: Optional[str] = None) -> None:
-        super().__init__(default_solver, _numpy_array)
+        # PICOS accepts scipy CSR matrices as affine constants. Keep the
+        # backend's graph operators sparse; dense conversion is only needed
+        # for the data-loading helpers used by older callers.
+        super().__init__(default_solver, csr_matrix)
 
     def _load(self):
         import picos

corneto/datasets/__init__.py

@@ -0,0 +1,5 @@
+"""Small, versioned datasets fetched on demand."""
+
+from corneto.datasets._fetch import DatasetError, DatasetIntegrityError, fetch_dataset
+
+__all__ = ["DatasetError", "DatasetIntegrityError", "fetch_dataset"]

corneto/datasets/_catalog.py

@@ -0,0 +1,61 @@
+"""Built-in dataset metadata for the CORNETO dataset loader."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Mapping
+
+
+@dataclass(frozen=True)
+class DatasetSpec:
+    """Immutable metadata required to locate and validate one dataset revision."""
+
+    name: str
+    version: str
+    remote_url: str | None
+    archive_sha256: str
+    file_sha256: Mapping[str, str]
+    provenance: str = ""
+    license: str = ""
+
+    @property
+    def relative_path(self) -> Path:
+        return Path(self.name) / self.version
+
+
+_SACHS_SPEC = DatasetSpec(
+    name="sachs",
+    version="v1",
+    # This path becomes available when the dataset revision is published. The
+    # checksum prevents a mutable hosting path from silently changing data.
+    remote_url="https://raw.githubusercontent.com/saezlab/corneto/main/datasets/sachs/v1/sachs-v1.tar.gz",
+    archive_sha256="78c23eb79223a9b03cea9f4ce4e832151be3c99c54b31e2ee3ea7ec983bdaca3",
+    file_sha256={
+        "measurements.tsv": "a40650c0fe6aa6581dca556202cd4e94fb0dfa1e6d121c710123bbc164509bcd",
+        "condition_manifest.csv": "9f3083b5a2b20b8b2d4cbfab1f00153765c400fd76c94c5392c81b004ebf6fcb",
+        "README.md": "979d829cbc105873c38b00d99733d0563c190bbb44f873b4697f13d8acccd777",
+    },
+    provenance="https://doi.org/10.5281/zenodo.7681811",
+    license="CC BY 4.0",
+)
+
+DATASETS: dict[tuple[str, str], DatasetSpec] = {
+    (_SACHS_SPEC.name, _SACHS_SPEC.version): _SACHS_SPEC,
+}
+
+
+def _validate_component(value: str, label: str) -> None:
+    if not isinstance(value, str) or not value or value in {".", ".."} or Path(value).name != value:
+        raise ValueError(f"{label} must be a single non-empty path component.")
+
+
+def get_dataset_spec(name: str, version: str) -> DatasetSpec:
+    """Return the registered specification for a dataset revision."""
+    _validate_component(name, "name")
+    _validate_component(version, "version")
+    try:
+        return DATASETS[(name, version)]
+    except KeyError as error:
+        known = ", ".join(f"{dataset}:{revision}" for dataset, revision in sorted(DATASETS))
+        raise ValueError(f"Unknown dataset revision {name!r}, {version!r}; known revisions: {known}.") from error