Part 3 - pow: remove pyethash C extension, always use pure Python ethash

.gitignore

@@ -8,6 +8,10 @@ __pycache__/
 
 # C extensions
 *.so
+*.pyd
+
+# Cython generated
+ethereum/pow/ethash_cy.c
 
 # qkchash binaries
 qkchash/qkchash

README.md

@@ -71,9 +71,19 @@ To install the required modules for the project. Under `pyquarkchain` dir where
 # you may want to set the following if cryptography complains about header files: (https://github.com/pyca/cryptography/issues/3489)
 # export CPPFLAGS=-I/usr/local/opt/openssl/include
 # export LDFLAGS=-L/usr/local/opt/openssl/lib
-pip install -e .
+pip install -r requirements.txt
+python setup.py build_ext --inplace
 ```
 
+The second command builds the optional native extensions that accelerate Ethash PoW verification:
+
+- **Cython extension** (`ethash_cy`): requires a C compiler — ~30x speedup
+- **Rust extension** (`ethash_rs`): requires [Rust/cargo](https://rustup.rs) — ~112x speedup
+
+Both are built in-place by `python setup.py build_ext --inplace`. If either build is skipped (compiler or Rust not available), the pure-Python fallback is used automatically.
+
+The auto-detection order is `ethash_rs` → `ethash_cy` → pure Python. Override with the `ETHASH_LIB` environment variable (`ethash_rs`, `ethash_cy`, or `ethash`).
+
 Once all the modules are installed, try running all the unit tests under `pyquarkchain`
 
 ```

ethereum/pow/ethash.py

@@ -1,94 +1,166 @@
-import copy
+import importlib
+import os
+import numpy as np
 from functools import lru_cache
 from typing import Callable, Dict, List
 
-from ethereum.pow.ethash_utils import *
+from ethereum.pow.ethash_utils import (
+    ethash_sha3_512, ethash_sha3_256,
+    FNV_PRIME, HASH_BYTES, WORD_BYTES, MIX_BYTES,
+    DATASET_PARENTS, CACHE_ROUNDS, ACCESSES, EPOCH_LENGTH,
+)
+
+_FNV_PRIME = np.uint32(FNV_PRIME)
+
+# ---------------------------------------------------------------------------
+# ETHASH_LIB selects the implementation used for PoW verification.
+#   "ethash"    — pure-Python + numpy (always available)
+#   "ethash_cy" — Cython + C keccak  (requires python setup.py build_ext)
+#   "ethash_rs" — Rust + tiny-keccak (requires python setup.py build_ext)
+# Default: auto-detect best available (ethash_rs → ethash_cy → ethash)
+# ---------------------------------------------------------------------------
+_impl_hashimoto_light = None
+_impl_mkcache = None
+_mix_parents_fn = None
+
+ETHASH_LIB = os.environ.get("ETHASH_LIB", "auto")
+
+if ETHASH_LIB == "auto":
+    # Check symbol presence to avoid false-positives from namespace packages
+    # (e.g. the ethash_rs/ Cargo source directory looks like a package but has
+    # no compiled symbols until the extension is built).
+    _REQUIRED = {"ethash_rs": "rs_hashimoto_light", "ethash_cy": "cy_hashimoto_light"}
+    for _candidate in ("ethash_rs", "ethash_cy"):
+        try:
+            _mod = importlib.import_module(f"ethereum.pow.{_candidate}")
+            if hasattr(_mod, _REQUIRED[_candidate]):
+                ETHASH_LIB = _candidate
+                break
+        except ImportError:
+            continue
+    else:
+        ETHASH_LIB = "ethash"
+
+if ETHASH_LIB == "ethash_cy":
+    from ethereum.pow.ethash_cy import cy_hashimoto_light as _impl_hashimoto_light
+    from ethereum.pow.ethash_cy import cy_mkcache as _impl_mkcache
+    from ethereum.pow.ethash_cy import mix_parents as _mix_parents_fn
+
+elif ETHASH_LIB == "ethash_rs":
+    from ethereum.pow.ethash_rs import rs_hashimoto_light as _impl_hashimoto_light
+    from ethereum.pow.ethash_rs import rs_mkcache as _impl_mkcache
+    from ethereum.pow.ethash_rs import mix_parents as _mix_parents_fn
+
+elif ETHASH_LIB != "ethash":
+    raise ValueError(f"Unknown ETHASH_LIB={ETHASH_LIB!r}. "
+                     f"Use 'ethash', 'ethash_cy', 'ethash_rs', or 'auto'.")
+
+print(f"[ethash] using implementation: {ETHASH_LIB}")
+
 
 cache_seeds = [b"\x00" * 32]  # type: List[bytes]
 
 
-def mkcache(cache_size: int, block_number) -> List[List[int]]:
+@lru_cache(10)
+def _get_cache(seed: bytes, n: int) -> np.ndarray:
+    """Returns cache as uint32 ndarray of shape (n, 16)."""
+    if _impl_mkcache is not None:
+        return _impl_mkcache(np.frombuffer(seed, dtype=np.uint8), n)
+    o = np.empty((n, 16), dtype=np.uint32)
+    o[0] = ethash_sha3_512(seed)
+    for i in range(1, n):
+        o[i] = ethash_sha3_512(o[i - 1])
+    for _ in range(CACHE_ROUNDS):
+        for i in range(n):
+            v = int(o[i, 0]) % n
+            xored = o[(i - 1 + n) % n] ^ o[v]
+            o[i] = ethash_sha3_512(xored)
+    return o
+
+
+def mkcache(cache_size: int, block_number) -> np.ndarray:
     while len(cache_seeds) <= block_number // EPOCH_LENGTH:
-        new_seed = serialize_hash(ethash_sha3_256(cache_seeds[-1]))
+        new_seed = ethash_sha3_256(cache_seeds[-1]).tobytes()
         cache_seeds.append(new_seed)
 
     seed = cache_seeds[block_number // EPOCH_LENGTH]
     return _get_cache(seed, cache_size // HASH_BYTES)
 
 
-@lru_cache(10)
-def _get_cache(seed, n) -> List[List[int]]:
-    # Sequentially produce the initial dataset
-    o = [ethash_sha3_512(seed)]
-    for i in range(1, n):
-        o.append(ethash_sha3_512(o[-1]))
-
-    # Use a low-round version of randmemohash
-    for _ in range(CACHE_ROUNDS):
-        for i in range(n):
-            v = o[i][0] % n
-            o[i] = ethash_sha3_512(list(map(xor, o[(i - 1 + n) % n], o[v])))
-
-    return o
+def hashimoto_light(
+    full_size: int, cache: np.ndarray, header: bytes, nonce: bytes
+) -> Dict:
+    if _impl_hashimoto_light is not None:
+        return _impl_hashimoto_light(
+            full_size, cache,
+            np.frombuffer(header, dtype=np.uint8),
+            np.frombuffer(nonce, dtype=np.uint8),
+        )
+    return hashimoto(header, nonce, full_size, lambda x: calc_dataset_item(cache, x))
 
 
-def calc_dataset_item(cache: List[List[int]], i: int) -> List[int]:
+def calc_dataset_item(cache: np.ndarray, i: int) -> np.ndarray:
     n = len(cache)
-    r = HASH_BYTES // WORD_BYTES
-    # initialize the mix
-    mix = copy.copy(cache[i % n])  # type: List[int]
+    mix = cache[i % n].copy()
     mix[0] ^= i
     mix = ethash_sha3_512(mix)
-    # fnv it with a lot of random cache nodes based on i
-    for j in range(DATASET_PARENTS):
-        cache_index = fnv(i ^ j, mix[j % r])
-        mix = list(map(fnv, mix, cache[cache_index % n]))
+    if _mix_parents_fn is not None:
+        _mix_parents_fn(mix, cache, i)
+    else:
+        r = HASH_BYTES // WORD_BYTES   # 16
+        # uint32 overflow is intentional in FNV arithmetic
+        with np.errstate(over="ignore"):
+            for j in range(DATASET_PARENTS):
+                cache_index = ((i ^ j) * FNV_PRIME ^ int(mix[j % r])) & 0xFFFFFFFF
+                mix *= _FNV_PRIME
+                mix ^= cache[cache_index % n]
     return ethash_sha3_512(mix)
 
 
-def calc_dataset(full_size, cache) -> List[List[int]]:
-    o = []
-    for i in range(full_size // HASH_BYTES):
-        o.append(calc_dataset_item(cache, i))
-    return o
+def calc_dataset(full_size, cache: np.ndarray) -> np.ndarray:
+    rows = full_size // HASH_BYTES
+    out = np.empty((rows, 16), dtype=np.uint32)
+    for i in range(rows):
+        out[i] = calc_dataset_item(cache, i)
+    return out
 
 
 def hashimoto(
     header: bytes,
     nonce: bytes,
     full_size: int,
-    dataset_lookup: Callable[[int], List[int]],
+    dataset_lookup: Callable[[int], np.ndarray],
 ) -> Dict:
     n = full_size // HASH_BYTES
     w = MIX_BYTES // WORD_BYTES
     mixhashes = MIX_BYTES // HASH_BYTES
-    # combine header+nonce into a 64 byte seed
+
     s = ethash_sha3_512(header + nonce[::-1])
-    mix = []
-    for _ in range(MIX_BYTES // HASH_BYTES):
-        mix.extend(s)
-    # mix in random dataset nodes
-    for i in range(ACCESSES):
-        p = fnv(i ^ s[0], mix[i % w]) % (n // mixhashes) * mixhashes
-        newdata = []
-        for j in range(mixhashes):
-            newdata.extend(dataset_lookup(p + j))
-        mix = list(map(fnv, mix, newdata))
-    # compress mix
-    cmix = []
-    for i in range(0, len(mix), 4):
-        cmix.append(fnv(fnv(fnv(mix[i], mix[i + 1]), mix[i + 2]), mix[i + 3]))
+    mix = np.tile(s, mixhashes)
+    s0 = int(s[0])
+    newdata = np.empty(w, dtype=np.uint32)
+
+    # uint32 overflow is intentional in FNV arithmetic
+    with np.errstate(over="ignore"):
+        for i in range(ACCESSES):
+            p = ((i ^ s0) * FNV_PRIME ^ int(mix[i % w])) & 0xFFFFFFFF
+            p = p % (n // mixhashes) * mixhashes
+            for j in range(mixhashes):
+                newdata[j * 16:(j + 1) * 16] = dataset_lookup(p + j)
+            mix *= _FNV_PRIME
+            mix ^= newdata
+
+        mix_r = mix.reshape(-1, 4)
+        cmix = mix_r[:, 0] * _FNV_PRIME ^ mix_r[:, 1]
+        cmix = cmix * _FNV_PRIME ^ mix_r[:, 2]
+        cmix = cmix * _FNV_PRIME ^ mix_r[:, 3]
+
+    s_cmix = np.concatenate([s, cmix])
     return {
-        b"mix digest": serialize_hash(cmix),
-        b"result": serialize_hash(ethash_sha3_256(s + cmix)),
+        b"mix digest": cmix.tobytes(),
+        b"result": ethash_sha3_256(s_cmix).tobytes(),
     }
 
 
-def hashimoto_light(
-    full_size: int, cache: List[List[int]], header: bytes, nonce: bytes
-) -> Dict:
-    return hashimoto(header, nonce, full_size, lambda x: calc_dataset_item(cache, x))
-
-
-def hashimoto_full(dataset: List[List[int]], header: bytes, nonce: bytes) -> Dict:
+def hashimoto_full(dataset: np.ndarray, header: bytes, nonce: bytes) -> Dict:
     return hashimoto(header, nonce, len(dataset) * HASH_BYTES, lambda x: dataset[x])