mteb/Makefile at main · embeddings-benchmark/mteb · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
install:
	@echo "--- 🚀 Installing project dependencies ---"
	uv sync --frozen --extra image --group dev

install-for-tests:
	@echo "--- 🚀 Installing project dependencies for test ---"
	@echo "This ensures that the project is not installed in editable mode"
	uv sync --extra bm25s --extra image --extra audio --extra leaderboard --extra faiss-cpu --extra github --extra api --group dev --frozen

lint:
	@echo "--- 🧹 Running linters ---"
	uv run --no-sync ruff format .
	uv run --no-sync ruff check . --fix --exit-non-zero-on-fix
	uv run --no-sync typos

lint-check:
	@echo "--- 🧹 Check is project is linted ---"
	uv run --no-sync ruff format . --check
	uv run --no-sync ruff check .
	uv run --no-sync typos --diff

test:
	@echo "--- 🧪 Running tests ---"
	uv run --no-sync --group test pytest -n auto -m "not (test_datasets or leaderboard_stability or test_reference_models)"

install-core:
	@echo "--- 🚀 Installing mteb-core, the minimal install without torch ---"
	@echo "This rewrites pyproject.toml, so only run it in a throwaway checkout"
	uv venv
	uv pip install packaging
	uv run --no-sync python scripts/build_core_package.py --edit-pyproject
	uv pip install ".[api,leaderboard]" --group test

test-core:
	@echo "--- 🧪 Running the tests that must pass without torch ---"
	uv run --no-sync python -c "import importlib.util; assert importlib.util.find_spec('torch') is None, 'torch must not be installed'"
	uv run --no-sync pytest -n auto -m core


test-with-coverage:
	@echo "--- 🧪 Running tests with coverage ---"
	uv run --no-sync --group test pytest -n auto --cov-report=term-missing --cov-config=pyproject.toml --cov=mteb

pr:
	@echo "--- 🚀 Running requirements for a PR ---"
	make lint
	make test

build-docs: build-docs-overview
	@echo "--- 📚 Building documentation ---"
	uv run --no-sync --group docs zensical build --clean --strict


build-docs-overview:
	@echo "--- 📚 Building documentation overview ---"
	uv run --no-sync --group docs python docs/overview/create_available_tasks.py
	uv run --no-sync --group docs python docs/overview/create_available_models.py
	uv run --no-sync --group docs python docs/overview/create_available_benchmarks.py


serve-docs:
	@echo "--- 📚 Serving documentation ---"
	uv run --no-sync --group docs zensical serve


install-for-model-load-test:
	@echo "--- 🚀 Installing project dependencies for model load test ---"
	uv sync --extra pylate --group dev --frozen

model-load-test:
	@echo "--- 🚀 Running model load test ---"
	uv run --no-sync python scripts/tools/extract_model_names.py $(BASE_BRANCH) --return_one_model_name_per_file
	uv run --no-sync python tests/test_models/model_loading.py --model_name_file scripts/tools/model_names.txt


dataset-load-test:
	@echo "--- 🚀 Running dataset load test ---"
	uv run --no-sync --group test pytest -m test_datasets

dataset-load-test-pr:
	@echo "--- 🚀 Running dataset load test for PR ---"
	eval "$$(uv run --no-sync python -m scripts.tools.extract_datasets $(BASE_BRANCH))" && uv run --no-sync --group test pytest -m test_datasets

citation-check:
	@echo "--- 🔍 Checking that the cited works exist ---"
	uv run --no-sync --group test python tests/test_validate_metadata/check_citations.py $(CITATION_CHECK_ARGS)

reference-model-test:
	@echo "--- Running reference model coverage test ---"
	uv run --no-sync --group test pytest -m test_reference_models -v

run-leaderboard:
	@echo "--- 🚀 Running leaderboard locally ---"
	uv run --no-sync --extra leaderboard python -m mteb leaderboard

serve-api:
	@echo "--- 🚀 Serving leaderboard API on http://localhost:$(or $(PORT),8000) ---"
	@echo "Set PRELOAD=1 to warm every benchmark summary on startup."
	uv run --no-sync --extra api uvicorn mteb.api.app:app --host $(or $(HOST),127.0.0.1) --port $(or $(PORT),8000)

format-citations:
	@echo "--- 🧹 Formatting citations ---"
	uv run --no-sync python scripts/tools/format_citations.py benchmarks
	uv run --no-sync python scripts/tools/format_citations.py tasks


.PHONY: check
check: ## Run code quality tools.
	@echo "--- 🧹 Running code quality tools ---"
	@uv run --no-sync pre-commit run -a

.PHONY: typecheck
typecheck:
	@echo "--- 🔍 Running type checks ---"
	uv run --no-sync mypy mteb