Skip to content

Commit 973d560

Browse files
authored
Merge pull request #2029 from max-svistunov/lcore-2337-migrate-legacy-config-to-unified
LCORE-2337: Migration tool — dumb-mode lift-and-shift
2 parents e3f8757 + 65b68ba commit 973d560

4 files changed

Lines changed: 314 additions & 4 deletions

File tree

‎src/lightspeed_stack.py‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import constants
1111
from configuration import configuration
1212
from constants import LIGHTSPEED_STACK_LOG_LEVEL_ENV_VAR
13+
from llama_stack_configuration import migrate_config_dumb
1314
from log import get_logger, setup_logging
1415
from runners.quota_scheduler import start_quota_scheduler
1516
from runners.uvicorn import start_uvicorn
@@ -75,6 +76,31 @@ def create_argument_parser() -> ArgumentParser:
7576
f"{constants.DEFAULT_SYNTHESIZED_CONFIG_PATH})",
7677
default=None,
7778
)
79+
parser.add_argument(
80+
"--migrate-config",
81+
dest="migrate_config",
82+
help="migrate a legacy two-file config to a unified single file and "
83+
"exit. Lifts the run.yaml given by --run-yaml into the "
84+
"llama_stack.config.native_override of the -c lightspeed-stack.yaml "
85+
"and writes the result to --migrate-output. Replace literal secrets "
86+
"with ${env.VAR} references before or after migrating.",
87+
action="store_true",
88+
default=False,
89+
)
90+
parser.add_argument(
91+
"--run-yaml",
92+
dest="run_yaml",
93+
help="path to the legacy Llama Stack run.yaml to migrate "
94+
"(used with --migrate-config)",
95+
default=None,
96+
)
97+
parser.add_argument(
98+
"--migrate-output",
99+
dest="migrate_output",
100+
help="path to write the unified lightspeed-stack.yaml "
101+
"(used with --migrate-config)",
102+
default=None,
103+
)
78104

79105
return parser
80106

@@ -90,9 +116,10 @@ def main() -> None:
90116
configuration.json and exits (exits with status 1 on failure).
91117
- If --dump-schema is provided, writes the active configuration schema to
92118
schema.json and exits (exits with status 1 on failure).
93-
- If --generate-llama-stack-configuration is provided, generates and stores
94-
the Llama Stack configuration to the specified output file and exits
95-
(exits with status 1 on failure).
119+
- If --migrate-config is provided, migrates the legacy two-file config
120+
(--run-yaml plus the -c lightspeed-stack.yaml) into a unified single
121+
file at --migrate-output and exits (status 1 on failure or missing
122+
flags).
96123
- Otherwise, sets LIGHTSPEED_STACK_CONFIG_PATH for worker processes, starts
97124
the quota scheduler, and starts the Uvicorn web service.
98125
@@ -108,6 +135,24 @@ def main() -> None:
108135
os.environ[LIGHTSPEED_STACK_LOG_LEVEL_ENV_VAR] = "DEBUG"
109136
setup_logging()
110137

138+
# --migrate-config converts a legacy two-file config to a unified single
139+
# file and exits. It reads the legacy files raw (the -c config still uses
140+
# the legacy library_client_config_path shape), so it must run before
141+
# load_configuration, which would validate against the current schema.
142+
if args.migrate_config:
143+
if args.run_yaml is None or args.migrate_output is None:
144+
logger.error("--migrate-config requires --run-yaml and --migrate-output")
145+
raise SystemExit(1)
146+
try:
147+
migrate_config_dumb(args.run_yaml, args.config_file, args.migrate_output)
148+
logger.info(
149+
"Migrated unified configuration written to %s", args.migrate_output
150+
)
151+
except Exception as e:
152+
logger.error("Failed to migrate configuration: %s", e)
153+
raise SystemExit(1) from e
154+
return
155+
111156
configuration.load_configuration(args.config_file)
112157
logger.info("Configuration: %s", configuration.configuration)
113158
logger.info(

‎src/llama_stack_configuration.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
operator-facing artifact.
1717
"""
1818

19+
# pylint: disable=too-many-lines
20+
1921
import copy
2022
import os
2123
from argparse import ArgumentParser
@@ -900,6 +902,86 @@ def synthesize_to_file(
900902
logger.info("Wrote synthesized Llama Stack configuration to %s (mode 0600)", path)
901903

902904

905+
# =============================================================================
906+
# Migration: legacy two-file config -> unified single file (LCORE-2337)
907+
# =============================================================================
908+
909+
910+
def migrate_config_dumb(
911+
run_yaml_path: str,
912+
lightspeed_yaml_path: str,
913+
output_path: str,
914+
) -> None:
915+
"""Migrate a legacy two-file config to a unified single file (dumb mode).
916+
917+
"Dumb" lift-and-shift: the operator's ``lightspeed-stack.yaml`` is kept
918+
verbatim except for its ``llama_stack`` section, where
919+
``library_client_config_path`` is dropped and replaced by a unified
920+
``config`` block that lifts the *entire* legacy ``run.yaml`` body into
921+
``native_override`` with ``baseline: empty``. Synthesizing the result then
922+
starts from an empty baseline and deep-merges only the lifted run.yaml, so
923+
it reproduces the original run.yaml (Decision T7) — a lossless round-trip,
924+
without trying to factor anything into high-level sections (that "smart"
925+
mode is deferred future work).
926+
927+
All other ``lightspeed-stack.yaml`` content (name, service, byok_rag, …) is
928+
preserved untouched, so any existing enrichment keeps working in unified
929+
mode exactly as it did in legacy mode.
930+
931+
Parameters:
932+
run_yaml_path: Path to the legacy Llama Stack ``run.yaml``.
933+
lightspeed_yaml_path: Path to the legacy ``lightspeed-stack.yaml``.
934+
output_path: Path to write the unified ``lightspeed-stack.yaml``.
935+
936+
Returns:
937+
None.
938+
939+
Raises:
940+
OSError: If an input file cannot be read or the output cannot be
941+
written.
942+
yaml.YAMLError: If an input file is not valid YAML.
943+
ValueError: If either input file does not parse to a mapping (e.g. an
944+
empty or comment-only file).
945+
"""
946+
with open(run_yaml_path, "r", encoding="utf-8") as file:
947+
run_yaml = yaml.safe_load(file)
948+
with open(lightspeed_yaml_path, "r", encoding="utf-8") as file:
949+
lcs_config = yaml.safe_load(file)
950+
951+
# An empty or comment-only YAML file parses to None; fail with a clear
952+
# message rather than a downstream AttributeError/TypeError.
953+
if not isinstance(lcs_config, dict):
954+
raise ValueError(
955+
f"{lightspeed_yaml_path} did not parse to a mapping; cannot migrate."
956+
)
957+
if not isinstance(run_yaml, dict):
958+
raise ValueError(f"{run_yaml_path} did not parse to a mapping; cannot migrate.")
959+
960+
# Preserve the whole lightspeed-stack.yaml; only rewrite the llama_stack
961+
# section: drop the legacy path, add the unified config block.
962+
llama_stack = dict(lcs_config.get("llama_stack") or {})
963+
llama_stack.pop("library_client_config_path", None)
964+
llama_stack["config"] = {
965+
"baseline": "empty",
966+
"native_override": run_yaml,
967+
}
968+
lcs_config["llama_stack"] = llama_stack
969+
970+
logger.info(
971+
"Migrating legacy config (%s + %s) to unified %s",
972+
lightspeed_yaml_path,
973+
run_yaml_path,
974+
output_path,
975+
)
976+
# The lifted run.yaml may carry literal secrets, so write owner-only (0600),
977+
# matching synthesize_to_file. O_CREAT's mode only applies on create, so
978+
# chmod after the write also tightens a pre-existing file.
979+
fd = os.open(output_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
980+
with os.fdopen(fd, "w", encoding="utf-8") as file:
981+
yaml.dump(lcs_config, file, Dumper=YamlDumper, default_flow_style=False)
982+
os.chmod(output_path, 0o600)
983+
984+
903985
# =============================================================================
904986
# Main Generation Function (service/container mode only)
905987
# =============================================================================

‎tests/unit/test_lightspeed_stack.py‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
"""Unit tests for functions defined in src/lightspeed_stack.py."""
22

3-
from lightspeed_stack import create_argument_parser
3+
from pathlib import Path
4+
5+
import pytest
6+
import yaml
7+
8+
from lightspeed_stack import create_argument_parser, main
49

510

611
def test_create_argument_parser() -> None:
@@ -14,3 +19,81 @@ def test_create_argument_parser() -> None:
1419
arg_parser = create_argument_parser()
1520
# nothing more to test w/o actual parsing is done
1621
assert arg_parser is not None
22+
23+
24+
def test_argument_parser_accepts_migrate_flags() -> None:
25+
"""The parser accepts --migrate-config, --run-yaml, and --migrate-output."""
26+
args = create_argument_parser().parse_args(
27+
[
28+
"--migrate-config",
29+
"--run-yaml",
30+
"run.yaml",
31+
"-c",
32+
"lightspeed-stack.yaml",
33+
"--migrate-output",
34+
"unified.yaml",
35+
]
36+
)
37+
assert args.migrate_config is True
38+
assert args.run_yaml == "run.yaml"
39+
assert args.config_file == "lightspeed-stack.yaml"
40+
assert args.migrate_output == "unified.yaml"
41+
42+
43+
def test_main_migrate_config_writes_unified_file(
44+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
45+
) -> None:
46+
"""`--migrate-config` migrates the legacy pair and exits without starting the service."""
47+
run_path = tmp_path / "run.yaml"
48+
run_path.write_text(
49+
yaml.dump({"version": 2, "apis": ["inference"]}), encoding="utf-8"
50+
)
51+
lcs_path = tmp_path / "lightspeed-stack.yaml"
52+
lcs_path.write_text(
53+
yaml.dump(
54+
{
55+
"name": "LCS",
56+
"llama_stack": {
57+
"use_as_library_client": True,
58+
"library_client_config_path": "run.yaml",
59+
},
60+
}
61+
),
62+
encoding="utf-8",
63+
)
64+
out_path = tmp_path / "unified.yaml"
65+
monkeypatch.setattr(
66+
"sys.argv",
67+
[
68+
"lightspeed-stack",
69+
"--migrate-config",
70+
"--run-yaml",
71+
str(run_path),
72+
"-c",
73+
str(lcs_path),
74+
"--migrate-output",
75+
str(out_path),
76+
],
77+
)
78+
79+
main() # returns early, never starts uvicorn
80+
81+
migrated = yaml.safe_load(out_path.read_text(encoding="utf-8"))
82+
assert "library_client_config_path" not in migrated["llama_stack"]
83+
assert migrated["llama_stack"]["config"]["baseline"] == "empty"
84+
assert migrated["llama_stack"]["config"]["native_override"] == {
85+
"version": 2,
86+
"apis": ["inference"],
87+
}
88+
89+
90+
def test_main_migrate_config_requires_run_yaml_and_output(
91+
monkeypatch: pytest.MonkeyPatch,
92+
) -> None:
93+
"""`--migrate-config` without --run-yaml / --migrate-output exits with status 1."""
94+
monkeypatch.setattr(
95+
"sys.argv",
96+
["lightspeed-stack", "--migrate-config", "-c", "lightspeed-stack.yaml"],
97+
)
98+
with pytest.raises(SystemExit):
99+
main()

‎tests/unit/test_llama_stack_synthesize.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
apply_high_level_inference,
2020
deep_merge_list_replace,
2121
load_default_baseline,
22+
migrate_config_dumb,
2223
synthesize_configuration,
2324
synthesize_to_file,
2425
)
@@ -369,3 +370,102 @@ def test_synthesize_to_file_tightens_perms_on_overwrite(tmp_path: Path) -> None:
369370
synthesize_to_file(lcs, str(out), str(tmp_path))
370371
assert stat.S_IMODE(os.stat(out).st_mode) == 0o600
371372
assert yaml.safe_load(out.read_text(encoding="utf-8")) == {"v": 3}
373+
374+
375+
# ---------------------------------------------------------------------------
376+
# migrate_config_dumb (LCORE-2337)
377+
# ---------------------------------------------------------------------------
378+
379+
380+
# A representative legacy run.yaml body with nested maps, lists, and env refs.
381+
_LEGACY_RUN_YAML: dict[str, Any] = {
382+
"version": 2,
383+
"apis": ["agents", "inference", "safety", "vector_io"],
384+
"providers": {
385+
"inference": [
386+
{
387+
"provider_id": "openai",
388+
"provider_type": "remote::openai",
389+
"config": {
390+
"api_key": "${env.OPENAI_API_KEY}",
391+
"allowed_models": ["gpt-4o-mini"],
392+
},
393+
},
394+
{
395+
"provider_id": "sentence-transformers",
396+
"provider_type": "inline::sentence-transformers",
397+
},
398+
],
399+
},
400+
"safety": {"default_shield_id": "llama-guard", "excluded_categories": []},
401+
}
402+
403+
404+
def _write_legacy_pair(tmp_path: Path) -> tuple[str, str]:
405+
"""Write a legacy run.yaml + lightspeed-stack.yaml pair, return their paths."""
406+
run_path = tmp_path / "run.yaml"
407+
run_path.write_text(yaml.dump(_LEGACY_RUN_YAML), encoding="utf-8")
408+
lcs = {
409+
"name": "LCS",
410+
"service": {"host": "localhost", "port": 8080},
411+
"llama_stack": {
412+
"use_as_library_client": True,
413+
"library_client_config_path": "run.yaml",
414+
},
415+
}
416+
lcs_path = tmp_path / "lightspeed-stack.yaml"
417+
lcs_path.write_text(yaml.dump(lcs), encoding="utf-8")
418+
return str(run_path), str(lcs_path)
419+
420+
421+
def test_migrate_config_dumb_structure(tmp_path: Path) -> None:
422+
"""Dumb migration lifts run.yaml into native_override and drops the legacy path."""
423+
run_path, lcs_path = _write_legacy_pair(tmp_path)
424+
out_path = tmp_path / "unified.yaml"
425+
migrate_config_dumb(run_path, lcs_path, str(out_path))
426+
427+
migrated = yaml.safe_load(out_path.read_text(encoding="utf-8"))
428+
# unrelated top-level content preserved
429+
assert migrated["name"] == "LCS"
430+
assert migrated["service"] == {"host": "localhost", "port": 8080}
431+
# legacy path dropped; use_as_library_client preserved
432+
assert "library_client_config_path" not in migrated["llama_stack"]
433+
assert migrated["llama_stack"]["use_as_library_client"] is True
434+
# whole run.yaml lifted into native_override with an empty baseline
435+
assert migrated["llama_stack"]["config"]["baseline"] == "empty"
436+
assert migrated["llama_stack"]["config"]["native_override"] == _LEGACY_RUN_YAML
437+
438+
439+
def test_migrate_then_synthesize_reproduces_run_yaml(tmp_path: Path) -> None:
440+
"""Round-trip: migrate -> synthesize reproduces the original run.yaml (R4)."""
441+
run_path, lcs_path = _write_legacy_pair(tmp_path)
442+
out_path = tmp_path / "unified.yaml"
443+
migrate_config_dumb(run_path, lcs_path, str(out_path))
444+
445+
migrated = yaml.safe_load(out_path.read_text(encoding="utf-8"))
446+
synthesized = synthesize_configuration(migrated)
447+
assert synthesized == _LEGACY_RUN_YAML
448+
449+
450+
def test_migrate_config_dumb_writes_output_0600(tmp_path: Path) -> None:
451+
"""The migrated file may carry lifted secrets, so it is written owner-only."""
452+
run_path, lcs_path = _write_legacy_pair(tmp_path)
453+
out_path = tmp_path / "unified.yaml"
454+
migrate_config_dumb(run_path, lcs_path, str(out_path))
455+
assert stat.S_IMODE(os.stat(out_path).st_mode) == 0o600
456+
457+
458+
def test_migrate_config_dumb_rejects_non_mapping_inputs(tmp_path: Path) -> None:
459+
"""An empty/comment-only input fails with a clear ValueError, not AttributeError."""
460+
run_path, lcs_path = _write_legacy_pair(tmp_path)
461+
out_path = str(tmp_path / "unified.yaml")
462+
463+
empty_lcs = tmp_path / "empty-lcs.yaml"
464+
empty_lcs.write_text("# only a comment\n", encoding="utf-8")
465+
with pytest.raises(ValueError, match="did not parse to a mapping"):
466+
migrate_config_dumb(run_path, str(empty_lcs), out_path)
467+
468+
empty_run = tmp_path / "empty-run.yaml"
469+
empty_run.write_text("", encoding="utf-8")
470+
with pytest.raises(ValueError, match="did not parse to a mapping"):
471+
migrate_config_dumb(str(empty_run), lcs_path, out_path)

0 commit comments

Comments
 (0)