Repository navigation
Expand file tree
/
Copy pathresolve_path.py
More file actions
63 lines (52 loc) · 1.59 KB
/
Copy pathresolve_path.py
File metadata and controls
63 lines (52 loc) · 1.59 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#!/usr/bin/env python3
"""Resolve a StreamingDataset-style path via litdata.streaming.resolver._resolve_dir.
Prints JSON: path, url, data_connection_id, index_json.
"""
from __future__ import annotations
import json
import os
import sys
from pathlib import Path
_ROOT = Path(__file__).resolve().parent.parent
_SRC = _ROOT / "src"
if _SRC.is_dir():
sys.path.insert(0, str(_SRC))
try:
from litdata.streaming.resolver import _resolve_dir
except ImportError as e:
print(
json.dumps({"error": (f"litdata resolver unavailable; install litdata or set PYTHONPATH to src. {e}")}),
file=sys.stderr,
)
sys.exit(2)
def _index_json(path: str | None, url: str | None, user: str) -> str:
u = user.rstrip("/")
if u.endswith("index.json"):
return u
if path:
p = path.rstrip("/")
if p.endswith("index.json"):
return p
return os.path.join(p, "index.json")
if url:
base = url.rstrip("/")
if base.endswith("index.json"):
return base
return base + "/index.json"
return u + "/index.json"
def main() -> None:
"""Print JSON for the resolved dataset directory and index.json path."""
if len(sys.argv) != 2:
print("usage: resolve_path.py <dataset-path-or-index.json>", file=sys.stderr)
sys.exit(1)
user = sys.argv[1]
d = _resolve_dir(user)
out = {
"path": d.path,
"url": d.url,
"data_connection_id": d.data_connection_id,
"index_json": _index_json(d.path, d.url, user),
}
print(json.dumps(out))
if __name__ == "__main__":
main()