Repository navigation
Expand file tree
/
Copy pathsummarize_data.py
More file actions
79 lines (65 loc) · 3 KB
/
Copy pathsummarize_data.py
File metadata and controls
79 lines (65 loc) · 3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
import argparse
from datasets import load_from_disk
from collections import Counter
from tqdm.auto import tqdm
import textwrap
import os
def summarize_dataset(raw_dataset_dir: str):
"""
Loads the "raw" DatasetDict from disk and prints a summary of its splits.
For each split, it calculates:
1. Total number of tokens.
2. The count of each feature combination type ('combo_feats').
"""
print(f"🔄 Loading 'raw' dataset from '{raw_dataset_dir}'...")
try:
# We expect this to be the "raw" output from datawriter.py
dataset_dict = load_from_disk(raw_dataset_dir)
print("Dataset loaded successfully.")
except FileNotFoundError:
print(f"Error: Dataset not found at '{raw_dataset_dir}'")
return
# Loop through each split (train, val, test) and generate a summary
for split_name, dset in dataset_dict.items():
# --- Print Header for the split ---
print("\n" + "="*50)
print(f"Summary for '{split_name}' split")
print(f" Number of rows: {len(dset):,}")
print("="*50)
# --- 1. Calculate Total Number of Tokens ---
if "input_ids" in dset.column_names:
print("⏳ Calculating total number of tokens...")
# This is the fast, parallel way to count tokens
token_count_ds = dset.map(
lambda x: {"num_tokens": len(x['input_ids'])},
num_proc=max(1, os.cpu_count() - 2)
)
total_tokens = sum(token_count_ds['num_tokens'])
print(f" Total Tokens: {total_tokens:,}")
else:
print(" (Skipping token count as 'input_ids' column not found)")
# --- 2. Count Occurrences of each `combo_feats` ---
if "combo_feats" in dset.column_names:
print(f"⏳ Counting feature combinations for '{split_name}' split...")
combo_counts = Counter(
tuple(sorted(feats)) for feats in tqdm(dset["combo_feats"], desc=f"Counting {split_name}")
)
print(" Feature Combination Counts:")
# Sort the combinations by count in descending order for a clean report
sorted_combos = sorted(combo_counts.items(), key=lambda item: item[1], reverse=True)
for combo, count in sorted_combos:
combo_str = textwrap.shorten(str(combo), width=50, placeholder="...")
print(f" - {combo_str:<50} : {count:10,d} rows")
else:
print(" (Skipping feature counts as 'combo_feats' column not found)")
print("\n Summary complete.\n")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Summarize a Hugging Face dataset generated by datawriter.py.")
parser.add_argument(
"--raw_dataset_dir",
type=str,
required=True,
help="Path to the 'raw' output directory from datawriter.py (the one with text columns)."
)
args = parser.parse_args()
summarize_dataset(args.raw_dataset_dir)