Skip to content

Commit f8a9aa2

Browse files
authored
Merge pull request #82 from eic/Try-to-allow-file-wildcards
Add path globbing to allow wildcards in file names. Looping of hepmc3 sources and multiple input files
2 parents 38fddcf + 6bfc469 commit f8a9aa2

4 files changed

Lines changed: 78 additions & 34 deletions

File tree

‎include/CommandLineParser.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,13 @@ class CommandLineParser {
4040
*/
4141
static std::vector<std::string> splitCommaSeparated(const std::string& value);
4242

43+
/**
44+
* Expand glob patterns in file paths (e.g., "*.root") to actual file lists
45+
* @param patterns Vector of glob patterns
46+
* @return Vector of expanded file paths
47+
*/
48+
static std::vector<std::string> expandGlobPatterns(const std::vector<std::string>& patterns);
49+
4350
/**
4451
* Find or create a source configuration by name
4552
* @param sources Vector of source configurations

‎include/HepMC3DataSource.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,13 @@ class HepMC3DataSource : public DataSource {
4343
std::shared_ptr<HepMC3::ReaderRootTree> reader_;
4444

4545
// Current event
46-
HepMC3::GenEvent current_event_;
46+
HepMC3::GenEvent current_event_;
47+
size_t m_current_file; // Index of the current input file being processed
48+
size_t m_total_files; // Total number of input files
49+
long long m_still_to_skip; // Number of events still to skip
4750

4851
// Private helper methods
49-
void openInputFiles();
52+
void openNextFile();
5053
void cleanup();
5154

5255
// Format-specific vertex extraction from HepMC3 events (overrides base class)

‎src/CommandLineParser.cc‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include <algorithm>
66
#include <cctype>
77
#include <getopt.h>
8+
#include <glob.h>
89

910
void CommandLineParser::printUsage(const char* program_name) {
1011
std::cout << "Usage: " << program_name << " [options] input_file1 [input_file2 ...]\n"
@@ -75,6 +76,35 @@ std::vector<std::string> CommandLineParser::splitCommaSeparated(const std::strin
7576
return result;
7677
}
7778

79+
std::vector<std::string> CommandLineParser::expandGlobPatterns(const std::vector<std::string>& patterns) {
80+
std::vector<std::string> expanded_files;
81+
for (const auto& pattern : patterns) {
82+
glob_t glob_result{};
83+
int ret = glob(pattern.c_str(), GLOB_TILDE, nullptr, &glob_result);
84+
85+
if (ret == 0) {
86+
// matched files
87+
for (size_t i = 0; i < glob_result.gl_pathc; ++i) {
88+
expanded_files.emplace_back(glob_result.gl_pathv[i]);
89+
}
90+
} else if (ret == GLOB_NOMATCH) {
91+
bool has_wildcard = pattern.find_first_of("*?[") != std::string::npos;
92+
93+
if (has_wildcard) {
94+
std::cerr << "Warning: No files matched wildcard pattern: " << pattern << std::endl;
95+
} else {
96+
// treat as literal filename (local or XRootD)
97+
expanded_files.emplace_back(pattern);
98+
}
99+
} else {
100+
std::cerr << "Error: glob() failed for pattern: " << pattern << std::endl;
101+
}
102+
103+
globfree(&glob_result);
104+
}
105+
return expanded_files;
106+
}
107+
78108
SourceConfig* CommandLineParser::findOrCreateSource(std::vector<SourceConfig>& sources, const std::string& name) {
79109
for (auto& source : sources) {
80110
if (source.name == name) {
@@ -117,7 +147,8 @@ bool CommandLineParser::handleSourceOption(std::vector<SourceConfig>& sources, c
117147
SourceConfig* source = findOrCreateSource(sources, source_name);
118148

119149
if (property == "input_files") {
120-
source->input_files = splitCommaSeparated(value);
150+
std::vector<std::string> files = splitCommaSeparated(value);
151+
source->input_files = expandGlobPatterns(files);
121152
} else if (property == "skip") {
122153
source->skip = std::stoi(value);
123154
} else if (property == "frequency") {
@@ -167,7 +198,8 @@ void CommandLineParser::loadYAMLConfig(const std::string& config_file, MergerCon
167198
SourceConfig source;
168199
if (source_yaml["input_files"]) {
169200
for (const auto& f : source_yaml["input_files"]) {
170-
source.input_files.push_back(f.as<std::string>());
201+
std::vector<std::string> expanded_files = expandGlobPatterns({f.as<std::string>()});
202+
source.input_files.insert(source.input_files.end(), expanded_files.begin(), expanded_files.end());
171203
}
172204
}
173205
if (source_yaml["name"]) source.name = source_yaml["name"].as<std::string>();

‎src/HepMC3DataSource.cc‎

Lines changed: 32 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -8,35 +8,38 @@ HepMC3DataSource::HepMC3DataSource(const SourceConfig& config, size_t source_ind
88
total_entries_ = 0;
99
current_entry_index_ = 0;
1010
entries_needed_ = 0;
11-
openInputFiles();
11+
m_still_to_skip = config_->skip;
12+
m_current_file = 0;
13+
m_total_files = config_->input_files.size();
14+
openNextFile();
1215
}
1316

1417
HepMC3DataSource::~HepMC3DataSource() {
1518
cleanup();
1619
}
1720

18-
void HepMC3DataSource::openInputFiles() {
19-
if (config_->input_files.empty()) {
20-
throw std::runtime_error("No input files specified for source: " + config_->name);
21-
}
22-
23-
// For now, support only single file (could be extended to support multiple files)
24-
if (config_->input_files.size() > 1) {
25-
std::cout << "Warning: HepMC3DataSource currently supports only the first input file. "
26-
<< "Using: " << config_->input_files[0] << std::endl;
21+
void HepMC3DataSource::openNextFile() {
22+
if (m_current_file >= config_->input_files.size()) {
23+
if (!config_->repeat_on_eof) {
24+
throw std::runtime_error("No more input files to process for source: " + config_->name);
25+
} else {
26+
std::cout << "Reached end of input files for source: " << config_->name
27+
<< ". Repeating from the first file." << std::endl;
28+
m_current_file = 0; // Loop back to the first file if repeat_on_eof is true
29+
}
2730
}
28-
29-
const std::string& input_file = config_->input_files[0];
30-
31+
32+
const std::string& input_file = config_->input_files[m_current_file];
33+
3134
// Validate file extension
3235
if (input_file.find(".hepmc3.tree.root") == std::string::npos) {
3336
throw std::runtime_error(
3437
"HepMC3DataSource only supports .hepmc3.tree.root format. Got: " + input_file
3538
);
3639
}
37-
40+
3841
std::cout << "Opening HepMC3 file: " << input_file << std::endl;
39-
42+
4043
// Create ReaderRootTree explicitly for ROOT tree format
4144
reader_ = std::make_shared<HepMC3::ReaderRootTree>(input_file);
4245
if (!reader_) {
@@ -45,27 +48,26 @@ void HepMC3DataSource::openInputFiles() {
4548

4649
// Get entry count directly from TTree
4750
total_entries_ = reader_->m_tree->GetEntries();
48-
51+
4952
std::cout << "Found " << total_entries_ << " events in HepMC3 file" << std::endl;
50-
53+
5154
current_entry_index_ = 0;
5255

5356
// Skip initial events if specified
54-
long long skipN = config_->skip;
55-
if (skipN > 0) {
56-
std::cout << "Skipping first " << skipN << " events for source " << config_->name << " as per configuration" << std::endl;
57-
if (config_->skip >= total_entries_) {
58-
if(!config_->repeat_on_eof) {
59-
throw std::runtime_error("Skip value exceeds total entries in source: " + config_->name);
60-
} else {
61-
std::cout << "Warning: Skip value exceeds total entries, but repeat_on_eof is true. Will loop back to start after reaching EOF." << std::endl;
62-
skipN = skipN % total_entries_; // Wrap around if repeat_on_eof is true
63-
}
57+
if (m_still_to_skip > 0) {
58+
std::cout << "Skipping first " << m_still_to_skip << " events for source " << config_->name << " as per configuration" << std::endl;
59+
if (m_still_to_skip >= total_entries_) {
60+
std::cout << "Warning: Skip value exceeds total entries, moving to next file." << std::endl;
61+
m_still_to_skip = m_still_to_skip - total_entries_; // Wrap around if repeat_on_eof is true
62+
m_current_file++;
63+
openNextFile(); // Open next file if available
64+
return;
65+
} else {
66+
reader_->skip(m_still_to_skip);
67+
current_entry_index_ += m_still_to_skip;
68+
m_still_to_skip = 0; // Reset skip counter after skipping
6469
}
65-
reader_->skip(skipN);
66-
current_entry_index_ += skipN;
6770
}
68-
6971
}
7072

7173
bool HepMC3DataSource::hasMoreEntries() const {

0 commit comments

Comments
 (0)