Skip to content

Commit b4cc6ec

Browse files
jesgumjesgum
authored
[ALICE3] Refactor alice3 configuration parser (#18145)
1 parent 915acae commit b4cc6ec

6 files changed

Lines changed: 272 additions & 205 deletions

File tree

‎ALICE3/Core/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
o2physics_add_library(ALICE3Core
1313
SOURCES TrackUtilities.cxx
14+
ConfigurationParser.cxx
1415
FlatLutEntry.cxx
1516
FlatTrackSmearer.cxx
1617
GeometryContainer.cxx

‎ALICE3/Core/ConfigurationParser.cxx‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
///
12+
/// \file ConfigurationParser.cxx
13+
/// \brief Utilities to access and parse TEnv configuration files for the ALICE3 fast simulation
14+
/// \author Nicolò Jacazio, Università del Piemonte Orientale (IT)
15+
///
16+
17+
#include "ConfigurationParser.h"
18+
19+
#include <CCDB/BasicCCDBManager.h>
20+
#include <Framework/Logger.h>
21+
22+
#include <TEnv.h>
23+
#include <THashList.h>
24+
#include <TString.h>
25+
#include <TSystem.h>
26+
27+
#include <sys/file.h>
28+
#include <sys/stat.h>
29+
30+
#include <algorithm>
31+
#include <cerrno>
32+
#include <chrono>
33+
#include <cstdio>
34+
#include <fstream>
35+
#include <map>
36+
#include <string>
37+
#include <thread>
38+
#include <vector>
39+
40+
#include <fcntl.h>
41+
#include <unistd.h>
42+
43+
namespace o2::fastsim
44+
{
45+
46+
std::map<std::string, std::map<std::string, std::string>> ConfigurationParser::parseTEnvConfiguration(std::string& filename, std::vector<std::string>& entries)
47+
{
48+
std::map<std::string, std::map<std::string, std::string>> configMap;
49+
filename = gSystem->ExpandPathName(filename.c_str());
50+
LOG(info) << "Parsing TEnv configuration file: " << filename;
51+
TEnv env(filename.c_str());
52+
THashList* table = env.GetTable();
53+
entries.clear();
54+
for (int i = 0; i < table->GetEntries(); ++i) {
55+
const std::string key = table->At(i)->GetName();
56+
// key should contain exactly one dot
57+
if (key.find('.') == std::string::npos || key.find('.') != key.rfind('.')) {
58+
LOG(fatal) << "Key " << key << " does not contain exactly one dot";
59+
continue;
60+
}
61+
const std::string firstPart = key.substr(0, key.find('.'));
62+
if (std::find(entries.begin(), entries.end(), firstPart) == entries.end()) {
63+
entries.push_back(firstPart);
64+
}
65+
}
66+
env.Print();
67+
// Entries
68+
for (const auto& entry : entries) {
69+
LOG(info) << " Reading entry " << entry;
70+
for (int i = 0; i < table->GetEntries(); ++i) {
71+
const std::string key = table->At(i)->GetName();
72+
if (key.find(entry + ".") == 0) {
73+
const std::string paramName = key.substr(key.find('.') + 1);
74+
const std::string value = env.GetValue(key.c_str(), "");
75+
configMap[entry][paramName] = value;
76+
}
77+
}
78+
}
79+
return configMap;
80+
}
81+
82+
std::string ConfigurationParser::accessFile(const std::string& path, const std::string& downloadPath, o2::ccdb::BasicCCDBManager* ccdb, int timeoutSeconds)
83+
{
84+
if (path.rfind("ccdb:", 0) == 0) {
85+
const std::string ccdbPath = path.substr(5); // remove "ccdb:" prefix
86+
const std::string localPath = Form("%s/%s/snapshot.root", downloadPath.c_str(), ccdbPath.c_str());
87+
const std::string lockFile = localPath + ".lock";
88+
const std::string doneFile = localPath + ".done";
89+
90+
// Create directory structure if it doesn't exist
91+
std::string dirPath = localPath.substr(0, localPath.find_last_of('/'));
92+
gSystem->mkdir(dirPath.c_str(), true);
93+
94+
// Check if file is already fully downloaded
95+
struct stat buffer;
96+
if (stat(doneFile.c_str(), &buffer) == 0) {
97+
LOG(info) << " --- Geometry configuration file already exists: " << localPath << ". Skipping download.";
98+
return localPath;
99+
}
100+
101+
// Acquire file lock for inter-process synchronization
102+
int lockFd = open(lockFile.c_str(), O_CREAT | O_RDWR, 0666);
103+
if (lockFd == -1) {
104+
LOG(error) << " --- Failed to create lock file: " << lockFile;
105+
return localPath;
106+
}
107+
108+
// Try to acquire exclusive lock (non-blocking)
109+
LOG(info) << " --- Attempting to acquire lock for: " << localPath;
110+
int lockResult = flock(lockFd, LOCK_EX | LOCK_NB);
111+
112+
if (lockResult == -1 && errno == EWOULDBLOCK) {
113+
// Lock is held by another process - wait up to 10 minutes for download to complete
114+
LOG(info) << " --- Lock is held by another process. Waiting for download to complete (up to 10 minutes)...";
115+
close(lockFd);
116+
117+
const auto startTime = std::chrono::steady_clock::now();
118+
const auto timeout = std::chrono::minutes(10);
119+
const auto checkInterval = std::chrono::seconds(5);
120+
121+
while (true) {
122+
// Check if download is complete
123+
if (stat(doneFile.c_str(), &buffer) == 0) {
124+
LOG(info) << " --- Geometry configuration file was downloaded by another process: " << localPath;
125+
return localPath;
126+
}
127+
128+
// Check timeout
129+
auto elapsed = std::chrono::steady_clock::now() - startTime;
130+
if (elapsed >= timeout) {
131+
LOG(fatal) << " --- Timeout waiting for file download: " << localPath << ". Waited for 10 minutes.";
132+
return localPath;
133+
}
134+
135+
// Wait before checking again
136+
std::this_thread::sleep_for(checkInterval);
137+
}
138+
} else if (lockResult == -1) {
139+
LOG(error) << " --- Failed to acquire lock for: " << lockFile;
140+
close(lockFd);
141+
return localPath;
142+
}
143+
144+
// Lock acquired successfully - double-check if file was downloaded while we were trying
145+
if (stat(doneFile.c_str(), &buffer) == 0) {
146+
LOG(info) << " --- Geometry configuration file was downloaded by another process: " << localPath;
147+
flock(lockFd, LOCK_UN);
148+
close(lockFd);
149+
return localPath;
150+
}
151+
152+
// File does not exist, retrieve from CCDB
153+
LOG(info) << " --- CCDB source detected for " << path;
154+
std::map<std::string, std::string> metadata;
155+
bool status = ccdb->getCCDBAccessor().retrieveBlob(ccdbPath, downloadPath, metadata, 1);
156+
if (!status) {
157+
flock(lockFd, LOCK_UN);
158+
close(lockFd);
159+
LOG(fatal) << " --- Failed to retrieve configuration from CCDB for path: " << ccdbPath;
160+
return "";
161+
}
162+
LOG(info) << " --- Retrieved configuration from CCDB to: " << localPath;
163+
164+
// Verify the integrity of the downloaded file
165+
if (stat(localPath.c_str(), &buffer) != 0) {
166+
flock(lockFd, LOCK_UN);
167+
close(lockFd);
168+
LOG(fatal) << " --- Downloaded file does not exist or is not accessible: " << localPath;
169+
return "";
170+
}
171+
if (buffer.st_size == 0) {
172+
flock(lockFd, LOCK_UN);
173+
close(lockFd);
174+
LOG(fatal) << " --- Downloaded file is empty: " << localPath;
175+
return "";
176+
}
177+
LOG(info) << " --- File integrity verified. Size: " << buffer.st_size << " bytes";
178+
179+
// Create done marker file to indicate successful download
180+
std::ofstream doneMarker(doneFile);
181+
doneMarker.close();
182+
183+
// Release lock
184+
flock(lockFd, LOCK_UN);
185+
close(lockFd);
186+
187+
// If timeout is specified, schedule file deletion after timeout
188+
if (timeoutSeconds > 0) {
189+
LOG(info) << " --- Deleting configuration file after timeout: " << localPath;
190+
std::thread deletionThread([localPath, doneFile, timeoutSeconds]() {
191+
LOG(info) << " --- Operating deletion of configuration file after timeout: " << localPath;
192+
std::this_thread::sleep_for(std::chrono::seconds(timeoutSeconds));
193+
if (std::remove(localPath.c_str()) == 0) {
194+
LOG(info) << " --- File deleted successfully: " << localPath;
195+
} else {
196+
LOG(warning) << " --- Failed to delete file: " << localPath;
197+
}
198+
// Also remove the done marker file
199+
if (std::remove(doneFile.c_str()) == 0) {
200+
LOG(info) << " --- Done marker deleted: " << doneFile;
201+
}
202+
});
203+
deletionThread.detach();
204+
}
205+
206+
return localPath;
207+
}
208+
return path;
209+
}
210+
211+
} // namespace o2::fastsim

‎ALICE3/Core/ConfigurationParser.h‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
///
12+
/// \file ConfigurationParser.h
13+
/// \brief Utilities to access and parse TEnv configuration files for the ALICE3 fast simulation
14+
/// \author Nicolò Jacazio, Università del Piemonte Orientale (IT)
15+
///
16+
17+
#ifndef ALICE3_CORE_CONFIGURATIONPARSER_H_
18+
#define ALICE3_CORE_CONFIGURATIONPARSER_H_
19+
20+
#include <CCDB/BasicCCDBManager.h>
21+
22+
#include <map>
23+
#include <string>
24+
#include <vector>
25+
26+
namespace o2::fastsim
27+
{
28+
29+
class ConfigurationParser
30+
{
31+
public:
32+
/**
33+
* @brief Parses a TEnv configuration file with keys of the form "<entry>.<parameter>" and returns the key-value pairs split per entry
34+
* @param filename Path to the TEnv configuration file
35+
* @param entries Vector to store the order of the entries as they appear in the file
36+
* @return A map where each key is an entry name and the value is another map of key-value pairs for that entry
37+
*/
38+
static std::map<std::string, std::map<std::string, std::string>> parseTEnvConfiguration(std::string& filename, std::vector<std::string>& entries);
39+
40+
/**
41+
* @brief Accesses a file given its path, which can be either a local path or a ccdb path (starting with "ccdb:"). In the first case it returns the local path, in the second it retrieves the file from ccdb and returns the local path to the retrieved file.
42+
* @param path The path to the file, either local or ccdb (starting with "ccdb:")
43+
* @param downloadPath The local path where to download the file if it's a ccdb path. Default is "/tmp/GeometryContainer/"
44+
* @param ccdb Pointer to the CCDB manager to use for retrieving the file if it's a ccdb path. Must be set when path is a ccdb path.
45+
* @param timeoutSeconds If positive, then this function will wait for these seconds after download before removing the downloaded file.
46+
* @return The local path to the file, either the original local path or the path to the retrieved file from ccdb
47+
*/
48+
static std::string accessFile(const std::string& path, const std::string& downloadPath = "/tmp/GeometryContainer/", o2::ccdb::BasicCCDBManager* ccdb = nullptr, int timeoutSeconds = 0);
49+
};
50+
51+
} // namespace o2::fastsim
52+
53+
#endif // ALICE3_CORE_CONFIGURATIONPARSER_H_

‎ALICE3/Core/FlatTrackSmearer.cxx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
#include "FlatTrackSmearer.h"
1313

14+
#include "ALICE3/Core/ConfigurationParser.h"
1415
#include "ALICE3/Core/FlatLutEntry.h"
1516
#include "ALICE3/Core/GeometryContainer.h"
1617

@@ -96,7 +97,7 @@ bool TrackSmearer::loadTable(int pdg, const char* filename, bool forceReload)
9697
}
9798

9899
LOGF(info, "Loading %s LUT file: '%s'", getParticleName(pdg), filename);
99-
const std::string localFilename = o2::fastsim::GeometryEntry::accessFile(filename, "./.ALICE3/LUTs/", mCcdbManager, 10);
100+
const std::string localFilename = o2::fastsim::ConfigurationParser::accessFile(filename, "./.ALICE3/LUTs/", mCcdbManager, o2::fastsim::GeometryContainer::cleanLutWhenLoaded() ? 10 : 0);
100101

101102
std::ifstream lutFile(localFilename, std::ifstream::binary);
102103
if (!lutFile.is_open()) {

0 commit comments

Comments
 (0)