Skip to content

Please consider the following formatting changes to #15927 #155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alibuild wants to merge 3 commits into from
Closed

Please consider the following formatting changes to #15927 #155

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
763aeac
Allow overriding vectors in ConfigParamRegistry by replacing a subtree
aalkin Oct 1, 2026
c26537c
Allow erasing parameter subtrees in ConfigParamRegistry
aalkin Oct 2, 2026
42dcb5a
Please consider the following formatting changes
alibuild Oct 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Framework/Core/include/Framework/ConfigParamRegistry.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ class ConfigParamRegistry

void override(const char* key, ConfigValueType auto const& val) const;

// Remove an option (a leaf or a whole subtree) from the active store, no-op if it is missing
void erase(const char* key) const;

// Load extra parameters discovered while we process data
void loadExtra(std::vector<ConfigParamSpec>& extras);

Expand Down
59 changes: 46 additions & 13 deletions Framework/Core/src/ConfigParamRegistry.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
#include "Framework/VariantPropertyTreeHelpers.h"
#include "Framework/Array2D.h"

#include <string_view>
#include <type_traits>

namespace o2::framework
{

Expand Down Expand Up @@ -67,6 +70,19 @@ auto getImpl(boost::property_tree::ptree& tree, const char* key)
{
return labeledArrayFromBranch<typename T::element_t>(tree.get_child(key));
}

// ptree::erase only removes direct children, so the parent of a nested key has to be looked up first
void eraseImpl(boost::property_tree::ptree& tree, std::string_view key)
{
auto pos = key.rfind('.');
if (pos == std::string_view::npos) {
tree.erase(std::string(key));
return;
}
if (auto parent = tree.get_child_optional(std::string(key.substr(0, pos)))) {
parent->erase(std::string(key.substr(pos + 1)));
}
}
} // namespace

template <ConfigValueType T>
Expand All @@ -83,15 +99,32 @@ T ConfigParamRegistry::get(const char* key) const

void ConfigParamRegistry::override(const char* key, ConfigValueType auto const& val) const
{
using T = std::decay_t<decltype(val)>;
try {
mStore->store().put(key, val);
auto& store = mStore->store();
if constexpr (SimpleConfigValueType<T> || StringConfigValueType<T>) {
store.put(key, val);
} else if constexpr (VectorConfigValueType<T>) {
// arrays are stored as subtrees, which have to be replaced as a whole
store.put_child(key, vectorToBranch(val.data(), val.size()));
} else if constexpr (Array2DLike<T>) {
store.put_child(key, array2DToBranch(T(val)));
} else if constexpr (LabeledArrayLike<T>) {
store.put_child(key, labeledArrayToBranch(T(val)));
}
} catch (std::exception& e) {
throw std::invalid_argument(std::string("failed to store an option: ") + key + " (" + e.what() + ")");
} catch (...) {
throw std::invalid_argument(std::string("failed to store an option: ") + key);
}
}

void ConfigParamRegistry::erase(const char* key) const
{
eraseImpl(mStore->store(), key);
eraseImpl(mStore->provenanceTree(), key);
}

// Load extra parameters discovered while we process data
void ConfigParamRegistry::loadExtra(std::vector<ConfigParamSpec>& extras)
{
Expand Down Expand Up @@ -143,16 +176,16 @@ template void ConfigParamRegistry::override(const char* key, double const&) cons
template void ConfigParamRegistry::override(const char* key, std::string const&) const;
template void ConfigParamRegistry::override(const char* key, bool const&) const;

// template void ConfigParamRegistry::override(char const* key, LabeledArray<std::string> const&) const;
// template void ConfigParamRegistry::override(char const* key, LabeledArray<double> const&) const;
// template void ConfigParamRegistry::override(char const* key, LabeledArray<float> const&) const;
// template void ConfigParamRegistry::override(char const* key, LabeledArray<int> const&) const;
// template void ConfigParamRegistry::override(char const* key, Array2D<std::string> const&) const;
// template void ConfigParamRegistry::override(char const* key, Array2D<double> const&) const;
// template void ConfigParamRegistry::override(char const* key, Array2D<float> const&) const;
// template void ConfigParamRegistry::override(char const* key, Array2D<int> const&) const;
// template void ConfigParamRegistry::override(char const* key, std::vector<std::string> const&) const;
// template void ConfigParamRegistry::override(char const* key, std::vector<double> const&) const;
// template void ConfigParamRegistry::override(char const* key, std::vector<float> const&) const;
// template void ConfigParamRegistry::override(char const* key, std::vector<int> const&) const;
template void ConfigParamRegistry::override(char const* key, LabeledArray<std::string> const&) const;
template void ConfigParamRegistry::override(char const* key, LabeledArray<double> const&) const;
template void ConfigParamRegistry::override(char const* key, LabeledArray<float> const&) const;
template void ConfigParamRegistry::override(char const* key, LabeledArray<int> const&) const;
template void ConfigParamRegistry::override(char const* key, Array2D<std::string> const&) const;
template void ConfigParamRegistry::override(char const* key, Array2D<double> const&) const;
template void ConfigParamRegistry::override(char const* key, Array2D<float> const&) const;
template void ConfigParamRegistry::override(char const* key, Array2D<int> const&) const;
template void ConfigParamRegistry::override(char const* key, std::vector<std::string> const&) const;
template void ConfigParamRegistry::override(char const* key, std::vector<double> const&) const;
template void ConfigParamRegistry::override(char const* key, std::vector<float> const&) const;
template void ConfigParamRegistry::override(char const* key, std::vector<int> const&) const;
} // namespace o2::framework
138 changes: 138 additions & 0 deletions Framework/Core/test/test_ConfigParamRegistry.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
#include <catch_amalgamated.hpp>
#include "Framework/FairOptionsRetriever.h"
#include "Framework/ConfigParamRegistry.h"
#include "Framework/ConfigParamStore.h"
#include "Framework/Array2D.h"

#include <fairmq/ProgOptions.h>
#include <boost/program_options.hpp>
Expand Down Expand Up @@ -119,3 +121,139 @@ TEST_CASE("TestConfigParamRegistry")
REQUIRE(obj.x == 1);
REQUIRE(obj.y == 2.f);
}

TEST_CASE("TestConfigParamRegistryOverride")
{
float m[3][4] = {{1, 2, 3, 4}, {5, 6, 7, 8}, {9, 10, 11, 12}};
int l[2][2] = {{1, 2}, {3, 4}};
// a Variant cannot be created from a temporary vector, so the defaults are named
std::vector<int> defaultVector{1, 2, 3, 4, 5};
std::vector<std::string> defaultStringVector{"a", "b", "c"};
std::vector<double> defaultNestedVector{0.5, 1.5};
// the store keeps a reference to the specs, which have to outlive it
std::vector<ConfigParamSpec> specs{
ConfigParamSpec{"anInt", VariantType::Int, 1, {"an int option"}},
ConfigParamSpec{"aString", VariantType::String, "foo", {"a string option"}},
ConfigParamSpec{"aVector", VariantType::ArrayInt, defaultVector, {"an int array option"}},
ConfigParamSpec{"aStringVector", VariantType::ArrayString, defaultStringVector, {"a string array option"}},
ConfigParamSpec{"anArray2D", VariantType::Array2DFloat, Array2D<float>{&m[0][0], 3, 4}, {"a 2D array option"}},
ConfigParamSpec{"aLabeledArray", VariantType::LabeledArrayInt, LabeledArray<int>{&l[0][0], 2, 2, {"r1", "r2"}, {"c1", "c2"}}, {"a labeled array option"}},
ConfigParamSpec{"aNested.x", VariantType::Int, 2, {"an int option, nested in an object"}},
ConfigParamSpec{"aNested.vec", VariantType::ArrayDouble, defaultNestedVector, {"a double array option, nested in an object"}},
};
auto store = std::make_unique<ConfigParamStore>(specs, std::vector<std::unique_ptr<ParamRetriever>>{});
store->preload();
store->activate();
ConfigParamRegistry registry(std::move(store));
REQUIRE(registry.get<std::vector<int>>("aVector").size() == 5);

// leaf values keep working
registry.override("anInt", 10);
REQUIRE(registry.get<int>("anInt") == 10);

// a shorter vector replaces the whole subtree, no stale elements are left
registry.override("aVector", std::vector<int>{7, 8});
REQUIRE(registry.get<std::vector<int>>("aVector") == std::vector<int>{7, 8});
registry.override("aStringVector", std::vector<std::string>{"x"});
REQUIRE(registry.get<std::vector<std::string>>("aStringVector") == std::vector<std::string>{"x"});

// a smaller 2D array replaces the whole subtree
float m2[1][2] = {{0.25, 0.75}};
registry.override("anArray2D", Array2D<float>{&m2[0][0], 1, 2});
auto a2d = registry.get<Array2D<float>>("anArray2D");
REQUIRE(a2d.rows == 1);
REQUIRE(a2d.cols == 2);
REQUIRE(a2d(0, 1) == 0.75f);

// labels are replaced together with the values
int l2[1][3] = {{5, 6, 7}};
registry.override("aLabeledArray", LabeledArray<int>{&l2[0][0], 1, 3, {"row"}, {"x", "y", "z"}});
auto la = registry.get<LabeledArray<int>>("aLabeledArray");
REQUIRE(la.getLabelsRows() == std::vector<std::string>{"row"});
REQUIRE(la.getLabelsCols() == std::vector<std::string>{"x", "y", "z"});
REQUIRE(la.get("row", "z") == 7);
// an unlabeled value is stored as it is, without merging the previous labels
registry.override("aLabeledArray", LabeledArray<int>{&l[0][0], 2, 2});
la = registry.get<LabeledArray<int>>("aLabeledArray");
REQUIRE(la.getLabelsRows().empty());
REQUIRE(la.getLabelsCols().empty());
REQUIRE(la.get(1U, 0U) == 3);

// nested subtrees are replaced without touching their siblings
registry.override("aNested.vec", std::vector<double>{2.5});
REQUIRE(registry.get<std::vector<double>>("aNested.vec") == std::vector<double>{2.5});
REQUIRE(registry.get<int>("aNested.x") == 2);
REQUIRE(registry.get<std::string>("aString") == "foo");
REQUIRE(registry.get<std::vector<int>>("aVector") == std::vector<int>{7, 8});

// a key which does not exist yet is created
REQUIRE(registry.hasOption("aNewVector") == false);
registry.override("aNewVector", std::vector<float>{1.f, 2.f});
REQUIRE(registry.hasOption("aNewVector"));
REQUIRE(registry.get<std::vector<float>>("aNewVector") == std::vector<float>{1.f, 2.f});

// a leaf value can be replaced by a subtree
registry.override("aString", std::vector<std::string>{"now", "an", "array"});
REQUIRE(registry.get<std::vector<std::string>>("aString") == std::vector<std::string>{"now", "an", "array"});
}

TEST_CASE("TestConfigParamRegistryErase")
{
int l[2][2] = {{1, 2}, {3, 4}};
// a Variant cannot be created from a temporary vector, so the defaults are named
std::vector<int> defaultVector{1, 2, 3, 4, 5};
std::vector<std::string> defaultStringVector{"a", "b", "c"};
std::vector<double> defaultNestedVector{0.5, 1.5};
// the store keeps a reference to the specs, which have to outlive it
std::vector<ConfigParamSpec> specs{
ConfigParamSpec{"anInt", VariantType::Int, 1, {"an int option"}},
ConfigParamSpec{"aString", VariantType::String, "foo", {"a string option"}},
ConfigParamSpec{"aVector", VariantType::ArrayInt, defaultVector, {"an int array option"}},
ConfigParamSpec{"aStringVector", VariantType::ArrayString, defaultStringVector, {"a string array option"}},
ConfigParamSpec{"aLabeledArray", VariantType::LabeledArrayInt, LabeledArray<int>{&l[0][0], 2, 2, {"r1", "r2"}, {"c1", "c2"}}, {"a labeled array option"}},
ConfigParamSpec{"aNested.x", VariantType::Int, 2, {"an int option, nested in an object"}},
ConfigParamSpec{"aNested.vec", VariantType::ArrayDouble, defaultNestedVector, {"a double array option, nested in an object"}},
};
auto store = std::make_unique<ConfigParamStore>(specs, std::vector<std::unique_ptr<ParamRetriever>>{});
store->preload();
store->activate();
ConfigParamRegistry registry(std::move(store));

// a leaf value is removed
REQUIRE(registry.hasOption("anInt"));
registry.erase("anInt");
REQUIRE(registry.hasOption("anInt") == false);
REQUIRE(registry.isSet("anInt") == false);
REQUIRE(registry.isDefault("anInt") == false);
REQUIRE_THROWS_AS(registry.get<int>("anInt"), std::invalid_argument);

// arrays are removed as whole subtrees
registry.erase("aVector");
REQUIRE(registry.hasOption("aVector") == false);
REQUIRE(registry.isSet("aVector") == false);
registry.erase("aLabeledArray");
REQUIRE(registry.hasOption("aLabeledArray") == false);
REQUIRE(registry.isSet("aLabeledArray") == false);

// a nested key is removed without touching its siblings
registry.erase("aNested.vec");
REQUIRE(registry.hasOption("aNested.vec") == false);
REQUIRE(registry.get<int>("aNested.x") == 2);

// other options are untouched
REQUIRE(registry.get<std::string>("aString") == "foo");
REQUIRE(registry.get<std::vector<std::string>>("aStringVector") == std::vector<std::string>{"a", "b", "c"});

// missing keys are ignored
REQUIRE_NOTHROW(registry.erase("anInt"));
REQUIRE_NOTHROW(registry.erase("noSuchKey"));
REQUIRE_NOTHROW(registry.erase("noSuchParent.x"));
REQUIRE_NOTHROW(registry.erase("aNested.noSuchKey"));
REQUIRE(registry.get<int>("aNested.x") == 2);

// an erased key can be stored again
registry.override("anInt", 3);
REQUIRE(registry.get<int>("anInt") == 3);
registry.override("aVector", std::vector<int>{7, 8});
REQUIRE(registry.get<std::vector<int>>("aVector") == std::vector<int>{7, 8});
}
10 changes: 5 additions & 5 deletions Framework/Core/test/test_WorkflowSerialization.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,28 +27,28 @@ TEST_CASE("TestVerifyWorkflowSerialization")
ConfigParamSpec{"bUrl", VariantType::String, "foo/foo", {"Another InputSpec option"}}, //
}}}, //
{OutputSpec{{"bar"}, "C", "D", 2, Lifetime::Timeframe}}, //
AlgorithmSpec{[](ProcessingContext& /*ctx*/) {}}, //
AlgorithmSpec{[](ProcessingContext& /*ctx*/) {}}, //
{ //
ConfigParamSpec{"aInt", VariantType::Int, 0, {"An Int"}}, //
ConfigParamSpec{"aFloat", VariantType::Float, 1.3, {"A Float"}}, //
ConfigParamSpec{"aBool", VariantType::Bool, true, {"A Bool"}}, //
ConfigParamSpec{"aString", VariantType::String, "some string", {"A String"}}, //
ConfigParamSpec{"aLabeledArray", VariantType::LabeledArrayInt, labeled, {"A LabeledArray"}}}}, // //
ConfigParamSpec{"aLabeledArray", VariantType::LabeledArrayInt, labeled, {"A LabeledArray"}}}}, // //
DataProcessorSpec{"B", //
{InputSpec{"foo", "C", "D"}}, //
{ //
OutputSpec{{"bar1"}, "E", "F", 0}, //
OutputSpec{{"bar2"}, "E", "F", 1}}, //
AlgorithmSpec{[](ProcessingContext& /*ctx*/) {}}, //
AlgorithmSpec{[](ProcessingContext& /*ctx*/) {}}, //
{}}, //
DataProcessorSpec{"C", {}, //
{ //
OutputSpec{{"bar"}, "G", "H"}}, //
AlgorithmSpec{[](ProcessingContext& /*ctx*/) {}}, //
AlgorithmSpec{[](ProcessingContext& /*ctx*/) {}}, //
{}}, //
DataProcessorSpec{"D", {InputSpec{"foo", {"C", "D"}}}, //
{OutputSpec{{"bar"}, {"I", "L"}}}, //
AlgorithmSpec{[](ProcessingContext& /*ctx*/) {}}, //
AlgorithmSpec{[](ProcessingContext& /*ctx*/) {}}, //
{}, //
CommonServices::defaultServices(), //
{{"label a"}, {"label \"b\""}},
Expand Down
Loading