m_strat.api.project_management.parameters_manager

  1import copy
  2import os
  3from typing import Any
  4from uuid import uuid4
  5
  6from omegaconf import DictConfig, ListConfig, OmegaConf
  7
  8
  9class ParametersManager:
 10    """Manages parameters of YAML files to build contexts parsed as (context, __) and parameters of Execution nodes, parsed as(_, params  )"""
 11
 12    def __init__(
 13        self,
 14        root_dir: str,
 15        node_param_path: str = "params/nodes.yaml",
 16        ignores: list[str] = None,
 17    ):
 18        """Create a Paramter Manager instance
 19        Args:
 20            root_dir (str): Where the Yaml files for the project reside.
 21            node_param_path (str, optional): File for the node parameters of this project. Defaults to "params/nodes.yaml".
 22            ignores (list[str], optional): Files that are not be referenced for this project. Defaults to ["params/nodes.yaml"].
 23        """
 24
 25        self.root_dir_abs = root_dir
 26        self.ignores = ignores if ignores is not None else ["params/nodes.yaml"]
 27
 28        self.snapshots = {}
 29        self.node_params = {}
 30        self.node_param_path = node_param_path
 31        self.scan_dir()
 32        if os.path.exists(node_param_path):
 33            node_conf = OmegaConf.load(node_param_path)
 34            self.node_params = OmegaConf.to_object(node_conf)
 35        self.node_params["EMPTY"] = {}
 36        if node_param_path not in self.ignores:
 37            ignores.append(node_param_path)
 38
 39    @staticmethod
 40    def node_params_is_valid(
 41        schema: dict[str, type], params: dict[str, Any], allow_none=False
 42    ):
 43        """Determines whether parameter types in schema are matched in params
 44        Args:
 45            schema (dict[str,type]): The BlueprintNode Schema
 46            params (dict[str, Any]): The node params to be used with the blueprint node
 47            allow_none (bool, optional): Whether 'None' is allowed within the schema. Defaults to False.
 48        Returns:
 49            bool: True if the params are valid given the schema description.
 50        """
 51
 52        params_valid = len(schema) == len(params)
 53
 54        for s_key, s_type in schema.items():
 55            param_val = params.get(s_key, None)
 56            if param_val is None:
 57                params_valid &= allow_none
 58                continue
 59            params_valid &= isinstance(param_val, s_type)
 60
 61        return params_valid
 62
 63    def update_node_param_field(self, param_id, param_key, param_value):
 64        assert param_id != "EMPTY", "Cannot modify default EMPTY Params"
 65        n_params = self.node_params.get(param_id, None)
 66        if not n_params:
 67            return False
 68        n_params[param_key] = param_value
 69        return True
 70
 71    def add_node_params(self, param_id: str, params: dict):
 72        assert self.node_params.get(param_id, None) is None, (
 73            "parameter label already exists"
 74        )
 75        self.node_params[param_id] = copy.deepcopy(params)
 76
 77    def get_node_params(self, param_id: str, safe_cpy: bool = True):
 78        """Gets the node params via param_id
 79        Args:
 80            param_id (str): the node params name/ identifier
 81            safe_cpy (bool, optional): If True uses copy.deepcopy to produce final output, to prevent mutation. Defaults to True.
 82        Returns:
 83            dict[str,Any]: the node params requested
 84        NB: These are the params used in some_blueprint_execution(_ , THESE_PARAMS)
 85        """
 86        params = self.node_params.get(param_id, {})
 87        return copy.deepcopy(params) if safe_cpy else params
 88
 89    def remove_node_params(self, param_id: str):
 90        """
 91        Removes node_param
 92        Args:
 93            param_lbl (str): _description_
 94        Returns:
 95            _type_: _description_
 96        """
 97        assert param_id != "EMPTY", "EMPTY params required for system functionality"
 98        param_file = self.node_params.pop(param_id, None)
 99
100        return param_file
101
102    def save_node_params(self, rel_path=None):
103        """Saves the current node params
104        Args:
105            rel_path (str, optional): The relative path where the .yaml file is saved. Defaults to internal value.
106        """
107        if rel_path is None:
108            rel_path = self.node_param_path
109        self.primitives_to_yaml(self.node_params, rel_path, False)
110
111    def to_node_param(self, path=None, name=None):
112        assert path is not None or name is not None, "No param value specified"
113        if name and path is None:
114            path = self.yamls.get(name, None)
115            if not path:
116                return False
117
118        config = OmegaConf.load(path)
119        new_node_dict = dict(OmegaConf.to_container(config))
120        self.node_params = new_node_dict
121        return True
122
123    def get_all_node_param_ids(self):
124        """Gets all current node param ids
125        Returns:
126            lst[str]: a list of all param ids currently registerd with this manager
127        """
128        return [str(k) for k in self.node_params.keys()]
129
130    def scan_dir(self):
131        """Scans the the directory which the manager was intiated with and refreshes it's parameter registry"""
132        self.yamls = {}
133        for dir_path, _, filenames in os.walk(self.root_dir_abs):
134            for fn in filenames:
135                if (
136                    not fn.endswith(".yml") and not fn.endswith(".yaml")
137                ) or fn in self.ignores:
138                    continue
139                file_path = os.path.join(dir_path, fn)
140                name = self.get_name_from_path(file_path)
141                self.yamls[name] = file_path
142
143    def build_param_context(
144        self,
145        param_cascade_names: list[str],
146        overwrite: bool = True,
147        as_omega_conf: bool = True,
148        snapshot_name: str = None,
149    ):
150        """Generates a merged (and unresolved) parameter configuration from the list of parameter ids given
151        Args:
152            param_cascade_names (list[str]): list of parameter ids
153            overwrite (bool, optional): If true the values towards the end of the list will overwrite values at the beginning. Defaults to True.
154            as_omega_conf (bool, optional): If true returns values as OmegConf obj else returns python primitive (either dict or list dependent on file). Defaults to True.
155            snapshot_name (str, optional): If supplied, adds the parameters to snapshot registry. Defaults to None.
156
157        Returns:
158            OmegaConf|list|dict: The resultant Omega Configs for the merge.
159        """
160        file_list = [self.yamls.get(f) for f in param_cascade_names]
161        assert all(file_list), "Some yaml files/file references are missing, aborting"
162        config_lst = [OmegaConf.load(f) for f in file_list]
163        out_config = (
164            OmegaConf.merge(*config_lst)
165            if overwrite
166            else OmegaConf.merge(*reversed(config_lst))
167        )
168        if snapshot_name is not None:
169            self.snapshots[snapshot_name] = out_config.copy()
170
171        if as_omega_conf:
172            OmegaConf.resolve(out_config)
173            return out_config
174        else:
175            return OmegaConf.to_container(out_config, resolve=True)
176
177    def upsert_param(
178        self, param_name: str, val: Any, target_name: str = "params.global"
179    ):
180        """Adds a single parameter to the yaml file via target_name
181        Args:
182            param_name (str): name of the parameter attribute
183            val (Any): the new value to be upserted for that param
184            target_name (str, optional): target name of yaml file (identifier). Defaults to "params.global".
185        """
186        config_pth = self.yamls.get(target_name, None)
187        assert config_pth is not None, f"Config yml {target_name} does not exist"
188        config = OmegaConf.load(config_pth)
189        OmegaConf.update(config, param_name, val)
190        OmegaConf.save(config, config_pth)
191
192    def get_name_from_path(self, path):
193        """Generates the name of a yaml file from the path
194        Args:
195            path (str): the relative or absolute path of the yaml file
196        Returns:
197            str: the parameter file name/id
198        NB: will not function correctly if given a file from outside of the directory
199        """
200        return (
201            os.path.relpath(path, self.root_dir_abs)
202            .removesuffix(".yml")
203            .removesuffix(".yaml")
204            .replace(os.sep, ".")
205            .strip()
206        )
207
208    def snapshot_params(
209        self,
210        params: ListConfig | DictConfig | dict | list,
211        snapshot_name: str = None,
212    ):
213        """Saves the params supplied as parameter snapshots (within memory only)
214        Args:
215            params (ListConfig | DictConfig | dict | list): The parameters to be saved
216            snapshot_name (str, optional): Name of snapshot, uuid4 autogenerated if none supplied. Defaults to None.
217        """
218        cfg = (
219            OmegaConf.copy(params)
220            if type(params) is ListConfig or type(params) is DictConfig
221            else OmegaConf.create(params)
222        )
223        OmegaConf.resolve(cfg)
224        if snapshot_name is None:
225            snapshot_name = uuid4().hex
226        self.snapshots[snapshot_name] = cfg
227
228    def get_snapshot(self, snapshot_name: str):
229        return OmegaConf.copy(self.snapshots[snapshot_name])
230
231    def write_snapshot_to_yaml(
232        self, snapshot_name: str, path: str, overwrite_okay=False
233    ):
234        """
235        Given a snapshot name, writes - it to disk as a yaml file
236        Args:
237            snapshot_name (str): the name of the snapshot to be written
238            path (str): The path to be save the yaml file to
239            overwrite_okay (bool, optional): If true will overwrite any previously existing file. Defaults to False.
240        """
241        assert path.endswith("yml") or path.endswith("yaml"), (
242            f"invalid path supplied: {path}"
243        )
244        yaml_name = self.get_name_from_path(path)
245        assert overwrite_okay or (
246            not os.path.exists(path) and self.yamls.get(yaml_name, None) is None
247        ), "Yaml file already exists {path}, overwrite blocked"
248        ss = self.snapshots.get(snapshot_name, None)
249        assert ss is not None, f"Snapshot: {snapshot_name}, not Found"
250        OmegaConf.save(ss, path)
251        self.yamls[yaml_name] = path
252
253    def primitives_to_yaml(
254        self, params: dict | list, path: str, register_in_memory=False
255    ):
256        """Saves the data given (dict or list) and saves them as params
257        Args:
258            params (dict | list): the parameter data to be saved
259            path (str): the relative path of the data
260            register_in_memory (bool, optional):If true adds the param reference using the autoname feature to the manager. Defaults to False.
261        """
262        assert path.endswith(".yml") or path.endswith(".yaml"), (
263            f"Must be a YAML file type, got {path}"
264        )
265        OmegaConf.save(params, path)
266        if register_in_memory:
267            name = self.get_name_from_path(path)
268            self.yamls[name] = path
269
270    def remove_param(self, param_name: str, target_name="params.global"):
271        """Removes a paramter from the target name
272        Args:
273            param_name (str): the name of the value being removed
274            target_name (str, optional): parameter id of the set to be changed. Defaults to "params.global".
275        """
276        config_path = self.yamls.get(target_name, None)
277        assert config_path is not None, f"Config yml {target_name} does not exist"
278        config = OmegaConf.load(config_path)
279        parent = config
280        keys = param_name.split(".")
281
282        for key in keys[:-1]:
283            parent = parent[key]
284        del parent[keys[-1]]
285        OmegaConf.save(config, config_path)
class ParametersManager:
 10class ParametersManager:
 11    """Manages parameters of YAML files to build contexts parsed as (context, __) and parameters of Execution nodes, parsed as(_, params  )"""
 12
 13    def __init__(
 14        self,
 15        root_dir: str,
 16        node_param_path: str = "params/nodes.yaml",
 17        ignores: list[str] = None,
 18    ):
 19        """Create a Paramter Manager instance
 20        Args:
 21            root_dir (str): Where the Yaml files for the project reside.
 22            node_param_path (str, optional): File for the node parameters of this project. Defaults to "params/nodes.yaml".
 23            ignores (list[str], optional): Files that are not be referenced for this project. Defaults to ["params/nodes.yaml"].
 24        """
 25
 26        self.root_dir_abs = root_dir
 27        self.ignores = ignores if ignores is not None else ["params/nodes.yaml"]
 28
 29        self.snapshots = {}
 30        self.node_params = {}
 31        self.node_param_path = node_param_path
 32        self.scan_dir()
 33        if os.path.exists(node_param_path):
 34            node_conf = OmegaConf.load(node_param_path)
 35            self.node_params = OmegaConf.to_object(node_conf)
 36        self.node_params["EMPTY"] = {}
 37        if node_param_path not in self.ignores:
 38            ignores.append(node_param_path)
 39
 40    @staticmethod
 41    def node_params_is_valid(
 42        schema: dict[str, type], params: dict[str, Any], allow_none=False
 43    ):
 44        """Determines whether parameter types in schema are matched in params
 45        Args:
 46            schema (dict[str,type]): The BlueprintNode Schema
 47            params (dict[str, Any]): The node params to be used with the blueprint node
 48            allow_none (bool, optional): Whether 'None' is allowed within the schema. Defaults to False.
 49        Returns:
 50            bool: True if the params are valid given the schema description.
 51        """
 52
 53        params_valid = len(schema) == len(params)
 54
 55        for s_key, s_type in schema.items():
 56            param_val = params.get(s_key, None)
 57            if param_val is None:
 58                params_valid &= allow_none
 59                continue
 60            params_valid &= isinstance(param_val, s_type)
 61
 62        return params_valid
 63
 64    def update_node_param_field(self, param_id, param_key, param_value):
 65        assert param_id != "EMPTY", "Cannot modify default EMPTY Params"
 66        n_params = self.node_params.get(param_id, None)
 67        if not n_params:
 68            return False
 69        n_params[param_key] = param_value
 70        return True
 71
 72    def add_node_params(self, param_id: str, params: dict):
 73        assert self.node_params.get(param_id, None) is None, (
 74            "parameter label already exists"
 75        )
 76        self.node_params[param_id] = copy.deepcopy(params)
 77
 78    def get_node_params(self, param_id: str, safe_cpy: bool = True):
 79        """Gets the node params via param_id
 80        Args:
 81            param_id (str): the node params name/ identifier
 82            safe_cpy (bool, optional): If True uses copy.deepcopy to produce final output, to prevent mutation. Defaults to True.
 83        Returns:
 84            dict[str,Any]: the node params requested
 85        NB: These are the params used in some_blueprint_execution(_ , THESE_PARAMS)
 86        """
 87        params = self.node_params.get(param_id, {})
 88        return copy.deepcopy(params) if safe_cpy else params
 89
 90    def remove_node_params(self, param_id: str):
 91        """
 92        Removes node_param
 93        Args:
 94            param_lbl (str): _description_
 95        Returns:
 96            _type_: _description_
 97        """
 98        assert param_id != "EMPTY", "EMPTY params required for system functionality"
 99        param_file = self.node_params.pop(param_id, None)
100
101        return param_file
102
103    def save_node_params(self, rel_path=None):
104        """Saves the current node params
105        Args:
106            rel_path (str, optional): The relative path where the .yaml file is saved. Defaults to internal value.
107        """
108        if rel_path is None:
109            rel_path = self.node_param_path
110        self.primitives_to_yaml(self.node_params, rel_path, False)
111
112    def to_node_param(self, path=None, name=None):
113        assert path is not None or name is not None, "No param value specified"
114        if name and path is None:
115            path = self.yamls.get(name, None)
116            if not path:
117                return False
118
119        config = OmegaConf.load(path)
120        new_node_dict = dict(OmegaConf.to_container(config))
121        self.node_params = new_node_dict
122        return True
123
124    def get_all_node_param_ids(self):
125        """Gets all current node param ids
126        Returns:
127            lst[str]: a list of all param ids currently registerd with this manager
128        """
129        return [str(k) for k in self.node_params.keys()]
130
131    def scan_dir(self):
132        """Scans the the directory which the manager was intiated with and refreshes it's parameter registry"""
133        self.yamls = {}
134        for dir_path, _, filenames in os.walk(self.root_dir_abs):
135            for fn in filenames:
136                if (
137                    not fn.endswith(".yml") and not fn.endswith(".yaml")
138                ) or fn in self.ignores:
139                    continue
140                file_path = os.path.join(dir_path, fn)
141                name = self.get_name_from_path(file_path)
142                self.yamls[name] = file_path
143
144    def build_param_context(
145        self,
146        param_cascade_names: list[str],
147        overwrite: bool = True,
148        as_omega_conf: bool = True,
149        snapshot_name: str = None,
150    ):
151        """Generates a merged (and unresolved) parameter configuration from the list of parameter ids given
152        Args:
153            param_cascade_names (list[str]): list of parameter ids
154            overwrite (bool, optional): If true the values towards the end of the list will overwrite values at the beginning. Defaults to True.
155            as_omega_conf (bool, optional): If true returns values as OmegConf obj else returns python primitive (either dict or list dependent on file). Defaults to True.
156            snapshot_name (str, optional): If supplied, adds the parameters to snapshot registry. Defaults to None.
157
158        Returns:
159            OmegaConf|list|dict: The resultant Omega Configs for the merge.
160        """
161        file_list = [self.yamls.get(f) for f in param_cascade_names]
162        assert all(file_list), "Some yaml files/file references are missing, aborting"
163        config_lst = [OmegaConf.load(f) for f in file_list]
164        out_config = (
165            OmegaConf.merge(*config_lst)
166            if overwrite
167            else OmegaConf.merge(*reversed(config_lst))
168        )
169        if snapshot_name is not None:
170            self.snapshots[snapshot_name] = out_config.copy()
171
172        if as_omega_conf:
173            OmegaConf.resolve(out_config)
174            return out_config
175        else:
176            return OmegaConf.to_container(out_config, resolve=True)
177
178    def upsert_param(
179        self, param_name: str, val: Any, target_name: str = "params.global"
180    ):
181        """Adds a single parameter to the yaml file via target_name
182        Args:
183            param_name (str): name of the parameter attribute
184            val (Any): the new value to be upserted for that param
185            target_name (str, optional): target name of yaml file (identifier). Defaults to "params.global".
186        """
187        config_pth = self.yamls.get(target_name, None)
188        assert config_pth is not None, f"Config yml {target_name} does not exist"
189        config = OmegaConf.load(config_pth)
190        OmegaConf.update(config, param_name, val)
191        OmegaConf.save(config, config_pth)
192
193    def get_name_from_path(self, path):
194        """Generates the name of a yaml file from the path
195        Args:
196            path (str): the relative or absolute path of the yaml file
197        Returns:
198            str: the parameter file name/id
199        NB: will not function correctly if given a file from outside of the directory
200        """
201        return (
202            os.path.relpath(path, self.root_dir_abs)
203            .removesuffix(".yml")
204            .removesuffix(".yaml")
205            .replace(os.sep, ".")
206            .strip()
207        )
208
209    def snapshot_params(
210        self,
211        params: ListConfig | DictConfig | dict | list,
212        snapshot_name: str = None,
213    ):
214        """Saves the params supplied as parameter snapshots (within memory only)
215        Args:
216            params (ListConfig | DictConfig | dict | list): The parameters to be saved
217            snapshot_name (str, optional): Name of snapshot, uuid4 autogenerated if none supplied. Defaults to None.
218        """
219        cfg = (
220            OmegaConf.copy(params)
221            if type(params) is ListConfig or type(params) is DictConfig
222            else OmegaConf.create(params)
223        )
224        OmegaConf.resolve(cfg)
225        if snapshot_name is None:
226            snapshot_name = uuid4().hex
227        self.snapshots[snapshot_name] = cfg
228
229    def get_snapshot(self, snapshot_name: str):
230        return OmegaConf.copy(self.snapshots[snapshot_name])
231
232    def write_snapshot_to_yaml(
233        self, snapshot_name: str, path: str, overwrite_okay=False
234    ):
235        """
236        Given a snapshot name, writes - it to disk as a yaml file
237        Args:
238            snapshot_name (str): the name of the snapshot to be written
239            path (str): The path to be save the yaml file to
240            overwrite_okay (bool, optional): If true will overwrite any previously existing file. Defaults to False.
241        """
242        assert path.endswith("yml") or path.endswith("yaml"), (
243            f"invalid path supplied: {path}"
244        )
245        yaml_name = self.get_name_from_path(path)
246        assert overwrite_okay or (
247            not os.path.exists(path) and self.yamls.get(yaml_name, None) is None
248        ), "Yaml file already exists {path}, overwrite blocked"
249        ss = self.snapshots.get(snapshot_name, None)
250        assert ss is not None, f"Snapshot: {snapshot_name}, not Found"
251        OmegaConf.save(ss, path)
252        self.yamls[yaml_name] = path
253
254    def primitives_to_yaml(
255        self, params: dict | list, path: str, register_in_memory=False
256    ):
257        """Saves the data given (dict or list) and saves them as params
258        Args:
259            params (dict | list): the parameter data to be saved
260            path (str): the relative path of the data
261            register_in_memory (bool, optional):If true adds the param reference using the autoname feature to the manager. Defaults to False.
262        """
263        assert path.endswith(".yml") or path.endswith(".yaml"), (
264            f"Must be a YAML file type, got {path}"
265        )
266        OmegaConf.save(params, path)
267        if register_in_memory:
268            name = self.get_name_from_path(path)
269            self.yamls[name] = path
270
271    def remove_param(self, param_name: str, target_name="params.global"):
272        """Removes a paramter from the target name
273        Args:
274            param_name (str): the name of the value being removed
275            target_name (str, optional): parameter id of the set to be changed. Defaults to "params.global".
276        """
277        config_path = self.yamls.get(target_name, None)
278        assert config_path is not None, f"Config yml {target_name} does not exist"
279        config = OmegaConf.load(config_path)
280        parent = config
281        keys = param_name.split(".")
282
283        for key in keys[:-1]:
284            parent = parent[key]
285        del parent[keys[-1]]
286        OmegaConf.save(config, config_path)

Manages parameters of YAML files to build contexts parsed as (context, __) and parameters of Execution nodes, parsed as(_, params )

ParametersManager( root_dir: str, node_param_path: str = 'params/nodes.yaml', ignores: list[str] = None)
13    def __init__(
14        self,
15        root_dir: str,
16        node_param_path: str = "params/nodes.yaml",
17        ignores: list[str] = None,
18    ):
19        """Create a Paramter Manager instance
20        Args:
21            root_dir (str): Where the Yaml files for the project reside.
22            node_param_path (str, optional): File for the node parameters of this project. Defaults to "params/nodes.yaml".
23            ignores (list[str], optional): Files that are not be referenced for this project. Defaults to ["params/nodes.yaml"].
24        """
25
26        self.root_dir_abs = root_dir
27        self.ignores = ignores if ignores is not None else ["params/nodes.yaml"]
28
29        self.snapshots = {}
30        self.node_params = {}
31        self.node_param_path = node_param_path
32        self.scan_dir()
33        if os.path.exists(node_param_path):
34            node_conf = OmegaConf.load(node_param_path)
35            self.node_params = OmegaConf.to_object(node_conf)
36        self.node_params["EMPTY"] = {}
37        if node_param_path not in self.ignores:
38            ignores.append(node_param_path)

Create a Paramter Manager instance Args: root_dir (str): Where the Yaml files for the project reside. node_param_path (str, optional): File for the node parameters of this project. Defaults to "params/nodes.yaml". ignores (list[str], optional): Files that are not be referenced for this project. Defaults to ["params/nodes.yaml"].

root_dir_abs
ignores
snapshots
node_params
node_param_path
@staticmethod
def node_params_is_valid( schema: dict[str, type], params: dict[str, typing.Any], allow_none=False):
40    @staticmethod
41    def node_params_is_valid(
42        schema: dict[str, type], params: dict[str, Any], allow_none=False
43    ):
44        """Determines whether parameter types in schema are matched in params
45        Args:
46            schema (dict[str,type]): The BlueprintNode Schema
47            params (dict[str, Any]): The node params to be used with the blueprint node
48            allow_none (bool, optional): Whether 'None' is allowed within the schema. Defaults to False.
49        Returns:
50            bool: True if the params are valid given the schema description.
51        """
52
53        params_valid = len(schema) == len(params)
54
55        for s_key, s_type in schema.items():
56            param_val = params.get(s_key, None)
57            if param_val is None:
58                params_valid &= allow_none
59                continue
60            params_valid &= isinstance(param_val, s_type)
61
62        return params_valid

Determines whether parameter types in schema are matched in params Args: schema (dict[str,type]): The BlueprintNode Schema params (dict[str, Any]): The node params to be used with the blueprint node allow_none (bool, optional): Whether 'None' is allowed within the schema. Defaults to False. Returns: bool: True if the params are valid given the schema description.

def update_node_param_field(self, param_id, param_key, param_value):
64    def update_node_param_field(self, param_id, param_key, param_value):
65        assert param_id != "EMPTY", "Cannot modify default EMPTY Params"
66        n_params = self.node_params.get(param_id, None)
67        if not n_params:
68            return False
69        n_params[param_key] = param_value
70        return True
def add_node_params(self, param_id: str, params: dict):
72    def add_node_params(self, param_id: str, params: dict):
73        assert self.node_params.get(param_id, None) is None, (
74            "parameter label already exists"
75        )
76        self.node_params[param_id] = copy.deepcopy(params)
def get_node_params(self, param_id: str, safe_cpy: bool = True):
78    def get_node_params(self, param_id: str, safe_cpy: bool = True):
79        """Gets the node params via param_id
80        Args:
81            param_id (str): the node params name/ identifier
82            safe_cpy (bool, optional): If True uses copy.deepcopy to produce final output, to prevent mutation. Defaults to True.
83        Returns:
84            dict[str,Any]: the node params requested
85        NB: These are the params used in some_blueprint_execution(_ , THESE_PARAMS)
86        """
87        params = self.node_params.get(param_id, {})
88        return copy.deepcopy(params) if safe_cpy else params

Gets the node params via param_id Args: param_id (str): the node params name/ identifier safe_cpy (bool, optional): If True uses copy.deepcopy to produce final output, to prevent mutation. Defaults to True. Returns: dict[str,Any]: the node params requested NB: These are the params used in some_blueprint_execution(_ , THESE_PARAMS)

def remove_node_params(self, param_id: str):
 90    def remove_node_params(self, param_id: str):
 91        """
 92        Removes node_param
 93        Args:
 94            param_lbl (str): _description_
 95        Returns:
 96            _type_: _description_
 97        """
 98        assert param_id != "EMPTY", "EMPTY params required for system functionality"
 99        param_file = self.node_params.pop(param_id, None)
100
101        return param_file

Removes node_param Args: param_lbl (str): _description_ Returns: _type_: _description_

def save_node_params(self, rel_path=None):
103    def save_node_params(self, rel_path=None):
104        """Saves the current node params
105        Args:
106            rel_path (str, optional): The relative path where the .yaml file is saved. Defaults to internal value.
107        """
108        if rel_path is None:
109            rel_path = self.node_param_path
110        self.primitives_to_yaml(self.node_params, rel_path, False)

Saves the current node params Args: rel_path (str, optional): The relative path where the .yaml file is saved. Defaults to internal value.

def to_node_param(self, path=None, name=None):
112    def to_node_param(self, path=None, name=None):
113        assert path is not None or name is not None, "No param value specified"
114        if name and path is None:
115            path = self.yamls.get(name, None)
116            if not path:
117                return False
118
119        config = OmegaConf.load(path)
120        new_node_dict = dict(OmegaConf.to_container(config))
121        self.node_params = new_node_dict
122        return True
def get_all_node_param_ids(self):
124    def get_all_node_param_ids(self):
125        """Gets all current node param ids
126        Returns:
127            lst[str]: a list of all param ids currently registerd with this manager
128        """
129        return [str(k) for k in self.node_params.keys()]

Gets all current node param ids Returns: lst[str]: a list of all param ids currently registerd with this manager

def scan_dir(self):
131    def scan_dir(self):
132        """Scans the the directory which the manager was intiated with and refreshes it's parameter registry"""
133        self.yamls = {}
134        for dir_path, _, filenames in os.walk(self.root_dir_abs):
135            for fn in filenames:
136                if (
137                    not fn.endswith(".yml") and not fn.endswith(".yaml")
138                ) or fn in self.ignores:
139                    continue
140                file_path = os.path.join(dir_path, fn)
141                name = self.get_name_from_path(file_path)
142                self.yamls[name] = file_path

Scans the the directory which the manager was intiated with and refreshes it's parameter registry

def build_param_context( self, param_cascade_names: list[str], overwrite: bool = True, as_omega_conf: bool = True, snapshot_name: str = None):
144    def build_param_context(
145        self,
146        param_cascade_names: list[str],
147        overwrite: bool = True,
148        as_omega_conf: bool = True,
149        snapshot_name: str = None,
150    ):
151        """Generates a merged (and unresolved) parameter configuration from the list of parameter ids given
152        Args:
153            param_cascade_names (list[str]): list of parameter ids
154            overwrite (bool, optional): If true the values towards the end of the list will overwrite values at the beginning. Defaults to True.
155            as_omega_conf (bool, optional): If true returns values as OmegConf obj else returns python primitive (either dict or list dependent on file). Defaults to True.
156            snapshot_name (str, optional): If supplied, adds the parameters to snapshot registry. Defaults to None.
157
158        Returns:
159            OmegaConf|list|dict: The resultant Omega Configs for the merge.
160        """
161        file_list = [self.yamls.get(f) for f in param_cascade_names]
162        assert all(file_list), "Some yaml files/file references are missing, aborting"
163        config_lst = [OmegaConf.load(f) for f in file_list]
164        out_config = (
165            OmegaConf.merge(*config_lst)
166            if overwrite
167            else OmegaConf.merge(*reversed(config_lst))
168        )
169        if snapshot_name is not None:
170            self.snapshots[snapshot_name] = out_config.copy()
171
172        if as_omega_conf:
173            OmegaConf.resolve(out_config)
174            return out_config
175        else:
176            return OmegaConf.to_container(out_config, resolve=True)

Generates a merged (and unresolved) parameter configuration from the list of parameter ids given Args: param_cascade_names (list[str]): list of parameter ids overwrite (bool, optional): If true the values towards the end of the list will overwrite values at the beginning. Defaults to True. as_omega_conf (bool, optional): If true returns values as OmegConf obj else returns python primitive (either dict or list dependent on file). Defaults to True. snapshot_name (str, optional): If supplied, adds the parameters to snapshot registry. Defaults to None.

Returns: OmegaConf|list|dict: The resultant Omega Configs for the merge.

def upsert_param(self, param_name: str, val: Any, target_name: str = 'params.global'):
178    def upsert_param(
179        self, param_name: str, val: Any, target_name: str = "params.global"
180    ):
181        """Adds a single parameter to the yaml file via target_name
182        Args:
183            param_name (str): name of the parameter attribute
184            val (Any): the new value to be upserted for that param
185            target_name (str, optional): target name of yaml file (identifier). Defaults to "params.global".
186        """
187        config_pth = self.yamls.get(target_name, None)
188        assert config_pth is not None, f"Config yml {target_name} does not exist"
189        config = OmegaConf.load(config_pth)
190        OmegaConf.update(config, param_name, val)
191        OmegaConf.save(config, config_pth)

Adds a single parameter to the yaml file via target_name Args: param_name (str): name of the parameter attribute val (Any): the new value to be upserted for that param target_name (str, optional): target name of yaml file (identifier). Defaults to "params.global".

def get_name_from_path(self, path):
193    def get_name_from_path(self, path):
194        """Generates the name of a yaml file from the path
195        Args:
196            path (str): the relative or absolute path of the yaml file
197        Returns:
198            str: the parameter file name/id
199        NB: will not function correctly if given a file from outside of the directory
200        """
201        return (
202            os.path.relpath(path, self.root_dir_abs)
203            .removesuffix(".yml")
204            .removesuffix(".yaml")
205            .replace(os.sep, ".")
206            .strip()
207        )

Generates the name of a yaml file from the path Args: path (str): the relative or absolute path of the yaml file Returns: str: the parameter file name/id NB: will not function correctly if given a file from outside of the directory

def snapshot_params( self, params: omegaconf.listconfig.ListConfig | omegaconf.dictconfig.DictConfig | dict | list, snapshot_name: str = None):
209    def snapshot_params(
210        self,
211        params: ListConfig | DictConfig | dict | list,
212        snapshot_name: str = None,
213    ):
214        """Saves the params supplied as parameter snapshots (within memory only)
215        Args:
216            params (ListConfig | DictConfig | dict | list): The parameters to be saved
217            snapshot_name (str, optional): Name of snapshot, uuid4 autogenerated if none supplied. Defaults to None.
218        """
219        cfg = (
220            OmegaConf.copy(params)
221            if type(params) is ListConfig or type(params) is DictConfig
222            else OmegaConf.create(params)
223        )
224        OmegaConf.resolve(cfg)
225        if snapshot_name is None:
226            snapshot_name = uuid4().hex
227        self.snapshots[snapshot_name] = cfg

Saves the params supplied as parameter snapshots (within memory only) Args: params (ListConfig | DictConfig | dict | list): The parameters to be saved snapshot_name (str, optional): Name of snapshot, uuid4 autogenerated if none supplied. Defaults to None.

def get_snapshot(self, snapshot_name: str):
229    def get_snapshot(self, snapshot_name: str):
230        return OmegaConf.copy(self.snapshots[snapshot_name])
def write_snapshot_to_yaml(self, snapshot_name: str, path: str, overwrite_okay=False):
232    def write_snapshot_to_yaml(
233        self, snapshot_name: str, path: str, overwrite_okay=False
234    ):
235        """
236        Given a snapshot name, writes - it to disk as a yaml file
237        Args:
238            snapshot_name (str): the name of the snapshot to be written
239            path (str): The path to be save the yaml file to
240            overwrite_okay (bool, optional): If true will overwrite any previously existing file. Defaults to False.
241        """
242        assert path.endswith("yml") or path.endswith("yaml"), (
243            f"invalid path supplied: {path}"
244        )
245        yaml_name = self.get_name_from_path(path)
246        assert overwrite_okay or (
247            not os.path.exists(path) and self.yamls.get(yaml_name, None) is None
248        ), "Yaml file already exists {path}, overwrite blocked"
249        ss = self.snapshots.get(snapshot_name, None)
250        assert ss is not None, f"Snapshot: {snapshot_name}, not Found"
251        OmegaConf.save(ss, path)
252        self.yamls[yaml_name] = path

Given a snapshot name, writes - it to disk as a yaml file Args: snapshot_name (str): the name of the snapshot to be written path (str): The path to be save the yaml file to overwrite_okay (bool, optional): If true will overwrite any previously existing file. Defaults to False.

def primitives_to_yaml(self, params: dict | list, path: str, register_in_memory=False):
254    def primitives_to_yaml(
255        self, params: dict | list, path: str, register_in_memory=False
256    ):
257        """Saves the data given (dict or list) and saves them as params
258        Args:
259            params (dict | list): the parameter data to be saved
260            path (str): the relative path of the data
261            register_in_memory (bool, optional):If true adds the param reference using the autoname feature to the manager. Defaults to False.
262        """
263        assert path.endswith(".yml") or path.endswith(".yaml"), (
264            f"Must be a YAML file type, got {path}"
265        )
266        OmegaConf.save(params, path)
267        if register_in_memory:
268            name = self.get_name_from_path(path)
269            self.yamls[name] = path

Saves the data given (dict or list) and saves them as params Args: params (dict | list): the parameter data to be saved path (str): the relative path of the data register_in_memory (bool, optional):If true adds the param reference using the autoname feature to the manager. Defaults to False.

def remove_param(self, param_name: str, target_name='params.global'):
271    def remove_param(self, param_name: str, target_name="params.global"):
272        """Removes a paramter from the target name
273        Args:
274            param_name (str): the name of the value being removed
275            target_name (str, optional): parameter id of the set to be changed. Defaults to "params.global".
276        """
277        config_path = self.yamls.get(target_name, None)
278        assert config_path is not None, f"Config yml {target_name} does not exist"
279        config = OmegaConf.load(config_path)
280        parent = config
281        keys = param_name.split(".")
282
283        for key in keys[:-1]:
284            parent = parent[key]
285        del parent[keys[-1]]
286        OmegaConf.save(config, config_path)

Removes a paramter from the target name Args: param_name (str): the name of the value being removed target_name (str, optional): parameter id of the set to be changed. Defaults to "params.global".