m_strat.api.project_management.parameters_manager
1import copy 2import os 3from typing import Any 4from uuid import uuid4 5 6from omegaconf import DictConfig, ListConfig, OmegaConf 7 8 9class ParametersManager: 10 """Manages parameters of YAML files to build contexts parsed as (context, __) and parameters of Execution nodes, parsed as(_, params )""" 11 12 def __init__( 13 self, 14 root_dir: str, 15 node_param_path: str = "params/nodes.yaml", 16 ignores: list[str] = None, 17 ): 18 """Create a Paramter Manager instance 19 Args: 20 root_dir (str): Where the Yaml files for the project reside. 21 node_param_path (str, optional): File for the node parameters of this project. Defaults to "params/nodes.yaml". 22 ignores (list[str], optional): Files that are not be referenced for this project. Defaults to ["params/nodes.yaml"]. 23 """ 24 25 self.root_dir_abs = root_dir 26 self.ignores = ignores if ignores is not None else ["params/nodes.yaml"] 27 28 self.snapshots = {} 29 self.node_params = {} 30 self.node_param_path = node_param_path 31 self.scan_dir() 32 if os.path.exists(node_param_path): 33 node_conf = OmegaConf.load(node_param_path) 34 self.node_params = OmegaConf.to_object(node_conf) 35 self.node_params["EMPTY"] = {} 36 if node_param_path not in self.ignores: 37 ignores.append(node_param_path) 38 39 @staticmethod 40 def node_params_is_valid( 41 schema: dict[str, type], params: dict[str, Any], allow_none=False 42 ): 43 """Determines whether parameter types in schema are matched in params 44 Args: 45 schema (dict[str,type]): The BlueprintNode Schema 46 params (dict[str, Any]): The node params to be used with the blueprint node 47 allow_none (bool, optional): Whether 'None' is allowed within the schema. Defaults to False. 48 Returns: 49 bool: True if the params are valid given the schema description. 50 """ 51 52 params_valid = len(schema) == len(params) 53 54 for s_key, s_type in schema.items(): 55 param_val = params.get(s_key, None) 56 if param_val is None: 57 params_valid &= allow_none 58 continue 59 params_valid &= isinstance(param_val, s_type) 60 61 return params_valid 62 63 def update_node_param_field(self, param_id, param_key, param_value): 64 assert param_id != "EMPTY", "Cannot modify default EMPTY Params" 65 n_params = self.node_params.get(param_id, None) 66 if not n_params: 67 return False 68 n_params[param_key] = param_value 69 return True 70 71 def add_node_params(self, param_id: str, params: dict): 72 assert self.node_params.get(param_id, None) is None, ( 73 "parameter label already exists" 74 ) 75 self.node_params[param_id] = copy.deepcopy(params) 76 77 def get_node_params(self, param_id: str, safe_cpy: bool = True): 78 """Gets the node params via param_id 79 Args: 80 param_id (str): the node params name/ identifier 81 safe_cpy (bool, optional): If True uses copy.deepcopy to produce final output, to prevent mutation. Defaults to True. 82 Returns: 83 dict[str,Any]: the node params requested 84 NB: These are the params used in some_blueprint_execution(_ , THESE_PARAMS) 85 """ 86 params = self.node_params.get(param_id, {}) 87 return copy.deepcopy(params) if safe_cpy else params 88 89 def remove_node_params(self, param_id: str): 90 """ 91 Removes node_param 92 Args: 93 param_lbl (str): _description_ 94 Returns: 95 _type_: _description_ 96 """ 97 assert param_id != "EMPTY", "EMPTY params required for system functionality" 98 param_file = self.node_params.pop(param_id, None) 99 100 return param_file 101 102 def save_node_params(self, rel_path=None): 103 """Saves the current node params 104 Args: 105 rel_path (str, optional): The relative path where the .yaml file is saved. Defaults to internal value. 106 """ 107 if rel_path is None: 108 rel_path = self.node_param_path 109 self.primitives_to_yaml(self.node_params, rel_path, False) 110 111 def to_node_param(self, path=None, name=None): 112 assert path is not None or name is not None, "No param value specified" 113 if name and path is None: 114 path = self.yamls.get(name, None) 115 if not path: 116 return False 117 118 config = OmegaConf.load(path) 119 new_node_dict = dict(OmegaConf.to_container(config)) 120 self.node_params = new_node_dict 121 return True 122 123 def get_all_node_param_ids(self): 124 """Gets all current node param ids 125 Returns: 126 lst[str]: a list of all param ids currently registerd with this manager 127 """ 128 return [str(k) for k in self.node_params.keys()] 129 130 def scan_dir(self): 131 """Scans the the directory which the manager was intiated with and refreshes it's parameter registry""" 132 self.yamls = {} 133 for dir_path, _, filenames in os.walk(self.root_dir_abs): 134 for fn in filenames: 135 if ( 136 not fn.endswith(".yml") and not fn.endswith(".yaml") 137 ) or fn in self.ignores: 138 continue 139 file_path = os.path.join(dir_path, fn) 140 name = self.get_name_from_path(file_path) 141 self.yamls[name] = file_path 142 143 def build_param_context( 144 self, 145 param_cascade_names: list[str], 146 overwrite: bool = True, 147 as_omega_conf: bool = True, 148 snapshot_name: str = None, 149 ): 150 """Generates a merged (and unresolved) parameter configuration from the list of parameter ids given 151 Args: 152 param_cascade_names (list[str]): list of parameter ids 153 overwrite (bool, optional): If true the values towards the end of the list will overwrite values at the beginning. Defaults to True. 154 as_omega_conf (bool, optional): If true returns values as OmegConf obj else returns python primitive (either dict or list dependent on file). Defaults to True. 155 snapshot_name (str, optional): If supplied, adds the parameters to snapshot registry. Defaults to None. 156 157 Returns: 158 OmegaConf|list|dict: The resultant Omega Configs for the merge. 159 """ 160 file_list = [self.yamls.get(f) for f in param_cascade_names] 161 assert all(file_list), "Some yaml files/file references are missing, aborting" 162 config_lst = [OmegaConf.load(f) for f in file_list] 163 out_config = ( 164 OmegaConf.merge(*config_lst) 165 if overwrite 166 else OmegaConf.merge(*reversed(config_lst)) 167 ) 168 if snapshot_name is not None: 169 self.snapshots[snapshot_name] = out_config.copy() 170 171 if as_omega_conf: 172 OmegaConf.resolve(out_config) 173 return out_config 174 else: 175 return OmegaConf.to_container(out_config, resolve=True) 176 177 def upsert_param( 178 self, param_name: str, val: Any, target_name: str = "params.global" 179 ): 180 """Adds a single parameter to the yaml file via target_name 181 Args: 182 param_name (str): name of the parameter attribute 183 val (Any): the new value to be upserted for that param 184 target_name (str, optional): target name of yaml file (identifier). Defaults to "params.global". 185 """ 186 config_pth = self.yamls.get(target_name, None) 187 assert config_pth is not None, f"Config yml {target_name} does not exist" 188 config = OmegaConf.load(config_pth) 189 OmegaConf.update(config, param_name, val) 190 OmegaConf.save(config, config_pth) 191 192 def get_name_from_path(self, path): 193 """Generates the name of a yaml file from the path 194 Args: 195 path (str): the relative or absolute path of the yaml file 196 Returns: 197 str: the parameter file name/id 198 NB: will not function correctly if given a file from outside of the directory 199 """ 200 return ( 201 os.path.relpath(path, self.root_dir_abs) 202 .removesuffix(".yml") 203 .removesuffix(".yaml") 204 .replace(os.sep, ".") 205 .strip() 206 ) 207 208 def snapshot_params( 209 self, 210 params: ListConfig | DictConfig | dict | list, 211 snapshot_name: str = None, 212 ): 213 """Saves the params supplied as parameter snapshots (within memory only) 214 Args: 215 params (ListConfig | DictConfig | dict | list): The parameters to be saved 216 snapshot_name (str, optional): Name of snapshot, uuid4 autogenerated if none supplied. Defaults to None. 217 """ 218 cfg = ( 219 OmegaConf.copy(params) 220 if type(params) is ListConfig or type(params) is DictConfig 221 else OmegaConf.create(params) 222 ) 223 OmegaConf.resolve(cfg) 224 if snapshot_name is None: 225 snapshot_name = uuid4().hex 226 self.snapshots[snapshot_name] = cfg 227 228 def get_snapshot(self, snapshot_name: str): 229 return OmegaConf.copy(self.snapshots[snapshot_name]) 230 231 def write_snapshot_to_yaml( 232 self, snapshot_name: str, path: str, overwrite_okay=False 233 ): 234 """ 235 Given a snapshot name, writes - it to disk as a yaml file 236 Args: 237 snapshot_name (str): the name of the snapshot to be written 238 path (str): The path to be save the yaml file to 239 overwrite_okay (bool, optional): If true will overwrite any previously existing file. Defaults to False. 240 """ 241 assert path.endswith("yml") or path.endswith("yaml"), ( 242 f"invalid path supplied: {path}" 243 ) 244 yaml_name = self.get_name_from_path(path) 245 assert overwrite_okay or ( 246 not os.path.exists(path) and self.yamls.get(yaml_name, None) is None 247 ), "Yaml file already exists {path}, overwrite blocked" 248 ss = self.snapshots.get(snapshot_name, None) 249 assert ss is not None, f"Snapshot: {snapshot_name}, not Found" 250 OmegaConf.save(ss, path) 251 self.yamls[yaml_name] = path 252 253 def primitives_to_yaml( 254 self, params: dict | list, path: str, register_in_memory=False 255 ): 256 """Saves the data given (dict or list) and saves them as params 257 Args: 258 params (dict | list): the parameter data to be saved 259 path (str): the relative path of the data 260 register_in_memory (bool, optional):If true adds the param reference using the autoname feature to the manager. Defaults to False. 261 """ 262 assert path.endswith(".yml") or path.endswith(".yaml"), ( 263 f"Must be a YAML file type, got {path}" 264 ) 265 OmegaConf.save(params, path) 266 if register_in_memory: 267 name = self.get_name_from_path(path) 268 self.yamls[name] = path 269 270 def remove_param(self, param_name: str, target_name="params.global"): 271 """Removes a paramter from the target name 272 Args: 273 param_name (str): the name of the value being removed 274 target_name (str, optional): parameter id of the set to be changed. Defaults to "params.global". 275 """ 276 config_path = self.yamls.get(target_name, None) 277 assert config_path is not None, f"Config yml {target_name} does not exist" 278 config = OmegaConf.load(config_path) 279 parent = config 280 keys = param_name.split(".") 281 282 for key in keys[:-1]: 283 parent = parent[key] 284 del parent[keys[-1]] 285 OmegaConf.save(config, config_path)
10class ParametersManager: 11 """Manages parameters of YAML files to build contexts parsed as (context, __) and parameters of Execution nodes, parsed as(_, params )""" 12 13 def __init__( 14 self, 15 root_dir: str, 16 node_param_path: str = "params/nodes.yaml", 17 ignores: list[str] = None, 18 ): 19 """Create a Paramter Manager instance 20 Args: 21 root_dir (str): Where the Yaml files for the project reside. 22 node_param_path (str, optional): File for the node parameters of this project. Defaults to "params/nodes.yaml". 23 ignores (list[str], optional): Files that are not be referenced for this project. Defaults to ["params/nodes.yaml"]. 24 """ 25 26 self.root_dir_abs = root_dir 27 self.ignores = ignores if ignores is not None else ["params/nodes.yaml"] 28 29 self.snapshots = {} 30 self.node_params = {} 31 self.node_param_path = node_param_path 32 self.scan_dir() 33 if os.path.exists(node_param_path): 34 node_conf = OmegaConf.load(node_param_path) 35 self.node_params = OmegaConf.to_object(node_conf) 36 self.node_params["EMPTY"] = {} 37 if node_param_path not in self.ignores: 38 ignores.append(node_param_path) 39 40 @staticmethod 41 def node_params_is_valid( 42 schema: dict[str, type], params: dict[str, Any], allow_none=False 43 ): 44 """Determines whether parameter types in schema are matched in params 45 Args: 46 schema (dict[str,type]): The BlueprintNode Schema 47 params (dict[str, Any]): The node params to be used with the blueprint node 48 allow_none (bool, optional): Whether 'None' is allowed within the schema. Defaults to False. 49 Returns: 50 bool: True if the params are valid given the schema description. 51 """ 52 53 params_valid = len(schema) == len(params) 54 55 for s_key, s_type in schema.items(): 56 param_val = params.get(s_key, None) 57 if param_val is None: 58 params_valid &= allow_none 59 continue 60 params_valid &= isinstance(param_val, s_type) 61 62 return params_valid 63 64 def update_node_param_field(self, param_id, param_key, param_value): 65 assert param_id != "EMPTY", "Cannot modify default EMPTY Params" 66 n_params = self.node_params.get(param_id, None) 67 if not n_params: 68 return False 69 n_params[param_key] = param_value 70 return True 71 72 def add_node_params(self, param_id: str, params: dict): 73 assert self.node_params.get(param_id, None) is None, ( 74 "parameter label already exists" 75 ) 76 self.node_params[param_id] = copy.deepcopy(params) 77 78 def get_node_params(self, param_id: str, safe_cpy: bool = True): 79 """Gets the node params via param_id 80 Args: 81 param_id (str): the node params name/ identifier 82 safe_cpy (bool, optional): If True uses copy.deepcopy to produce final output, to prevent mutation. Defaults to True. 83 Returns: 84 dict[str,Any]: the node params requested 85 NB: These are the params used in some_blueprint_execution(_ , THESE_PARAMS) 86 """ 87 params = self.node_params.get(param_id, {}) 88 return copy.deepcopy(params) if safe_cpy else params 89 90 def remove_node_params(self, param_id: str): 91 """ 92 Removes node_param 93 Args: 94 param_lbl (str): _description_ 95 Returns: 96 _type_: _description_ 97 """ 98 assert param_id != "EMPTY", "EMPTY params required for system functionality" 99 param_file = self.node_params.pop(param_id, None) 100 101 return param_file 102 103 def save_node_params(self, rel_path=None): 104 """Saves the current node params 105 Args: 106 rel_path (str, optional): The relative path where the .yaml file is saved. Defaults to internal value. 107 """ 108 if rel_path is None: 109 rel_path = self.node_param_path 110 self.primitives_to_yaml(self.node_params, rel_path, False) 111 112 def to_node_param(self, path=None, name=None): 113 assert path is not None or name is not None, "No param value specified" 114 if name and path is None: 115 path = self.yamls.get(name, None) 116 if not path: 117 return False 118 119 config = OmegaConf.load(path) 120 new_node_dict = dict(OmegaConf.to_container(config)) 121 self.node_params = new_node_dict 122 return True 123 124 def get_all_node_param_ids(self): 125 """Gets all current node param ids 126 Returns: 127 lst[str]: a list of all param ids currently registerd with this manager 128 """ 129 return [str(k) for k in self.node_params.keys()] 130 131 def scan_dir(self): 132 """Scans the the directory which the manager was intiated with and refreshes it's parameter registry""" 133 self.yamls = {} 134 for dir_path, _, filenames in os.walk(self.root_dir_abs): 135 for fn in filenames: 136 if ( 137 not fn.endswith(".yml") and not fn.endswith(".yaml") 138 ) or fn in self.ignores: 139 continue 140 file_path = os.path.join(dir_path, fn) 141 name = self.get_name_from_path(file_path) 142 self.yamls[name] = file_path 143 144 def build_param_context( 145 self, 146 param_cascade_names: list[str], 147 overwrite: bool = True, 148 as_omega_conf: bool = True, 149 snapshot_name: str = None, 150 ): 151 """Generates a merged (and unresolved) parameter configuration from the list of parameter ids given 152 Args: 153 param_cascade_names (list[str]): list of parameter ids 154 overwrite (bool, optional): If true the values towards the end of the list will overwrite values at the beginning. Defaults to True. 155 as_omega_conf (bool, optional): If true returns values as OmegConf obj else returns python primitive (either dict or list dependent on file). Defaults to True. 156 snapshot_name (str, optional): If supplied, adds the parameters to snapshot registry. Defaults to None. 157 158 Returns: 159 OmegaConf|list|dict: The resultant Omega Configs for the merge. 160 """ 161 file_list = [self.yamls.get(f) for f in param_cascade_names] 162 assert all(file_list), "Some yaml files/file references are missing, aborting" 163 config_lst = [OmegaConf.load(f) for f in file_list] 164 out_config = ( 165 OmegaConf.merge(*config_lst) 166 if overwrite 167 else OmegaConf.merge(*reversed(config_lst)) 168 ) 169 if snapshot_name is not None: 170 self.snapshots[snapshot_name] = out_config.copy() 171 172 if as_omega_conf: 173 OmegaConf.resolve(out_config) 174 return out_config 175 else: 176 return OmegaConf.to_container(out_config, resolve=True) 177 178 def upsert_param( 179 self, param_name: str, val: Any, target_name: str = "params.global" 180 ): 181 """Adds a single parameter to the yaml file via target_name 182 Args: 183 param_name (str): name of the parameter attribute 184 val (Any): the new value to be upserted for that param 185 target_name (str, optional): target name of yaml file (identifier). Defaults to "params.global". 186 """ 187 config_pth = self.yamls.get(target_name, None) 188 assert config_pth is not None, f"Config yml {target_name} does not exist" 189 config = OmegaConf.load(config_pth) 190 OmegaConf.update(config, param_name, val) 191 OmegaConf.save(config, config_pth) 192 193 def get_name_from_path(self, path): 194 """Generates the name of a yaml file from the path 195 Args: 196 path (str): the relative or absolute path of the yaml file 197 Returns: 198 str: the parameter file name/id 199 NB: will not function correctly if given a file from outside of the directory 200 """ 201 return ( 202 os.path.relpath(path, self.root_dir_abs) 203 .removesuffix(".yml") 204 .removesuffix(".yaml") 205 .replace(os.sep, ".") 206 .strip() 207 ) 208 209 def snapshot_params( 210 self, 211 params: ListConfig | DictConfig | dict | list, 212 snapshot_name: str = None, 213 ): 214 """Saves the params supplied as parameter snapshots (within memory only) 215 Args: 216 params (ListConfig | DictConfig | dict | list): The parameters to be saved 217 snapshot_name (str, optional): Name of snapshot, uuid4 autogenerated if none supplied. Defaults to None. 218 """ 219 cfg = ( 220 OmegaConf.copy(params) 221 if type(params) is ListConfig or type(params) is DictConfig 222 else OmegaConf.create(params) 223 ) 224 OmegaConf.resolve(cfg) 225 if snapshot_name is None: 226 snapshot_name = uuid4().hex 227 self.snapshots[snapshot_name] = cfg 228 229 def get_snapshot(self, snapshot_name: str): 230 return OmegaConf.copy(self.snapshots[snapshot_name]) 231 232 def write_snapshot_to_yaml( 233 self, snapshot_name: str, path: str, overwrite_okay=False 234 ): 235 """ 236 Given a snapshot name, writes - it to disk as a yaml file 237 Args: 238 snapshot_name (str): the name of the snapshot to be written 239 path (str): The path to be save the yaml file to 240 overwrite_okay (bool, optional): If true will overwrite any previously existing file. Defaults to False. 241 """ 242 assert path.endswith("yml") or path.endswith("yaml"), ( 243 f"invalid path supplied: {path}" 244 ) 245 yaml_name = self.get_name_from_path(path) 246 assert overwrite_okay or ( 247 not os.path.exists(path) and self.yamls.get(yaml_name, None) is None 248 ), "Yaml file already exists {path}, overwrite blocked" 249 ss = self.snapshots.get(snapshot_name, None) 250 assert ss is not None, f"Snapshot: {snapshot_name}, not Found" 251 OmegaConf.save(ss, path) 252 self.yamls[yaml_name] = path 253 254 def primitives_to_yaml( 255 self, params: dict | list, path: str, register_in_memory=False 256 ): 257 """Saves the data given (dict or list) and saves them as params 258 Args: 259 params (dict | list): the parameter data to be saved 260 path (str): the relative path of the data 261 register_in_memory (bool, optional):If true adds the param reference using the autoname feature to the manager. Defaults to False. 262 """ 263 assert path.endswith(".yml") or path.endswith(".yaml"), ( 264 f"Must be a YAML file type, got {path}" 265 ) 266 OmegaConf.save(params, path) 267 if register_in_memory: 268 name = self.get_name_from_path(path) 269 self.yamls[name] = path 270 271 def remove_param(self, param_name: str, target_name="params.global"): 272 """Removes a paramter from the target name 273 Args: 274 param_name (str): the name of the value being removed 275 target_name (str, optional): parameter id of the set to be changed. Defaults to "params.global". 276 """ 277 config_path = self.yamls.get(target_name, None) 278 assert config_path is not None, f"Config yml {target_name} does not exist" 279 config = OmegaConf.load(config_path) 280 parent = config 281 keys = param_name.split(".") 282 283 for key in keys[:-1]: 284 parent = parent[key] 285 del parent[keys[-1]] 286 OmegaConf.save(config, config_path)
Manages parameters of YAML files to build contexts parsed as (context, __) and parameters of Execution nodes, parsed as(_, params )
13 def __init__( 14 self, 15 root_dir: str, 16 node_param_path: str = "params/nodes.yaml", 17 ignores: list[str] = None, 18 ): 19 """Create a Paramter Manager instance 20 Args: 21 root_dir (str): Where the Yaml files for the project reside. 22 node_param_path (str, optional): File for the node parameters of this project. Defaults to "params/nodes.yaml". 23 ignores (list[str], optional): Files that are not be referenced for this project. Defaults to ["params/nodes.yaml"]. 24 """ 25 26 self.root_dir_abs = root_dir 27 self.ignores = ignores if ignores is not None else ["params/nodes.yaml"] 28 29 self.snapshots = {} 30 self.node_params = {} 31 self.node_param_path = node_param_path 32 self.scan_dir() 33 if os.path.exists(node_param_path): 34 node_conf = OmegaConf.load(node_param_path) 35 self.node_params = OmegaConf.to_object(node_conf) 36 self.node_params["EMPTY"] = {} 37 if node_param_path not in self.ignores: 38 ignores.append(node_param_path)
Create a Paramter Manager instance Args: root_dir (str): Where the Yaml files for the project reside. node_param_path (str, optional): File for the node parameters of this project. Defaults to "params/nodes.yaml". ignores (list[str], optional): Files that are not be referenced for this project. Defaults to ["params/nodes.yaml"].
40 @staticmethod 41 def node_params_is_valid( 42 schema: dict[str, type], params: dict[str, Any], allow_none=False 43 ): 44 """Determines whether parameter types in schema are matched in params 45 Args: 46 schema (dict[str,type]): The BlueprintNode Schema 47 params (dict[str, Any]): The node params to be used with the blueprint node 48 allow_none (bool, optional): Whether 'None' is allowed within the schema. Defaults to False. 49 Returns: 50 bool: True if the params are valid given the schema description. 51 """ 52 53 params_valid = len(schema) == len(params) 54 55 for s_key, s_type in schema.items(): 56 param_val = params.get(s_key, None) 57 if param_val is None: 58 params_valid &= allow_none 59 continue 60 params_valid &= isinstance(param_val, s_type) 61 62 return params_valid
Determines whether parameter types in schema are matched in params Args: schema (dict[str,type]): The BlueprintNode Schema params (dict[str, Any]): The node params to be used with the blueprint node allow_none (bool, optional): Whether 'None' is allowed within the schema. Defaults to False. Returns: bool: True if the params are valid given the schema description.
78 def get_node_params(self, param_id: str, safe_cpy: bool = True): 79 """Gets the node params via param_id 80 Args: 81 param_id (str): the node params name/ identifier 82 safe_cpy (bool, optional): If True uses copy.deepcopy to produce final output, to prevent mutation. Defaults to True. 83 Returns: 84 dict[str,Any]: the node params requested 85 NB: These are the params used in some_blueprint_execution(_ , THESE_PARAMS) 86 """ 87 params = self.node_params.get(param_id, {}) 88 return copy.deepcopy(params) if safe_cpy else params
Gets the node params via param_id Args: param_id (str): the node params name/ identifier safe_cpy (bool, optional): If True uses copy.deepcopy to produce final output, to prevent mutation. Defaults to True. Returns: dict[str,Any]: the node params requested NB: These are the params used in some_blueprint_execution(_ , THESE_PARAMS)
90 def remove_node_params(self, param_id: str): 91 """ 92 Removes node_param 93 Args: 94 param_lbl (str): _description_ 95 Returns: 96 _type_: _description_ 97 """ 98 assert param_id != "EMPTY", "EMPTY params required for system functionality" 99 param_file = self.node_params.pop(param_id, None) 100 101 return param_file
Removes node_param Args: param_lbl (str): _description_ Returns: _type_: _description_
103 def save_node_params(self, rel_path=None): 104 """Saves the current node params 105 Args: 106 rel_path (str, optional): The relative path where the .yaml file is saved. Defaults to internal value. 107 """ 108 if rel_path is None: 109 rel_path = self.node_param_path 110 self.primitives_to_yaml(self.node_params, rel_path, False)
Saves the current node params Args: rel_path (str, optional): The relative path where the .yaml file is saved. Defaults to internal value.
112 def to_node_param(self, path=None, name=None): 113 assert path is not None or name is not None, "No param value specified" 114 if name and path is None: 115 path = self.yamls.get(name, None) 116 if not path: 117 return False 118 119 config = OmegaConf.load(path) 120 new_node_dict = dict(OmegaConf.to_container(config)) 121 self.node_params = new_node_dict 122 return True
124 def get_all_node_param_ids(self): 125 """Gets all current node param ids 126 Returns: 127 lst[str]: a list of all param ids currently registerd with this manager 128 """ 129 return [str(k) for k in self.node_params.keys()]
Gets all current node param ids Returns: lst[str]: a list of all param ids currently registerd with this manager
131 def scan_dir(self): 132 """Scans the the directory which the manager was intiated with and refreshes it's parameter registry""" 133 self.yamls = {} 134 for dir_path, _, filenames in os.walk(self.root_dir_abs): 135 for fn in filenames: 136 if ( 137 not fn.endswith(".yml") and not fn.endswith(".yaml") 138 ) or fn in self.ignores: 139 continue 140 file_path = os.path.join(dir_path, fn) 141 name = self.get_name_from_path(file_path) 142 self.yamls[name] = file_path
Scans the the directory which the manager was intiated with and refreshes it's parameter registry
144 def build_param_context( 145 self, 146 param_cascade_names: list[str], 147 overwrite: bool = True, 148 as_omega_conf: bool = True, 149 snapshot_name: str = None, 150 ): 151 """Generates a merged (and unresolved) parameter configuration from the list of parameter ids given 152 Args: 153 param_cascade_names (list[str]): list of parameter ids 154 overwrite (bool, optional): If true the values towards the end of the list will overwrite values at the beginning. Defaults to True. 155 as_omega_conf (bool, optional): If true returns values as OmegConf obj else returns python primitive (either dict or list dependent on file). Defaults to True. 156 snapshot_name (str, optional): If supplied, adds the parameters to snapshot registry. Defaults to None. 157 158 Returns: 159 OmegaConf|list|dict: The resultant Omega Configs for the merge. 160 """ 161 file_list = [self.yamls.get(f) for f in param_cascade_names] 162 assert all(file_list), "Some yaml files/file references are missing, aborting" 163 config_lst = [OmegaConf.load(f) for f in file_list] 164 out_config = ( 165 OmegaConf.merge(*config_lst) 166 if overwrite 167 else OmegaConf.merge(*reversed(config_lst)) 168 ) 169 if snapshot_name is not None: 170 self.snapshots[snapshot_name] = out_config.copy() 171 172 if as_omega_conf: 173 OmegaConf.resolve(out_config) 174 return out_config 175 else: 176 return OmegaConf.to_container(out_config, resolve=True)
Generates a merged (and unresolved) parameter configuration from the list of parameter ids given Args: param_cascade_names (list[str]): list of parameter ids overwrite (bool, optional): If true the values towards the end of the list will overwrite values at the beginning. Defaults to True. as_omega_conf (bool, optional): If true returns values as OmegConf obj else returns python primitive (either dict or list dependent on file). Defaults to True. snapshot_name (str, optional): If supplied, adds the parameters to snapshot registry. Defaults to None.
Returns: OmegaConf|list|dict: The resultant Omega Configs for the merge.
178 def upsert_param( 179 self, param_name: str, val: Any, target_name: str = "params.global" 180 ): 181 """Adds a single parameter to the yaml file via target_name 182 Args: 183 param_name (str): name of the parameter attribute 184 val (Any): the new value to be upserted for that param 185 target_name (str, optional): target name of yaml file (identifier). Defaults to "params.global". 186 """ 187 config_pth = self.yamls.get(target_name, None) 188 assert config_pth is not None, f"Config yml {target_name} does not exist" 189 config = OmegaConf.load(config_pth) 190 OmegaConf.update(config, param_name, val) 191 OmegaConf.save(config, config_pth)
Adds a single parameter to the yaml file via target_name Args: param_name (str): name of the parameter attribute val (Any): the new value to be upserted for that param target_name (str, optional): target name of yaml file (identifier). Defaults to "params.global".
193 def get_name_from_path(self, path): 194 """Generates the name of a yaml file from the path 195 Args: 196 path (str): the relative or absolute path of the yaml file 197 Returns: 198 str: the parameter file name/id 199 NB: will not function correctly if given a file from outside of the directory 200 """ 201 return ( 202 os.path.relpath(path, self.root_dir_abs) 203 .removesuffix(".yml") 204 .removesuffix(".yaml") 205 .replace(os.sep, ".") 206 .strip() 207 )
Generates the name of a yaml file from the path Args: path (str): the relative or absolute path of the yaml file Returns: str: the parameter file name/id NB: will not function correctly if given a file from outside of the directory
209 def snapshot_params( 210 self, 211 params: ListConfig | DictConfig | dict | list, 212 snapshot_name: str = None, 213 ): 214 """Saves the params supplied as parameter snapshots (within memory only) 215 Args: 216 params (ListConfig | DictConfig | dict | list): The parameters to be saved 217 snapshot_name (str, optional): Name of snapshot, uuid4 autogenerated if none supplied. Defaults to None. 218 """ 219 cfg = ( 220 OmegaConf.copy(params) 221 if type(params) is ListConfig or type(params) is DictConfig 222 else OmegaConf.create(params) 223 ) 224 OmegaConf.resolve(cfg) 225 if snapshot_name is None: 226 snapshot_name = uuid4().hex 227 self.snapshots[snapshot_name] = cfg
Saves the params supplied as parameter snapshots (within memory only) Args: params (ListConfig | DictConfig | dict | list): The parameters to be saved snapshot_name (str, optional): Name of snapshot, uuid4 autogenerated if none supplied. Defaults to None.
232 def write_snapshot_to_yaml( 233 self, snapshot_name: str, path: str, overwrite_okay=False 234 ): 235 """ 236 Given a snapshot name, writes - it to disk as a yaml file 237 Args: 238 snapshot_name (str): the name of the snapshot to be written 239 path (str): The path to be save the yaml file to 240 overwrite_okay (bool, optional): If true will overwrite any previously existing file. Defaults to False. 241 """ 242 assert path.endswith("yml") or path.endswith("yaml"), ( 243 f"invalid path supplied: {path}" 244 ) 245 yaml_name = self.get_name_from_path(path) 246 assert overwrite_okay or ( 247 not os.path.exists(path) and self.yamls.get(yaml_name, None) is None 248 ), "Yaml file already exists {path}, overwrite blocked" 249 ss = self.snapshots.get(snapshot_name, None) 250 assert ss is not None, f"Snapshot: {snapshot_name}, not Found" 251 OmegaConf.save(ss, path) 252 self.yamls[yaml_name] = path
Given a snapshot name, writes - it to disk as a yaml file Args: snapshot_name (str): the name of the snapshot to be written path (str): The path to be save the yaml file to overwrite_okay (bool, optional): If true will overwrite any previously existing file. Defaults to False.
254 def primitives_to_yaml( 255 self, params: dict | list, path: str, register_in_memory=False 256 ): 257 """Saves the data given (dict or list) and saves them as params 258 Args: 259 params (dict | list): the parameter data to be saved 260 path (str): the relative path of the data 261 register_in_memory (bool, optional):If true adds the param reference using the autoname feature to the manager. Defaults to False. 262 """ 263 assert path.endswith(".yml") or path.endswith(".yaml"), ( 264 f"Must be a YAML file type, got {path}" 265 ) 266 OmegaConf.save(params, path) 267 if register_in_memory: 268 name = self.get_name_from_path(path) 269 self.yamls[name] = path
Saves the data given (dict or list) and saves them as params Args: params (dict | list): the parameter data to be saved path (str): the relative path of the data register_in_memory (bool, optional):If true adds the param reference using the autoname feature to the manager. Defaults to False.
271 def remove_param(self, param_name: str, target_name="params.global"): 272 """Removes a paramter from the target name 273 Args: 274 param_name (str): the name of the value being removed 275 target_name (str, optional): parameter id of the set to be changed. Defaults to "params.global". 276 """ 277 config_path = self.yamls.get(target_name, None) 278 assert config_path is not None, f"Config yml {target_name} does not exist" 279 config = OmegaConf.load(config_path) 280 parent = config 281 keys = param_name.split(".") 282 283 for key in keys[:-1]: 284 parent = parent[key] 285 del parent[keys[-1]] 286 OmegaConf.save(config, config_path)
Removes a paramter from the target name Args: param_name (str): the name of the value being removed target_name (str, optional): parameter id of the set to be changed. Defaults to "params.global".