m_strat.api.project_management.project_manager
1import concurrent.futures 2import json 3import os 4from collections import deque 5from contextlib import chdir 6from shutil import copy as copy_file 7from typing import Any 8 9from loguru import logger 10 11from api.dag import BlueprintRegister, GraphManager 12from api.dag.executor import _EXECUTORS 13from api.ecs import ComponentDispatcher, ECS_Register 14 15from .parameters_manager import ParametersManager 16 17FOLDER_STRUCTURE = [ 18 "datasets", 19 "registers", 20 "blueprints", 21 "executions", 22 "params", 23 "configs", 24 "exec_graphs", 25 "results", 26] 27 28DEFAULT_FILES = [ 29 "project.yaml", 30 "readme.md", 31 "params/global.yaml", 32 "params/ignores.yaml", 33 "configs/bp_files_meta.jsonl", 34 "configs/executors_dict.json", 35] 36 37_ALLOWED_ACCESS = { 38 ComponentDispatcher, 39 ECS_Register, 40 BlueprintRegister, 41 GraphManager, 42 ParametersManager, 43} 44 45_TEMPLATES_DIR = os.path.join( 46 os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "templates" 47) 48 49 50class ProjectManager: 51 """The Main enty point for m-strat, used to manage the whole m-strat project""" 52 53 def __init__( 54 self, dir_path: str, load: bool = False, undo_limit: int = 100, max_workers=4 55 ): 56 """Create a project manager instance, designed to be sustained in memory while in use 57 Args: 58 dir_path (str): The absolute path of the root working directory 59 load (bool, optional): If true attempts to load the project, if false creates a new one. Defaults to False. 60 undo_limit (int, optional): How many items can be undone (commands are stored in memory) . Defaults to 100. 61 NB: Having mutliple instances running on the same project will produce errors 62 """ 63 assert os.path.isabs(dir_path), ( 64 "The Path must be absolute for project directories" 65 ) 66 self.project_path = dir_path 67 self.commands = deque([], undo_limit) 68 self.command_limit = undo_limit 69 if load: 70 self.load_project() 71 else: 72 assert len(os.listdir(dir_path)) == 0, ( 73 "Project folder must be empty for init" 74 ) # Warn instead? 75 self.init_project_structure() 76 self.api_thread_pool = concurrent.futures.ThreadPoolExecutor( 77 max_workers=max_workers 78 ) 79 80 def copy_gen_templates(self, template_dir: str = None): 81 """Copies a generation template (the directory and files at the location to the project path) 82 NB: only files with _template within the name are copied, only folders with template files within them will be made 83 Args: 84 template_dir (str, optional): Where to copy the directory if None uses the default embedded folder . Defaults to None. 85 """ 86 if template_dir is None: 87 template_dir = _TEMPLATES_DIR 88 89 for dir_path, _, fns in os.walk(template_dir): 90 for fn in fns: 91 if "template" not in fn or ( 92 fn.startswith("_") 93 and not fn.startswith( 94 "_template" 95 ) # .param type kind of problem, _template.param 96 ): 97 continue 98 rel_dir = os.path.relpath(dir_path, template_dir) 99 if rel_dir == "[root]" or rel_dir == ".": 100 copy_dest = self.project_path 101 else: 102 copy_dest = os.path.join(self.project_path, rel_dir) 103 os.makedirs(copy_dest, exist_ok=True) 104 105 copy_fn = fn.replace("_template", "") 106 copy_file(os.path.join(dir_path, fn), os.path.join(copy_dest, copy_fn)) 107 logger.info("Template generation completed") 108 109 def init_project_structure(self): 110 """Initiates the project structure, automatically called if load false on object inits""" 111 logger.info(f"Generating Project files at: {self.project_path}") 112 with chdir(self.project_path): 113 for folder in FOLDER_STRUCTURE: 114 os.makedirs(folder, exist_ok=True) 115 for file in DEFAULT_FILES: 116 with open(file, "w", encoding="utf-8") as f_obj: 117 if file.endswith(".json"): 118 f_obj.write("{}") 119 logger.info( 120 "Basic folders and empty files completed, copying premade templates" 121 ) 122 self.copy_gen_templates() 123 self.cd = ComponentDispatcher() 124 self.ecs = ECS_Register("registers", dispatcher=self.cd) 125 self.graphs = { 126 "Data": GraphManager(name="Data"), 127 "Model": GraphManager(name="Model"), 128 "Analysis": GraphManager(name="Analysis"), 129 } 130 self.executors = {} 131 self.bp_reg = BlueprintRegister() 132 self.bp_reg.add_from_folder("blueprints") 133 self.param_manager = ParametersManager(self.project_path) 134 self.commandables = { 135 "cd": self.cd, 136 "ecs": self.ecs, 137 "param_manager": self.param_manager, 138 "bp_reg": self.bp_reg, 139 "graphs": self.graphs, 140 "executors": self.executors, 141 } 142 self.save_project() # this generates all the config files automatically 143 144 def add_graph(self, name: str): 145 """Adds a graph to the project 146 Args: 147 name (str): The name of the graph given. 148 """ 149 assert isinstance(name, str), "Invalid Graph Name, must be a string" 150 assert len(name) > 0, "Invalid graph Name, must be at least one character" 151 assert self.graphs.get(name, None) is None, "Graph Name Already exists" 152 self.graphs[name] = GraphManager(name=name, dir="exec_graphs", load=False) 153 logger.info(f"Graph '{name} was added to the project") 154 155 def rename_graph(self, source_name: str, new_name: str): 156 """Renames a graph 157 (a bit heavy but gets the job done!) 158 Args: 159 source_name (str): the name of original graph 160 new_name (str): the new name of the graph 161 """ 162 self.duplicate_graph(source_name, new_name) 163 self.delete_graph(source_name) 164 logger.info(f"Graph {source_name} was renamed to {new_name}") 165 166 def delete_graph(self, name: str): 167 """Deletes the given graph 168 Args: 169 name (str): The name of the graph to be deleted 170 """ 171 assert isinstance(name, str), "Invalid Graph Name, must be a string" 172 graph = self.graphs.pop(name, None) 173 self.executors = { 174 k: v for k, v in self.executors.items() if v["graph_name"] != name 175 } 176 if graph: 177 path = graph.get_file_path(graph.name, graph.dir) 178 with chdir(self.project_path): 179 if os.path.exists(path): 180 os.remove(path) 181 graph = None 182 logger.info( 183 f"Graph '{name}' was deleted along with it's configuration files" 184 ) 185 else: 186 logger.info(f"Attempted to delete graph '{name}' but no graph was found") 187 188 def duplicate_graph(self, source_name: str, copy_name: str): 189 """Duplicates a graph 190 Args: 191 source_name (str): The name of the graph to be copied 192 copy_name (str): the new name of the graph clone 193 """ 194 assert isinstance(source_name, str) and isinstance(copy_name, str), ( 195 "Graph source and copy name must be string" 196 ) 197 assert len(source_name) > 0 and len(copy_name) > 0, ( 198 "Graph source and copy names must be 1 character or more" 199 ) 200 assert self.graphs.get(copy_name, None) is None, ( 201 f"Destination name, {copy_name} already exists" 202 ) 203 204 origin_graph = self.graphs.get(source_name, None) 205 assert origin_graph is not None, ( 206 f"Source Name ,{source_name} does not exist in current context" 207 ) 208 self.graphs[copy_name] = origin_graph.duplicate(copy_name) 209 logger.info(f"'{source_name}' graph sucessfully duplicated as '{copy_name}'") 210 211 def load_project(self): 212 """Loads the project (done automaticaly when project manager initiated)""" 213 logger.info(f"Loading project: {self.project_path}") 214 try: 215 with chdir(self.project_path): 216 self.cd = ComponentDispatcher() 217 self.ecs = ECS_Register("registers", dispatcher=self.cd) 218 self.param_manager = ParametersManager(self.project_path) 219 self.bp_reg = BlueprintRegister() 220 221 self.graphs = {} 222 for dir_path, _, fns in os.walk("exec_graphs"): 223 for fn in fns: 224 if not fn.startswith("gm_") or not fn.endswith(".json"): 225 continue 226 f_p = os.path.join(dir_path, fn) 227 new_graph = GraphManager(name=None, load_path=f_p, load=True) 228 self.graphs[new_graph.name] = new_graph 229 self.executors = {} 230 231 with open("configs/executors_dict.json", encoding="utf-8") as f: 232 j_data = json.load(f) 233 self.executors = j_data if j_data else {} 234 235 self.commandables = { 236 "cd": self.cd, 237 "ecs": self.ecs, 238 "param_manager": self.param_manager, 239 "bp_reg": self.bp_reg, 240 "graphs": self.graphs, 241 "executors": self.executors, 242 } 243 self.bp_reg.add_from_folder("blueprints") 244 except Exception as e: 245 logger.error(f"Project loading failed:\n{e}") 246 raise e 247 logger.info(f"Project loaded sucessfully ({self.project_path})") 248 249 def save_project(self): 250 """Saves the whole project in it's current state""" 251 with chdir(self.project_path): 252 self.ecs._save_register() 253 self.param_manager.save_node_params() 254 self.bp_reg.save_meta_file() 255 for g in self.graphs.values(): 256 g.save() 257 with open("configs/executors_dict.json", "w", encoding="utf-8") as f: 258 json.dump(self.executors, f) 259 logger.info(f"Project saved: {self.project_path}") 260 261 @staticmethod 262 def is_valid_command_dict(params: dict): 263 """Returns true if the command dictionary given is valid""" 264 type_set = { 265 "target_obj": str, 266 "do_params": dict, 267 "undo_params": dict, 268 "target_func": str, 269 "undo_func": str, 270 } 271 is_valid = all(p in type_set.keys() for p in params.keys()) 272 is_valid &= all(type(v) is type_set.get(k, None) for k, v in params.items()) 273 is_valid &= ( 274 params.get("target_obj", None) is not None 275 and params.get("target_func", None) is not None 276 ) 277 return is_valid 278 279 def execute_command(self, **params): 280 """Executes a command to a commandable target (Any manager or Register) 281 Args: 282 target_obj (str): The commandable's name (full name e.g. 'graphs.graph-name') or pm/self for project manager 283 target_func (str): The name of callable to be executed within the class 284 do_params(dict[str,Any]): The parameters needed to execute the command parsed via **params syntax. 285 undo_func(str, optional): The name of the function to execute an undoing of the original command 286 undo_params(dict[str, any]): The parameters needed to execute an undoing parsed via **params syntax. 287 Returns: 288 Any : any values resulted from the given command (if available else None) 289 """ 290 try: 291 assert self.is_valid_command_dict(params), ( 292 "Execute params failed validation" 293 ) 294 target_obj = params.get("target_obj", None) 295 if target_obj.lower() in ["self", "pm", "projectmanager"]: 296 commandable = self 297 target_obj = "self" # project manager is also a commandable 298 else: 299 commandable = self.commandables 300 if "." in target_obj: 301 embeds = target_obj.split(".") 302 commandable = self.commandables 303 for embed in embeds: 304 commandable = commandable[embed] 305 else: 306 commandable = commandable[target_obj] 307 308 cm_t = type(commandable) 309 assert cm_t in _ALLOWED_ACCESS, ( 310 f"Attempted to access item that is not commandable: {cm_t}" 311 ) 312 313 target_func = params["target_func"] 314 assert not target_func.startswith("_"), ( 315 f"Attempted to access a private method: {target_func}" 316 ) 317 318 assert target_func in cm_t.__dict__, ( 319 f"Attempted to acces a function not implemented on the child class: {target_func}" 320 ) 321 func = getattr(commandable, target_func) 322 assert callable(func), f"Designated attribute is not callable {target_func}" 323 324 func_result = None 325 do_params = params.get("do_params", {}) 326 # don't log getters, a few places can use them in loops! 327 is_logworthy = not target_func.startswith("get") 328 329 if is_logworthy: 330 logger.info( 331 f"Performing {target_obj}.{target_func}() with params: [|] {do_params} [|]" 332 ) 333 undo_func = params.get("undo_func", None) 334 335 if undo_func: 336 undo_func_name = params.get("undo_func") 337 assert callable(getattr(commandable, undo_func_name, None)), ( 338 f"Undo function is not a callable: {params['undo_func']}, command aborted" 339 ) 340 assert not undo_func_name.startswith("_"), ( 341 "Undo function supplied is private, command aborted" 342 ) 343 command_log = dict(params) 344 command_log.pop("do_params") # delete do_params to avoid memory leak 345 self.commands.append(command_log) 346 if is_logworthy: 347 logger.debug( 348 f"Undo detected and added for {target_obj}.{target_func}-> {params['undo_func']}" 349 ) 350 if len(self.commands) > self.command_limit: 351 self.commands.popleft() 352 elif is_logworthy: 353 logger.debug(f"No Undo method detected for {target_obj}.{target_func}") 354 355 with chdir(self.project_path): 356 func_result = func(**do_params) 357 if is_logworthy: 358 logger.info("Command Executed Sucessfully") 359 except Exception as e: 360 logger.error(f"Command failed for {target_obj}->{target_func}(): {e} ") 361 return None 362 363 return func_result 364 365 def undo_last_command(self): 366 """undoes the most recent command if there are any""" 367 if not self.commands: 368 logger.info("Tried to undo but No Undo Commands available") 369 return 370 371 last_cmd = self.commands.pop() 372 commandable_name = last_cmd["target_obj"] 373 if commandable_name == "self": 374 commandable = self 375 else: 376 commandable = self.commandables 377 if "." in commandable_name: 378 embeds = commandable_name.split(".") 379 commandable = self.commandables 380 for embed in embeds: 381 commandable = commandable[embed] 382 else: 383 commandable = commandable[commandable_name] 384 385 undo_func_name = last_cmd["undo_func"] 386 undo_func = getattr(commandable, undo_func_name) 387 logger.info( 388 f"Undoing {'pm' if commandable_name == 'self' else commandable_name}.{last_cmd['target_func']} USING {undo_func_name}()" 389 ) 390 try: 391 with chdir(self.project_path): 392 undo_func(**last_cmd["undo_params"]) 393 except Exception as e: 394 logger.error( 395 f"Command failed for {'pm' if commandable_name == 'self' else commandable_name}->{undo_func_name}(): {e} " 396 ) 397 398 def create_executor( 399 self, 400 graph_name: str, 401 exec_lbl: str, 402 executor_type_name: str, 403 start_node_id="start", 404 ctx_build_kwargs: dict[str, Any] = None, 405 ): 406 """Creates an execution context within the project manager 407 Args: 408 graph_name (str): The name of the graph to be create the execution from 409 exec_lbl (str): The given name of the execution to identify. 410 executor_type_name (str): The name of the Executor to be used 411 start_node_id (str, optional): The node from which execution begins at. Defaults to "start". 412 ctx_build_kwargs (dict[str, Any], optional): .Defaults to None. 413 """ 414 assert self.graphs.get(graph_name, None) is not None, "graph does not exist" 415 assert self.executors.get(exec_lbl, None) is None, ( 416 "Execution with the same label already exists" 417 ) 418 if ctx_build_kwargs is None: 419 ctx_build_kwargs = {"param_cascade_names": ["params.global"]} 420 421 self.executors[exec_lbl] = { 422 "exec_func_type": executor_type_name, 423 "graph_name": graph_name, 424 "context_args": ctx_build_kwargs, 425 "start_node": start_node_id, 426 } 427 428 def run_executor(self, executor_lbl: str, run_async: bool = True): 429 """Simple wrapper to allow for async using conccurrent.futures""" 430 logger.info( 431 f"Saving project to {self.project_path} before execution (and refreshing blueprints!)" 432 ) 433 with chdir(self.project_path): 434 self.save_project() 435 self.bp_reg.refresh_blueprints() 436 logger.info( 437 f"Execution starting {executor_lbl} {'Asychronously' if run_async else 'Synchronously \nWARNING! This will likely block the user interface!'} " 438 ) 439 if run_async: 440 return self.api_thread_pool.submit(self._execute_graph, executor_lbl) 441 else: 442 return self._execute_graph(executor_lbl) 443 444 def _execute_graph(self, executor_lbl: str): 445 """Runs the executor 446 Args: 447 executor_lbl (str): the name of the executor to be used 448 Returns: 449 dict[str, Any]: returns the final context generated from the execution 450 Raises: 451 ValueError: If anything required for execution (label, executor type, target graph) is not found 452 """ 453 execution_target = None 454 try: 455 executor_info = self.executors.get(executor_lbl, None) 456 assert bool(executor_info), ( 457 f"Did not find executor labelled {executor_lbl} within project, cannot run!" 458 ) 459 460 execution_target = _EXECUTORS.get(executor_info["exec_func_type"], None) 461 assert bool(execution_target), ( 462 f"Did not find executor of type {execution_target} within project, cannot run!" 463 ) 464 465 target_graph = self.graphs.get(executor_info["graph_name"], None) 466 assert bool(target_graph), ( 467 f"Target Graph not Found: {target_graph}, cannot run!" 468 ) 469 470 execution_graph = target_graph.build_execution( 471 executor_info.get("start_node", "start") 472 ) 473 logger.debug( 474 f"Execution graph built {'->'.join([n.id for n in execution_graph])}" 475 ) 476 exec_targets = [] 477 478 for node in execution_graph: 479 if node.id == "start": 480 continue 481 bp = self.bp_reg.get_blueprint(node.blueprint_name) 482 node_param = self.param_manager.get_node_params(node.param_id) 483 exec_targets.append((bp, node_param, node)) 484 ctx_args = executor_info["context_args"] 485 init_ctx = ( 486 self.param_manager.build_param_context(**ctx_args, as_omega_conf=False) 487 if ctx_args 488 else {} 489 ) 490 logger.debug(f"Context built:\n {init_ctx}") 491 logger.info("Execution Starting...") 492 with chdir(self.project_path): 493 out_ctx = execution_target(exec_targets, init_ctx) 494 logger.info("Execution Finished") 495 except Exception as e: 496 logger.error( 497 f"Execution of {executor_lbl} using {execution_target} failed:\n {e}" 498 ) 499 return {} 500 501 return out_ctx 502 503 504_ALLOWED_ACCESS.add(ProjectManager)
51class ProjectManager: 52 """The Main enty point for m-strat, used to manage the whole m-strat project""" 53 54 def __init__( 55 self, dir_path: str, load: bool = False, undo_limit: int = 100, max_workers=4 56 ): 57 """Create a project manager instance, designed to be sustained in memory while in use 58 Args: 59 dir_path (str): The absolute path of the root working directory 60 load (bool, optional): If true attempts to load the project, if false creates a new one. Defaults to False. 61 undo_limit (int, optional): How many items can be undone (commands are stored in memory) . Defaults to 100. 62 NB: Having mutliple instances running on the same project will produce errors 63 """ 64 assert os.path.isabs(dir_path), ( 65 "The Path must be absolute for project directories" 66 ) 67 self.project_path = dir_path 68 self.commands = deque([], undo_limit) 69 self.command_limit = undo_limit 70 if load: 71 self.load_project() 72 else: 73 assert len(os.listdir(dir_path)) == 0, ( 74 "Project folder must be empty for init" 75 ) # Warn instead? 76 self.init_project_structure() 77 self.api_thread_pool = concurrent.futures.ThreadPoolExecutor( 78 max_workers=max_workers 79 ) 80 81 def copy_gen_templates(self, template_dir: str = None): 82 """Copies a generation template (the directory and files at the location to the project path) 83 NB: only files with _template within the name are copied, only folders with template files within them will be made 84 Args: 85 template_dir (str, optional): Where to copy the directory if None uses the default embedded folder . Defaults to None. 86 """ 87 if template_dir is None: 88 template_dir = _TEMPLATES_DIR 89 90 for dir_path, _, fns in os.walk(template_dir): 91 for fn in fns: 92 if "template" not in fn or ( 93 fn.startswith("_") 94 and not fn.startswith( 95 "_template" 96 ) # .param type kind of problem, _template.param 97 ): 98 continue 99 rel_dir = os.path.relpath(dir_path, template_dir) 100 if rel_dir == "[root]" or rel_dir == ".": 101 copy_dest = self.project_path 102 else: 103 copy_dest = os.path.join(self.project_path, rel_dir) 104 os.makedirs(copy_dest, exist_ok=True) 105 106 copy_fn = fn.replace("_template", "") 107 copy_file(os.path.join(dir_path, fn), os.path.join(copy_dest, copy_fn)) 108 logger.info("Template generation completed") 109 110 def init_project_structure(self): 111 """Initiates the project structure, automatically called if load false on object inits""" 112 logger.info(f"Generating Project files at: {self.project_path}") 113 with chdir(self.project_path): 114 for folder in FOLDER_STRUCTURE: 115 os.makedirs(folder, exist_ok=True) 116 for file in DEFAULT_FILES: 117 with open(file, "w", encoding="utf-8") as f_obj: 118 if file.endswith(".json"): 119 f_obj.write("{}") 120 logger.info( 121 "Basic folders and empty files completed, copying premade templates" 122 ) 123 self.copy_gen_templates() 124 self.cd = ComponentDispatcher() 125 self.ecs = ECS_Register("registers", dispatcher=self.cd) 126 self.graphs = { 127 "Data": GraphManager(name="Data"), 128 "Model": GraphManager(name="Model"), 129 "Analysis": GraphManager(name="Analysis"), 130 } 131 self.executors = {} 132 self.bp_reg = BlueprintRegister() 133 self.bp_reg.add_from_folder("blueprints") 134 self.param_manager = ParametersManager(self.project_path) 135 self.commandables = { 136 "cd": self.cd, 137 "ecs": self.ecs, 138 "param_manager": self.param_manager, 139 "bp_reg": self.bp_reg, 140 "graphs": self.graphs, 141 "executors": self.executors, 142 } 143 self.save_project() # this generates all the config files automatically 144 145 def add_graph(self, name: str): 146 """Adds a graph to the project 147 Args: 148 name (str): The name of the graph given. 149 """ 150 assert isinstance(name, str), "Invalid Graph Name, must be a string" 151 assert len(name) > 0, "Invalid graph Name, must be at least one character" 152 assert self.graphs.get(name, None) is None, "Graph Name Already exists" 153 self.graphs[name] = GraphManager(name=name, dir="exec_graphs", load=False) 154 logger.info(f"Graph '{name} was added to the project") 155 156 def rename_graph(self, source_name: str, new_name: str): 157 """Renames a graph 158 (a bit heavy but gets the job done!) 159 Args: 160 source_name (str): the name of original graph 161 new_name (str): the new name of the graph 162 """ 163 self.duplicate_graph(source_name, new_name) 164 self.delete_graph(source_name) 165 logger.info(f"Graph {source_name} was renamed to {new_name}") 166 167 def delete_graph(self, name: str): 168 """Deletes the given graph 169 Args: 170 name (str): The name of the graph to be deleted 171 """ 172 assert isinstance(name, str), "Invalid Graph Name, must be a string" 173 graph = self.graphs.pop(name, None) 174 self.executors = { 175 k: v for k, v in self.executors.items() if v["graph_name"] != name 176 } 177 if graph: 178 path = graph.get_file_path(graph.name, graph.dir) 179 with chdir(self.project_path): 180 if os.path.exists(path): 181 os.remove(path) 182 graph = None 183 logger.info( 184 f"Graph '{name}' was deleted along with it's configuration files" 185 ) 186 else: 187 logger.info(f"Attempted to delete graph '{name}' but no graph was found") 188 189 def duplicate_graph(self, source_name: str, copy_name: str): 190 """Duplicates a graph 191 Args: 192 source_name (str): The name of the graph to be copied 193 copy_name (str): the new name of the graph clone 194 """ 195 assert isinstance(source_name, str) and isinstance(copy_name, str), ( 196 "Graph source and copy name must be string" 197 ) 198 assert len(source_name) > 0 and len(copy_name) > 0, ( 199 "Graph source and copy names must be 1 character or more" 200 ) 201 assert self.graphs.get(copy_name, None) is None, ( 202 f"Destination name, {copy_name} already exists" 203 ) 204 205 origin_graph = self.graphs.get(source_name, None) 206 assert origin_graph is not None, ( 207 f"Source Name ,{source_name} does not exist in current context" 208 ) 209 self.graphs[copy_name] = origin_graph.duplicate(copy_name) 210 logger.info(f"'{source_name}' graph sucessfully duplicated as '{copy_name}'") 211 212 def load_project(self): 213 """Loads the project (done automaticaly when project manager initiated)""" 214 logger.info(f"Loading project: {self.project_path}") 215 try: 216 with chdir(self.project_path): 217 self.cd = ComponentDispatcher() 218 self.ecs = ECS_Register("registers", dispatcher=self.cd) 219 self.param_manager = ParametersManager(self.project_path) 220 self.bp_reg = BlueprintRegister() 221 222 self.graphs = {} 223 for dir_path, _, fns in os.walk("exec_graphs"): 224 for fn in fns: 225 if not fn.startswith("gm_") or not fn.endswith(".json"): 226 continue 227 f_p = os.path.join(dir_path, fn) 228 new_graph = GraphManager(name=None, load_path=f_p, load=True) 229 self.graphs[new_graph.name] = new_graph 230 self.executors = {} 231 232 with open("configs/executors_dict.json", encoding="utf-8") as f: 233 j_data = json.load(f) 234 self.executors = j_data if j_data else {} 235 236 self.commandables = { 237 "cd": self.cd, 238 "ecs": self.ecs, 239 "param_manager": self.param_manager, 240 "bp_reg": self.bp_reg, 241 "graphs": self.graphs, 242 "executors": self.executors, 243 } 244 self.bp_reg.add_from_folder("blueprints") 245 except Exception as e: 246 logger.error(f"Project loading failed:\n{e}") 247 raise e 248 logger.info(f"Project loaded sucessfully ({self.project_path})") 249 250 def save_project(self): 251 """Saves the whole project in it's current state""" 252 with chdir(self.project_path): 253 self.ecs._save_register() 254 self.param_manager.save_node_params() 255 self.bp_reg.save_meta_file() 256 for g in self.graphs.values(): 257 g.save() 258 with open("configs/executors_dict.json", "w", encoding="utf-8") as f: 259 json.dump(self.executors, f) 260 logger.info(f"Project saved: {self.project_path}") 261 262 @staticmethod 263 def is_valid_command_dict(params: dict): 264 """Returns true if the command dictionary given is valid""" 265 type_set = { 266 "target_obj": str, 267 "do_params": dict, 268 "undo_params": dict, 269 "target_func": str, 270 "undo_func": str, 271 } 272 is_valid = all(p in type_set.keys() for p in params.keys()) 273 is_valid &= all(type(v) is type_set.get(k, None) for k, v in params.items()) 274 is_valid &= ( 275 params.get("target_obj", None) is not None 276 and params.get("target_func", None) is not None 277 ) 278 return is_valid 279 280 def execute_command(self, **params): 281 """Executes a command to a commandable target (Any manager or Register) 282 Args: 283 target_obj (str): The commandable's name (full name e.g. 'graphs.graph-name') or pm/self for project manager 284 target_func (str): The name of callable to be executed within the class 285 do_params(dict[str,Any]): The parameters needed to execute the command parsed via **params syntax. 286 undo_func(str, optional): The name of the function to execute an undoing of the original command 287 undo_params(dict[str, any]): The parameters needed to execute an undoing parsed via **params syntax. 288 Returns: 289 Any : any values resulted from the given command (if available else None) 290 """ 291 try: 292 assert self.is_valid_command_dict(params), ( 293 "Execute params failed validation" 294 ) 295 target_obj = params.get("target_obj", None) 296 if target_obj.lower() in ["self", "pm", "projectmanager"]: 297 commandable = self 298 target_obj = "self" # project manager is also a commandable 299 else: 300 commandable = self.commandables 301 if "." in target_obj: 302 embeds = target_obj.split(".") 303 commandable = self.commandables 304 for embed in embeds: 305 commandable = commandable[embed] 306 else: 307 commandable = commandable[target_obj] 308 309 cm_t = type(commandable) 310 assert cm_t in _ALLOWED_ACCESS, ( 311 f"Attempted to access item that is not commandable: {cm_t}" 312 ) 313 314 target_func = params["target_func"] 315 assert not target_func.startswith("_"), ( 316 f"Attempted to access a private method: {target_func}" 317 ) 318 319 assert target_func in cm_t.__dict__, ( 320 f"Attempted to acces a function not implemented on the child class: {target_func}" 321 ) 322 func = getattr(commandable, target_func) 323 assert callable(func), f"Designated attribute is not callable {target_func}" 324 325 func_result = None 326 do_params = params.get("do_params", {}) 327 # don't log getters, a few places can use them in loops! 328 is_logworthy = not target_func.startswith("get") 329 330 if is_logworthy: 331 logger.info( 332 f"Performing {target_obj}.{target_func}() with params: [|] {do_params} [|]" 333 ) 334 undo_func = params.get("undo_func", None) 335 336 if undo_func: 337 undo_func_name = params.get("undo_func") 338 assert callable(getattr(commandable, undo_func_name, None)), ( 339 f"Undo function is not a callable: {params['undo_func']}, command aborted" 340 ) 341 assert not undo_func_name.startswith("_"), ( 342 "Undo function supplied is private, command aborted" 343 ) 344 command_log = dict(params) 345 command_log.pop("do_params") # delete do_params to avoid memory leak 346 self.commands.append(command_log) 347 if is_logworthy: 348 logger.debug( 349 f"Undo detected and added for {target_obj}.{target_func}-> {params['undo_func']}" 350 ) 351 if len(self.commands) > self.command_limit: 352 self.commands.popleft() 353 elif is_logworthy: 354 logger.debug(f"No Undo method detected for {target_obj}.{target_func}") 355 356 with chdir(self.project_path): 357 func_result = func(**do_params) 358 if is_logworthy: 359 logger.info("Command Executed Sucessfully") 360 except Exception as e: 361 logger.error(f"Command failed for {target_obj}->{target_func}(): {e} ") 362 return None 363 364 return func_result 365 366 def undo_last_command(self): 367 """undoes the most recent command if there are any""" 368 if not self.commands: 369 logger.info("Tried to undo but No Undo Commands available") 370 return 371 372 last_cmd = self.commands.pop() 373 commandable_name = last_cmd["target_obj"] 374 if commandable_name == "self": 375 commandable = self 376 else: 377 commandable = self.commandables 378 if "." in commandable_name: 379 embeds = commandable_name.split(".") 380 commandable = self.commandables 381 for embed in embeds: 382 commandable = commandable[embed] 383 else: 384 commandable = commandable[commandable_name] 385 386 undo_func_name = last_cmd["undo_func"] 387 undo_func = getattr(commandable, undo_func_name) 388 logger.info( 389 f"Undoing {'pm' if commandable_name == 'self' else commandable_name}.{last_cmd['target_func']} USING {undo_func_name}()" 390 ) 391 try: 392 with chdir(self.project_path): 393 undo_func(**last_cmd["undo_params"]) 394 except Exception as e: 395 logger.error( 396 f"Command failed for {'pm' if commandable_name == 'self' else commandable_name}->{undo_func_name}(): {e} " 397 ) 398 399 def create_executor( 400 self, 401 graph_name: str, 402 exec_lbl: str, 403 executor_type_name: str, 404 start_node_id="start", 405 ctx_build_kwargs: dict[str, Any] = None, 406 ): 407 """Creates an execution context within the project manager 408 Args: 409 graph_name (str): The name of the graph to be create the execution from 410 exec_lbl (str): The given name of the execution to identify. 411 executor_type_name (str): The name of the Executor to be used 412 start_node_id (str, optional): The node from which execution begins at. Defaults to "start". 413 ctx_build_kwargs (dict[str, Any], optional): .Defaults to None. 414 """ 415 assert self.graphs.get(graph_name, None) is not None, "graph does not exist" 416 assert self.executors.get(exec_lbl, None) is None, ( 417 "Execution with the same label already exists" 418 ) 419 if ctx_build_kwargs is None: 420 ctx_build_kwargs = {"param_cascade_names": ["params.global"]} 421 422 self.executors[exec_lbl] = { 423 "exec_func_type": executor_type_name, 424 "graph_name": graph_name, 425 "context_args": ctx_build_kwargs, 426 "start_node": start_node_id, 427 } 428 429 def run_executor(self, executor_lbl: str, run_async: bool = True): 430 """Simple wrapper to allow for async using conccurrent.futures""" 431 logger.info( 432 f"Saving project to {self.project_path} before execution (and refreshing blueprints!)" 433 ) 434 with chdir(self.project_path): 435 self.save_project() 436 self.bp_reg.refresh_blueprints() 437 logger.info( 438 f"Execution starting {executor_lbl} {'Asychronously' if run_async else 'Synchronously \nWARNING! This will likely block the user interface!'} " 439 ) 440 if run_async: 441 return self.api_thread_pool.submit(self._execute_graph, executor_lbl) 442 else: 443 return self._execute_graph(executor_lbl) 444 445 def _execute_graph(self, executor_lbl: str): 446 """Runs the executor 447 Args: 448 executor_lbl (str): the name of the executor to be used 449 Returns: 450 dict[str, Any]: returns the final context generated from the execution 451 Raises: 452 ValueError: If anything required for execution (label, executor type, target graph) is not found 453 """ 454 execution_target = None 455 try: 456 executor_info = self.executors.get(executor_lbl, None) 457 assert bool(executor_info), ( 458 f"Did not find executor labelled {executor_lbl} within project, cannot run!" 459 ) 460 461 execution_target = _EXECUTORS.get(executor_info["exec_func_type"], None) 462 assert bool(execution_target), ( 463 f"Did not find executor of type {execution_target} within project, cannot run!" 464 ) 465 466 target_graph = self.graphs.get(executor_info["graph_name"], None) 467 assert bool(target_graph), ( 468 f"Target Graph not Found: {target_graph}, cannot run!" 469 ) 470 471 execution_graph = target_graph.build_execution( 472 executor_info.get("start_node", "start") 473 ) 474 logger.debug( 475 f"Execution graph built {'->'.join([n.id for n in execution_graph])}" 476 ) 477 exec_targets = [] 478 479 for node in execution_graph: 480 if node.id == "start": 481 continue 482 bp = self.bp_reg.get_blueprint(node.blueprint_name) 483 node_param = self.param_manager.get_node_params(node.param_id) 484 exec_targets.append((bp, node_param, node)) 485 ctx_args = executor_info["context_args"] 486 init_ctx = ( 487 self.param_manager.build_param_context(**ctx_args, as_omega_conf=False) 488 if ctx_args 489 else {} 490 ) 491 logger.debug(f"Context built:\n {init_ctx}") 492 logger.info("Execution Starting...") 493 with chdir(self.project_path): 494 out_ctx = execution_target(exec_targets, init_ctx) 495 logger.info("Execution Finished") 496 except Exception as e: 497 logger.error( 498 f"Execution of {executor_lbl} using {execution_target} failed:\n {e}" 499 ) 500 return {} 501 502 return out_ctx
The Main enty point for m-strat, used to manage the whole m-strat project
54 def __init__( 55 self, dir_path: str, load: bool = False, undo_limit: int = 100, max_workers=4 56 ): 57 """Create a project manager instance, designed to be sustained in memory while in use 58 Args: 59 dir_path (str): The absolute path of the root working directory 60 load (bool, optional): If true attempts to load the project, if false creates a new one. Defaults to False. 61 undo_limit (int, optional): How many items can be undone (commands are stored in memory) . Defaults to 100. 62 NB: Having mutliple instances running on the same project will produce errors 63 """ 64 assert os.path.isabs(dir_path), ( 65 "The Path must be absolute for project directories" 66 ) 67 self.project_path = dir_path 68 self.commands = deque([], undo_limit) 69 self.command_limit = undo_limit 70 if load: 71 self.load_project() 72 else: 73 assert len(os.listdir(dir_path)) == 0, ( 74 "Project folder must be empty for init" 75 ) # Warn instead? 76 self.init_project_structure() 77 self.api_thread_pool = concurrent.futures.ThreadPoolExecutor( 78 max_workers=max_workers 79 )
Create a project manager instance, designed to be sustained in memory while in use Args: dir_path (str): The absolute path of the root working directory load (bool, optional): If true attempts to load the project, if false creates a new one. Defaults to False. undo_limit (int, optional): How many items can be undone (commands are stored in memory) . Defaults to 100. NB: Having mutliple instances running on the same project will produce errors
81 def copy_gen_templates(self, template_dir: str = None): 82 """Copies a generation template (the directory and files at the location to the project path) 83 NB: only files with _template within the name are copied, only folders with template files within them will be made 84 Args: 85 template_dir (str, optional): Where to copy the directory if None uses the default embedded folder . Defaults to None. 86 """ 87 if template_dir is None: 88 template_dir = _TEMPLATES_DIR 89 90 for dir_path, _, fns in os.walk(template_dir): 91 for fn in fns: 92 if "template" not in fn or ( 93 fn.startswith("_") 94 and not fn.startswith( 95 "_template" 96 ) # .param type kind of problem, _template.param 97 ): 98 continue 99 rel_dir = os.path.relpath(dir_path, template_dir) 100 if rel_dir == "[root]" or rel_dir == ".": 101 copy_dest = self.project_path 102 else: 103 copy_dest = os.path.join(self.project_path, rel_dir) 104 os.makedirs(copy_dest, exist_ok=True) 105 106 copy_fn = fn.replace("_template", "") 107 copy_file(os.path.join(dir_path, fn), os.path.join(copy_dest, copy_fn)) 108 logger.info("Template generation completed")
Copies a generation template (the directory and files at the location to the project path) NB: only files with _template within the name are copied, only folders with template files within them will be made Args: template_dir (str, optional): Where to copy the directory if None uses the default embedded folder . Defaults to None.
110 def init_project_structure(self): 111 """Initiates the project structure, automatically called if load false on object inits""" 112 logger.info(f"Generating Project files at: {self.project_path}") 113 with chdir(self.project_path): 114 for folder in FOLDER_STRUCTURE: 115 os.makedirs(folder, exist_ok=True) 116 for file in DEFAULT_FILES: 117 with open(file, "w", encoding="utf-8") as f_obj: 118 if file.endswith(".json"): 119 f_obj.write("{}") 120 logger.info( 121 "Basic folders and empty files completed, copying premade templates" 122 ) 123 self.copy_gen_templates() 124 self.cd = ComponentDispatcher() 125 self.ecs = ECS_Register("registers", dispatcher=self.cd) 126 self.graphs = { 127 "Data": GraphManager(name="Data"), 128 "Model": GraphManager(name="Model"), 129 "Analysis": GraphManager(name="Analysis"), 130 } 131 self.executors = {} 132 self.bp_reg = BlueprintRegister() 133 self.bp_reg.add_from_folder("blueprints") 134 self.param_manager = ParametersManager(self.project_path) 135 self.commandables = { 136 "cd": self.cd, 137 "ecs": self.ecs, 138 "param_manager": self.param_manager, 139 "bp_reg": self.bp_reg, 140 "graphs": self.graphs, 141 "executors": self.executors, 142 } 143 self.save_project() # this generates all the config files automatically
Initiates the project structure, automatically called if load false on object inits
145 def add_graph(self, name: str): 146 """Adds a graph to the project 147 Args: 148 name (str): The name of the graph given. 149 """ 150 assert isinstance(name, str), "Invalid Graph Name, must be a string" 151 assert len(name) > 0, "Invalid graph Name, must be at least one character" 152 assert self.graphs.get(name, None) is None, "Graph Name Already exists" 153 self.graphs[name] = GraphManager(name=name, dir="exec_graphs", load=False) 154 logger.info(f"Graph '{name} was added to the project")
Adds a graph to the project Args: name (str): The name of the graph given.
156 def rename_graph(self, source_name: str, new_name: str): 157 """Renames a graph 158 (a bit heavy but gets the job done!) 159 Args: 160 source_name (str): the name of original graph 161 new_name (str): the new name of the graph 162 """ 163 self.duplicate_graph(source_name, new_name) 164 self.delete_graph(source_name) 165 logger.info(f"Graph {source_name} was renamed to {new_name}")
Renames a graph (a bit heavy but gets the job done!) Args: source_name (str): the name of original graph new_name (str): the new name of the graph
167 def delete_graph(self, name: str): 168 """Deletes the given graph 169 Args: 170 name (str): The name of the graph to be deleted 171 """ 172 assert isinstance(name, str), "Invalid Graph Name, must be a string" 173 graph = self.graphs.pop(name, None) 174 self.executors = { 175 k: v for k, v in self.executors.items() if v["graph_name"] != name 176 } 177 if graph: 178 path = graph.get_file_path(graph.name, graph.dir) 179 with chdir(self.project_path): 180 if os.path.exists(path): 181 os.remove(path) 182 graph = None 183 logger.info( 184 f"Graph '{name}' was deleted along with it's configuration files" 185 ) 186 else: 187 logger.info(f"Attempted to delete graph '{name}' but no graph was found")
Deletes the given graph Args: name (str): The name of the graph to be deleted
189 def duplicate_graph(self, source_name: str, copy_name: str): 190 """Duplicates a graph 191 Args: 192 source_name (str): The name of the graph to be copied 193 copy_name (str): the new name of the graph clone 194 """ 195 assert isinstance(source_name, str) and isinstance(copy_name, str), ( 196 "Graph source and copy name must be string" 197 ) 198 assert len(source_name) > 0 and len(copy_name) > 0, ( 199 "Graph source and copy names must be 1 character or more" 200 ) 201 assert self.graphs.get(copy_name, None) is None, ( 202 f"Destination name, {copy_name} already exists" 203 ) 204 205 origin_graph = self.graphs.get(source_name, None) 206 assert origin_graph is not None, ( 207 f"Source Name ,{source_name} does not exist in current context" 208 ) 209 self.graphs[copy_name] = origin_graph.duplicate(copy_name) 210 logger.info(f"'{source_name}' graph sucessfully duplicated as '{copy_name}'")
Duplicates a graph Args: source_name (str): The name of the graph to be copied copy_name (str): the new name of the graph clone
212 def load_project(self): 213 """Loads the project (done automaticaly when project manager initiated)""" 214 logger.info(f"Loading project: {self.project_path}") 215 try: 216 with chdir(self.project_path): 217 self.cd = ComponentDispatcher() 218 self.ecs = ECS_Register("registers", dispatcher=self.cd) 219 self.param_manager = ParametersManager(self.project_path) 220 self.bp_reg = BlueprintRegister() 221 222 self.graphs = {} 223 for dir_path, _, fns in os.walk("exec_graphs"): 224 for fn in fns: 225 if not fn.startswith("gm_") or not fn.endswith(".json"): 226 continue 227 f_p = os.path.join(dir_path, fn) 228 new_graph = GraphManager(name=None, load_path=f_p, load=True) 229 self.graphs[new_graph.name] = new_graph 230 self.executors = {} 231 232 with open("configs/executors_dict.json", encoding="utf-8") as f: 233 j_data = json.load(f) 234 self.executors = j_data if j_data else {} 235 236 self.commandables = { 237 "cd": self.cd, 238 "ecs": self.ecs, 239 "param_manager": self.param_manager, 240 "bp_reg": self.bp_reg, 241 "graphs": self.graphs, 242 "executors": self.executors, 243 } 244 self.bp_reg.add_from_folder("blueprints") 245 except Exception as e: 246 logger.error(f"Project loading failed:\n{e}") 247 raise e 248 logger.info(f"Project loaded sucessfully ({self.project_path})")
Loads the project (done automaticaly when project manager initiated)
250 def save_project(self): 251 """Saves the whole project in it's current state""" 252 with chdir(self.project_path): 253 self.ecs._save_register() 254 self.param_manager.save_node_params() 255 self.bp_reg.save_meta_file() 256 for g in self.graphs.values(): 257 g.save() 258 with open("configs/executors_dict.json", "w", encoding="utf-8") as f: 259 json.dump(self.executors, f) 260 logger.info(f"Project saved: {self.project_path}")
Saves the whole project in it's current state
262 @staticmethod 263 def is_valid_command_dict(params: dict): 264 """Returns true if the command dictionary given is valid""" 265 type_set = { 266 "target_obj": str, 267 "do_params": dict, 268 "undo_params": dict, 269 "target_func": str, 270 "undo_func": str, 271 } 272 is_valid = all(p in type_set.keys() for p in params.keys()) 273 is_valid &= all(type(v) is type_set.get(k, None) for k, v in params.items()) 274 is_valid &= ( 275 params.get("target_obj", None) is not None 276 and params.get("target_func", None) is not None 277 ) 278 return is_valid
Returns true if the command dictionary given is valid
280 def execute_command(self, **params): 281 """Executes a command to a commandable target (Any manager or Register) 282 Args: 283 target_obj (str): The commandable's name (full name e.g. 'graphs.graph-name') or pm/self for project manager 284 target_func (str): The name of callable to be executed within the class 285 do_params(dict[str,Any]): The parameters needed to execute the command parsed via **params syntax. 286 undo_func(str, optional): The name of the function to execute an undoing of the original command 287 undo_params(dict[str, any]): The parameters needed to execute an undoing parsed via **params syntax. 288 Returns: 289 Any : any values resulted from the given command (if available else None) 290 """ 291 try: 292 assert self.is_valid_command_dict(params), ( 293 "Execute params failed validation" 294 ) 295 target_obj = params.get("target_obj", None) 296 if target_obj.lower() in ["self", "pm", "projectmanager"]: 297 commandable = self 298 target_obj = "self" # project manager is also a commandable 299 else: 300 commandable = self.commandables 301 if "." in target_obj: 302 embeds = target_obj.split(".") 303 commandable = self.commandables 304 for embed in embeds: 305 commandable = commandable[embed] 306 else: 307 commandable = commandable[target_obj] 308 309 cm_t = type(commandable) 310 assert cm_t in _ALLOWED_ACCESS, ( 311 f"Attempted to access item that is not commandable: {cm_t}" 312 ) 313 314 target_func = params["target_func"] 315 assert not target_func.startswith("_"), ( 316 f"Attempted to access a private method: {target_func}" 317 ) 318 319 assert target_func in cm_t.__dict__, ( 320 f"Attempted to acces a function not implemented on the child class: {target_func}" 321 ) 322 func = getattr(commandable, target_func) 323 assert callable(func), f"Designated attribute is not callable {target_func}" 324 325 func_result = None 326 do_params = params.get("do_params", {}) 327 # don't log getters, a few places can use them in loops! 328 is_logworthy = not target_func.startswith("get") 329 330 if is_logworthy: 331 logger.info( 332 f"Performing {target_obj}.{target_func}() with params: [|] {do_params} [|]" 333 ) 334 undo_func = params.get("undo_func", None) 335 336 if undo_func: 337 undo_func_name = params.get("undo_func") 338 assert callable(getattr(commandable, undo_func_name, None)), ( 339 f"Undo function is not a callable: {params['undo_func']}, command aborted" 340 ) 341 assert not undo_func_name.startswith("_"), ( 342 "Undo function supplied is private, command aborted" 343 ) 344 command_log = dict(params) 345 command_log.pop("do_params") # delete do_params to avoid memory leak 346 self.commands.append(command_log) 347 if is_logworthy: 348 logger.debug( 349 f"Undo detected and added for {target_obj}.{target_func}-> {params['undo_func']}" 350 ) 351 if len(self.commands) > self.command_limit: 352 self.commands.popleft() 353 elif is_logworthy: 354 logger.debug(f"No Undo method detected for {target_obj}.{target_func}") 355 356 with chdir(self.project_path): 357 func_result = func(**do_params) 358 if is_logworthy: 359 logger.info("Command Executed Sucessfully") 360 except Exception as e: 361 logger.error(f"Command failed for {target_obj}->{target_func}(): {e} ") 362 return None 363 364 return func_result
Executes a command to a commandable target (Any manager or Register) Args: target_obj (str): The commandable's name (full name e.g. 'graphs.graph-name') or pm/self for project manager target_func (str): The name of callable to be executed within the class do_params(dict[str,Any]): The parameters needed to execute the command parsed via **params syntax. undo_func(str, optional): The name of the function to execute an undoing of the original command undo_params(dict[str, any]): The parameters needed to execute an undoing parsed via **params syntax. Returns: Any : any values resulted from the given command (if available else None)
366 def undo_last_command(self): 367 """undoes the most recent command if there are any""" 368 if not self.commands: 369 logger.info("Tried to undo but No Undo Commands available") 370 return 371 372 last_cmd = self.commands.pop() 373 commandable_name = last_cmd["target_obj"] 374 if commandable_name == "self": 375 commandable = self 376 else: 377 commandable = self.commandables 378 if "." in commandable_name: 379 embeds = commandable_name.split(".") 380 commandable = self.commandables 381 for embed in embeds: 382 commandable = commandable[embed] 383 else: 384 commandable = commandable[commandable_name] 385 386 undo_func_name = last_cmd["undo_func"] 387 undo_func = getattr(commandable, undo_func_name) 388 logger.info( 389 f"Undoing {'pm' if commandable_name == 'self' else commandable_name}.{last_cmd['target_func']} USING {undo_func_name}()" 390 ) 391 try: 392 with chdir(self.project_path): 393 undo_func(**last_cmd["undo_params"]) 394 except Exception as e: 395 logger.error( 396 f"Command failed for {'pm' if commandable_name == 'self' else commandable_name}->{undo_func_name}(): {e} " 397 )
undoes the most recent command if there are any
399 def create_executor( 400 self, 401 graph_name: str, 402 exec_lbl: str, 403 executor_type_name: str, 404 start_node_id="start", 405 ctx_build_kwargs: dict[str, Any] = None, 406 ): 407 """Creates an execution context within the project manager 408 Args: 409 graph_name (str): The name of the graph to be create the execution from 410 exec_lbl (str): The given name of the execution to identify. 411 executor_type_name (str): The name of the Executor to be used 412 start_node_id (str, optional): The node from which execution begins at. Defaults to "start". 413 ctx_build_kwargs (dict[str, Any], optional): .Defaults to None. 414 """ 415 assert self.graphs.get(graph_name, None) is not None, "graph does not exist" 416 assert self.executors.get(exec_lbl, None) is None, ( 417 "Execution with the same label already exists" 418 ) 419 if ctx_build_kwargs is None: 420 ctx_build_kwargs = {"param_cascade_names": ["params.global"]} 421 422 self.executors[exec_lbl] = { 423 "exec_func_type": executor_type_name, 424 "graph_name": graph_name, 425 "context_args": ctx_build_kwargs, 426 "start_node": start_node_id, 427 }
Creates an execution context within the project manager Args: graph_name (str): The name of the graph to be create the execution from exec_lbl (str): The given name of the execution to identify. executor_type_name (str): The name of the Executor to be used start_node_id (str, optional): The node from which execution begins at. Defaults to "start". ctx_build_kwargs (dict[str, Any], optional): .Defaults to None.
429 def run_executor(self, executor_lbl: str, run_async: bool = True): 430 """Simple wrapper to allow for async using conccurrent.futures""" 431 logger.info( 432 f"Saving project to {self.project_path} before execution (and refreshing blueprints!)" 433 ) 434 with chdir(self.project_path): 435 self.save_project() 436 self.bp_reg.refresh_blueprints() 437 logger.info( 438 f"Execution starting {executor_lbl} {'Asychronously' if run_async else 'Synchronously \nWARNING! This will likely block the user interface!'} " 439 ) 440 if run_async: 441 return self.api_thread_pool.submit(self._execute_graph, executor_lbl) 442 else: 443 return self._execute_graph(executor_lbl)
Simple wrapper to allow for async using conccurrent.futures