m_strat.api.project_management.project_manager

  1import concurrent.futures
  2import json
  3import os
  4from collections import deque
  5from contextlib import chdir
  6from shutil import copy as copy_file
  7from typing import Any
  8
  9from loguru import logger
 10
 11from api.dag import BlueprintRegister, GraphManager
 12from api.dag.executor import _EXECUTORS
 13from api.ecs import ComponentDispatcher, ECS_Register
 14
 15from .parameters_manager import ParametersManager
 16
 17FOLDER_STRUCTURE = [
 18    "datasets",
 19    "registers",
 20    "blueprints",
 21    "executions",
 22    "params",
 23    "configs",
 24    "exec_graphs",
 25    "results",
 26]
 27
 28DEFAULT_FILES = [
 29    "project.yaml",
 30    "readme.md",
 31    "params/global.yaml",
 32    "params/ignores.yaml",
 33    "configs/bp_files_meta.jsonl",
 34    "configs/executors_dict.json",
 35]
 36
 37_ALLOWED_ACCESS = {
 38    ComponentDispatcher,
 39    ECS_Register,
 40    BlueprintRegister,
 41    GraphManager,
 42    ParametersManager,
 43}
 44
 45_TEMPLATES_DIR = os.path.join(
 46    os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "templates"
 47)
 48
 49
 50class ProjectManager:
 51    """The Main enty point for m-strat, used to manage the whole m-strat project"""
 52
 53    def __init__(
 54        self, dir_path: str, load: bool = False, undo_limit: int = 100, max_workers=4
 55    ):
 56        """Create a project manager instance, designed to be sustained in memory while in use
 57        Args:
 58            dir_path (str): The absolute path of the root working directory
 59            load (bool, optional): If true attempts to load the project, if false creates a new one. Defaults to False.
 60            undo_limit (int, optional): How many items can be undone (commands are stored in memory) . Defaults to 100.
 61        NB: Having mutliple instances running on the same project will produce errors
 62        """
 63        assert os.path.isabs(dir_path), (
 64            "The Path must be absolute for project directories"
 65        )
 66        self.project_path = dir_path
 67        self.commands = deque([], undo_limit)
 68        self.command_limit = undo_limit
 69        if load:
 70            self.load_project()
 71        else:
 72            assert len(os.listdir(dir_path)) == 0, (
 73                "Project folder must be empty for init"
 74            )  # Warn instead?
 75            self.init_project_structure()
 76        self.api_thread_pool = concurrent.futures.ThreadPoolExecutor(
 77            max_workers=max_workers
 78        )
 79
 80    def copy_gen_templates(self, template_dir: str = None):
 81        """Copies a generation template (the directory and files at the location to the project path)
 82        NB: only files with _template within the name are copied, only folders with template files within them will be made
 83        Args:
 84            template_dir (str, optional): Where to copy the directory if None uses the default embedded folder . Defaults to None.
 85        """
 86        if template_dir is None:
 87            template_dir = _TEMPLATES_DIR
 88
 89        for dir_path, _, fns in os.walk(template_dir):
 90            for fn in fns:
 91                if "template" not in fn or (
 92                    fn.startswith("_")
 93                    and not fn.startswith(
 94                        "_template"
 95                    )  # .param type kind of problem, _template.param
 96                ):
 97                    continue
 98                rel_dir = os.path.relpath(dir_path, template_dir)
 99                if rel_dir == "[root]" or rel_dir == ".":
100                    copy_dest = self.project_path
101                else:
102                    copy_dest = os.path.join(self.project_path, rel_dir)
103                os.makedirs(copy_dest, exist_ok=True)
104
105                copy_fn = fn.replace("_template", "")
106                copy_file(os.path.join(dir_path, fn), os.path.join(copy_dest, copy_fn))
107        logger.info("Template generation completed")
108
109    def init_project_structure(self):
110        """Initiates the project structure, automatically called if load false on object inits"""
111        logger.info(f"Generating Project files at: {self.project_path}")
112        with chdir(self.project_path):
113            for folder in FOLDER_STRUCTURE:
114                os.makedirs(folder, exist_ok=True)
115            for file in DEFAULT_FILES:
116                with open(file, "w", encoding="utf-8") as f_obj:
117                    if file.endswith(".json"):
118                        f_obj.write("{}")
119            logger.info(
120                "Basic folders and empty files completed, copying premade templates"
121            )
122            self.copy_gen_templates()
123            self.cd = ComponentDispatcher()
124            self.ecs = ECS_Register("registers", dispatcher=self.cd)
125            self.graphs = {
126                "Data": GraphManager(name="Data"),
127                "Model": GraphManager(name="Model"),
128                "Analysis": GraphManager(name="Analysis"),
129            }
130            self.executors = {}
131            self.bp_reg = BlueprintRegister()
132            self.bp_reg.add_from_folder("blueprints")
133            self.param_manager = ParametersManager(self.project_path)
134            self.commandables = {
135                "cd": self.cd,
136                "ecs": self.ecs,
137                "param_manager": self.param_manager,
138                "bp_reg": self.bp_reg,
139                "graphs": self.graphs,
140                "executors": self.executors,
141            }
142        self.save_project()  # this generates all the config files automatically
143
144    def add_graph(self, name: str):
145        """Adds a graph to the project
146        Args:
147            name (str): The name of the graph given.
148        """
149        assert isinstance(name, str), "Invalid Graph Name, must be a string"
150        assert len(name) > 0, "Invalid graph Name, must be at least one character"
151        assert self.graphs.get(name, None) is None, "Graph Name Already exists"
152        self.graphs[name] = GraphManager(name=name, dir="exec_graphs", load=False)
153        logger.info(f"Graph '{name} was added to the project")
154
155    def rename_graph(self, source_name: str, new_name: str):
156        """Renames a graph
157        (a bit heavy but gets the job done!)
158        Args:
159            source_name (str): the name of original graph
160            new_name (str): the new name of the graph
161        """
162        self.duplicate_graph(source_name, new_name)
163        self.delete_graph(source_name)
164        logger.info(f"Graph {source_name} was renamed to {new_name}")
165
166    def delete_graph(self, name: str):
167        """Deletes the given graph
168        Args:
169            name (str): The name of the graph to be deleted
170        """
171        assert isinstance(name, str), "Invalid Graph Name, must be a string"
172        graph = self.graphs.pop(name, None)
173        self.executors = {
174            k: v for k, v in self.executors.items() if v["graph_name"] != name
175        }
176        if graph:
177            path = graph.get_file_path(graph.name, graph.dir)
178            with chdir(self.project_path):
179                if os.path.exists(path):
180                    os.remove(path)
181            graph = None
182            logger.info(
183                f"Graph '{name}' was deleted along with it's configuration files"
184            )
185        else:
186            logger.info(f"Attempted to delete graph '{name}' but no graph was found")
187
188    def duplicate_graph(self, source_name: str, copy_name: str):
189        """Duplicates a graph
190        Args:
191            source_name (str): The name of the graph to be copied
192            copy_name (str): the new name of the graph clone
193        """
194        assert isinstance(source_name, str) and isinstance(copy_name, str), (
195            "Graph source and copy name must be string"
196        )
197        assert len(source_name) > 0 and len(copy_name) > 0, (
198            "Graph source and copy names must be 1 character or more"
199        )
200        assert self.graphs.get(copy_name, None) is None, (
201            f"Destination name, {copy_name} already exists"
202        )
203
204        origin_graph = self.graphs.get(source_name, None)
205        assert origin_graph is not None, (
206            f"Source Name ,{source_name} does not exist in current context"
207        )
208        self.graphs[copy_name] = origin_graph.duplicate(copy_name)
209        logger.info(f"'{source_name}' graph sucessfully duplicated as '{copy_name}'")
210
211    def load_project(self):
212        """Loads the project (done automaticaly when project manager initiated)"""
213        logger.info(f"Loading project: {self.project_path}")
214        try:
215            with chdir(self.project_path):
216                self.cd = ComponentDispatcher()
217                self.ecs = ECS_Register("registers", dispatcher=self.cd)
218                self.param_manager = ParametersManager(self.project_path)
219                self.bp_reg = BlueprintRegister()
220
221                self.graphs = {}
222                for dir_path, _, fns in os.walk("exec_graphs"):
223                    for fn in fns:
224                        if not fn.startswith("gm_") or not fn.endswith(".json"):
225                            continue
226                        f_p = os.path.join(dir_path, fn)
227                        new_graph = GraphManager(name=None, load_path=f_p, load=True)
228                        self.graphs[new_graph.name] = new_graph
229                self.executors = {}
230
231                with open("configs/executors_dict.json", encoding="utf-8") as f:
232                    j_data = json.load(f)
233                    self.executors = j_data if j_data else {}
234
235                self.commandables = {
236                    "cd": self.cd,
237                    "ecs": self.ecs,
238                    "param_manager": self.param_manager,
239                    "bp_reg": self.bp_reg,
240                    "graphs": self.graphs,
241                    "executors": self.executors,
242                }
243                self.bp_reg.add_from_folder("blueprints")
244        except Exception as e:
245            logger.error(f"Project loading failed:\n{e}")
246            raise e
247        logger.info(f"Project loaded sucessfully ({self.project_path})")
248
249    def save_project(self):
250        """Saves the whole project in it's current state"""
251        with chdir(self.project_path):
252            self.ecs._save_register()
253            self.param_manager.save_node_params()
254            self.bp_reg.save_meta_file()
255            for g in self.graphs.values():
256                g.save()
257            with open("configs/executors_dict.json", "w", encoding="utf-8") as f:
258                json.dump(self.executors, f)
259        logger.info(f"Project saved: {self.project_path}")
260
261    @staticmethod
262    def is_valid_command_dict(params: dict):
263        """Returns true if the command dictionary given  is valid"""
264        type_set = {
265            "target_obj": str,
266            "do_params": dict,
267            "undo_params": dict,
268            "target_func": str,
269            "undo_func": str,
270        }
271        is_valid = all(p in type_set.keys() for p in params.keys())
272        is_valid &= all(type(v) is type_set.get(k, None) for k, v in params.items())
273        is_valid &= (
274            params.get("target_obj", None) is not None
275            and params.get("target_func", None) is not None
276        )
277        return is_valid
278
279    def execute_command(self, **params):
280        """Executes a command to a commandable target (Any manager or Register)
281        Args:
282            target_obj (str): The commandable's name (full name e.g. 'graphs.graph-name') or pm/self for project manager
283            target_func (str): The name of callable to be executed within the class
284            do_params(dict[str,Any]): The parameters needed to execute the command parsed via **params syntax.
285            undo_func(str, optional): The name of the function to execute an undoing of the original command
286            undo_params(dict[str, any]): The parameters needed to execute an undoing parsed via **params syntax.
287        Returns:
288            Any : any values resulted from the given command (if available else None)
289        """
290        try:
291            assert self.is_valid_command_dict(params), (
292                "Execute params failed validation"
293            )
294            target_obj = params.get("target_obj", None)
295            if target_obj.lower() in ["self", "pm", "projectmanager"]:
296                commandable = self
297                target_obj = "self"  # project manager is also a commandable
298            else:
299                commandable = self.commandables
300                if "." in target_obj:
301                    embeds = target_obj.split(".")
302                    commandable = self.commandables
303                    for embed in embeds:
304                        commandable = commandable[embed]
305                else:
306                    commandable = commandable[target_obj]
307
308            cm_t = type(commandable)
309            assert cm_t in _ALLOWED_ACCESS, (
310                f"Attempted to access item that is not commandable: {cm_t}"
311            )
312
313            target_func = params["target_func"]
314            assert not target_func.startswith("_"), (
315                f"Attempted to access a private method: {target_func}"
316            )
317
318            assert target_func in cm_t.__dict__, (
319                f"Attempted to acces a function not implemented on the child class: {target_func}"
320            )
321            func = getattr(commandable, target_func)
322            assert callable(func), f"Designated attribute is not callable {target_func}"
323
324            func_result = None
325            do_params = params.get("do_params", {})
326            # don't log getters, a few places can use them in loops!
327            is_logworthy = not target_func.startswith("get")
328
329            if is_logworthy:
330                logger.info(
331                    f"Performing {target_obj}.{target_func}() with params: [|]  {do_params}  [|]"
332                )
333            undo_func = params.get("undo_func", None)
334
335            if undo_func:
336                undo_func_name = params.get("undo_func")
337                assert callable(getattr(commandable, undo_func_name, None)), (
338                    f"Undo function is not a callable: {params['undo_func']}, command aborted"
339                )
340                assert not undo_func_name.startswith("_"), (
341                    "Undo function supplied is private, command aborted"
342                )
343                command_log = dict(params)
344                command_log.pop("do_params")  # delete do_params to avoid memory leak
345                self.commands.append(command_log)
346                if is_logworthy:
347                    logger.debug(
348                        f"Undo detected and added for {target_obj}.{target_func}-> {params['undo_func']}"
349                    )
350                if len(self.commands) > self.command_limit:
351                    self.commands.popleft()
352            elif is_logworthy:
353                logger.debug(f"No Undo method detected for {target_obj}.{target_func}")
354
355            with chdir(self.project_path):
356                func_result = func(**do_params)
357            if is_logworthy:
358                logger.info("Command Executed Sucessfully")
359        except Exception as e:
360            logger.error(f"Command failed for {target_obj}->{target_func}(): {e} ")
361            return None
362
363        return func_result
364
365    def undo_last_command(self):
366        """undoes the most recent command if there are any"""
367        if not self.commands:
368            logger.info("Tried to undo but No Undo Commands available")
369            return
370
371        last_cmd = self.commands.pop()
372        commandable_name = last_cmd["target_obj"]
373        if commandable_name == "self":
374            commandable = self
375        else:
376            commandable = self.commandables
377            if "." in commandable_name:
378                embeds = commandable_name.split(".")
379                commandable = self.commandables
380                for embed in embeds:
381                    commandable = commandable[embed]
382            else:
383                commandable = commandable[commandable_name]
384
385        undo_func_name = last_cmd["undo_func"]
386        undo_func = getattr(commandable, undo_func_name)
387        logger.info(
388            f"Undoing {'pm' if commandable_name == 'self' else commandable_name}.{last_cmd['target_func']} USING {undo_func_name}()"
389        )
390        try:
391            with chdir(self.project_path):
392                undo_func(**last_cmd["undo_params"])
393        except Exception as e:
394            logger.error(
395                f"Command failed for {'pm' if commandable_name == 'self' else commandable_name}->{undo_func_name}(): {e} "
396            )
397
398    def create_executor(
399        self,
400        graph_name: str,
401        exec_lbl: str,
402        executor_type_name: str,
403        start_node_id="start",
404        ctx_build_kwargs: dict[str, Any] = None,
405    ):
406        """Creates an execution context within the project manager
407        Args:
408            graph_name (str): The name of the graph to be create the execution from
409            exec_lbl (str): The given name of the execution to identify.
410            executor_type_name (str): The name of the Executor to be used
411            start_node_id (str, optional): The node from which execution begins at. Defaults to "start".
412            ctx_build_kwargs (dict[str, Any], optional): .Defaults to None.
413        """
414        assert self.graphs.get(graph_name, None) is not None, "graph does not exist"
415        assert self.executors.get(exec_lbl, None) is None, (
416            "Execution with the same label already exists"
417        )
418        if ctx_build_kwargs is None:
419            ctx_build_kwargs = {"param_cascade_names": ["params.global"]}
420
421        self.executors[exec_lbl] = {
422            "exec_func_type": executor_type_name,
423            "graph_name": graph_name,
424            "context_args": ctx_build_kwargs,
425            "start_node": start_node_id,
426        }
427
428    def run_executor(self, executor_lbl: str, run_async: bool = True):
429        """Simple wrapper to allow for async using conccurrent.futures"""
430        logger.info(
431            f"Saving project to {self.project_path} before execution (and refreshing blueprints!)"
432        )
433        with chdir(self.project_path):
434            self.save_project()
435            self.bp_reg.refresh_blueprints()
436        logger.info(
437            f"Execution starting {executor_lbl} {'Asychronously' if run_async else 'Synchronously \nWARNING! This will likely block the user interface!'} "
438        )
439        if run_async:
440            return self.api_thread_pool.submit(self._execute_graph, executor_lbl)
441        else:
442            return self._execute_graph(executor_lbl)
443
444    def _execute_graph(self, executor_lbl: str):
445        """Runs the executor
446        Args:
447            executor_lbl (str): the name of the executor to be used
448        Returns:
449            dict[str, Any]: returns the final context generated from the execution
450        Raises:
451            ValueError: If anything required for execution (label, executor type, target graph) is not found
452        """
453        execution_target = None
454        try:
455            executor_info = self.executors.get(executor_lbl, None)
456            assert bool(executor_info), (
457                f"Did not find executor labelled {executor_lbl} within project, cannot run!"
458            )
459
460            execution_target = _EXECUTORS.get(executor_info["exec_func_type"], None)
461            assert bool(execution_target), (
462                f"Did not find executor of type {execution_target} within project, cannot run!"
463            )
464
465            target_graph = self.graphs.get(executor_info["graph_name"], None)
466            assert bool(target_graph), (
467                f"Target Graph not Found: {target_graph}, cannot run!"
468            )
469
470            execution_graph = target_graph.build_execution(
471                executor_info.get("start_node", "start")
472            )
473            logger.debug(
474                f"Execution graph built {'->'.join([n.id for n in execution_graph])}"
475            )
476            exec_targets = []
477
478            for node in execution_graph:
479                if node.id == "start":
480                    continue
481                bp = self.bp_reg.get_blueprint(node.blueprint_name)
482                node_param = self.param_manager.get_node_params(node.param_id)
483                exec_targets.append((bp, node_param, node))
484            ctx_args = executor_info["context_args"]
485            init_ctx = (
486                self.param_manager.build_param_context(**ctx_args, as_omega_conf=False)
487                if ctx_args
488                else {}
489            )
490            logger.debug(f"Context built:\n {init_ctx}")
491            logger.info("Execution Starting...")
492            with chdir(self.project_path):
493                out_ctx = execution_target(exec_targets, init_ctx)
494            logger.info("Execution Finished")
495        except Exception as e:
496            logger.error(
497                f"Execution of {executor_lbl} using {execution_target} failed:\n {e}"
498            )
499            return {}
500
501        return out_ctx
502
503
504_ALLOWED_ACCESS.add(ProjectManager)
FOLDER_STRUCTURE = ['datasets', 'registers', 'blueprints', 'executions', 'params', 'configs', 'exec_graphs', 'results']
DEFAULT_FILES = ['project.yaml', 'readme.md', 'params/global.yaml', 'params/ignores.yaml', 'configs/bp_files_meta.jsonl', 'configs/executors_dict.json']
class ProjectManager:
 51class ProjectManager:
 52    """The Main enty point for m-strat, used to manage the whole m-strat project"""
 53
 54    def __init__(
 55        self, dir_path: str, load: bool = False, undo_limit: int = 100, max_workers=4
 56    ):
 57        """Create a project manager instance, designed to be sustained in memory while in use
 58        Args:
 59            dir_path (str): The absolute path of the root working directory
 60            load (bool, optional): If true attempts to load the project, if false creates a new one. Defaults to False.
 61            undo_limit (int, optional): How many items can be undone (commands are stored in memory) . Defaults to 100.
 62        NB: Having mutliple instances running on the same project will produce errors
 63        """
 64        assert os.path.isabs(dir_path), (
 65            "The Path must be absolute for project directories"
 66        )
 67        self.project_path = dir_path
 68        self.commands = deque([], undo_limit)
 69        self.command_limit = undo_limit
 70        if load:
 71            self.load_project()
 72        else:
 73            assert len(os.listdir(dir_path)) == 0, (
 74                "Project folder must be empty for init"
 75            )  # Warn instead?
 76            self.init_project_structure()
 77        self.api_thread_pool = concurrent.futures.ThreadPoolExecutor(
 78            max_workers=max_workers
 79        )
 80
 81    def copy_gen_templates(self, template_dir: str = None):
 82        """Copies a generation template (the directory and files at the location to the project path)
 83        NB: only files with _template within the name are copied, only folders with template files within them will be made
 84        Args:
 85            template_dir (str, optional): Where to copy the directory if None uses the default embedded folder . Defaults to None.
 86        """
 87        if template_dir is None:
 88            template_dir = _TEMPLATES_DIR
 89
 90        for dir_path, _, fns in os.walk(template_dir):
 91            for fn in fns:
 92                if "template" not in fn or (
 93                    fn.startswith("_")
 94                    and not fn.startswith(
 95                        "_template"
 96                    )  # .param type kind of problem, _template.param
 97                ):
 98                    continue
 99                rel_dir = os.path.relpath(dir_path, template_dir)
100                if rel_dir == "[root]" or rel_dir == ".":
101                    copy_dest = self.project_path
102                else:
103                    copy_dest = os.path.join(self.project_path, rel_dir)
104                os.makedirs(copy_dest, exist_ok=True)
105
106                copy_fn = fn.replace("_template", "")
107                copy_file(os.path.join(dir_path, fn), os.path.join(copy_dest, copy_fn))
108        logger.info("Template generation completed")
109
110    def init_project_structure(self):
111        """Initiates the project structure, automatically called if load false on object inits"""
112        logger.info(f"Generating Project files at: {self.project_path}")
113        with chdir(self.project_path):
114            for folder in FOLDER_STRUCTURE:
115                os.makedirs(folder, exist_ok=True)
116            for file in DEFAULT_FILES:
117                with open(file, "w", encoding="utf-8") as f_obj:
118                    if file.endswith(".json"):
119                        f_obj.write("{}")
120            logger.info(
121                "Basic folders and empty files completed, copying premade templates"
122            )
123            self.copy_gen_templates()
124            self.cd = ComponentDispatcher()
125            self.ecs = ECS_Register("registers", dispatcher=self.cd)
126            self.graphs = {
127                "Data": GraphManager(name="Data"),
128                "Model": GraphManager(name="Model"),
129                "Analysis": GraphManager(name="Analysis"),
130            }
131            self.executors = {}
132            self.bp_reg = BlueprintRegister()
133            self.bp_reg.add_from_folder("blueprints")
134            self.param_manager = ParametersManager(self.project_path)
135            self.commandables = {
136                "cd": self.cd,
137                "ecs": self.ecs,
138                "param_manager": self.param_manager,
139                "bp_reg": self.bp_reg,
140                "graphs": self.graphs,
141                "executors": self.executors,
142            }
143        self.save_project()  # this generates all the config files automatically
144
145    def add_graph(self, name: str):
146        """Adds a graph to the project
147        Args:
148            name (str): The name of the graph given.
149        """
150        assert isinstance(name, str), "Invalid Graph Name, must be a string"
151        assert len(name) > 0, "Invalid graph Name, must be at least one character"
152        assert self.graphs.get(name, None) is None, "Graph Name Already exists"
153        self.graphs[name] = GraphManager(name=name, dir="exec_graphs", load=False)
154        logger.info(f"Graph '{name} was added to the project")
155
156    def rename_graph(self, source_name: str, new_name: str):
157        """Renames a graph
158        (a bit heavy but gets the job done!)
159        Args:
160            source_name (str): the name of original graph
161            new_name (str): the new name of the graph
162        """
163        self.duplicate_graph(source_name, new_name)
164        self.delete_graph(source_name)
165        logger.info(f"Graph {source_name} was renamed to {new_name}")
166
167    def delete_graph(self, name: str):
168        """Deletes the given graph
169        Args:
170            name (str): The name of the graph to be deleted
171        """
172        assert isinstance(name, str), "Invalid Graph Name, must be a string"
173        graph = self.graphs.pop(name, None)
174        self.executors = {
175            k: v for k, v in self.executors.items() if v["graph_name"] != name
176        }
177        if graph:
178            path = graph.get_file_path(graph.name, graph.dir)
179            with chdir(self.project_path):
180                if os.path.exists(path):
181                    os.remove(path)
182            graph = None
183            logger.info(
184                f"Graph '{name}' was deleted along with it's configuration files"
185            )
186        else:
187            logger.info(f"Attempted to delete graph '{name}' but no graph was found")
188
189    def duplicate_graph(self, source_name: str, copy_name: str):
190        """Duplicates a graph
191        Args:
192            source_name (str): The name of the graph to be copied
193            copy_name (str): the new name of the graph clone
194        """
195        assert isinstance(source_name, str) and isinstance(copy_name, str), (
196            "Graph source and copy name must be string"
197        )
198        assert len(source_name) > 0 and len(copy_name) > 0, (
199            "Graph source and copy names must be 1 character or more"
200        )
201        assert self.graphs.get(copy_name, None) is None, (
202            f"Destination name, {copy_name} already exists"
203        )
204
205        origin_graph = self.graphs.get(source_name, None)
206        assert origin_graph is not None, (
207            f"Source Name ,{source_name} does not exist in current context"
208        )
209        self.graphs[copy_name] = origin_graph.duplicate(copy_name)
210        logger.info(f"'{source_name}' graph sucessfully duplicated as '{copy_name}'")
211
212    def load_project(self):
213        """Loads the project (done automaticaly when project manager initiated)"""
214        logger.info(f"Loading project: {self.project_path}")
215        try:
216            with chdir(self.project_path):
217                self.cd = ComponentDispatcher()
218                self.ecs = ECS_Register("registers", dispatcher=self.cd)
219                self.param_manager = ParametersManager(self.project_path)
220                self.bp_reg = BlueprintRegister()
221
222                self.graphs = {}
223                for dir_path, _, fns in os.walk("exec_graphs"):
224                    for fn in fns:
225                        if not fn.startswith("gm_") or not fn.endswith(".json"):
226                            continue
227                        f_p = os.path.join(dir_path, fn)
228                        new_graph = GraphManager(name=None, load_path=f_p, load=True)
229                        self.graphs[new_graph.name] = new_graph
230                self.executors = {}
231
232                with open("configs/executors_dict.json", encoding="utf-8") as f:
233                    j_data = json.load(f)
234                    self.executors = j_data if j_data else {}
235
236                self.commandables = {
237                    "cd": self.cd,
238                    "ecs": self.ecs,
239                    "param_manager": self.param_manager,
240                    "bp_reg": self.bp_reg,
241                    "graphs": self.graphs,
242                    "executors": self.executors,
243                }
244                self.bp_reg.add_from_folder("blueprints")
245        except Exception as e:
246            logger.error(f"Project loading failed:\n{e}")
247            raise e
248        logger.info(f"Project loaded sucessfully ({self.project_path})")
249
250    def save_project(self):
251        """Saves the whole project in it's current state"""
252        with chdir(self.project_path):
253            self.ecs._save_register()
254            self.param_manager.save_node_params()
255            self.bp_reg.save_meta_file()
256            for g in self.graphs.values():
257                g.save()
258            with open("configs/executors_dict.json", "w", encoding="utf-8") as f:
259                json.dump(self.executors, f)
260        logger.info(f"Project saved: {self.project_path}")
261
262    @staticmethod
263    def is_valid_command_dict(params: dict):
264        """Returns true if the command dictionary given  is valid"""
265        type_set = {
266            "target_obj": str,
267            "do_params": dict,
268            "undo_params": dict,
269            "target_func": str,
270            "undo_func": str,
271        }
272        is_valid = all(p in type_set.keys() for p in params.keys())
273        is_valid &= all(type(v) is type_set.get(k, None) for k, v in params.items())
274        is_valid &= (
275            params.get("target_obj", None) is not None
276            and params.get("target_func", None) is not None
277        )
278        return is_valid
279
280    def execute_command(self, **params):
281        """Executes a command to a commandable target (Any manager or Register)
282        Args:
283            target_obj (str): The commandable's name (full name e.g. 'graphs.graph-name') or pm/self for project manager
284            target_func (str): The name of callable to be executed within the class
285            do_params(dict[str,Any]): The parameters needed to execute the command parsed via **params syntax.
286            undo_func(str, optional): The name of the function to execute an undoing of the original command
287            undo_params(dict[str, any]): The parameters needed to execute an undoing parsed via **params syntax.
288        Returns:
289            Any : any values resulted from the given command (if available else None)
290        """
291        try:
292            assert self.is_valid_command_dict(params), (
293                "Execute params failed validation"
294            )
295            target_obj = params.get("target_obj", None)
296            if target_obj.lower() in ["self", "pm", "projectmanager"]:
297                commandable = self
298                target_obj = "self"  # project manager is also a commandable
299            else:
300                commandable = self.commandables
301                if "." in target_obj:
302                    embeds = target_obj.split(".")
303                    commandable = self.commandables
304                    for embed in embeds:
305                        commandable = commandable[embed]
306                else:
307                    commandable = commandable[target_obj]
308
309            cm_t = type(commandable)
310            assert cm_t in _ALLOWED_ACCESS, (
311                f"Attempted to access item that is not commandable: {cm_t}"
312            )
313
314            target_func = params["target_func"]
315            assert not target_func.startswith("_"), (
316                f"Attempted to access a private method: {target_func}"
317            )
318
319            assert target_func in cm_t.__dict__, (
320                f"Attempted to acces a function not implemented on the child class: {target_func}"
321            )
322            func = getattr(commandable, target_func)
323            assert callable(func), f"Designated attribute is not callable {target_func}"
324
325            func_result = None
326            do_params = params.get("do_params", {})
327            # don't log getters, a few places can use them in loops!
328            is_logworthy = not target_func.startswith("get")
329
330            if is_logworthy:
331                logger.info(
332                    f"Performing {target_obj}.{target_func}() with params: [|]  {do_params}  [|]"
333                )
334            undo_func = params.get("undo_func", None)
335
336            if undo_func:
337                undo_func_name = params.get("undo_func")
338                assert callable(getattr(commandable, undo_func_name, None)), (
339                    f"Undo function is not a callable: {params['undo_func']}, command aborted"
340                )
341                assert not undo_func_name.startswith("_"), (
342                    "Undo function supplied is private, command aborted"
343                )
344                command_log = dict(params)
345                command_log.pop("do_params")  # delete do_params to avoid memory leak
346                self.commands.append(command_log)
347                if is_logworthy:
348                    logger.debug(
349                        f"Undo detected and added for {target_obj}.{target_func}-> {params['undo_func']}"
350                    )
351                if len(self.commands) > self.command_limit:
352                    self.commands.popleft()
353            elif is_logworthy:
354                logger.debug(f"No Undo method detected for {target_obj}.{target_func}")
355
356            with chdir(self.project_path):
357                func_result = func(**do_params)
358            if is_logworthy:
359                logger.info("Command Executed Sucessfully")
360        except Exception as e:
361            logger.error(f"Command failed for {target_obj}->{target_func}(): {e} ")
362            return None
363
364        return func_result
365
366    def undo_last_command(self):
367        """undoes the most recent command if there are any"""
368        if not self.commands:
369            logger.info("Tried to undo but No Undo Commands available")
370            return
371
372        last_cmd = self.commands.pop()
373        commandable_name = last_cmd["target_obj"]
374        if commandable_name == "self":
375            commandable = self
376        else:
377            commandable = self.commandables
378            if "." in commandable_name:
379                embeds = commandable_name.split(".")
380                commandable = self.commandables
381                for embed in embeds:
382                    commandable = commandable[embed]
383            else:
384                commandable = commandable[commandable_name]
385
386        undo_func_name = last_cmd["undo_func"]
387        undo_func = getattr(commandable, undo_func_name)
388        logger.info(
389            f"Undoing {'pm' if commandable_name == 'self' else commandable_name}.{last_cmd['target_func']} USING {undo_func_name}()"
390        )
391        try:
392            with chdir(self.project_path):
393                undo_func(**last_cmd["undo_params"])
394        except Exception as e:
395            logger.error(
396                f"Command failed for {'pm' if commandable_name == 'self' else commandable_name}->{undo_func_name}(): {e} "
397            )
398
399    def create_executor(
400        self,
401        graph_name: str,
402        exec_lbl: str,
403        executor_type_name: str,
404        start_node_id="start",
405        ctx_build_kwargs: dict[str, Any] = None,
406    ):
407        """Creates an execution context within the project manager
408        Args:
409            graph_name (str): The name of the graph to be create the execution from
410            exec_lbl (str): The given name of the execution to identify.
411            executor_type_name (str): The name of the Executor to be used
412            start_node_id (str, optional): The node from which execution begins at. Defaults to "start".
413            ctx_build_kwargs (dict[str, Any], optional): .Defaults to None.
414        """
415        assert self.graphs.get(graph_name, None) is not None, "graph does not exist"
416        assert self.executors.get(exec_lbl, None) is None, (
417            "Execution with the same label already exists"
418        )
419        if ctx_build_kwargs is None:
420            ctx_build_kwargs = {"param_cascade_names": ["params.global"]}
421
422        self.executors[exec_lbl] = {
423            "exec_func_type": executor_type_name,
424            "graph_name": graph_name,
425            "context_args": ctx_build_kwargs,
426            "start_node": start_node_id,
427        }
428
429    def run_executor(self, executor_lbl: str, run_async: bool = True):
430        """Simple wrapper to allow for async using conccurrent.futures"""
431        logger.info(
432            f"Saving project to {self.project_path} before execution (and refreshing blueprints!)"
433        )
434        with chdir(self.project_path):
435            self.save_project()
436            self.bp_reg.refresh_blueprints()
437        logger.info(
438            f"Execution starting {executor_lbl} {'Asychronously' if run_async else 'Synchronously \nWARNING! This will likely block the user interface!'} "
439        )
440        if run_async:
441            return self.api_thread_pool.submit(self._execute_graph, executor_lbl)
442        else:
443            return self._execute_graph(executor_lbl)
444
445    def _execute_graph(self, executor_lbl: str):
446        """Runs the executor
447        Args:
448            executor_lbl (str): the name of the executor to be used
449        Returns:
450            dict[str, Any]: returns the final context generated from the execution
451        Raises:
452            ValueError: If anything required for execution (label, executor type, target graph) is not found
453        """
454        execution_target = None
455        try:
456            executor_info = self.executors.get(executor_lbl, None)
457            assert bool(executor_info), (
458                f"Did not find executor labelled {executor_lbl} within project, cannot run!"
459            )
460
461            execution_target = _EXECUTORS.get(executor_info["exec_func_type"], None)
462            assert bool(execution_target), (
463                f"Did not find executor of type {execution_target} within project, cannot run!"
464            )
465
466            target_graph = self.graphs.get(executor_info["graph_name"], None)
467            assert bool(target_graph), (
468                f"Target Graph not Found: {target_graph}, cannot run!"
469            )
470
471            execution_graph = target_graph.build_execution(
472                executor_info.get("start_node", "start")
473            )
474            logger.debug(
475                f"Execution graph built {'->'.join([n.id for n in execution_graph])}"
476            )
477            exec_targets = []
478
479            for node in execution_graph:
480                if node.id == "start":
481                    continue
482                bp = self.bp_reg.get_blueprint(node.blueprint_name)
483                node_param = self.param_manager.get_node_params(node.param_id)
484                exec_targets.append((bp, node_param, node))
485            ctx_args = executor_info["context_args"]
486            init_ctx = (
487                self.param_manager.build_param_context(**ctx_args, as_omega_conf=False)
488                if ctx_args
489                else {}
490            )
491            logger.debug(f"Context built:\n {init_ctx}")
492            logger.info("Execution Starting...")
493            with chdir(self.project_path):
494                out_ctx = execution_target(exec_targets, init_ctx)
495            logger.info("Execution Finished")
496        except Exception as e:
497            logger.error(
498                f"Execution of {executor_lbl} using {execution_target} failed:\n {e}"
499            )
500            return {}
501
502        return out_ctx

The Main enty point for m-strat, used to manage the whole m-strat project

ProjectManager( dir_path: str, load: bool = False, undo_limit: int = 100, max_workers=4)
54    def __init__(
55        self, dir_path: str, load: bool = False, undo_limit: int = 100, max_workers=4
56    ):
57        """Create a project manager instance, designed to be sustained in memory while in use
58        Args:
59            dir_path (str): The absolute path of the root working directory
60            load (bool, optional): If true attempts to load the project, if false creates a new one. Defaults to False.
61            undo_limit (int, optional): How many items can be undone (commands are stored in memory) . Defaults to 100.
62        NB: Having mutliple instances running on the same project will produce errors
63        """
64        assert os.path.isabs(dir_path), (
65            "The Path must be absolute for project directories"
66        )
67        self.project_path = dir_path
68        self.commands = deque([], undo_limit)
69        self.command_limit = undo_limit
70        if load:
71            self.load_project()
72        else:
73            assert len(os.listdir(dir_path)) == 0, (
74                "Project folder must be empty for init"
75            )  # Warn instead?
76            self.init_project_structure()
77        self.api_thread_pool = concurrent.futures.ThreadPoolExecutor(
78            max_workers=max_workers
79        )

Create a project manager instance, designed to be sustained in memory while in use Args: dir_path (str): The absolute path of the root working directory load (bool, optional): If true attempts to load the project, if false creates a new one. Defaults to False. undo_limit (int, optional): How many items can be undone (commands are stored in memory) . Defaults to 100. NB: Having mutliple instances running on the same project will produce errors

project_path
commands
command_limit
api_thread_pool
def copy_gen_templates(self, template_dir: str = None):
 81    def copy_gen_templates(self, template_dir: str = None):
 82        """Copies a generation template (the directory and files at the location to the project path)
 83        NB: only files with _template within the name are copied, only folders with template files within them will be made
 84        Args:
 85            template_dir (str, optional): Where to copy the directory if None uses the default embedded folder . Defaults to None.
 86        """
 87        if template_dir is None:
 88            template_dir = _TEMPLATES_DIR
 89
 90        for dir_path, _, fns in os.walk(template_dir):
 91            for fn in fns:
 92                if "template" not in fn or (
 93                    fn.startswith("_")
 94                    and not fn.startswith(
 95                        "_template"
 96                    )  # .param type kind of problem, _template.param
 97                ):
 98                    continue
 99                rel_dir = os.path.relpath(dir_path, template_dir)
100                if rel_dir == "[root]" or rel_dir == ".":
101                    copy_dest = self.project_path
102                else:
103                    copy_dest = os.path.join(self.project_path, rel_dir)
104                os.makedirs(copy_dest, exist_ok=True)
105
106                copy_fn = fn.replace("_template", "")
107                copy_file(os.path.join(dir_path, fn), os.path.join(copy_dest, copy_fn))
108        logger.info("Template generation completed")

Copies a generation template (the directory and files at the location to the project path) NB: only files with _template within the name are copied, only folders with template files within them will be made Args: template_dir (str, optional): Where to copy the directory if None uses the default embedded folder . Defaults to None.

def init_project_structure(self):
110    def init_project_structure(self):
111        """Initiates the project structure, automatically called if load false on object inits"""
112        logger.info(f"Generating Project files at: {self.project_path}")
113        with chdir(self.project_path):
114            for folder in FOLDER_STRUCTURE:
115                os.makedirs(folder, exist_ok=True)
116            for file in DEFAULT_FILES:
117                with open(file, "w", encoding="utf-8") as f_obj:
118                    if file.endswith(".json"):
119                        f_obj.write("{}")
120            logger.info(
121                "Basic folders and empty files completed, copying premade templates"
122            )
123            self.copy_gen_templates()
124            self.cd = ComponentDispatcher()
125            self.ecs = ECS_Register("registers", dispatcher=self.cd)
126            self.graphs = {
127                "Data": GraphManager(name="Data"),
128                "Model": GraphManager(name="Model"),
129                "Analysis": GraphManager(name="Analysis"),
130            }
131            self.executors = {}
132            self.bp_reg = BlueprintRegister()
133            self.bp_reg.add_from_folder("blueprints")
134            self.param_manager = ParametersManager(self.project_path)
135            self.commandables = {
136                "cd": self.cd,
137                "ecs": self.ecs,
138                "param_manager": self.param_manager,
139                "bp_reg": self.bp_reg,
140                "graphs": self.graphs,
141                "executors": self.executors,
142            }
143        self.save_project()  # this generates all the config files automatically

Initiates the project structure, automatically called if load false on object inits

def add_graph(self, name: str):
145    def add_graph(self, name: str):
146        """Adds a graph to the project
147        Args:
148            name (str): The name of the graph given.
149        """
150        assert isinstance(name, str), "Invalid Graph Name, must be a string"
151        assert len(name) > 0, "Invalid graph Name, must be at least one character"
152        assert self.graphs.get(name, None) is None, "Graph Name Already exists"
153        self.graphs[name] = GraphManager(name=name, dir="exec_graphs", load=False)
154        logger.info(f"Graph '{name} was added to the project")

Adds a graph to the project Args: name (str): The name of the graph given.

def rename_graph(self, source_name: str, new_name: str):
156    def rename_graph(self, source_name: str, new_name: str):
157        """Renames a graph
158        (a bit heavy but gets the job done!)
159        Args:
160            source_name (str): the name of original graph
161            new_name (str): the new name of the graph
162        """
163        self.duplicate_graph(source_name, new_name)
164        self.delete_graph(source_name)
165        logger.info(f"Graph {source_name} was renamed to {new_name}")

Renames a graph (a bit heavy but gets the job done!) Args: source_name (str): the name of original graph new_name (str): the new name of the graph

def delete_graph(self, name: str):
167    def delete_graph(self, name: str):
168        """Deletes the given graph
169        Args:
170            name (str): The name of the graph to be deleted
171        """
172        assert isinstance(name, str), "Invalid Graph Name, must be a string"
173        graph = self.graphs.pop(name, None)
174        self.executors = {
175            k: v for k, v in self.executors.items() if v["graph_name"] != name
176        }
177        if graph:
178            path = graph.get_file_path(graph.name, graph.dir)
179            with chdir(self.project_path):
180                if os.path.exists(path):
181                    os.remove(path)
182            graph = None
183            logger.info(
184                f"Graph '{name}' was deleted along with it's configuration files"
185            )
186        else:
187            logger.info(f"Attempted to delete graph '{name}' but no graph was found")

Deletes the given graph Args: name (str): The name of the graph to be deleted

def duplicate_graph(self, source_name: str, copy_name: str):
189    def duplicate_graph(self, source_name: str, copy_name: str):
190        """Duplicates a graph
191        Args:
192            source_name (str): The name of the graph to be copied
193            copy_name (str): the new name of the graph clone
194        """
195        assert isinstance(source_name, str) and isinstance(copy_name, str), (
196            "Graph source and copy name must be string"
197        )
198        assert len(source_name) > 0 and len(copy_name) > 0, (
199            "Graph source and copy names must be 1 character or more"
200        )
201        assert self.graphs.get(copy_name, None) is None, (
202            f"Destination name, {copy_name} already exists"
203        )
204
205        origin_graph = self.graphs.get(source_name, None)
206        assert origin_graph is not None, (
207            f"Source Name ,{source_name} does not exist in current context"
208        )
209        self.graphs[copy_name] = origin_graph.duplicate(copy_name)
210        logger.info(f"'{source_name}' graph sucessfully duplicated as '{copy_name}'")

Duplicates a graph Args: source_name (str): The name of the graph to be copied copy_name (str): the new name of the graph clone

def load_project(self):
212    def load_project(self):
213        """Loads the project (done automaticaly when project manager initiated)"""
214        logger.info(f"Loading project: {self.project_path}")
215        try:
216            with chdir(self.project_path):
217                self.cd = ComponentDispatcher()
218                self.ecs = ECS_Register("registers", dispatcher=self.cd)
219                self.param_manager = ParametersManager(self.project_path)
220                self.bp_reg = BlueprintRegister()
221
222                self.graphs = {}
223                for dir_path, _, fns in os.walk("exec_graphs"):
224                    for fn in fns:
225                        if not fn.startswith("gm_") or not fn.endswith(".json"):
226                            continue
227                        f_p = os.path.join(dir_path, fn)
228                        new_graph = GraphManager(name=None, load_path=f_p, load=True)
229                        self.graphs[new_graph.name] = new_graph
230                self.executors = {}
231
232                with open("configs/executors_dict.json", encoding="utf-8") as f:
233                    j_data = json.load(f)
234                    self.executors = j_data if j_data else {}
235
236                self.commandables = {
237                    "cd": self.cd,
238                    "ecs": self.ecs,
239                    "param_manager": self.param_manager,
240                    "bp_reg": self.bp_reg,
241                    "graphs": self.graphs,
242                    "executors": self.executors,
243                }
244                self.bp_reg.add_from_folder("blueprints")
245        except Exception as e:
246            logger.error(f"Project loading failed:\n{e}")
247            raise e
248        logger.info(f"Project loaded sucessfully ({self.project_path})")

Loads the project (done automaticaly when project manager initiated)

def save_project(self):
250    def save_project(self):
251        """Saves the whole project in it's current state"""
252        with chdir(self.project_path):
253            self.ecs._save_register()
254            self.param_manager.save_node_params()
255            self.bp_reg.save_meta_file()
256            for g in self.graphs.values():
257                g.save()
258            with open("configs/executors_dict.json", "w", encoding="utf-8") as f:
259                json.dump(self.executors, f)
260        logger.info(f"Project saved: {self.project_path}")

Saves the whole project in it's current state

@staticmethod
def is_valid_command_dict(params: dict):
262    @staticmethod
263    def is_valid_command_dict(params: dict):
264        """Returns true if the command dictionary given  is valid"""
265        type_set = {
266            "target_obj": str,
267            "do_params": dict,
268            "undo_params": dict,
269            "target_func": str,
270            "undo_func": str,
271        }
272        is_valid = all(p in type_set.keys() for p in params.keys())
273        is_valid &= all(type(v) is type_set.get(k, None) for k, v in params.items())
274        is_valid &= (
275            params.get("target_obj", None) is not None
276            and params.get("target_func", None) is not None
277        )
278        return is_valid

Returns true if the command dictionary given is valid

def execute_command(self, **params):
280    def execute_command(self, **params):
281        """Executes a command to a commandable target (Any manager or Register)
282        Args:
283            target_obj (str): The commandable's name (full name e.g. 'graphs.graph-name') or pm/self for project manager
284            target_func (str): The name of callable to be executed within the class
285            do_params(dict[str,Any]): The parameters needed to execute the command parsed via **params syntax.
286            undo_func(str, optional): The name of the function to execute an undoing of the original command
287            undo_params(dict[str, any]): The parameters needed to execute an undoing parsed via **params syntax.
288        Returns:
289            Any : any values resulted from the given command (if available else None)
290        """
291        try:
292            assert self.is_valid_command_dict(params), (
293                "Execute params failed validation"
294            )
295            target_obj = params.get("target_obj", None)
296            if target_obj.lower() in ["self", "pm", "projectmanager"]:
297                commandable = self
298                target_obj = "self"  # project manager is also a commandable
299            else:
300                commandable = self.commandables
301                if "." in target_obj:
302                    embeds = target_obj.split(".")
303                    commandable = self.commandables
304                    for embed in embeds:
305                        commandable = commandable[embed]
306                else:
307                    commandable = commandable[target_obj]
308
309            cm_t = type(commandable)
310            assert cm_t in _ALLOWED_ACCESS, (
311                f"Attempted to access item that is not commandable: {cm_t}"
312            )
313
314            target_func = params["target_func"]
315            assert not target_func.startswith("_"), (
316                f"Attempted to access a private method: {target_func}"
317            )
318
319            assert target_func in cm_t.__dict__, (
320                f"Attempted to acces a function not implemented on the child class: {target_func}"
321            )
322            func = getattr(commandable, target_func)
323            assert callable(func), f"Designated attribute is not callable {target_func}"
324
325            func_result = None
326            do_params = params.get("do_params", {})
327            # don't log getters, a few places can use them in loops!
328            is_logworthy = not target_func.startswith("get")
329
330            if is_logworthy:
331                logger.info(
332                    f"Performing {target_obj}.{target_func}() with params: [|]  {do_params}  [|]"
333                )
334            undo_func = params.get("undo_func", None)
335
336            if undo_func:
337                undo_func_name = params.get("undo_func")
338                assert callable(getattr(commandable, undo_func_name, None)), (
339                    f"Undo function is not a callable: {params['undo_func']}, command aborted"
340                )
341                assert not undo_func_name.startswith("_"), (
342                    "Undo function supplied is private, command aborted"
343                )
344                command_log = dict(params)
345                command_log.pop("do_params")  # delete do_params to avoid memory leak
346                self.commands.append(command_log)
347                if is_logworthy:
348                    logger.debug(
349                        f"Undo detected and added for {target_obj}.{target_func}-> {params['undo_func']}"
350                    )
351                if len(self.commands) > self.command_limit:
352                    self.commands.popleft()
353            elif is_logworthy:
354                logger.debug(f"No Undo method detected for {target_obj}.{target_func}")
355
356            with chdir(self.project_path):
357                func_result = func(**do_params)
358            if is_logworthy:
359                logger.info("Command Executed Sucessfully")
360        except Exception as e:
361            logger.error(f"Command failed for {target_obj}->{target_func}(): {e} ")
362            return None
363
364        return func_result

Executes a command to a commandable target (Any manager or Register) Args: target_obj (str): The commandable's name (full name e.g. 'graphs.graph-name') or pm/self for project manager target_func (str): The name of callable to be executed within the class do_params(dict[str,Any]): The parameters needed to execute the command parsed via **params syntax. undo_func(str, optional): The name of the function to execute an undoing of the original command undo_params(dict[str, any]): The parameters needed to execute an undoing parsed via **params syntax. Returns: Any : any values resulted from the given command (if available else None)

def undo_last_command(self):
366    def undo_last_command(self):
367        """undoes the most recent command if there are any"""
368        if not self.commands:
369            logger.info("Tried to undo but No Undo Commands available")
370            return
371
372        last_cmd = self.commands.pop()
373        commandable_name = last_cmd["target_obj"]
374        if commandable_name == "self":
375            commandable = self
376        else:
377            commandable = self.commandables
378            if "." in commandable_name:
379                embeds = commandable_name.split(".")
380                commandable = self.commandables
381                for embed in embeds:
382                    commandable = commandable[embed]
383            else:
384                commandable = commandable[commandable_name]
385
386        undo_func_name = last_cmd["undo_func"]
387        undo_func = getattr(commandable, undo_func_name)
388        logger.info(
389            f"Undoing {'pm' if commandable_name == 'self' else commandable_name}.{last_cmd['target_func']} USING {undo_func_name}()"
390        )
391        try:
392            with chdir(self.project_path):
393                undo_func(**last_cmd["undo_params"])
394        except Exception as e:
395            logger.error(
396                f"Command failed for {'pm' if commandable_name == 'self' else commandable_name}->{undo_func_name}(): {e} "
397            )

undoes the most recent command if there are any

def create_executor( self, graph_name: str, exec_lbl: str, executor_type_name: str, start_node_id='start', ctx_build_kwargs: dict[str, typing.Any] = None):
399    def create_executor(
400        self,
401        graph_name: str,
402        exec_lbl: str,
403        executor_type_name: str,
404        start_node_id="start",
405        ctx_build_kwargs: dict[str, Any] = None,
406    ):
407        """Creates an execution context within the project manager
408        Args:
409            graph_name (str): The name of the graph to be create the execution from
410            exec_lbl (str): The given name of the execution to identify.
411            executor_type_name (str): The name of the Executor to be used
412            start_node_id (str, optional): The node from which execution begins at. Defaults to "start".
413            ctx_build_kwargs (dict[str, Any], optional): .Defaults to None.
414        """
415        assert self.graphs.get(graph_name, None) is not None, "graph does not exist"
416        assert self.executors.get(exec_lbl, None) is None, (
417            "Execution with the same label already exists"
418        )
419        if ctx_build_kwargs is None:
420            ctx_build_kwargs = {"param_cascade_names": ["params.global"]}
421
422        self.executors[exec_lbl] = {
423            "exec_func_type": executor_type_name,
424            "graph_name": graph_name,
425            "context_args": ctx_build_kwargs,
426            "start_node": start_node_id,
427        }

Creates an execution context within the project manager Args: graph_name (str): The name of the graph to be create the execution from exec_lbl (str): The given name of the execution to identify. executor_type_name (str): The name of the Executor to be used start_node_id (str, optional): The node from which execution begins at. Defaults to "start". ctx_build_kwargs (dict[str, Any], optional): .Defaults to None.

def run_executor(self, executor_lbl: str, run_async: bool = True):
429    def run_executor(self, executor_lbl: str, run_async: bool = True):
430        """Simple wrapper to allow for async using conccurrent.futures"""
431        logger.info(
432            f"Saving project to {self.project_path} before execution (and refreshing blueprints!)"
433        )
434        with chdir(self.project_path):
435            self.save_project()
436            self.bp_reg.refresh_blueprints()
437        logger.info(
438            f"Execution starting {executor_lbl} {'Asychronously' if run_async else 'Synchronously \nWARNING! This will likely block the user interface!'} "
439        )
440        if run_async:
441            return self.api_thread_pool.submit(self._execute_graph, executor_lbl)
442        else:
443            return self._execute_graph(executor_lbl)

Simple wrapper to allow for async using conccurrent.futures