m_strat.api.dag.blueprint_register

  1import importlib.util
  2import inspect
  3import json
  4import os
  5
  6from api.dag.node import BlueprintNode
  7from api.logger import m_log
  8
  9
 10class BlueprintRegister:
 11    """Manages Blueprint Nodes in a project"""
 12
 13    def __init__(self, meta_file_path: str = "configs/bp_files_meta.jsonl"):
 14        self.blueprints = {}
 15        self.schemas = {}
 16        self.files_found = []
 17
 18        if meta_file_path:
 19            self.load_from_meta_file(meta_file_path)
 20        self.meta_file_path = meta_file_path
 21
 22    def register_node(
 23        self, name: str, func: callable, schema: dict, overwrite: bool = True
 24    ):
 25        """Adds a method to the project registry
 26        Args:
 27            name (str): The new name of the blueprint node being created (can be given via BlueprintNode decorator)
 28            func (callable): The function that will be used to execute the node logic
 29            schema (dict[str, type]): Used to guide the UI and create an automated form in the application, not enforced
 30            overwrite (bool, optional): Allows a node to be overwritten if the name already exists. Defaults to True.
 31        """
 32        assert overwrite or self.blueprints.get(name, None) is None, (
 33            f"blueprint of '{name}' already exists, overwrite not allowed"
 34        )
 35        self.blueprints[name] = func
 36        self.schemas[name] = schema
 37
 38    def unregister_node(self, name: str):
 39        """Remove a node from the registry by name
 40        Returns:
 41           tuple[dict,dict] : blueprint callable and schema respectively
 42        """
 43        return (self.blueprints.pop(name, None), self.schemas.pop(name, None))
 44
 45    def get_blueprint(self, name: str):
 46        """Get the callable of a blueprint by name"""
 47        return self.blueprints.get(name, None)
 48
 49    def refresh_blueprints(self):
 50        self.blueprints = {}
 51        self.schemas = {}
 52        old_files_found = list(self.files_found)
 53        self.files_found = []
 54        for f in old_files_found:
 55            self.add_from_file(f)
 56
 57    def save_meta_file(self, meta_file_path="configs/bp_files_meta.jsonl"):
 58        """Generates a jsonl file of where all the blueprints were retrieved from, so they can be loaded easily
 59        Args:
 60            meta_file_path (str, optional): relative path of wheree this generated file will save. Defaults to "configs/bp_files_meta.jsonl".
 61        """
 62        if not self.files_found:
 63            m_log.info("Attempted to save blueprints, but no blueprints registered")
 64            return
 65        with open(meta_file_path, "w", encoding="utf-8") as meta_f:
 66            for bp_file_path in set(self.files_found):
 67                meta_f.write(json.dumps(bp_file_path) + "\n")
 68        m_log.info(f"Blueprint data saved {meta_file_path}")
 69
 70    def load_from_meta_file(self, meta_file_path: str = None):
 71        """Loads and use each line as a path to retrieve saved blueprint reference from py files .
 72        Args:
 73            meta_file_path (str, optional): The path where the meta file resides. Defaults to "configs/bp_files_meta.jsonl".
 74        Returns:
 75           list[str] : Returns a list of all the names of the blueprints that were added
 76        """
 77        m_log.info("Loading Blueprint Configuration Data...")
 78        if meta_file_path is None:
 79            meta_file_path = self.meta_file_path
 80        if not os.path.exists(meta_file_path):
 81            m_log.warning(f"bp_reg: path does not exist:: {meta_file_path}")
 82            return []
 83        added = []
 84        with open(meta_file_path, "r", encoding="utf8") as meta_f:
 85            for line in meta_f:
 86                if not line.strip():
 87                    continue
 88                try:
 89                    next_path = json.loads(line.strip())
 90                    added.extend(self.add_from_file(next_path))
 91                except:
 92                    pass
 93        return added
 94
 95    def get_schema(self, name: str):
 96        """Gets the (dict[str,type]) schema given a name of a Blueprint Node"""
 97        return self.schemas.get(name, None)
 98
 99    def add_from_file(self, file_path: str):
100        """uses importlib to check for any Blueprint Nodes and attempts to add them to the system
101        Args:
102            file_path (str): relative path of the .py blueprint file resides
103        Returns:
104            list[str] : Returns a list of all the names of the blueprints that were added
105        """
106        if not os.path.exists(file_path):
107            m_log.warning(f"bp_reg: path does not exist:: {file_path}")
108            return []
109
110        module_name = os.path.splitext(os.path.basename(file_path))[0]
111        spec = importlib.util.spec_from_file_location(module_name, file_path)
112        added = []
113        assert (
114            spec is not None and spec.loader
115        ), f"""Provided file does not have a spec or spec loader: ||{file_path} ||
116            This can happen because:
117            1. If the file exstension is wrong (not .py)
118            2. File permission issues such as file locked (in use elsewhere e.g )
119            3. Unfriendly path characters: path contains unusual unicode characters or file name is not python friendly
120            4. Some issue which I simply have not thought of in this moment.
121            """
122        module = importlib.util.module_from_spec(spec)
123        module.BlueprintNode = BlueprintNode
124        try:
125            spec.loader.exec_module(module)
126        except Exception as e:
127            # flagrantly assume the error is because the module should not be run like this!
128            m_log.debug(
129                f"Spec Loader did not like something when bp register used exec_module (can ignore typically): {e}"
130            )
131
132        for _, obj in inspect.getmembers(module, inspect.isfunction):
133            if getattr(obj, "__is_m_strat_node__", False):
134                self.register_node(
135                    obj.__m_strat_node_name__, obj, obj.__m_strat_node_schema__
136                )
137                added.append(obj.__m_strat_node_name__)
138
139        if len(added) > 0:
140            self.files_found.append(file_path)
141        return added
142
143    def add_from_folder(self, folder_path, add_subfolders=True):
144        """
145            From a directory, adds all Blueprint Nodes to the register
146        Args:
147            folder_path (str): The relative directory path
148            add_subfolders (bool, optional): Whether to add Blueprint Nodes from subdirectories (of any depth) . Defaults to True.
149        Returns:
150            list[str] : Returns a list of all the names of the blueprints that were added
151        """
152        if not os.path.isdir(folder_path):
153            m_log.warning(f"bp_reg: path does not exist:: {folder_path}")
154            return []
155        added_nodes = []
156        m_log.info(f"Adding folder blueprint folder: {folder_path}")
157        if add_subfolders:
158            for root, _, files in os.walk(folder_path):
159                for f_name in files:
160                    if f_name.endswith(".py") and not f_name.startswith("__"):
161                        full_path = os.path.join(root, f_name)
162                        next_added = []
163                        try:
164                            next_added = self.add_from_file(full_path)
165                        except Exception as e:
166                            m_log.error(
167                                f"Blueprint tried to add Blueprints from |{f_name}| in directory |{root}| but failed: \n {e} \n{'*' * 12}\n ...Skipping!"
168                            )
169                            continue
170
171                        added_nodes.extend(next_added)
172        else:
173            for f_name in os.listdir(folder_path):
174                if f_name.endswith(".py") and not f_name.startswith("__"):
175                    full_path = os.path.join(folder_path, f_name)
176                    if os.path.isfile(full_path):
177                        added_nodes.extend(self.add_from_file(full_path))
178        m_log.info(
179            f"Finished adding blueprint director, {len(added_nodes)} blueprint(s) found"
180        )
181        return added_nodes
182
183    def get_all_blueprints(self):
184        """Copies the blueprint callable dictionary, returning all blueprints
185        NB: Blueprints are given by reference, modifications will alter them!
186        """
187        return dict(self.blueprints)
188
189    def get_all_schemas(self):
190        """Copies the schema dictionary, returning all blueprint schemas
191        NB: Blueprints are given by reference, modifications will alter them!
192        """
193        return dict(self.schemas)
class BlueprintRegister:
 11class BlueprintRegister:
 12    """Manages Blueprint Nodes in a project"""
 13
 14    def __init__(self, meta_file_path: str = "configs/bp_files_meta.jsonl"):
 15        self.blueprints = {}
 16        self.schemas = {}
 17        self.files_found = []
 18
 19        if meta_file_path:
 20            self.load_from_meta_file(meta_file_path)
 21        self.meta_file_path = meta_file_path
 22
 23    def register_node(
 24        self, name: str, func: callable, schema: dict, overwrite: bool = True
 25    ):
 26        """Adds a method to the project registry
 27        Args:
 28            name (str): The new name of the blueprint node being created (can be given via BlueprintNode decorator)
 29            func (callable): The function that will be used to execute the node logic
 30            schema (dict[str, type]): Used to guide the UI and create an automated form in the application, not enforced
 31            overwrite (bool, optional): Allows a node to be overwritten if the name already exists. Defaults to True.
 32        """
 33        assert overwrite or self.blueprints.get(name, None) is None, (
 34            f"blueprint of '{name}' already exists, overwrite not allowed"
 35        )
 36        self.blueprints[name] = func
 37        self.schemas[name] = schema
 38
 39    def unregister_node(self, name: str):
 40        """Remove a node from the registry by name
 41        Returns:
 42           tuple[dict,dict] : blueprint callable and schema respectively
 43        """
 44        return (self.blueprints.pop(name, None), self.schemas.pop(name, None))
 45
 46    def get_blueprint(self, name: str):
 47        """Get the callable of a blueprint by name"""
 48        return self.blueprints.get(name, None)
 49
 50    def refresh_blueprints(self):
 51        self.blueprints = {}
 52        self.schemas = {}
 53        old_files_found = list(self.files_found)
 54        self.files_found = []
 55        for f in old_files_found:
 56            self.add_from_file(f)
 57
 58    def save_meta_file(self, meta_file_path="configs/bp_files_meta.jsonl"):
 59        """Generates a jsonl file of where all the blueprints were retrieved from, so they can be loaded easily
 60        Args:
 61            meta_file_path (str, optional): relative path of wheree this generated file will save. Defaults to "configs/bp_files_meta.jsonl".
 62        """
 63        if not self.files_found:
 64            m_log.info("Attempted to save blueprints, but no blueprints registered")
 65            return
 66        with open(meta_file_path, "w", encoding="utf-8") as meta_f:
 67            for bp_file_path in set(self.files_found):
 68                meta_f.write(json.dumps(bp_file_path) + "\n")
 69        m_log.info(f"Blueprint data saved {meta_file_path}")
 70
 71    def load_from_meta_file(self, meta_file_path: str = None):
 72        """Loads and use each line as a path to retrieve saved blueprint reference from py files .
 73        Args:
 74            meta_file_path (str, optional): The path where the meta file resides. Defaults to "configs/bp_files_meta.jsonl".
 75        Returns:
 76           list[str] : Returns a list of all the names of the blueprints that were added
 77        """
 78        m_log.info("Loading Blueprint Configuration Data...")
 79        if meta_file_path is None:
 80            meta_file_path = self.meta_file_path
 81        if not os.path.exists(meta_file_path):
 82            m_log.warning(f"bp_reg: path does not exist:: {meta_file_path}")
 83            return []
 84        added = []
 85        with open(meta_file_path, "r", encoding="utf8") as meta_f:
 86            for line in meta_f:
 87                if not line.strip():
 88                    continue
 89                try:
 90                    next_path = json.loads(line.strip())
 91                    added.extend(self.add_from_file(next_path))
 92                except:
 93                    pass
 94        return added
 95
 96    def get_schema(self, name: str):
 97        """Gets the (dict[str,type]) schema given a name of a Blueprint Node"""
 98        return self.schemas.get(name, None)
 99
100    def add_from_file(self, file_path: str):
101        """uses importlib to check for any Blueprint Nodes and attempts to add them to the system
102        Args:
103            file_path (str): relative path of the .py blueprint file resides
104        Returns:
105            list[str] : Returns a list of all the names of the blueprints that were added
106        """
107        if not os.path.exists(file_path):
108            m_log.warning(f"bp_reg: path does not exist:: {file_path}")
109            return []
110
111        module_name = os.path.splitext(os.path.basename(file_path))[0]
112        spec = importlib.util.spec_from_file_location(module_name, file_path)
113        added = []
114        assert (
115            spec is not None and spec.loader
116        ), f"""Provided file does not have a spec or spec loader: ||{file_path} ||
117            This can happen because:
118            1. If the file exstension is wrong (not .py)
119            2. File permission issues such as file locked (in use elsewhere e.g )
120            3. Unfriendly path characters: path contains unusual unicode characters or file name is not python friendly
121            4. Some issue which I simply have not thought of in this moment.
122            """
123        module = importlib.util.module_from_spec(spec)
124        module.BlueprintNode = BlueprintNode
125        try:
126            spec.loader.exec_module(module)
127        except Exception as e:
128            # flagrantly assume the error is because the module should not be run like this!
129            m_log.debug(
130                f"Spec Loader did not like something when bp register used exec_module (can ignore typically): {e}"
131            )
132
133        for _, obj in inspect.getmembers(module, inspect.isfunction):
134            if getattr(obj, "__is_m_strat_node__", False):
135                self.register_node(
136                    obj.__m_strat_node_name__, obj, obj.__m_strat_node_schema__
137                )
138                added.append(obj.__m_strat_node_name__)
139
140        if len(added) > 0:
141            self.files_found.append(file_path)
142        return added
143
144    def add_from_folder(self, folder_path, add_subfolders=True):
145        """
146            From a directory, adds all Blueprint Nodes to the register
147        Args:
148            folder_path (str): The relative directory path
149            add_subfolders (bool, optional): Whether to add Blueprint Nodes from subdirectories (of any depth) . Defaults to True.
150        Returns:
151            list[str] : Returns a list of all the names of the blueprints that were added
152        """
153        if not os.path.isdir(folder_path):
154            m_log.warning(f"bp_reg: path does not exist:: {folder_path}")
155            return []
156        added_nodes = []
157        m_log.info(f"Adding folder blueprint folder: {folder_path}")
158        if add_subfolders:
159            for root, _, files in os.walk(folder_path):
160                for f_name in files:
161                    if f_name.endswith(".py") and not f_name.startswith("__"):
162                        full_path = os.path.join(root, f_name)
163                        next_added = []
164                        try:
165                            next_added = self.add_from_file(full_path)
166                        except Exception as e:
167                            m_log.error(
168                                f"Blueprint tried to add Blueprints from |{f_name}| in directory |{root}| but failed: \n {e} \n{'*' * 12}\n ...Skipping!"
169                            )
170                            continue
171
172                        added_nodes.extend(next_added)
173        else:
174            for f_name in os.listdir(folder_path):
175                if f_name.endswith(".py") and not f_name.startswith("__"):
176                    full_path = os.path.join(folder_path, f_name)
177                    if os.path.isfile(full_path):
178                        added_nodes.extend(self.add_from_file(full_path))
179        m_log.info(
180            f"Finished adding blueprint director, {len(added_nodes)} blueprint(s) found"
181        )
182        return added_nodes
183
184    def get_all_blueprints(self):
185        """Copies the blueprint callable dictionary, returning all blueprints
186        NB: Blueprints are given by reference, modifications will alter them!
187        """
188        return dict(self.blueprints)
189
190    def get_all_schemas(self):
191        """Copies the schema dictionary, returning all blueprint schemas
192        NB: Blueprints are given by reference, modifications will alter them!
193        """
194        return dict(self.schemas)

Manages Blueprint Nodes in a project

BlueprintRegister(meta_file_path: str = 'configs/bp_files_meta.jsonl')
14    def __init__(self, meta_file_path: str = "configs/bp_files_meta.jsonl"):
15        self.blueprints = {}
16        self.schemas = {}
17        self.files_found = []
18
19        if meta_file_path:
20            self.load_from_meta_file(meta_file_path)
21        self.meta_file_path = meta_file_path
blueprints
schemas
files_found
meta_file_path
def register_node( self, name: str, func: <built-in function callable>, schema: dict, overwrite: bool = True):
23    def register_node(
24        self, name: str, func: callable, schema: dict, overwrite: bool = True
25    ):
26        """Adds a method to the project registry
27        Args:
28            name (str): The new name of the blueprint node being created (can be given via BlueprintNode decorator)
29            func (callable): The function that will be used to execute the node logic
30            schema (dict[str, type]): Used to guide the UI and create an automated form in the application, not enforced
31            overwrite (bool, optional): Allows a node to be overwritten if the name already exists. Defaults to True.
32        """
33        assert overwrite or self.blueprints.get(name, None) is None, (
34            f"blueprint of '{name}' already exists, overwrite not allowed"
35        )
36        self.blueprints[name] = func
37        self.schemas[name] = schema

Adds a method to the project registry Args: name (str): The new name of the blueprint node being created (can be given via BlueprintNode decorator) func (callable): The function that will be used to execute the node logic schema (dict[str, type]): Used to guide the UI and create an automated form in the application, not enforced overwrite (bool, optional): Allows a node to be overwritten if the name already exists. Defaults to True.

def unregister_node(self, name: str):
39    def unregister_node(self, name: str):
40        """Remove a node from the registry by name
41        Returns:
42           tuple[dict,dict] : blueprint callable and schema respectively
43        """
44        return (self.blueprints.pop(name, None), self.schemas.pop(name, None))

Remove a node from the registry by name Returns: tuple[dict,dict] : blueprint callable and schema respectively

def get_blueprint(self, name: str):
46    def get_blueprint(self, name: str):
47        """Get the callable of a blueprint by name"""
48        return self.blueprints.get(name, None)

Get the callable of a blueprint by name

def refresh_blueprints(self):
50    def refresh_blueprints(self):
51        self.blueprints = {}
52        self.schemas = {}
53        old_files_found = list(self.files_found)
54        self.files_found = []
55        for f in old_files_found:
56            self.add_from_file(f)
def save_meta_file(self, meta_file_path='configs/bp_files_meta.jsonl'):
58    def save_meta_file(self, meta_file_path="configs/bp_files_meta.jsonl"):
59        """Generates a jsonl file of where all the blueprints were retrieved from, so they can be loaded easily
60        Args:
61            meta_file_path (str, optional): relative path of wheree this generated file will save. Defaults to "configs/bp_files_meta.jsonl".
62        """
63        if not self.files_found:
64            m_log.info("Attempted to save blueprints, but no blueprints registered")
65            return
66        with open(meta_file_path, "w", encoding="utf-8") as meta_f:
67            for bp_file_path in set(self.files_found):
68                meta_f.write(json.dumps(bp_file_path) + "\n")
69        m_log.info(f"Blueprint data saved {meta_file_path}")

Generates a jsonl file of where all the blueprints were retrieved from, so they can be loaded easily Args: meta_file_path (str, optional): relative path of wheree this generated file will save. Defaults to "configs/bp_files_meta.jsonl".

def load_from_meta_file(self, meta_file_path: str = None):
71    def load_from_meta_file(self, meta_file_path: str = None):
72        """Loads and use each line as a path to retrieve saved blueprint reference from py files .
73        Args:
74            meta_file_path (str, optional): The path where the meta file resides. Defaults to "configs/bp_files_meta.jsonl".
75        Returns:
76           list[str] : Returns a list of all the names of the blueprints that were added
77        """
78        m_log.info("Loading Blueprint Configuration Data...")
79        if meta_file_path is None:
80            meta_file_path = self.meta_file_path
81        if not os.path.exists(meta_file_path):
82            m_log.warning(f"bp_reg: path does not exist:: {meta_file_path}")
83            return []
84        added = []
85        with open(meta_file_path, "r", encoding="utf8") as meta_f:
86            for line in meta_f:
87                if not line.strip():
88                    continue
89                try:
90                    next_path = json.loads(line.strip())
91                    added.extend(self.add_from_file(next_path))
92                except:
93                    pass
94        return added

Loads and use each line as a path to retrieve saved blueprint reference from py files . Args: meta_file_path (str, optional): The path where the meta file resides. Defaults to "configs/bp_files_meta.jsonl". Returns: list[str] : Returns a list of all the names of the blueprints that were added

def get_schema(self, name: str):
96    def get_schema(self, name: str):
97        """Gets the (dict[str,type]) schema given a name of a Blueprint Node"""
98        return self.schemas.get(name, None)

Gets the (dict[str,type]) schema given a name of a Blueprint Node

def add_from_file(self, file_path: str):
100    def add_from_file(self, file_path: str):
101        """uses importlib to check for any Blueprint Nodes and attempts to add them to the system
102        Args:
103            file_path (str): relative path of the .py blueprint file resides
104        Returns:
105            list[str] : Returns a list of all the names of the blueprints that were added
106        """
107        if not os.path.exists(file_path):
108            m_log.warning(f"bp_reg: path does not exist:: {file_path}")
109            return []
110
111        module_name = os.path.splitext(os.path.basename(file_path))[0]
112        spec = importlib.util.spec_from_file_location(module_name, file_path)
113        added = []
114        assert (
115            spec is not None and spec.loader
116        ), f"""Provided file does not have a spec or spec loader: ||{file_path} ||
117            This can happen because:
118            1. If the file exstension is wrong (not .py)
119            2. File permission issues such as file locked (in use elsewhere e.g )
120            3. Unfriendly path characters: path contains unusual unicode characters or file name is not python friendly
121            4. Some issue which I simply have not thought of in this moment.
122            """
123        module = importlib.util.module_from_spec(spec)
124        module.BlueprintNode = BlueprintNode
125        try:
126            spec.loader.exec_module(module)
127        except Exception as e:
128            # flagrantly assume the error is because the module should not be run like this!
129            m_log.debug(
130                f"Spec Loader did not like something when bp register used exec_module (can ignore typically): {e}"
131            )
132
133        for _, obj in inspect.getmembers(module, inspect.isfunction):
134            if getattr(obj, "__is_m_strat_node__", False):
135                self.register_node(
136                    obj.__m_strat_node_name__, obj, obj.__m_strat_node_schema__
137                )
138                added.append(obj.__m_strat_node_name__)
139
140        if len(added) > 0:
141            self.files_found.append(file_path)
142        return added

uses importlib to check for any Blueprint Nodes and attempts to add them to the system Args: file_path (str): relative path of the .py blueprint file resides Returns: list[str] : Returns a list of all the names of the blueprints that were added

def add_from_folder(self, folder_path, add_subfolders=True):
144    def add_from_folder(self, folder_path, add_subfolders=True):
145        """
146            From a directory, adds all Blueprint Nodes to the register
147        Args:
148            folder_path (str): The relative directory path
149            add_subfolders (bool, optional): Whether to add Blueprint Nodes from subdirectories (of any depth) . Defaults to True.
150        Returns:
151            list[str] : Returns a list of all the names of the blueprints that were added
152        """
153        if not os.path.isdir(folder_path):
154            m_log.warning(f"bp_reg: path does not exist:: {folder_path}")
155            return []
156        added_nodes = []
157        m_log.info(f"Adding folder blueprint folder: {folder_path}")
158        if add_subfolders:
159            for root, _, files in os.walk(folder_path):
160                for f_name in files:
161                    if f_name.endswith(".py") and not f_name.startswith("__"):
162                        full_path = os.path.join(root, f_name)
163                        next_added = []
164                        try:
165                            next_added = self.add_from_file(full_path)
166                        except Exception as e:
167                            m_log.error(
168                                f"Blueprint tried to add Blueprints from |{f_name}| in directory |{root}| but failed: \n {e} \n{'*' * 12}\n ...Skipping!"
169                            )
170                            continue
171
172                        added_nodes.extend(next_added)
173        else:
174            for f_name in os.listdir(folder_path):
175                if f_name.endswith(".py") and not f_name.startswith("__"):
176                    full_path = os.path.join(folder_path, f_name)
177                    if os.path.isfile(full_path):
178                        added_nodes.extend(self.add_from_file(full_path))
179        m_log.info(
180            f"Finished adding blueprint director, {len(added_nodes)} blueprint(s) found"
181        )
182        return added_nodes

From a directory, adds all Blueprint Nodes to the register Args: folder_path (str): The relative directory path add_subfolders (bool, optional): Whether to add Blueprint Nodes from subdirectories (of any depth) . Defaults to True. Returns: list[str] : Returns a list of all the names of the blueprints that were added

def get_all_blueprints(self):
184    def get_all_blueprints(self):
185        """Copies the blueprint callable dictionary, returning all blueprints
186        NB: Blueprints are given by reference, modifications will alter them!
187        """
188        return dict(self.blueprints)

Copies the blueprint callable dictionary, returning all blueprints NB: Blueprints are given by reference, modifications will alter them!

def get_all_schemas(self):
190    def get_all_schemas(self):
191        """Copies the schema dictionary, returning all blueprint schemas
192        NB: Blueprints are given by reference, modifications will alter them!
193        """
194        return dict(self.schemas)

Copies the schema dictionary, returning all blueprint schemas NB: Blueprints are given by reference, modifications will alter them!