m_strat.api.dag.blueprint_register
1import importlib.util 2import inspect 3import json 4import os 5 6from api.dag.node import BlueprintNode 7from api.logger import m_log 8 9 10class BlueprintRegister: 11 """Manages Blueprint Nodes in a project""" 12 13 def __init__(self, meta_file_path: str = "configs/bp_files_meta.jsonl"): 14 self.blueprints = {} 15 self.schemas = {} 16 self.files_found = [] 17 18 if meta_file_path: 19 self.load_from_meta_file(meta_file_path) 20 self.meta_file_path = meta_file_path 21 22 def register_node( 23 self, name: str, func: callable, schema: dict, overwrite: bool = True 24 ): 25 """Adds a method to the project registry 26 Args: 27 name (str): The new name of the blueprint node being created (can be given via BlueprintNode decorator) 28 func (callable): The function that will be used to execute the node logic 29 schema (dict[str, type]): Used to guide the UI and create an automated form in the application, not enforced 30 overwrite (bool, optional): Allows a node to be overwritten if the name already exists. Defaults to True. 31 """ 32 assert overwrite or self.blueprints.get(name, None) is None, ( 33 f"blueprint of '{name}' already exists, overwrite not allowed" 34 ) 35 self.blueprints[name] = func 36 self.schemas[name] = schema 37 38 def unregister_node(self, name: str): 39 """Remove a node from the registry by name 40 Returns: 41 tuple[dict,dict] : blueprint callable and schema respectively 42 """ 43 return (self.blueprints.pop(name, None), self.schemas.pop(name, None)) 44 45 def get_blueprint(self, name: str): 46 """Get the callable of a blueprint by name""" 47 return self.blueprints.get(name, None) 48 49 def refresh_blueprints(self): 50 self.blueprints = {} 51 self.schemas = {} 52 old_files_found = list(self.files_found) 53 self.files_found = [] 54 for f in old_files_found: 55 self.add_from_file(f) 56 57 def save_meta_file(self, meta_file_path="configs/bp_files_meta.jsonl"): 58 """Generates a jsonl file of where all the blueprints were retrieved from, so they can be loaded easily 59 Args: 60 meta_file_path (str, optional): relative path of wheree this generated file will save. Defaults to "configs/bp_files_meta.jsonl". 61 """ 62 if not self.files_found: 63 m_log.info("Attempted to save blueprints, but no blueprints registered") 64 return 65 with open(meta_file_path, "w", encoding="utf-8") as meta_f: 66 for bp_file_path in set(self.files_found): 67 meta_f.write(json.dumps(bp_file_path) + "\n") 68 m_log.info(f"Blueprint data saved {meta_file_path}") 69 70 def load_from_meta_file(self, meta_file_path: str = None): 71 """Loads and use each line as a path to retrieve saved blueprint reference from py files . 72 Args: 73 meta_file_path (str, optional): The path where the meta file resides. Defaults to "configs/bp_files_meta.jsonl". 74 Returns: 75 list[str] : Returns a list of all the names of the blueprints that were added 76 """ 77 m_log.info("Loading Blueprint Configuration Data...") 78 if meta_file_path is None: 79 meta_file_path = self.meta_file_path 80 if not os.path.exists(meta_file_path): 81 m_log.warning(f"bp_reg: path does not exist:: {meta_file_path}") 82 return [] 83 added = [] 84 with open(meta_file_path, "r", encoding="utf8") as meta_f: 85 for line in meta_f: 86 if not line.strip(): 87 continue 88 try: 89 next_path = json.loads(line.strip()) 90 added.extend(self.add_from_file(next_path)) 91 except: 92 pass 93 return added 94 95 def get_schema(self, name: str): 96 """Gets the (dict[str,type]) schema given a name of a Blueprint Node""" 97 return self.schemas.get(name, None) 98 99 def add_from_file(self, file_path: str): 100 """uses importlib to check for any Blueprint Nodes and attempts to add them to the system 101 Args: 102 file_path (str): relative path of the .py blueprint file resides 103 Returns: 104 list[str] : Returns a list of all the names of the blueprints that were added 105 """ 106 if not os.path.exists(file_path): 107 m_log.warning(f"bp_reg: path does not exist:: {file_path}") 108 return [] 109 110 module_name = os.path.splitext(os.path.basename(file_path))[0] 111 spec = importlib.util.spec_from_file_location(module_name, file_path) 112 added = [] 113 assert ( 114 spec is not None and spec.loader 115 ), f"""Provided file does not have a spec or spec loader: ||{file_path} || 116 This can happen because: 117 1. If the file exstension is wrong (not .py) 118 2. File permission issues such as file locked (in use elsewhere e.g ) 119 3. Unfriendly path characters: path contains unusual unicode characters or file name is not python friendly 120 4. Some issue which I simply have not thought of in this moment. 121 """ 122 module = importlib.util.module_from_spec(spec) 123 module.BlueprintNode = BlueprintNode 124 try: 125 spec.loader.exec_module(module) 126 except Exception as e: 127 # flagrantly assume the error is because the module should not be run like this! 128 m_log.debug( 129 f"Spec Loader did not like something when bp register used exec_module (can ignore typically): {e}" 130 ) 131 132 for _, obj in inspect.getmembers(module, inspect.isfunction): 133 if getattr(obj, "__is_m_strat_node__", False): 134 self.register_node( 135 obj.__m_strat_node_name__, obj, obj.__m_strat_node_schema__ 136 ) 137 added.append(obj.__m_strat_node_name__) 138 139 if len(added) > 0: 140 self.files_found.append(file_path) 141 return added 142 143 def add_from_folder(self, folder_path, add_subfolders=True): 144 """ 145 From a directory, adds all Blueprint Nodes to the register 146 Args: 147 folder_path (str): The relative directory path 148 add_subfolders (bool, optional): Whether to add Blueprint Nodes from subdirectories (of any depth) . Defaults to True. 149 Returns: 150 list[str] : Returns a list of all the names of the blueprints that were added 151 """ 152 if not os.path.isdir(folder_path): 153 m_log.warning(f"bp_reg: path does not exist:: {folder_path}") 154 return [] 155 added_nodes = [] 156 m_log.info(f"Adding folder blueprint folder: {folder_path}") 157 if add_subfolders: 158 for root, _, files in os.walk(folder_path): 159 for f_name in files: 160 if f_name.endswith(".py") and not f_name.startswith("__"): 161 full_path = os.path.join(root, f_name) 162 next_added = [] 163 try: 164 next_added = self.add_from_file(full_path) 165 except Exception as e: 166 m_log.error( 167 f"Blueprint tried to add Blueprints from |{f_name}| in directory |{root}| but failed: \n {e} \n{'*' * 12}\n ...Skipping!" 168 ) 169 continue 170 171 added_nodes.extend(next_added) 172 else: 173 for f_name in os.listdir(folder_path): 174 if f_name.endswith(".py") and not f_name.startswith("__"): 175 full_path = os.path.join(folder_path, f_name) 176 if os.path.isfile(full_path): 177 added_nodes.extend(self.add_from_file(full_path)) 178 m_log.info( 179 f"Finished adding blueprint director, {len(added_nodes)} blueprint(s) found" 180 ) 181 return added_nodes 182 183 def get_all_blueprints(self): 184 """Copies the blueprint callable dictionary, returning all blueprints 185 NB: Blueprints are given by reference, modifications will alter them! 186 """ 187 return dict(self.blueprints) 188 189 def get_all_schemas(self): 190 """Copies the schema dictionary, returning all blueprint schemas 191 NB: Blueprints are given by reference, modifications will alter them! 192 """ 193 return dict(self.schemas)
11class BlueprintRegister: 12 """Manages Blueprint Nodes in a project""" 13 14 def __init__(self, meta_file_path: str = "configs/bp_files_meta.jsonl"): 15 self.blueprints = {} 16 self.schemas = {} 17 self.files_found = [] 18 19 if meta_file_path: 20 self.load_from_meta_file(meta_file_path) 21 self.meta_file_path = meta_file_path 22 23 def register_node( 24 self, name: str, func: callable, schema: dict, overwrite: bool = True 25 ): 26 """Adds a method to the project registry 27 Args: 28 name (str): The new name of the blueprint node being created (can be given via BlueprintNode decorator) 29 func (callable): The function that will be used to execute the node logic 30 schema (dict[str, type]): Used to guide the UI and create an automated form in the application, not enforced 31 overwrite (bool, optional): Allows a node to be overwritten if the name already exists. Defaults to True. 32 """ 33 assert overwrite or self.blueprints.get(name, None) is None, ( 34 f"blueprint of '{name}' already exists, overwrite not allowed" 35 ) 36 self.blueprints[name] = func 37 self.schemas[name] = schema 38 39 def unregister_node(self, name: str): 40 """Remove a node from the registry by name 41 Returns: 42 tuple[dict,dict] : blueprint callable and schema respectively 43 """ 44 return (self.blueprints.pop(name, None), self.schemas.pop(name, None)) 45 46 def get_blueprint(self, name: str): 47 """Get the callable of a blueprint by name""" 48 return self.blueprints.get(name, None) 49 50 def refresh_blueprints(self): 51 self.blueprints = {} 52 self.schemas = {} 53 old_files_found = list(self.files_found) 54 self.files_found = [] 55 for f in old_files_found: 56 self.add_from_file(f) 57 58 def save_meta_file(self, meta_file_path="configs/bp_files_meta.jsonl"): 59 """Generates a jsonl file of where all the blueprints were retrieved from, so they can be loaded easily 60 Args: 61 meta_file_path (str, optional): relative path of wheree this generated file will save. Defaults to "configs/bp_files_meta.jsonl". 62 """ 63 if not self.files_found: 64 m_log.info("Attempted to save blueprints, but no blueprints registered") 65 return 66 with open(meta_file_path, "w", encoding="utf-8") as meta_f: 67 for bp_file_path in set(self.files_found): 68 meta_f.write(json.dumps(bp_file_path) + "\n") 69 m_log.info(f"Blueprint data saved {meta_file_path}") 70 71 def load_from_meta_file(self, meta_file_path: str = None): 72 """Loads and use each line as a path to retrieve saved blueprint reference from py files . 73 Args: 74 meta_file_path (str, optional): The path where the meta file resides. Defaults to "configs/bp_files_meta.jsonl". 75 Returns: 76 list[str] : Returns a list of all the names of the blueprints that were added 77 """ 78 m_log.info("Loading Blueprint Configuration Data...") 79 if meta_file_path is None: 80 meta_file_path = self.meta_file_path 81 if not os.path.exists(meta_file_path): 82 m_log.warning(f"bp_reg: path does not exist:: {meta_file_path}") 83 return [] 84 added = [] 85 with open(meta_file_path, "r", encoding="utf8") as meta_f: 86 for line in meta_f: 87 if not line.strip(): 88 continue 89 try: 90 next_path = json.loads(line.strip()) 91 added.extend(self.add_from_file(next_path)) 92 except: 93 pass 94 return added 95 96 def get_schema(self, name: str): 97 """Gets the (dict[str,type]) schema given a name of a Blueprint Node""" 98 return self.schemas.get(name, None) 99 100 def add_from_file(self, file_path: str): 101 """uses importlib to check for any Blueprint Nodes and attempts to add them to the system 102 Args: 103 file_path (str): relative path of the .py blueprint file resides 104 Returns: 105 list[str] : Returns a list of all the names of the blueprints that were added 106 """ 107 if not os.path.exists(file_path): 108 m_log.warning(f"bp_reg: path does not exist:: {file_path}") 109 return [] 110 111 module_name = os.path.splitext(os.path.basename(file_path))[0] 112 spec = importlib.util.spec_from_file_location(module_name, file_path) 113 added = [] 114 assert ( 115 spec is not None and spec.loader 116 ), f"""Provided file does not have a spec or spec loader: ||{file_path} || 117 This can happen because: 118 1. If the file exstension is wrong (not .py) 119 2. File permission issues such as file locked (in use elsewhere e.g ) 120 3. Unfriendly path characters: path contains unusual unicode characters or file name is not python friendly 121 4. Some issue which I simply have not thought of in this moment. 122 """ 123 module = importlib.util.module_from_spec(spec) 124 module.BlueprintNode = BlueprintNode 125 try: 126 spec.loader.exec_module(module) 127 except Exception as e: 128 # flagrantly assume the error is because the module should not be run like this! 129 m_log.debug( 130 f"Spec Loader did not like something when bp register used exec_module (can ignore typically): {e}" 131 ) 132 133 for _, obj in inspect.getmembers(module, inspect.isfunction): 134 if getattr(obj, "__is_m_strat_node__", False): 135 self.register_node( 136 obj.__m_strat_node_name__, obj, obj.__m_strat_node_schema__ 137 ) 138 added.append(obj.__m_strat_node_name__) 139 140 if len(added) > 0: 141 self.files_found.append(file_path) 142 return added 143 144 def add_from_folder(self, folder_path, add_subfolders=True): 145 """ 146 From a directory, adds all Blueprint Nodes to the register 147 Args: 148 folder_path (str): The relative directory path 149 add_subfolders (bool, optional): Whether to add Blueprint Nodes from subdirectories (of any depth) . Defaults to True. 150 Returns: 151 list[str] : Returns a list of all the names of the blueprints that were added 152 """ 153 if not os.path.isdir(folder_path): 154 m_log.warning(f"bp_reg: path does not exist:: {folder_path}") 155 return [] 156 added_nodes = [] 157 m_log.info(f"Adding folder blueprint folder: {folder_path}") 158 if add_subfolders: 159 for root, _, files in os.walk(folder_path): 160 for f_name in files: 161 if f_name.endswith(".py") and not f_name.startswith("__"): 162 full_path = os.path.join(root, f_name) 163 next_added = [] 164 try: 165 next_added = self.add_from_file(full_path) 166 except Exception as e: 167 m_log.error( 168 f"Blueprint tried to add Blueprints from |{f_name}| in directory |{root}| but failed: \n {e} \n{'*' * 12}\n ...Skipping!" 169 ) 170 continue 171 172 added_nodes.extend(next_added) 173 else: 174 for f_name in os.listdir(folder_path): 175 if f_name.endswith(".py") and not f_name.startswith("__"): 176 full_path = os.path.join(folder_path, f_name) 177 if os.path.isfile(full_path): 178 added_nodes.extend(self.add_from_file(full_path)) 179 m_log.info( 180 f"Finished adding blueprint director, {len(added_nodes)} blueprint(s) found" 181 ) 182 return added_nodes 183 184 def get_all_blueprints(self): 185 """Copies the blueprint callable dictionary, returning all blueprints 186 NB: Blueprints are given by reference, modifications will alter them! 187 """ 188 return dict(self.blueprints) 189 190 def get_all_schemas(self): 191 """Copies the schema dictionary, returning all blueprint schemas 192 NB: Blueprints are given by reference, modifications will alter them! 193 """ 194 return dict(self.schemas)
Manages Blueprint Nodes in a project
23 def register_node( 24 self, name: str, func: callable, schema: dict, overwrite: bool = True 25 ): 26 """Adds a method to the project registry 27 Args: 28 name (str): The new name of the blueprint node being created (can be given via BlueprintNode decorator) 29 func (callable): The function that will be used to execute the node logic 30 schema (dict[str, type]): Used to guide the UI and create an automated form in the application, not enforced 31 overwrite (bool, optional): Allows a node to be overwritten if the name already exists. Defaults to True. 32 """ 33 assert overwrite or self.blueprints.get(name, None) is None, ( 34 f"blueprint of '{name}' already exists, overwrite not allowed" 35 ) 36 self.blueprints[name] = func 37 self.schemas[name] = schema
Adds a method to the project registry Args: name (str): The new name of the blueprint node being created (can be given via BlueprintNode decorator) func (callable): The function that will be used to execute the node logic schema (dict[str, type]): Used to guide the UI and create an automated form in the application, not enforced overwrite (bool, optional): Allows a node to be overwritten if the name already exists. Defaults to True.
39 def unregister_node(self, name: str): 40 """Remove a node from the registry by name 41 Returns: 42 tuple[dict,dict] : blueprint callable and schema respectively 43 """ 44 return (self.blueprints.pop(name, None), self.schemas.pop(name, None))
Remove a node from the registry by name Returns: tuple[dict,dict] : blueprint callable and schema respectively
46 def get_blueprint(self, name: str): 47 """Get the callable of a blueprint by name""" 48 return self.blueprints.get(name, None)
Get the callable of a blueprint by name
58 def save_meta_file(self, meta_file_path="configs/bp_files_meta.jsonl"): 59 """Generates a jsonl file of where all the blueprints were retrieved from, so they can be loaded easily 60 Args: 61 meta_file_path (str, optional): relative path of wheree this generated file will save. Defaults to "configs/bp_files_meta.jsonl". 62 """ 63 if not self.files_found: 64 m_log.info("Attempted to save blueprints, but no blueprints registered") 65 return 66 with open(meta_file_path, "w", encoding="utf-8") as meta_f: 67 for bp_file_path in set(self.files_found): 68 meta_f.write(json.dumps(bp_file_path) + "\n") 69 m_log.info(f"Blueprint data saved {meta_file_path}")
Generates a jsonl file of where all the blueprints were retrieved from, so they can be loaded easily Args: meta_file_path (str, optional): relative path of wheree this generated file will save. Defaults to "configs/bp_files_meta.jsonl".
71 def load_from_meta_file(self, meta_file_path: str = None): 72 """Loads and use each line as a path to retrieve saved blueprint reference from py files . 73 Args: 74 meta_file_path (str, optional): The path where the meta file resides. Defaults to "configs/bp_files_meta.jsonl". 75 Returns: 76 list[str] : Returns a list of all the names of the blueprints that were added 77 """ 78 m_log.info("Loading Blueprint Configuration Data...") 79 if meta_file_path is None: 80 meta_file_path = self.meta_file_path 81 if not os.path.exists(meta_file_path): 82 m_log.warning(f"bp_reg: path does not exist:: {meta_file_path}") 83 return [] 84 added = [] 85 with open(meta_file_path, "r", encoding="utf8") as meta_f: 86 for line in meta_f: 87 if not line.strip(): 88 continue 89 try: 90 next_path = json.loads(line.strip()) 91 added.extend(self.add_from_file(next_path)) 92 except: 93 pass 94 return added
Loads and use each line as a path to retrieve saved blueprint reference from py files . Args: meta_file_path (str, optional): The path where the meta file resides. Defaults to "configs/bp_files_meta.jsonl". Returns: list[str] : Returns a list of all the names of the blueprints that were added
96 def get_schema(self, name: str): 97 """Gets the (dict[str,type]) schema given a name of a Blueprint Node""" 98 return self.schemas.get(name, None)
Gets the (dict[str,type]) schema given a name of a Blueprint Node
100 def add_from_file(self, file_path: str): 101 """uses importlib to check for any Blueprint Nodes and attempts to add them to the system 102 Args: 103 file_path (str): relative path of the .py blueprint file resides 104 Returns: 105 list[str] : Returns a list of all the names of the blueprints that were added 106 """ 107 if not os.path.exists(file_path): 108 m_log.warning(f"bp_reg: path does not exist:: {file_path}") 109 return [] 110 111 module_name = os.path.splitext(os.path.basename(file_path))[0] 112 spec = importlib.util.spec_from_file_location(module_name, file_path) 113 added = [] 114 assert ( 115 spec is not None and spec.loader 116 ), f"""Provided file does not have a spec or spec loader: ||{file_path} || 117 This can happen because: 118 1. If the file exstension is wrong (not .py) 119 2. File permission issues such as file locked (in use elsewhere e.g ) 120 3. Unfriendly path characters: path contains unusual unicode characters or file name is not python friendly 121 4. Some issue which I simply have not thought of in this moment. 122 """ 123 module = importlib.util.module_from_spec(spec) 124 module.BlueprintNode = BlueprintNode 125 try: 126 spec.loader.exec_module(module) 127 except Exception as e: 128 # flagrantly assume the error is because the module should not be run like this! 129 m_log.debug( 130 f"Spec Loader did not like something when bp register used exec_module (can ignore typically): {e}" 131 ) 132 133 for _, obj in inspect.getmembers(module, inspect.isfunction): 134 if getattr(obj, "__is_m_strat_node__", False): 135 self.register_node( 136 obj.__m_strat_node_name__, obj, obj.__m_strat_node_schema__ 137 ) 138 added.append(obj.__m_strat_node_name__) 139 140 if len(added) > 0: 141 self.files_found.append(file_path) 142 return added
uses importlib to check for any Blueprint Nodes and attempts to add them to the system Args: file_path (str): relative path of the .py blueprint file resides Returns: list[str] : Returns a list of all the names of the blueprints that were added
144 def add_from_folder(self, folder_path, add_subfolders=True): 145 """ 146 From a directory, adds all Blueprint Nodes to the register 147 Args: 148 folder_path (str): The relative directory path 149 add_subfolders (bool, optional): Whether to add Blueprint Nodes from subdirectories (of any depth) . Defaults to True. 150 Returns: 151 list[str] : Returns a list of all the names of the blueprints that were added 152 """ 153 if not os.path.isdir(folder_path): 154 m_log.warning(f"bp_reg: path does not exist:: {folder_path}") 155 return [] 156 added_nodes = [] 157 m_log.info(f"Adding folder blueprint folder: {folder_path}") 158 if add_subfolders: 159 for root, _, files in os.walk(folder_path): 160 for f_name in files: 161 if f_name.endswith(".py") and not f_name.startswith("__"): 162 full_path = os.path.join(root, f_name) 163 next_added = [] 164 try: 165 next_added = self.add_from_file(full_path) 166 except Exception as e: 167 m_log.error( 168 f"Blueprint tried to add Blueprints from |{f_name}| in directory |{root}| but failed: \n {e} \n{'*' * 12}\n ...Skipping!" 169 ) 170 continue 171 172 added_nodes.extend(next_added) 173 else: 174 for f_name in os.listdir(folder_path): 175 if f_name.endswith(".py") and not f_name.startswith("__"): 176 full_path = os.path.join(folder_path, f_name) 177 if os.path.isfile(full_path): 178 added_nodes.extend(self.add_from_file(full_path)) 179 m_log.info( 180 f"Finished adding blueprint director, {len(added_nodes)} blueprint(s) found" 181 ) 182 return added_nodes
From a directory, adds all Blueprint Nodes to the register Args: folder_path (str): The relative directory path add_subfolders (bool, optional): Whether to add Blueprint Nodes from subdirectories (of any depth) . Defaults to True. Returns: list[str] : Returns a list of all the names of the blueprints that were added
184 def get_all_blueprints(self): 185 """Copies the blueprint callable dictionary, returning all blueprints 186 NB: Blueprints are given by reference, modifications will alter them! 187 """ 188 return dict(self.blueprints)
Copies the blueprint callable dictionary, returning all blueprints NB: Blueprints are given by reference, modifications will alter them!
190 def get_all_schemas(self): 191 """Copies the schema dictionary, returning all blueprint schemas 192 NB: Blueprints are given by reference, modifications will alter them! 193 """ 194 return dict(self.schemas)
Copies the schema dictionary, returning all blueprint schemas NB: Blueprints are given by reference, modifications will alter them!