From a42676010ecc1f21438f81fa18d5d2223205e7e2 Mon Sep 17 00:00:00 2001 From: Bastian Schroll Date: Fri, 1 Mar 2019 12:09:12 +0100 Subject: [PATCH] some reworks - rework configYaml - rework router mechanism test - move plugin and module files --- boswatch/configYaml.py | 88 ++++++--------- boswatch/plugin/pluginManager.py | 16 +-- boswatch/router.py | 6 +- boswatch/utils/paths.py | 2 +- bw_client.py | 14 +-- bw_server.py | 16 +-- config/server.yaml | 27 +---- {boswatch/descriptor => module}/__init__.py | 0 {boswatch/descriptor => module}/descriptor.py | 0 {boswatch => module}/filter/__init__.py | 0 .../filter/doubleFilter.py | 6 +- module/module.py | 103 ++++++++++++++++++ module/template_module.py | 47 ++++++++ plugin/__init__.py | 2 + {boswatch/plugin => plugin}/plugin.py | 12 +- {plugins => plugin}/readme.md | 0 .../template.py => plugin/template_plugin.py | 11 +- plugins/template/template.yaml | 4 - router_test.py | 84 ++++++++++++++ 19 files changed, 317 insertions(+), 121 deletions(-) rename {boswatch/descriptor => module}/__init__.py (100%) rename {boswatch/descriptor => module}/descriptor.py (100%) rename {boswatch => module}/filter/__init__.py (100%) rename boswatch/filter/doubeFilter.py => module/filter/doubleFilter.py (94%) create mode 100644 module/module.py create mode 100644 module/template_module.py create mode 100644 plugin/__init__.py rename {boswatch/plugin => plugin}/plugin.py (94%) rename {plugins => plugin}/readme.md (100%) rename plugins/template/template.py => plugin/template_plugin.py (85%) delete mode 100644 plugins/template/template.yaml create mode 100644 router_test.py diff --git a/boswatch/configYaml.py b/boswatch/configYaml.py index 05dce6e..976e720 100644 --- a/boswatch/configYaml.py +++ b/boswatch/configYaml.py @@ -12,68 +12,52 @@ @file: configYaml.py @date: 27.02.2019 @author: Bastian Schroll -@description: Module for the configuration in yaml format +@description: Module for the configuration in YAML format """ import logging import yaml logging.debug("- %s loaded", __name__) -__sharePoints = {} +class ConfigYAML: -def loadConfigFile(configPath, sharePoint=""): - """!loads a given configuration + def __init__(self, config=None): + self.__config = config - @param configPath: Path to the config file - @param sharePoint: If you want to share the config set name here - @return python dict of config or None""" - logging.debug("load config file from: %s", configPath) - try: + def __iter__(self): + for item in self.__config: + if type(item) is list or type(item) is dict: + yield ConfigYAML(item) + else: + yield item - with open(configPath) as f: - # use safe_load instead load - config = yaml.safe_load(f) - if sharePoint: - _shareConfig(config, sharePoint) - return config - except: # pragma: no cover - logging.exception("cannot load config file") - return None + def __str__(self): + return str(self.__config) + def loadConfigFile(self, configPath): + """!loads a given configuration -def loadConfigSharepoint(sharePoint): - """!loads a given configuration from an sharepoint + @param configPath: Path to the config file + @return True or False""" + logging.debug("load config file from: %s", configPath) + try: + with open(configPath) as file: + # use safe_load instead load + self.__config = yaml.safe_load(file) + return True + except: # pragma: no cover + logging.exception("cannot load config file") + return False - @param sharePoint: Name of the sharepoint - @return python dict of config or None""" - try: - return __sharePoints[sharePoint] - except KeyError: - logging.error("no sharePoint named: %s", sharePoint) - except: # pragma: no cover - logging.exception("error while reading shared config") - return None - - -def _shareConfig(config, sharePoint): - """!Shares the configuration - - Shares the local _config to the class wide global _sharedConfig - @param config: Python dict of the configuration - @param sharePoint: Name of the global share point - @return True or False""" - if sharePoint in __sharePoints: - logging.error("cannot share config - name is always in use: %s", sharePoint) - return False - else: - __sharePoints[sharePoint] = config - logging.debug("add config sharePoint: %s", sharePoint) - return True - - -def getAllSharepoints(): - """!Return a python dict of all set sharepoints - - @return Sharepoint dict""" - return __sharePoints + def get(self, *args, default=None): + tmp = self.__config + try: + for arg in args: + tmp = tmp.get(arg, default) + if type(tmp) is list or type(tmp) is dict: + return ConfigYAML(tmp) + else: + return tmp + except AttributeError: + return default diff --git a/boswatch/plugin/pluginManager.py b/boswatch/plugin/pluginManager.py index ac5b46e..471c9cc 100644 --- a/boswatch/plugin/pluginManager.py +++ b/boswatch/plugin/pluginManager.py @@ -12,7 +12,7 @@ @file: pluginManager.py @date: 08.01.2018 @author: Bastian Schroll -@description: Plugin manager class to load and call the plugins +@description: Plugin manager class to load and call the plugin @todo must be mostly refactored """ import logging @@ -27,7 +27,7 @@ logging.debug("- %s loaded", __name__) class PluginManager: - """!Plugin manager class to load, manage and call the plugins + """!Plugin manager class to load, manage and call the plugin @todo refactor the class and add documentation""" @@ -37,7 +37,7 @@ class PluginManager: self._pluginList = [] def searchPluginDir(self): - logging.debug("search for plugins in: %s", paths.PLUGIN_PATH) + logging.debug("search for plugin in: %s", paths.PLUGIN_PATH) for name in os.listdir(paths.PLUGIN_PATH): location = os.path.join(paths.PLUGIN_PATH, name) @@ -45,7 +45,7 @@ class PluginManager: if not os.path.isdir(location) or not name + ".py" in os.listdir(location): continue - pluginPriority = self._config["plugins"][name] + pluginPriority = self._config["plugin"][name] if pluginPriority is None: logging.warning("no entry in server config for plugin: %s", name) @@ -60,7 +60,7 @@ class PluginManager: self._pluginList.sort(key=lambda x: x['pluginPriority'], reverse=True) def importAllPlugins(self): - logging.debug("importing all plugins") + logging.debug("importing all plugin") for item in self._pluginList: importPlugin = self._importPlugin(item["pluginName"]) if importPlugin: @@ -70,13 +70,13 @@ class PluginManager: def _importPlugin(pluginName): logging.debug("import plugin: %s", pluginName) try: - return importlib.import_module("plugins." + pluginName + "." + pluginName) + return importlib.import_module("plugin." + pluginName + "." + pluginName) except: logging.exception("error while loading plugin: %s", pluginName) return False def loadAllPlugins(self): - logging.debug("loading all plugins") + logging.debug("loading all plugin") for item in self._pluginList: item["pluginObject"] = None # todo del or none ??? item["pluginObject"] = item["pluginImport"].BoswatchPlugin() @@ -89,7 +89,7 @@ class PluginManager: self.printEndStats() def unloadAllPlugins(self): - logging.debug("unload all plugins") + logging.debug("unload all plugin") for item in self._pluginList: # todo del or None ??? del item["pluginObject"] # delete plugin object to force __del__() running diff --git a/boswatch/router.py b/boswatch/router.py index 5ed1a71..b4dc7e3 100644 --- a/boswatch/router.py +++ b/boswatch/router.py @@ -53,7 +53,7 @@ class Router: endpoint.call(bwPacket) -# modules +# module double = Module("double") descriptor = Module("descriptor") # boswatch plugins @@ -63,7 +63,7 @@ mysql = Plugin("mysql") Router1 = Router("R1") Router2 = Router("R2") -# Router 1 modules +# Router 1 module Router1.addModule(double) Router1.addModule(descriptor) Router1.addModule(double) @@ -74,7 +74,7 @@ Router1.addEndpoint(telegram) Router1.addEndpoint(mysql) Router1.addEndpoint(Router2) -# Router 2 modules +# Router 2 module Router2.addModule(double) Router2.addModule(descriptor) # Router 2 endpoints diff --git a/boswatch/utils/paths.py b/boswatch/utils/paths.py index d3f00a7..c20d1e1 100644 --- a/boswatch/utils/paths.py +++ b/boswatch/utils/paths.py @@ -34,7 +34,7 @@ else: LOG_PATH = ROOT_PATH + "log/" CONFIG_PATH = ROOT_PATH + "config/" -PLUGIN_PATH = ROOT_PATH + "plugins/" +PLUGIN_PATH = ROOT_PATH + "plugin/" CSV_PATH = ROOT_PATH + "csv/" BIN_PATH = ROOT_PATH + "_bin/" TEST_PATH = ROOT_PATH + "test/" diff --git a/bw_client.py b/bw_client.py index e355d27..5803be7 100644 --- a/bw_client.py +++ b/bw_client.py @@ -43,7 +43,7 @@ try: import time logging.debug("Import BOSWatch modules") - from boswatch import configYaml + from boswatch.configYaml import ConfigYAML from boswatch.network.client import TCPClient from boswatch.network.broadcast import BroadcastClient from boswatch.decoder.decoder import Decoder @@ -68,26 +68,26 @@ try: parser.add_argument("-c", "--config", help="Name to configuration File", required=True) args = parser.parse_args() - bwConfig = configYaml.loadConfigFile(paths.CONFIG_PATH + args.config, "clientConfig") - if bwConfig is None: + bwConfig = ConfigYAML() + if not bwConfig.loadConfigFile(paths.CONFIG_PATH + args.config): logging.error("cannot load config file") + exit(1) except: # pragma: no cover logging.exception("error occurred") exit(1) - # ############################# begin client system try: - if bwConfig["client"]["useBroadcast"]: + if bwConfig.get("client", "useBroadcast", default=False): broadcastClient = BroadcastClient() if broadcastClient.getConnInfo(): ip = broadcastClient.serverIP port = broadcastClient.serverPort else: - ip = bwConfig["server"]["ip"] - port = bwConfig["server"]["port"] + ip = bwConfig.get("server", "ip", default="127.0.0.1") + port = bwConfig.get("server", "port", default="8080") bwClient = TCPClient() if bwClient.connect(ip, port): diff --git a/bw_server.py b/bw_server.py index 7144e45..aa21002 100644 --- a/bw_server.py +++ b/bw_server.py @@ -35,7 +35,7 @@ except Exception as e: # pragma: no cover try: - logging.debug("Import python modules") + logging.debug("Import python module") import argparse logging.debug("- argparse") # following is temp for testing @@ -44,14 +44,14 @@ try: import threading import queue - logging.debug("Import BOSWatch modules") - from boswatch import configYaml + logging.debug("Import BOSWatch module") + from boswatch.configYaml import ConfigYAML from boswatch.network.server import TCPServer from boswatch.packet.packet import Packet from boswatch.utils import header from boswatch.network.broadcast import BroadcastServer except: # pragma: no cover - logging.exception("cannot import modules") + logging.exception("cannot import module") exit(1) try: @@ -69,19 +69,21 @@ try: parser.add_argument("-c", "--config", help="Name to configuration File", required=True) args = parser.parse_args() - bwConfig = configYaml.loadConfigFile(paths.CONFIG_PATH + args.config, "serverConfig") - if bwConfig is None: + bwConfig = ConfigYAML() + if not bwConfig.loadConfigFile(paths.CONFIG_PATH + args.config): logging.error("cannot load config file") + exit(1) except: # pragma: no cover logging.exception("error occurred") exit(1) +import router_test # ############################# begin server system try: - if bwConfig["server"]["useBroadcast"]: + if bwConfig.get("server", "useBroadcast", default=False): bcServer = BroadcastServer() bcServer.start() diff --git a/config/server.yaml b/config/server.yaml index fb1b90f..1c648ab 100644 --- a/config/server.yaml +++ b/config/server.yaml @@ -12,44 +12,29 @@ server: name: BW3 Server # name of the BW3 Server instance useBroadcast: no # serve server ip on broadcast request - -## here you can enable needed plugins -## 0 is disabled -## all greater than 0 enable the plugin -## the higher the number the earlier the plugin is called on alarm -## we call ist Plugin Prioority -plugins: - template: 1 - -filter: - doubleFilter: - maxEntry: 30 - ignoreTime: 10 - checkMsg: no - - alarmRouter: - Router 1 -- Router 2 router: - name: Router 1 route: - type: module - name: doubleFilter + name: template_module config: maxEntry: 30 ignoreTime: 10 checkMsg: no + - type: router + name: Router 2 - type: plugin - name: mysql + name: template_plugin config: user: test pass: test db: test - - type: router - name: Router 2 - name: Router 2 route: + - type: module + name: template_module diff --git a/boswatch/descriptor/__init__.py b/module/__init__.py similarity index 100% rename from boswatch/descriptor/__init__.py rename to module/__init__.py diff --git a/boswatch/descriptor/descriptor.py b/module/descriptor.py similarity index 100% rename from boswatch/descriptor/descriptor.py rename to module/descriptor.py diff --git a/boswatch/filter/__init__.py b/module/filter/__init__.py similarity index 100% rename from boswatch/filter/__init__.py rename to module/filter/__init__.py diff --git a/boswatch/filter/doubeFilter.py b/module/filter/doubleFilter.py similarity index 94% rename from boswatch/filter/doubeFilter.py rename to module/filter/doubleFilter.py index 68387e6..aa8ba9f 100644 --- a/boswatch/filter/doubeFilter.py +++ b/module/filter/doubleFilter.py @@ -18,17 +18,15 @@ import logging import time -from boswatch import configYaml - logging.debug("- %s loaded", __name__) class DoubleFilter: """!Double Filter Class""" - def __init__(self): + def __init__(self, config): """!init""" - self._config = configYaml.loadConfigSharepoint("serverConfig")["filter"]["doubleFilter"] + self._config = config self._filterLists = {} def filter(self, bwPacket): diff --git a/module/module.py b/module/module.py new file mode 100644 index 0000000..0e5ae6c --- /dev/null +++ b/module/module.py @@ -0,0 +1,103 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +"""! + ____ ____ ______ __ __ __ _____ + / __ )/ __ \/ ___/ | / /___ _/ /______/ /_ |__ / + / __ / / / /\__ \| | /| / / __ `/ __/ ___/ __ \ /_ < + / /_/ / /_/ /___/ /| |/ |/ / /_/ / /_/ /__/ / / / ___/ / +/_____/\____//____/ |__/|__/\__,_/\__/\___/_/ /_/ /____/ + German BOS Information Script + by Bastian Schroll + +@file: module.py +@date: 01.03.2019 +@author: Bastian Schroll +@description: Module main class to inherit +""" +import logging +import time + +logging.debug("- %s loaded", __name__) + + +class Module: + """!Main module class""" + + _modulesActive = 0 + + def __init__(self, moduleName, config): + """!init preload some needed locals and then call onLoad() directly""" + self._moduleName = moduleName + self.config = config + self._modulesActive += 1 + + # for time counting + self._cumTime = 0 + self._moduleTime = 0 + self._tmpTime = 0 + self._tmpTime = 0 + + # for statistics + self._runCount = 0 + self._moduleErrorCount = 0 + + logging.debug("[%s] onLoad()", moduleName) + self.onLoad() + + def __del__(self): + """!Destructor calls onUnload() directly""" + logging.debug("[%s] onUnload()", self._moduleName) + self._modulesActive -= 1 + self.onUnload() + + def _run(self, bwPacket): + """!start an rund of the module. + + @param bwPacket: A BOSWatch packet instance + @return bwPacket or False""" + self._runCount += 1 + logging.debug("[%s] run #%d", self._moduleName, self._runCount) + + self._tmpTime = time.time() + try: + logging.debug("[%s] doWork()", self._moduleName) + bwPacket = self.doWork(bwPacket) + except: + self._moduleErrorCount += 1 + logging.exception("[%s] alarm error", self._moduleName) + self._moduleTime = time.time() - self._tmpTime + + self._cumTime += self._moduleTime + self._endTime = time.time() + + logging.debug("[%s] took %0.3f seconds", self._moduleName, self._moduleTime) + + return bwPacket + + def _getStatistics(self): + """!Returns statistical information's from last module run + + @return Statistics as pyton dict""" + stats = {"runCount": self._runCount, + "cumTime": self._cumTime, + "moduleTime": self._moduleTime, + "moduleErrorCount": self._moduleErrorCount} + return stats + + def onLoad(self): + """!Called by import of the module + Must be inherit""" + pass + + def doWork(self, bwPacket): + """!Called module run + Must be inherit + + @param bwPacket: bwPacket instance""" + logging.warning("no functionality in module %s", self._moduleName) + + def onUnload(self): + """!Called by destruction of the module + Must be inherit""" + pass + diff --git a/module/template_module.py b/module/template_module.py new file mode 100644 index 0000000..15eb12e --- /dev/null +++ b/module/template_module.py @@ -0,0 +1,47 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +"""! + ____ ____ ______ __ __ __ _____ + / __ )/ __ \/ ___/ | / /___ _/ /______/ /_ |__ / + / __ / / / /\__ \| | /| / / __ `/ __/ ___/ __ \ /_ < + / /_/ / /_/ /___/ /| |/ |/ / /_/ / /_/ /__/ / / / ___/ / +/_____/\____//____/ |__/|__/\__,_/\__/\___/_/ /_/ /____/ + German BOS Information Script + by Bastian Schroll + +@file: template_module.py +@date: 01.03.2019 +@author: Bastian Schroll +@description: Template Module File +""" +import logging +from module.module import Module + +# ###################### # +# Custom plugin includes # + +# ###################### # + +logging.debug("- %s loaded", __name__) + + +class BoswatchModule(Module): + """!Description of the Module""" + def __init__(self, config): + """!Do not change anything here!""" + super().__init__(__name__, config) # you can access the config DICT by 'self._config' + + def onLoad(self): + """!Called by import of the plugin""" + pass + + def doWork(self, bwPacket): + """!start an rund of the module. + + @param bwPacket: A BOSWatch packet instance + @return bwPacket or False""" + return bwPacket + + def onUnload(self): + """!Called by destruction of the plugin""" + pass diff --git a/plugin/__init__.py b/plugin/__init__.py new file mode 100644 index 0000000..836e3e8 --- /dev/null +++ b/plugin/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- diff --git a/boswatch/plugin/plugin.py b/plugin/plugin.py similarity index 94% rename from boswatch/plugin/plugin.py rename to plugin/plugin.py index 339c85a..149cb51 100644 --- a/boswatch/plugin/plugin.py +++ b/plugin/plugin.py @@ -17,8 +17,6 @@ import logging import time -from boswatch.utils import paths -from boswatch import configYaml from boswatch.utils import wildcard logging.debug("- %s loaded", __name__) @@ -29,9 +27,10 @@ class Plugin: _pluginsActive = 0 - def __init__(self, pluginName): + def __init__(self, pluginName, config): """!init preload some needed locals and then call onLoad() directly""" self._pluginName = pluginName + self.config = config self._pluginsActive += 1 # to save the packet while alarm is running for other functions @@ -52,11 +51,6 @@ class Plugin: self._alarmErrorCount = 0 self._teardownErrorCount = 0 - if paths.fileExist(paths.PLUGIN_PATH + pluginName + "/" + pluginName + ".yaml"): - self.config = configYaml.loadConfigFile(paths.PLUGIN_PATH + pluginName + "/" + pluginName + ".yaml") - else: - logging.debug("no config for %s found", pluginName) - logging.debug("[%s] onLoad()", pluginName) self.onLoad() @@ -126,6 +120,8 @@ class Plugin: # logging.debug("- alarm: %0.2f sec.", self._alarmTime) # logging.debug("- teardown: %0.2f sec.", self._teardownTime) + return None + def _getStatistics(self): """!Returns statistical information's from last plugin run diff --git a/plugins/readme.md b/plugin/readme.md similarity index 100% rename from plugins/readme.md rename to plugin/readme.md diff --git a/plugins/template/template.py b/plugin/template_plugin.py similarity index 85% rename from plugins/template/template.py rename to plugin/template_plugin.py index de1b78a..99fb80c 100644 --- a/plugins/template/template.py +++ b/plugin/template_plugin.py @@ -9,13 +9,13 @@ German BOS Information Script by Bastian Schroll -@file: template.py +@file: template_module.py @date: 14.01.2018 @author: Bastian Schroll @description: Template Plugin File """ import logging -from boswatch.plugin.plugin import Plugin +from plugin.plugin import Plugin # ###################### # # Custom plugin includes # @@ -27,10 +27,9 @@ logging.debug("- %s loaded", __name__) class BoswatchPlugin(Plugin): """!Description of the Plugin""" - def __init__(self): - """!Do not change anything here except the PLUGIN NAME in the super() call""" - # PLEASE SET YOU PLUGIN NAME HERE !!!! - super().__init__("template") + def __init__(self, config): + """!Do not change anything here!""" + super().__init__(__name__, config) # you can access the config DICT by 'self._config' def onLoad(self): """!Called by import of the plugin""" diff --git a/plugins/template/template.yaml b/plugins/template/template.yaml deleted file mode 100644 index 4fde236..0000000 --- a/plugins/template/template.yaml +++ /dev/null @@ -1,4 +0,0 @@ -example: - string: Hello World! - bool: true - integer: 12 diff --git a/router_test.py b/router_test.py new file mode 100644 index 0000000..226768a --- /dev/null +++ b/router_test.py @@ -0,0 +1,84 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import logging + +from boswatch.configYaml import ConfigYAML +from boswatch.packet.packet import Packet + +import importlib +import copy + + +class Router: + def __init__(self, name): + self.__name = name + self.__route = [] + logging.debug("add new router: %s", self.__name) + + def addRoute(self, route): + logging.debug("[%s] add route: %s", self.__name, route) + self.__route.append(route) + + def call(self, bwPacket): + for call in self.__route: + logging.debug("[%s] -> run route: %s", self.__name, call) + bwPacket_tmp = call(copy.deepcopy(bwPacket)) # todo is deepcopy here right? + + if bwPacket_tmp is None: # returning None doesnt change the bwPacket + continue + + if bwPacket is False: # returning False stops the router immediately + logging.debug("[%s] stopped", self.__name) + break + + bwPacket = bwPacket_tmp + logging.debug("[%s] <- route returned: %s", self.__name, bwPacket) + logging.debug("[%s] ended", self.__name) + return bwPacket + + def showRoute(self): + logging.debug("[%s] internal route", self.__name) + for call in self.__route: + logging.debug(" - %s", call) + + +config = ConfigYAML() +config.loadConfigFile("config/server.yaml") + +routerList = {} +for router in config.get("router"): + routerList[router.get("name")] = Router(router.get("name")) + + +for router in config.get("router"): + for route in router.get("route"): + + if route.get("type") == "plugin": + importedFile = importlib.import_module(route.get("type") + "." + route.get("name")) + loadedClass = importedFile.BoswatchPlugin(route.get("config")) + routerList[router.get("name")].addRoute(loadedClass._run) + + elif route.get("type") == "module": + importedFile = importlib.import_module(route.get("type") + "." + route.get("name")) + loadedClass = importedFile.BoswatchModule(route.get("config")) + routerList[router.get("name")].addRoute(loadedClass._run) + + elif route.get("type") == "router": + routerList[router.get("name")].addRoute(routerList[route.get("name")].call) + +print() +print(routerList) +print() + +for router in routerList: + routerList[router].showRoute() + +print() + +bwPack = Packet("{'timestamp': 1551421020.9004176, 'mode': 'zvei', 'zvei': '12345'}") +for alaRouter in config.get("alarmRouter"): + routerList[str(alaRouter)].call(bwPack) + + +exit(0)