some reworks

- rework configYaml
- rework router mechanism test
- move plugin and module files
This commit is contained in:
Bastian Schroll 2019-03-01 12:09:12 +01:00
parent 71d87b080f
commit a42676010e
19 changed files with 317 additions and 121 deletions

View file

@ -12,68 +12,52 @@
@file: configYaml.py
@date: 27.02.2019
@author: Bastian Schroll
@description: Module for the configuration in yaml format
@description: Module for the configuration in YAML format
"""
import logging
import yaml
logging.debug("- %s loaded", __name__)
__sharePoints = {}
class ConfigYAML:
def loadConfigFile(configPath, sharePoint=""):
"""!loads a given configuration
def __init__(self, config=None):
self.__config = config
@param configPath: Path to the config file
@param sharePoint: If you want to share the config set name here
@return python dict of config or None"""
logging.debug("load config file from: %s", configPath)
try:
def __iter__(self):
for item in self.__config:
if type(item) is list or type(item) is dict:
yield ConfigYAML(item)
else:
yield item
with open(configPath) as f:
# use safe_load instead load
config = yaml.safe_load(f)
if sharePoint:
_shareConfig(config, sharePoint)
return config
except: # pragma: no cover
logging.exception("cannot load config file")
return None
def __str__(self):
return str(self.__config)
def loadConfigFile(self, configPath):
"""!loads a given configuration
def loadConfigSharepoint(sharePoint):
"""!loads a given configuration from an sharepoint
@param configPath: Path to the config file
@return True or False"""
logging.debug("load config file from: %s", configPath)
try:
with open(configPath) as file:
# use safe_load instead load
self.__config = yaml.safe_load(file)
return True
except: # pragma: no cover
logging.exception("cannot load config file")
return False
@param sharePoint: Name of the sharepoint
@return python dict of config or None"""
try:
return __sharePoints[sharePoint]
except KeyError:
logging.error("no sharePoint named: %s", sharePoint)
except: # pragma: no cover
logging.exception("error while reading shared config")
return None
def _shareConfig(config, sharePoint):
"""!Shares the configuration
Shares the local _config to the class wide global _sharedConfig
@param config: Python dict of the configuration
@param sharePoint: Name of the global share point
@return True or False"""
if sharePoint in __sharePoints:
logging.error("cannot share config - name is always in use: %s", sharePoint)
return False
else:
__sharePoints[sharePoint] = config
logging.debug("add config sharePoint: %s", sharePoint)
return True
def getAllSharepoints():
"""!Return a python dict of all set sharepoints
@return Sharepoint dict"""
return __sharePoints
def get(self, *args, default=None):
tmp = self.__config
try:
for arg in args:
tmp = tmp.get(arg, default)
if type(tmp) is list or type(tmp) is dict:
return ConfigYAML(tmp)
else:
return tmp
except AttributeError:
return default

View file

@ -1,2 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-

View file

@ -1,111 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""!
____ ____ ______ __ __ __ _____
/ __ )/ __ \/ ___/ | / /___ _/ /______/ /_ |__ /
/ __ / / / /\__ \| | /| / / __ `/ __/ ___/ __ \ /_ <
/ /_/ / /_/ /___/ /| |/ |/ / /_/ / /_/ /__/ / / / ___/ /
/_____/\____//____/ |__/|__/\__,_/\__/\___/_/ /_/ /____/
German BOS Information Script
by Bastian Schroll
@file: descriptor.py
@date: 07.01.2018
@author: Bastian Schroll
@description: Descriptor to load Descriptions from csv files
"""
import logging
import csv
import re
from boswatch.utils import paths
logging.debug("- %s loaded", __name__)
class Descriptor:
"""!CSV Descriptor class to load specific descriptions from CSV files, manage and serve them"""
def __init__(self):
"""!Initialise a private list for the DescriptionList objects"""
self._lists = {}
def loadDescription(self, csvType):
"""!Build a new description list from DescriptionList class
@param csvType: Name of the CSV file without `.csv`
@return True or False"""
bwDescriptionList = DescriptionList()
if bwDescriptionList.loadCSV(csvType):
self._lists[csvType] = bwDescriptionList
return True
return False
def addDescriptions(self, bwPacket):
"""!Add the short and long description to a bwPacket
@param bwPacket: bwPacket instance to add descriptions
@return True or False"""
logging.debug("add descriptions to bwPacket")
try:
bwPacket.set("shortDescription",
self._lists[bwPacket.get("mode")].getShortDescription(bwPacket.get(bwPacket.get("mode"))))
bwPacket.set("longDescription",
self._lists[bwPacket.get("mode")].getLongDescription(bwPacket.get(bwPacket.get("mode"))))
return True
except: # pragma: no cover
logging.exception("error while adding descriptions")
return False
class DescriptionList:
def __init__(self):
"""!Loads the given CSV file into internal list"""
logging.debug("create new descriptionList")
self._descriptionList = {}
def getShortDescription(self, checkId):
"""!Returns the short description of given id
@return short description or empty string"""
try:
return self._descriptionList[str(checkId)]["shortDescription"]
except:
return ""
def getLongDescription(self, checkId):
"""!Returns the long description of given id
@return long description or empty string"""
try:
return self._descriptionList[str(checkId)]["longDescription"]
except:
return ""
def loadCSV(self, csvType):
"""!Load descriptions from an csv file
@param csvType: Name of the CSV file without `.csv`
@return True or False"""
count = 0
logging.debug("loading csv file: %s", csvType)
csvPath = paths.CSV_PATH + csvType + ".csv"
try:
csvFile = open(csvPath, 'r', -1, 'utf-8')
reader = csv.DictReader(csvFile)
for line in reader:
if re.match("^[0-9]+[A-D]?$", line["id"], re.IGNORECASE):
self._descriptionList[line["id"]] = {"shortDescription": line["shortDescription"], "longDescription": line["longDescription"]}
logging.debug("- %s", line)
count += 1
logging.debug("%s entry's loaded", count)
return True
except FileNotFoundError:
logging.error("csv file not found: %s", csvPath)
return False
except: # pragma: no cover
logging.exception("error while loading descriptions")
return False

View file

@ -1,2 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-

View file

@ -1,77 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""!
____ ____ ______ __ __ __ _____
/ __ )/ __ \/ ___/ | / /___ _/ /______/ /_ |__ /
/ __ / / / /\__ \| | /| / / __ `/ __/ ___/ __ \ /_ <
/ /_/ / /_/ /___/ /| |/ |/ / /_/ / /_/ /__/ / / / ___/ /
/_____/\____//____/ |__/|__/\__,_/\__/\___/_/ /_/ /____/
German BOS Information Script
by Bastian Schroll
@file: doubleFilter.py
@date: 15.01.2018
@author: Bastian Schroll
@description: Class to implement a filter for double alarms
@todo test, refactor and document / check_msg is not implemented yet
"""
import logging
import time
from boswatch import configYaml
logging.debug("- %s loaded", __name__)
class DoubleFilter:
"""!Double Filter Class"""
def __init__(self):
"""!init"""
self._config = configYaml.loadConfigSharepoint("serverConfig")["filter"]["doubleFilter"]
self._filterLists = {}
def filter(self, bwPacket):
if bwPacket.get("mode") == "fms":
scanWord = "fms"
elif bwPacket.get("mode") == "pocsag":
scanWord = "ric"
elif bwPacket.get("mode") == "zvei":
scanWord = "zvei"
else:
logging.error("No Filter for '%s'", bwPacket)
return False
if not bwPacket.get("mode") in self._filterLists:
logging.debug("create new doubleFilter list for '%s'", bwPacket.get("mode"))
self._filterLists[bwPacket.get("mode")] = []
logging.debug("scanWord for '%s' is '%s'", bwPacket.get("mode"), scanWord)
return self._check(bwPacket, scanWord)
def _check(self, bwPacket, scanWord):
self._filterLists[bwPacket.get("mode")].insert(0, bwPacket)
# delete entries that are to old
counter = 0
for listPacket in self._filterLists[bwPacket.get("mode")][1:]: # [1:] skip first entry, thats the new one
if listPacket.get("timestamp") < (time.time() - self._config["ignoreTime"]):
self._filterLists[bwPacket.get("mode")].remove(listPacket)
counter += 1
if counter:
logging.debug("%d old entry(s) removed", counter)
# delete last entry if list is to big
if len(self._filterLists[bwPacket.get("mode")]) > self._config["maxEntry"]:
logging.debug("MaxEntry reached - delete oldest")
self._filterLists[bwPacket.get("mode")].pop()
for listPacket in self._filterLists[bwPacket.get("mode")][1:]: # [1:] skip first entry, thats the new one
if listPacket.get(scanWord) is bwPacket.get(scanWord):
logging.debug("found duplicate: %s", bwPacket.get(scanWord))
return False
logging.debug("doubleFilter ok")
return True

View file

@ -1,197 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""!
____ ____ ______ __ __ __ _____
/ __ )/ __ \/ ___/ | / /___ _/ /______/ /_ |__ /
/ __ / / / /\__ \| | /| / / __ `/ __/ ___/ __ \ /_ <
/ /_/ / /_/ /___/ /| |/ |/ / /_/ / /_/ /__/ / / / ___/ /
/_____/\____//____/ |__/|__/\__,_/\__/\___/_/ /_/ /____/
German BOS Information Script
by Bastian Schroll
@file: plugin.py
@date: 08.01.2018
@author: Bastian Schroll
@description: Plugin main class to inherit
"""
import logging
import time
from boswatch.utils import paths
from boswatch import configYaml
from boswatch.utils import wildcard
logging.debug("- %s loaded", __name__)
class Plugin:
"""!Main plugin class"""
_pluginsActive = 0
def __init__(self, pluginName):
"""!init preload some needed locals and then call onLoad() directly"""
self._pluginName = pluginName
self._pluginsActive += 1
# to save the packet while alarm is running for other functions
self._bwPacket = None
# for time counting
self._sumTime = 0
self._cumTime = 0
self._setupTime = 0
self._alarmTime = 0
self._teardownTime = 0
self._endTime = 0
self._tmpTime = 0
# for statistics
self._runCount = 0
self._setupErrorCount = 0
self._alarmErrorCount = 0
self._teardownErrorCount = 0
if paths.fileExist(paths.PLUGIN_PATH + pluginName + "/" + pluginName + ".yaml"):
self.config = configYaml.loadConfigFile(paths.PLUGIN_PATH + pluginName + "/" + pluginName + ".yaml")
else:
logging.debug("no config for %s found", pluginName)
logging.debug("[%s] onLoad()", pluginName)
self.onLoad()
def __del__(self):
"""!Destructor calls onUnload() directly"""
logging.debug("[%s] onUnload()", self._pluginName)
self._pluginsActive -= 1
self.onUnload()
def _run(self, bwPacket):
"""!start an complete running turn of an plugin.
Calls setup(), alarm() and teardown() in this order.
The alarm() method serves the BOSWatch packet to the plugin.
@param bwPacket: A BOSWatch packet instance"""
self._runCount += 1
logging.debug("[%s] run #%d", self._pluginName, self._runCount)
self._bwPacket = bwPacket
self._tmpTime = time.time()
try:
logging.debug("[%s] setup()", self._pluginName)
self.setup()
except:
self._setupErrorCount += 1
logging.exception("[%s] error in setup()", self._pluginName)
self._setupTime = time.time() - self._tmpTime
self._tmpTime = time.time()
try:
if bwPacket.get("mode") == "fms":
logging.debug("[%s] fms()", self._pluginName)
self.fms(bwPacket)
if bwPacket.get("mode") == "pocsag":
logging.debug("[%s] pocsag()", self._pluginName)
self.pocsag(bwPacket)
if bwPacket.get("mode") == "zvei":
logging.debug("[%s] zvei()", self._pluginName)
self.zvei(bwPacket)
if bwPacket.get("mode") == "msg":
logging.debug("[%s] msg()", self._pluginName)
self.msg(bwPacket)
except:
self._alarmErrorCount += 1
logging.exception("[%s] alarm error", self._pluginName)
self._alarmTime = time.time() - self._tmpTime
self._tmpTime = time.time()
try:
logging.debug("[%s] teardown()", self._pluginName)
self.teardown()
except:
self._teardownErrorCount += 1
logging.exception("[%s] error in teardown()", self._pluginName)
self._teardownTime = time.time() - self._tmpTime
self._sumTime = self._setupTime + self._alarmTime + self._teardownTime
self._cumTime += self._sumTime
self._endTime = time.time()
self._bwPacket = None
logging.debug("[%s] took %0.3f seconds", self._pluginName, self._sumTime)
# logging.debug("- setup: %0.2f sec.", self._setupTime)
# logging.debug("- alarm: %0.2f sec.", self._alarmTime)
# logging.debug("- teardown: %0.2f sec.", self._teardownTime)
def _getStatistics(self):
"""!Returns statistical information's from last plugin run
@return Statistics as pyton dict"""
stats = {"runCount": self._runCount,
"sumTime": self._sumTime,
"cumTime": self._cumTime,
"setupTime": self._setupTime,
"alarmTime": self._alarmTime,
"teardownTime": self._teardownTime,
"setupErrorCount": self._setupErrorCount,
"alarmErrorCount": self._alarmErrorCount,
"teardownErrorCount": self._teardownErrorCount}
return stats
def onLoad(self):
"""!Called by import of the plugin
Must be inherit"""
pass
def setup(self):
"""!Called before alarm
Must be inherit"""
pass
def fms(self, bwPacket):
"""!Called on FMS alarm
Must be inherit
@param bwPacket: bwPacket instance"""
logging.warning("ZVEI not implemented in %s", self._pluginName)
def pocsag(self, bwPacket):
"""!Called on POCSAG alarm
Must be inherit
@param bwPacket: bwPacket instance"""
logging.warning("POCSAG not implemented in %s", self._pluginName)
def zvei(self, bwPacket):
"""!Called on ZVEI alarm
Must be inherit
@param bwPacket: bwPacket instance"""
logging.warning("ZVEI not implemented in %s", self._pluginName)
def msg(self, bwPacket):
"""!Called on MSG packet
Must be inherit
@param bwPacket: bwPacket instance"""
logging.warning("MSG not implemented in %s", self._pluginName)
def teardown(self):
"""!Called after alarm
Must be inherit"""
pass
def onUnload(self):
"""!Called by destruction of the plugin
Must be inherit"""
pass
def parseWildcards(self, msg):
"""!Return the message with parsed wildcards"""
if self._bwPacket is None:
logging.warning("wildcard replacing not allowed - no bwPacket set")
return msg
return wildcard.replaceWildcards(self._bwPacket, msg)

View file

@ -12,7 +12,7 @@
@file: pluginManager.py
@date: 08.01.2018
@author: Bastian Schroll
@description: Plugin manager class to load and call the plugins
@description: Plugin manager class to load and call the plugin
@todo must be mostly refactored
"""
import logging
@ -27,7 +27,7 @@ logging.debug("- %s loaded", __name__)
class PluginManager:
"""!Plugin manager class to load, manage and call the plugins
"""!Plugin manager class to load, manage and call the plugin
@todo refactor the class and add documentation"""
@ -37,7 +37,7 @@ class PluginManager:
self._pluginList = []
def searchPluginDir(self):
logging.debug("search for plugins in: %s", paths.PLUGIN_PATH)
logging.debug("search for plugin in: %s", paths.PLUGIN_PATH)
for name in os.listdir(paths.PLUGIN_PATH):
location = os.path.join(paths.PLUGIN_PATH, name)
@ -45,7 +45,7 @@ class PluginManager:
if not os.path.isdir(location) or not name + ".py" in os.listdir(location):
continue
pluginPriority = self._config["plugins"][name]
pluginPriority = self._config["plugin"][name]
if pluginPriority is None:
logging.warning("no entry in server config for plugin: %s", name)
@ -60,7 +60,7 @@ class PluginManager:
self._pluginList.sort(key=lambda x: x['pluginPriority'], reverse=True)
def importAllPlugins(self):
logging.debug("importing all plugins")
logging.debug("importing all plugin")
for item in self._pluginList:
importPlugin = self._importPlugin(item["pluginName"])
if importPlugin:
@ -70,13 +70,13 @@ class PluginManager:
def _importPlugin(pluginName):
logging.debug("import plugin: %s", pluginName)
try:
return importlib.import_module("plugins." + pluginName + "." + pluginName)
return importlib.import_module("plugin." + pluginName + "." + pluginName)
except:
logging.exception("error while loading plugin: %s", pluginName)
return False
def loadAllPlugins(self):
logging.debug("loading all plugins")
logging.debug("loading all plugin")
for item in self._pluginList:
item["pluginObject"] = None # todo del or none ???
item["pluginObject"] = item["pluginImport"].BoswatchPlugin()
@ -89,7 +89,7 @@ class PluginManager:
self.printEndStats()
def unloadAllPlugins(self):
logging.debug("unload all plugins")
logging.debug("unload all plugin")
for item in self._pluginList:
# todo del or None ???
del item["pluginObject"] # delete plugin object to force __del__() running

View file

@ -53,7 +53,7 @@ class Router:
endpoint.call(bwPacket)
# modules
# module
double = Module("double")
descriptor = Module("descriptor")
# boswatch plugins
@ -63,7 +63,7 @@ mysql = Plugin("mysql")
Router1 = Router("R1")
Router2 = Router("R2")
# Router 1 modules
# Router 1 module
Router1.addModule(double)
Router1.addModule(descriptor)
Router1.addModule(double)
@ -74,7 +74,7 @@ Router1.addEndpoint(telegram)
Router1.addEndpoint(mysql)
Router1.addEndpoint(Router2)
# Router 2 modules
# Router 2 module
Router2.addModule(double)
Router2.addModule(descriptor)
# Router 2 endpoints

View file

@ -34,7 +34,7 @@ else:
LOG_PATH = ROOT_PATH + "log/"
CONFIG_PATH = ROOT_PATH + "config/"
PLUGIN_PATH = ROOT_PATH + "plugins/"
PLUGIN_PATH = ROOT_PATH + "plugin/"
CSV_PATH = ROOT_PATH + "csv/"
BIN_PATH = ROOT_PATH + "_bin/"
TEST_PATH = ROOT_PATH + "test/"