diff --git a/includes/descriptionList.py b/includes/descriptionList.py index fac53cc..3a5b662 100644 --- a/includes/descriptionList.py +++ b/includes/descriptionList.py @@ -14,7 +14,7 @@ import logging # Global logger import csv # for loading the description files from includes import globals # Global variables -from includes.helper import uft8Converter # UTF-8 converter +from includes.helper import stringConverter ## @@ -39,7 +39,7 @@ def loadCSV(typ, idField): # only import rows with an integer as id if row[idField].isdigit() == True: try: - resultList[row[idField]] = uft8Converter.convertToUTF8(row['description']) + resultList[row[idField]] = stringConverter.convertToUTF8(row['description']) except: # skip entry in case of an exception pass diff --git a/includes/globals.py b/includes/globals.py index a9b307d..ad3251e 100644 --- a/includes/globals.py +++ b/includes/globals.py @@ -10,7 +10,7 @@ Global variables # version info versionNr = "2.1-dev" -buildDate = "2015/07/29" +buildDate = "2015/07/30" # Global variables config = 0 diff --git a/includes/helper/stringConverter.py b/includes/helper/stringConverter.py new file mode 100644 index 0000000..4d2ee4b --- /dev/null +++ b/includes/helper/stringConverter.py @@ -0,0 +1,171 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# + +""" +little Helper for converting strings + +@author: Jens Herrmann +""" + +import logging + +# +# local helper function to decode a string... +# +def decodeString(inputString = ""): + """ + Returns given string as unicode + + @type string: String + @param string: String to convert to unicode + + @return: string in unicode + @exception: Exception if converting to unicode failed + """ + decodedString = "" + logging.debug("call decodeString('%s')", inputString) + # try to find out encoding: + encodings = ('utf-8', 'windows-1250', 'windows-1252', 'latin_1', 'cp850', 'cp852', 'iso8859_2', 'iso8859_15', 'mac_latin2', 'mac_roman') + for enc in encodings: + try: + decodedString = inputString.decode(enc) + logging.debug("-- string was encoded in: %s", enc) + break + except Exception: + # if exception for last encoding entry fail, raise exception + if enc == encodings[-1]: + logging.warning("no encoding found") + logging.debug("no encoding found", exc_info=True) + # no fixing possible, raise exception + raise + pass + return decodedString + + +def convertToUnicode(inputString = ""): + """ + Returns given string as unicode + + @type string: String + @param string: String to convert to unicode + + @return: string in unicode + @exception: Exception if converting to unicode failed + """ + + decodedString = "" + logging.debug("call convertToUnicode('%s')", inputString) + + # nothing to do if inputString is empty + if len(inputString) > 0: + # 1. check if integer + try: + if int(inputString): + logging.debug("-- integer") + # ... then return it + return inputString + except ValueError: + # ... no integer is okay... + pass + + # 2. Check if inputString is unicode... + if isinstance(inputString, unicode): + logging.debug("-- unicode") + return inputString + + try: + # try to decoding: + decodedString = decodeString(inputString) + except: + logging.warning("decoding string failed") + logging.debug("encoding string failed", exc_info=True) + # no fixing possible, raise exception + raise + return decodedString + + + +def convertToUTF8(inputString = ""): + """ + Returns given string in UTF-8 + + @type string: String + @param string: String to convert to UTF-8 + + @return: string in UTF-8 + @exception: Exception if converting to UTF-8 failed + """ + + uft8String = "" + logging.debug("call convertToUTF8('%s')", inputString) + + # nothing to do if inputString is empty + if len(inputString) > 0: + try: + # 1. check if integer + try: + if int(inputString): + logging.debug("-- integer") + # ... then return it + return inputString + except ValueError: + pass + + # 2. Check if inputString is unicode... + if isinstance(inputString, unicode): + logging.debug("-- unicode") + # ... then return it as UTF-8 + uft8String = decodedString.encode('UTF-8') + return uft8String + + # 2. check given inputString is already UTF-8... + decodedString = inputString.decode('UTF-8', 'strict') + # ... no UnicodeDecodeError exception, inputString ist UTF-8 + logging.debug("-- UTF-8") + return inputString + + except UnicodeDecodeError: + # inputString contains non-UTF-8 character + logging.debug("string contains non-UTF-8 characters: %s", inputString) + + try: + # try to decoding: + decodedString = decodeString(inputString) + except: + logging.warning("decoding string failed") + logging.debug("encoding string failed", exc_info=True) + # no fixing possible, raise exception + raise + + # inputString should now decoded... + + try: + # encode decodedString to UTF-8 + uft8String = decodedString.encode('UTF-8') + except: + logging.warning("encoding to UTF-8 failed") + logging.debug("encoding to UTF-8 failed", exc_info=True) + # no fixing possible, raise exception + raise + + # Now we must have an utf8-string, check it: + try: + uft8String.decode('UTF-8', 'strict') + logging.debug("string converting succeeded: %s", uft8String) + except: + logging.warning("converting to UTF-8 failed") + logging.debug("converting to UTF-8 failed", exc_info=True) + # no fixing possible, raise exception + raise + + # End of exception UnicodeDecodeError: check given string is already UTF-8 + pass + + except: + logging.warning("error checking given string") + logging.debug("error checking given string", exc_info=True) + # no fixing possible, raise exception + raise + + return uft8String \ No newline at end of file diff --git a/includes/helper/uft8Converter.py b/includes/helper/uft8Converter.py deleted file mode 100644 index 16fee3e..0000000 --- a/includes/helper/uft8Converter.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/python -# -*- coding: UTF-8 -*- -# - -""" -little Helper for converting strings - -@author: Jens Herrmann -""" - -import logging - - -def convertToUTF8(string = ""): - """ - Returns given string in UTF-8 - - @type string: String - @param string: String to convert to UTF-8 - - @return: string in UTF-8 - @exception: Exception if converting to UTF-8 failed - """ - - uft8String = "" - - # nothing to do if string is empty - if len(string) > 0: - try: - # check given string is already UTF-8, return - return string.decode('UTF-8', 'strict') - except UnicodeDecodeError: - # string contains non-UTF-8 character - logging.debug("string contains non-UTF-8 characters: %s", string) - - # try to find out encoding: - encodings = ('windows-1250', 'windows-1252', 'latin_1', 'cp850', 'cp852', 'iso8859_2', 'iso8859_15', 'mac_latin2', 'mac_roman') - for enc in encodings: - try: - string = string.decode(enc) - logging.debug("string was encoded in: %s", enc) - break - except Exception: - # if exception for last encoding entry fail, raise exception - if enc == encodings[-1]: - logging.warning("no encoding found") - logging.debug("no encoding found", exc_info=True) - # no fixing possible, raise exception - raise - pass - - # string should now decoded... - - try: - # encode decoded string to UTF-8 - uft8String = string.encode('UTF-8') - except: - logging.warning("encoding to UTF-8 failed") - logging.debug("encoding to UTF-8 failed", exc_info=True) - # no fixing possible, raise exception - raise - - # Now we must have an utf8-string, check it: - try: - uft8String.decode('UTF-8', 'strict') - logging.debug("string converting succeeded: %s", uft8String) - except: - logging.warning("converting to UTF-8 failed") - logging.debug("converting to UTF-8 failed", exc_info=True) - # no fixing possible, raise exception - raise - - # End of exception: check given string is already UTF-8 - pass - - return uft8String \ No newline at end of file diff --git a/plugins/firEmergency/firEmergency.py b/plugins/firEmergency/firEmergency.py index 10fe941..329120f 100644 --- a/plugins/firEmergency/firEmergency.py +++ b/plugins/firEmergency/firEmergency.py @@ -18,6 +18,8 @@ import socket from includes import globals # Global variables from includes.helper import configHandler +from includes.helper import stringConverter + ### # @@ -85,7 +87,8 @@ def run(typ,freq,data): elif typ == "ZVEI": logging.debug("ZVEI to firEmergency") try: - firXML = "\n
"+data["zvei"]+"
\n"+data["description"]+"\n"+data["zvei"]+" alarmiert.\n\n" + description = stringConverter.convertToUTF8(data["description"]) + firXML = "\n
"+data["zvei"]+"
\n"+description+"\n"+data["zvei"]+"\n\n" firSocket.send(firXML) except: logging.error("%s to firEmergency failed", typ) @@ -97,7 +100,9 @@ def run(typ,freq,data): logging.debug("POC to firEmergency") try: # !!! Subric+"XX" because of an Issuse in firEmergency !!! - firXML = "\n
"+data["ric"]+"
\n"+data["function"]+"XX\n"+data["description"]+"\n"+data["msg"]+"\n\n" + description = stringConverter.convertToUTF8(data["description"]) + msg = stringConverter.convertToUTF8(data["msg"]) + firXML = "\n
"+data["ric"]+"
\n"+data["function"]+"XX\n"+description+"\n"+msg+"\n\n" firSocket.send(firXML) except: logging.error("%s to firEmergency failed", typ) diff --git a/plugins/notifyMyAndroid/notifyMyAndroid.py b/plugins/notifyMyAndroid/notifyMyAndroid.py index e09a795..912865c 100644 --- a/plugins/notifyMyAndroid/notifyMyAndroid.py +++ b/plugins/notifyMyAndroid/notifyMyAndroid.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: UTF-8 -*- +# -*- coding: utf-8 -*- """ notifyMyAndroid-Plugin to dispatch FMS-, ZVEI- and POCSAG-messages via UDP/TCP @@ -20,7 +20,7 @@ from includes import globals # Global variables from includes.helper import configHandler from includes.helper import timeHandler -from includes.helper import uft8Converter # UTF-8 converter +from includes.helper import stringConverter from includes.pynma import pynma @@ -93,7 +93,7 @@ def onLoad(): # load config: configHandler.checkConfig("notifyMyAndroid") - application = globals.config.get("notifyMyAndroid","appName") + application = stringConverter.convertToUnicode(globals.config.get("notifyMyAndroid","appName")) usecsv = globals.config.getboolean("notifyMyAndroid","usecsv") # if no csv should use, we take the APIKey directly @@ -207,11 +207,12 @@ def run(typ,freq,data): logging.debug("Start %s to NMA", typ) try: # build event and msg - event = uft8Converter.convertToUTF8(data['description']) - msg = timeHandler.curtime() - if len(data['msg']) > 0: + # pyNMA expect strings are not in UTF-8 + event = stringConverter.convertToUnicode(data['description']) + msg = timeHandler.curtime() + if ("POC" in typ) and (len(data['msg']) > 0): msg += "\n" + data['msg'] - msg = uft8Converter.convertToUTF8(msg) + msg = stringConverter.convertToUnicode(msg) # if not using csv-import, all is simple... if usecsv == False: