diff --git a/boswatch.py b/boswatch.py index 9e63beb..7167b4c 100755 --- a/boswatch.py +++ b/boswatch.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- # """ BOSWatch diff --git a/csv/poc.csv b/csv/poc.csv index 44ddacb..c7c7893 100644 --- a/csv/poc.csv +++ b/csv/poc.csv @@ -7,4 +7,4 @@ ric,description # # !!! DO NOT delete the first line !!! # -1234567,"POCSAG testdata" +1234567,"POCSAG testdata üöäß" diff --git a/includes/MyTimedRotatingFileHandler.py b/includes/MyTimedRotatingFileHandler.py index bafd4a8..348d4d1 100644 --- a/includes/MyTimedRotatingFileHandler.py +++ b/includes/MyTimedRotatingFileHandler.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- # """ diff --git a/includes/__init__.py b/includes/__init__.py index e69de29..dcd3642 100644 --- a/includes/__init__.py +++ b/includes/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- diff --git a/includes/alarmHandler.py b/includes/alarmHandler.py index 212593e..6176a10 100644 --- a/includes/alarmHandler.py +++ b/includes/alarmHandler.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ Handler for the filter and plugins at an alarm diff --git a/includes/checkSubprocesses.py b/includes/checkSubprocesses.py index f76e1eb..43c91bd 100644 --- a/includes/checkSubprocesses.py +++ b/includes/checkSubprocesses.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- # """ Functions for checking the subprocesses rtl_fm and multimon-ng diff --git a/includes/converter.py b/includes/converter.py index b662509..c18b48a 100644 --- a/includes/converter.py +++ b/includes/converter.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- # """ diff --git a/includes/decoder.py b/includes/decoder.py index 6224314..d62604d 100644 --- a/includes/decoder.py +++ b/includes/decoder.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ Search for decode string and call the right decoder function diff --git a/includes/decoders/__init__.py b/includes/decoders/__init__.py index e69de29..dcd3642 100644 --- a/includes/decoders/__init__.py +++ b/includes/decoders/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- diff --git a/includes/decoders/fms.py b/includes/decoders/fms.py index 08c06f8..913119d 100644 --- a/includes/decoders/fms.py +++ b/includes/decoders/fms.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ FMS Decoder diff --git a/includes/decoders/poc.py b/includes/decoders/poc.py index b40dafe..20f393e 100644 --- a/includes/decoders/poc.py +++ b/includes/decoders/poc.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ POCSAG Decoder diff --git a/includes/decoders/zvei.py b/includes/decoders/zvei.py index ce6e9e2..21848f5 100644 --- a/includes/decoders/zvei.py +++ b/includes/decoders/zvei.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ ZVEI Decoder diff --git a/includes/descriptionList.py b/includes/descriptionList.py index 80b7ea7..3e7cb7f 100644 --- a/includes/descriptionList.py +++ b/includes/descriptionList.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ Function to expand the dataset with a description. @@ -15,6 +15,7 @@ import csv # for loading the description files from includes import globals # Global variables + ## # # Local function will load the csv-file @@ -36,7 +37,25 @@ def loadCSV(typ, idField): logging.debug(row) # only import rows with an integer as id if row[idField].isdigit() == True: - resultList[row[idField]] = row['description'] + # check if string contains non-utf8 characters + description = "" + try: + description = row['description'].decode('UTF-8', 'strict') + except UnicodeDecodeError: + # line contains non-utf8 character + logging.debug("row contains non-utf8 characters: %s", row['description']) + # try to find out codec: + encodings = ('windows-1250', 'windows-1252', 'iso-8859-1', 'iso-8859-15') + for enc in encodings: + try: + description = f.decode(enc) + break + except Exception: + pass + # encode in UTF-8 + description = description.encode('UTF-8') + pass + resultList[row[idField]] = description logging.debug("-- loading csv finished") except: logging.error("loading csvList for typ: %s failed", typ) diff --git a/includes/doubleFilter.py b/includes/doubleFilter.py index 710a777..ae7872b 100644 --- a/includes/doubleFilter.py +++ b/includes/doubleFilter.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ doubleFilter is the central function to filter out double alarms. diff --git a/includes/filter.py b/includes/filter.py index 77712a4..922dfa7 100644 --- a/includes/filter.py +++ b/includes/filter.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ Functions for the RegEX filter diff --git a/includes/globals.py b/includes/globals.py index 4d26d44..0ca275f 100644 --- a/includes/globals.py +++ b/includes/globals.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ Global variables @@ -31,6 +31,6 @@ ricDescribtionList = {} # function -> read only in script def getVers(mode="vers"): if mode == "vers": - return "2.0-RC" + return "2.0" elif mode == "date": - return " 2015/07/06" + return " 2015/07/13" diff --git a/includes/helper/__init__.py b/includes/helper/__init__.py index e69de29..dcd3642 100644 --- a/includes/helper/__init__.py +++ b/includes/helper/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- diff --git a/includes/helper/configHandler.py b/includes/helper/configHandler.py index 8dfe90e..2a21b4b 100644 --- a/includes/helper/configHandler.py +++ b/includes/helper/configHandler.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- # """ diff --git a/includes/helper/timeHandler.py b/includes/helper/timeHandler.py index 246a5ee..c030d5b 100644 --- a/includes/helper/timeHandler.py +++ b/includes/helper/timeHandler.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- # """ diff --git a/includes/helper/wildcardHandler.py b/includes/helper/wildcardHandler.py index 1b85b96..10040d3 100644 --- a/includes/helper/wildcardHandler.py +++ b/includes/helper/wildcardHandler.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- # """ diff --git a/includes/pluginLoader.py b/includes/pluginLoader.py index a79a779..95350cf 100644 --- a/includes/pluginLoader.py +++ b/includes/pluginLoader.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ Functions to Load and import the Plugins diff --git a/includes/shellHeader.py b/includes/shellHeader.py index 6cf4a29..28f8fd3 100644 --- a/includes/shellHeader.py +++ b/includes/shellHeader.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ Shows the header in shell if quiet mode is not active diff --git a/includes/signalHandler.py b/includes/signalHandler.py index 7afb978..825bd53 100644 --- a/includes/signalHandler.py +++ b/includes/signalHandler.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- # """ diff --git a/plugins/BosMon/BosMon.py b/plugins/BosMon/BosMon.py index 8ccb7b0..7be0116 100644 --- a/plugins/BosMon/BosMon.py +++ b/plugins/BosMon/BosMon.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ BOSWatch-Plugin to dispatch FMS-, ZVEI- and POCSAG - messages to BosMon diff --git a/plugins/MySQL/MySQL.py b/plugins/MySQL/MySQL.py index 39cab72..b480844 100644 --- a/plugins/MySQL/MySQL.py +++ b/plugins/MySQL/MySQL.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ MySQL-Plugin to dispatch FMS-, ZVEI- and POCSAG - messages to a MySQL database @@ -84,13 +84,13 @@ def run(typ,freq,data): logging.debug("Insert %s", typ) if typ == "FMS": - cursor.execute("INSERT INTO "+globals.config.get("MySQL","tableFMS")+" (time,fms,status,direction,directionText,tsi,description) VALUES (NOW(),%s,%s,%s,%s,%s,%s)",(data["fms"],data["status"],data["direction"],data["directionText"],data["tsi"],data["description"])) + cursor.execute("INSERT INTO "+globals.config.get("MySQL","tableFMS")+" (time, fms, status, direction, directionText, tsi, description) VALUES (NOW(),%s,%s,%s,%s,%s,%s)", (data["fms"], data["status"], data["direction"], data["directionText"], data["tsi"], data["description"])) elif typ == "ZVEI": - cursor.execute("INSERT INTO "+globals.config.get("MySQL","tableZVEI")+" (time,zvei,description) VALUES (NOW(),%s,%s)",(data["zvei"],data["description"])) + cursor.execute("INSERT INTO "+globals.config.get("MySQL","tableZVEI")+" (time, zvei, description) VALUES (NOW(),%s,%s)", (data["zvei"], data["description"])) elif typ == "POC": - cursor.execute("INSERT INTO "+globals.config.get("MySQL","tablePOC")+" (time,ric,funktion,funktionChar,msg,bitrate,description) VALUES (NOW(),%s,%s,%s,%s,%s,%s)",(data["ric"],data["function"],data["functionChar"],data["msg"],data["bitrate"],data["description"])) + cursor.execute("INSERT INTO "+globals.config.get("MySQL","tablePOC")+" (time, ric, function, functionChar, msg, bitrate, description) VALUES (NOW(),%s,%s,%s,%s,%s,%s)", (data["ric"], data["function"], data["functionChar"], data["msg"], data["bitrate"], data["description"])) else: logging.warning("Invalid Typ: %s", typ) diff --git a/plugins/MySQL/boswatch.sql b/plugins/MySQL/boswatch.sql index cf91616..edecfcd 100644 --- a/plugins/MySQL/boswatch.sql +++ b/plugins/MySQL/boswatch.sql @@ -32,12 +32,12 @@ SET time_zone = "+00:00"; CREATE TABLE IF NOT EXISTS `bos_fms` ( `id` int(11) NOT NULL AUTO_INCREMENT, `time` datetime NOT NULL, - `fms` varchar(8) NOT NULL, - `status` varchar(1) NOT NULL, - `direction` varchar(1) NOT NULL, - `directionText` text(10) NOT NULL, + `fms` char(8) NOT NULL, + `status` char(1) NOT NULL, + `direction` char(1) NOT NULL, + `directionText` char(10) NOT NULL, `tsi` varchar(3) NOT NULL, - `description` text NOT NULL, + `description` text, PRIMARY KEY (`ID`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; @@ -50,13 +50,13 @@ CREATE TABLE IF NOT EXISTS `bos_fms` ( CREATE TABLE IF NOT EXISTS `bos_pocsag` ( `id` int(11) NOT NULL AUTO_INCREMENT, `time` datetime NOT NULL, - `ric` varchar(7) NOT NULL DEFAULT '0', - `funktion` int(1) NOT NULL, - `funktionChar` text(1) NOT NULL, - `msg` text NOT NULL, - `bitrate` int(4) NOT NULL, - `description` text NOT NULL, - KEY `ID` (`ID`) + `ric` char(7) NOT NULL, + `function` int(1) NOT NULL, + `functionChar` char(1), + `bitrate` int(4), + `msg` text, + `description` text, + PRIMARY KEY `ID` (`ID`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; -- -------------------------------------------------------- @@ -68,11 +68,11 @@ CREATE TABLE IF NOT EXISTS `bos_pocsag` ( CREATE TABLE IF NOT EXISTS `bos_zvei` ( `id` int(11) NOT NULL AUTO_INCREMENT, `time` datetime NOT NULL, - `zvei` varchar(5) NOT NULL DEFAULT '0', - `description` text NOT NULL, + `zvei` char(5) NOT NULL, + `description` text, PRIMARY KEY (`ID`) ) ENGINE=MyISAM DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; -/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; \ No newline at end of file diff --git a/plugins/README.md b/plugins/README.md index f14a3b9..260d028 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -167,6 +167,7 @@ Thats the function of the alarm. Possible values are `FMS`, `ZVEI` or `POC` The reception frequency of the tuner in Hz #### 5.3 data[ ] +You can get an information with `data["INFO"]` In the data map are the folowing informations: **ZVEI:** diff --git a/plugins/eMail/eMail.py b/plugins/eMail/eMail.py index f7c7448..0b08a08 100644 --- a/plugins/eMail/eMail.py +++ b/plugins/eMail/eMail.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ eMail-Plugin to dispatch FMS-, ZVEI- and POCSAG - messages via eMail/SMTP diff --git a/plugins/firEmergency/firEmergency.py b/plugins/firEmergency/firEmergency.py index 84fefa3..10fe941 100644 --- a/plugins/firEmergency/firEmergency.py +++ b/plugins/firEmergency/firEmergency.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ firEmergency-Plugin to dispatch ZVEI- and POCSAG - messages to firEmergency diff --git a/plugins/httpRequest/httpRequest.py b/plugins/httpRequest/httpRequest.py index fd6815b..cbdb270 100644 --- a/plugins/httpRequest/httpRequest.py +++ b/plugins/httpRequest/httpRequest.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ httpRequest-Plugin to dispatch FMS-, ZVEI- and POCSAG - messages to an URL diff --git a/plugins/jsonSocket/jsonSocket.py b/plugins/jsonSocket/jsonSocket.py index 4bb7162..8298897 100644 --- a/plugins/jsonSocket/jsonSocket.py +++ b/plugins/jsonSocket/jsonSocket.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ jsonSocket-Plugin to dispatch FMS-, ZVEI- and POCSAG-messages via UDP/TCP diff --git a/plugins/template/template.py b/plugins/template/template.py index 148fb44..5e6ae42 100644 --- a/plugins/template/template.py +++ b/plugins/template/template.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- """ template plugin to show the function and usage of plugins diff --git a/serverExamples/jsonSocketServer.py b/serverExamples/jsonSocketServer.py index d659dd0..5c90cd5 100644 --- a/serverExamples/jsonSocketServer.py +++ b/serverExamples/jsonSocketServer.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: cp1252 -*- +# -*- coding: UTF-8 -*- # """