Merge pull request #1 from Schrolli91/develop

Develop

boswatch.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 #
 """
 BOSWatch

csv/poc.csv

@@ -7,4 +7,4 @@ ric,description
 #
 # !!! DO NOT delete the first line !!!
 #
-1234567,"POCSAG testdata"
+1234567,"POCSAG testdata üöäß"

includes/MyTimedRotatingFileHandler.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 #
 
 """

includes/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-

includes/alarmHandler.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 Handler for the filter and plugins at an alarm

includes/checkSubprocesses.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 #
 """
 Functions for checking the subprocesses rtl_fm and multimon-ng

includes/converter.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 #
 
 """

includes/decoder.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 Search for decode string and call the right decoder function

includes/decoders/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-

includes/decoders/fms.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 FMS Decoder

includes/decoders/poc.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 POCSAG Decoder

includes/decoders/zvei.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 ZVEI Decoder

includes/descriptionList.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 Function to expand the dataset with a description.
@@ -15,6 +15,7 @@ import csv # for loading the description files
 
 from includes import globals  # Global variables
 
+
 ##
 #
 # Local function will load the csv-file
@@ -36,7 +37,25 @@ def loadCSV(typ, idField):
 				logging.debug(row)
 				# only import rows with an integer as id
 				if row[idField].isdigit() == True:
-					resultList[row[idField]] = row['description']
+					# check if string contains non-utf8 characters
+					description = ""
+					try:
+						description = row['description'].decode('UTF-8', 'strict')
+					except UnicodeDecodeError:
+						# line contains non-utf8 character
+						logging.debug("row contains non-utf8 characters: %s", row['description'])
+						# try to find out codec:
+						encodings = ('windows-1250', 'windows-1252', 'iso-8859-1', 'iso-8859-15')
+						for enc in encodings:
+							try:
+								description = f.decode(enc)
+								break
+							except Exception:
+								pass
+						# encode in UTF-8
+						description = description.encode('UTF-8')
+						pass
+					resultList[row[idField]] = description
 		logging.debug("-- loading csv finished")
 	except:
 		logging.error("loading csvList for typ: %s failed", typ)

includes/doubleFilter.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 doubleFilter is the central function to filter out double alarms.

includes/filter.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 Functions for the RegEX filter

includes/globals.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 Global variables
@@ -31,6 +31,6 @@ ricDescribtionList  = {}
 # function -> read only in script
 def getVers(mode="vers"):
 	if mode == "vers":
-		return "2.0-RC"
+		return "2.0"
 	elif mode == "date":
-		return " 2015/07/06"
+		return " 2015/07/13"

includes/helper/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-

includes/helper/configHandler.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 #
 
 """

includes/helper/timeHandler.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 #
 
 """

includes/helper/wildcardHandler.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 #
 
 """

includes/pluginLoader.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 Functions to Load and import the Plugins

includes/shellHeader.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 Shows the header in shell if quiet mode is not active

includes/signalHandler.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 #
 
 """

plugins/BosMon/BosMon.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 BOSWatch-Plugin to dispatch FMS-, ZVEI- and POCSAG - messages to BosMon

plugins/MySQL/MySQL.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 MySQL-Plugin to dispatch FMS-, ZVEI- and POCSAG - messages to a MySQL database
@@ -84,13 +84,13 @@ def run(typ,freq,data):
 					logging.debug("Insert %s", typ)
 
 					if typ == "FMS":
-						cursor.execute("INSERT INTO "+globals.config.get("MySQL","tableFMS")+" (time,fms,status,direction,directionText,tsi,description) VALUES (NOW(),%s,%s,%s,%s,%s,%s)",(data["fms"],data["status"],data["direction"],data["directionText"],data["tsi"],data["description"]))
+						cursor.execute("INSERT INTO "+globals.config.get("MySQL","tableFMS")+" (time, fms, status, direction, directionText, tsi, description) VALUES (NOW(),%s,%s,%s,%s,%s,%s)", (data["fms"], data["status"], data["direction"], data["directionText"], data["tsi"], data["description"]))
 
 					elif typ == "ZVEI":
-						cursor.execute("INSERT INTO "+globals.config.get("MySQL","tableZVEI")+" (time,zvei,description) VALUES (NOW(),%s,%s)",(data["zvei"],data["description"]))
+						cursor.execute("INSERT INTO "+globals.config.get("MySQL","tableZVEI")+" (time, zvei, description) VALUES (NOW(),%s,%s)", (data["zvei"], data["description"]))
 
 					elif typ == "POC":
-						cursor.execute("INSERT INTO "+globals.config.get("MySQL","tablePOC")+" (time,ric,funktion,funktionChar,msg,bitrate,description) VALUES (NOW(),%s,%s,%s,%s,%s,%s)",(data["ric"],data["function"],data["functionChar"],data["msg"],data["bitrate"],data["description"]))
+						cursor.execute("INSERT INTO "+globals.config.get("MySQL","tablePOC")+" (time, ric, function, functionChar, msg, bitrate, description) VALUES (NOW(),%s,%s,%s,%s,%s,%s)", (data["ric"], data["function"], data["functionChar"], data["msg"], data["bitrate"], data["description"]))
 
 					else:
 						logging.warning("Invalid Typ: %s", typ)

plugins/MySQL/boswatch.sql

@@ -32,12 +32,12 @@ SET time_zone = "+00:00";
 CREATE TABLE IF NOT EXISTS `bos_fms` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `time` datetime NOT NULL,
-  `fms` varchar(8) NOT NULL,
-  `status` varchar(1) NOT NULL,
-  `direction` varchar(1) NOT NULL,
-	`directionText` text(10) NOT NULL,
+  `fms` char(8) NOT NULL,
+  `status` char(1) NOT NULL,
+  `direction` char(1) NOT NULL,
+  `directionText` char(10) NOT NULL,
   `tsi` varchar(3) NOT NULL,
-	`description` text NOT NULL,
+  `description` text,
   PRIMARY KEY (`ID`)
 ) ENGINE=MyISAM DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ;
 
@@ -50,13 +50,13 @@ CREATE TABLE IF NOT EXISTS `bos_fms` (
 CREATE TABLE IF NOT EXISTS `bos_pocsag` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `time` datetime NOT NULL,
-  `ric` varchar(7) NOT NULL DEFAULT '0',
-  `funktion` int(1) NOT NULL,
-	`funktionChar` text(1) NOT NULL,
-  `msg` text NOT NULL,
-	`bitrate` int(4) NOT NULL,
-	`description` text NOT NULL,
-  KEY `ID` (`ID`)
+  `ric` char(7) NOT NULL,
+  `function` int(1) NOT NULL,
+  `functionChar` char(1),
+  `bitrate` int(4),
+  `msg` text,
+  `description` text,
+  PRIMARY KEY `ID` (`ID`)
 ) ENGINE=MyISAM DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ;
 
 -- --------------------------------------------------------
@@ -68,11 +68,11 @@ CREATE TABLE IF NOT EXISTS `bos_pocsag` (
 CREATE TABLE IF NOT EXISTS `bos_zvei` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `time` datetime NOT NULL,
-  `zvei` varchar(5) NOT NULL DEFAULT '0',
-	`description` text NOT NULL,
+  `zvei` char(5) NOT NULL,
+  `description` text,
   PRIMARY KEY (`ID`)
 ) ENGINE=MyISAM DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ;
 
 /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
 /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
-/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;

plugins/README.md

@@ -167,6 +167,7 @@ Thats the function of the alarm. Possible values are `FMS`, `ZVEI` or `POC`
 The reception frequency of the tuner in Hz
 
 #### 5.3 data[ ]
+You can get an information with `data["INFO"]`
 In the data map are the folowing informations:
 
 **ZVEI:**

plugins/eMail/eMail.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 eMail-Plugin to dispatch FMS-, ZVEI- and POCSAG - messages via eMail/SMTP

plugins/firEmergency/firEmergency.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 firEmergency-Plugin to dispatch ZVEI- and POCSAG - messages to firEmergency

plugins/httpRequest/httpRequest.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 httpRequest-Plugin to dispatch FMS-, ZVEI- and POCSAG - messages to an URL

plugins/jsonSocket/jsonSocket.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 jsonSocket-Plugin to dispatch FMS-, ZVEI- and POCSAG-messages via UDP/TCP

plugins/template/template.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 
 """
 template plugin to show the function and usage of plugins

serverExamples/jsonSocketServer.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# -*- coding: cp1252 -*-
+# -*- coding: UTF-8 -*-
 #
 
 """