Schrolli91/BOSWatch
1
0
Fork 0
mirror of https://github.com/Schrolli91/BOSWatch.git synced 2026-04-04 14:07:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

convert csv-files to UTF-8 #63

Browse source
This commit is contained in:
JHCD 2015-07-14 16:49:14 +02:00
parent e829778db2
commit f3db4635c7
2 changed files with 80 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

21
includes/descriptionList.py
View file

@ -14,6 +14,7 @@ import logging # Global logger
import csv # for loading the description files
from includes import globals # Global variables
from includes.helper import uft8Converter # UTF-8 converter
##
@ -37,25 +38,11 @@ def loadCSV(typ, idField):
logging.debug(row)
# only import rows with an integer as id
if row[idField].isdigit() == True:
# check if string contains non-utf8 characters
description = ""
try:
description = row['description'].decode('UTF-8', 'strict')
except UnicodeDecodeError:
# line contains non-utf8 character
logging.debug("row contains non-utf8 characters: %s", row['description'])
# try to find out codec:
encodings = ('windows-1250', 'windows-1252', 'iso-8859-1', 'iso-8859-15')
for enc in encodings:
try:
description = f.decode(enc)
break
except Exception:
pass
# encode in UTF-8
description = description.encode('UTF-8')
resultList[row[idField]] = uft8Converter.convertToUTF8(row['description'])
except:
# skip entry in case of an exception
pass
resultList[row[idField]] = description
logging.debug("-- loading csv finished")
except:
logging.error("loading csvList for typ: %s failed", typ)