dh1tw/pyhamtools
1
0
Fork 0
mirror of https://github.com/dh1tw/pyhamtools.git synced 2025-12-06 06:52:00 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

corrected negative longitude problem and added clublog user lookup

Browse source
This commit is contained in:
Tobias Wellnitz, DH1TW 2018-01-27 15:11:46 +01:00
parent 2c37f422e1
commit dd6784ae09
15 changed files with 203 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
docs/source/changelog.rst
View file

@ -1,6 +1,19 @@
Changelog
---------
PyHamTools 0.6.0
================
23. January 2018
* BREAKING CHANGE: Longitude is now provided with the correct sign for all
lookup libraries. The AD1C cty format used by Countryfile and ClublogAPI
provide the longitude with the wrong sign. This is now covered and internally
corrected. East = positive longitude, West = negative longitude.
* Added a function to download the Clublog user list and the associated activity dates
* updated requirements for libraries used by pyhamtools
PyHamTools 0.5.6
================
@ -46,6 +59,7 @@ PyHamTools 0.5.2
14. April 2015
* catching another bug related to QRZ.com sessions
PyHamTools 0.5.1

2
docs/source/examples.rst
View file

@ -74,7 +74,7 @@ Now we can query the information conveniently through our Callinfo object:
'adif': 230,
'continent': 'EU',
'latitude': 51.0,
'longitude': -10.0,
'longitude': 10.0,
'cqz': 14,
'ituz': 28
}

20
pyhamtools/lookuplib.py
View file

@ -177,7 +177,7 @@ class LookupLib(object):
u'cqz': 32,
u'ituz': 56,
u'latitude': -12.48,
u'longitude': -177.08
u'longitude': 177.08
}
@ -255,7 +255,7 @@ class LookupLib(object):
{
'deleted': False,
'country': u'TURKMENISTAN',
'longitude': -58.4,
'longitude': 58.4,
'cqz': 17,
'prefix': u'EZ',
'latitude': 38.0,
@ -338,7 +338,7 @@ class LookupLib(object):
>>> print my_lookuplib.lookup_callsign("VK9XO", timestamp)
{
'country': u'CHRISTMAS ISLAND',
'longitude': -105.7,
'longitude': 105.7,
'cqz': 29,
'adif': 35,
'latitude': -10.5,
@ -500,7 +500,7 @@ class LookupLib(object):
{
'adif': 230,
'country': u'Fed. Rep. of Germany',
'longitude': -10.0,
'longitude': 10.0,
'cqz': 14,
'ituz': 28,
'latitude': 51.0,
@ -691,7 +691,7 @@ class LookupLib(object):
for item in jsonLookup:
if item == "Name": lookup[const.COUNTRY] = jsonLookup["Name"]
elif item == "DXCC": lookup[const.ADIF] = int(jsonLookup["DXCC"])
elif item == "Lon": lookup[const.LONGITUDE] = float(jsonLookup["Lon"])
elif item == "Lon": lookup[const.LONGITUDE] = float(jsonLookup["Lon"])*(-1)
elif item == "Lat": lookup[const.LATITUDE] = float(jsonLookup["Lat"])
elif item == "CQZ": lookup[const.CQZ] = int(jsonLookup["CQZ"])
elif item == "Continent": lookup[const.CONTINENT] = jsonLookup["Continent"]
@ -1172,7 +1172,7 @@ class LookupLib(object):
elif item.tag == "cont":
entity[const.CONTINENT] = unicode(item.text)
elif item.tag == "long":
entity[const.LONGITUDE] = float(item.text)*(-1)
entity[const.LONGITUDE] = float(item.text)
elif item.tag == "lat":
entity[const.LATITUDE] = float(item.text)
elif item.tag == "start":
@ -1220,7 +1220,7 @@ class LookupLib(object):
elif item.tag == "cont":
call_exception[const.CONTINENT] = unicode(item.text)
elif item.tag == "long":
call_exception[const.LONGITUDE] = float(item.text)*(-1)
call_exception[const.LONGITUDE] = float(item.text)
elif item.tag == "lat":
call_exception[const.LATITUDE] = float(item.text)
elif item.tag == "start":
@ -1261,7 +1261,7 @@ class LookupLib(object):
elif item.tag == "cont":
prefix[const.CONTINENT] = unicode(item.text)
elif item.tag == "long":
prefix[const.LONGITUDE] = float(item.text)*(-1)
prefix[const.LONGITUDE] = float(item.text)
elif item.tag == "lat":
prefix[const.LATITUDE] = float(item.text)
elif item.tag == "start":
@ -1383,7 +1383,7 @@ class LookupLib(object):
entry[const.ITUZ] = int(cty_list[item]["ITUZone"])
entry[const.CONTINENT] = unicode(cty_list[item]["Continent"])
entry[const.LATITUDE] = float(cty_list[item]["Latitude"])
entry[const.LONGITUDE] = float(cty_list[item]["Longitude"])
entry[const.LONGITUDE] = float(cty_list[item]["Longitude"])*(-1)
if cty_list[item]["ExactCallsign"]:
if call in exceptions_index.keys():
@ -1433,7 +1433,7 @@ class LookupLib(object):
else:
err_str = "HTTP Status Code: " + str(response.status_code) + " HTTP Response: " + str(response.text)
self._logger.error(err_str)
if response.text.strip() == error1 or response.text.strip() == error2:
if error1 in response.text.strip() or error2 in response.text.strip():
raise APIKeyMissingError
else:
raise LookupError(err_str)

96
pyhamtools/qsl.py
View file

@ -3,6 +3,9 @@ import re
import requests
import redis
import zipfile
import json
from io import BytesIO
from requests.exceptions import ConnectionError, HTTPError, Timeout
def get_lotw_users(**kwargs):
@ -16,6 +19,7 @@ def get_lotw_users(**kwargs):
Raises:
IOError: When network is unavailable, file can't be downloaded or processed
ValueError: Raised when data from file can't be read
Example:
@ -63,8 +67,96 @@ def get_lotw_users(**kwargs):
return lotw
def get_clublog_users(**kwargs):
"""Download the latest offical list of `Clublog`__ users.
Args:
url (str, optional): Download URL
Returns:
dict: Dictionary containing (if data available) the fields:
firstqso, lastqso, last-lotw, lastupload (datetime),
locator (string) and oqrs (boolean)
Raises:
IOError: When network is unavailable, file can't be downloaded or processed
Example:
The following example downloads the Clublog user list and returns a dictionary with the data of HC2/AL1O:
>>> from pyhamtools.qsl import get_clublog_users
>>> clublog = get_lotw_users()
>>> clublog['HC2/AL1O']
{'firstqso': datetime.datetime(2012, 1, 1, 19, 59, 27),
'last-lotw': datetime.datetime(2013, 5, 9, 1, 56, 23),
'lastqso': datetime.datetime(2013, 5, 5, 6, 39, 3),
'lastupload': datetime.datetime(2013, 5, 8, 15, 0, 6),
'oqrs': True}
.. _CLUBLOG: https://secure.clublog.org
__ CLUBLOG_
"""
url = ""
clublog = {}
try:
url = kwargs['url']
except KeyError:
url = "https://secure.clublog.org/clublog-users.json.zip"
try:
result = requests.get(url)
except (ConnectionError, HTTPError, Timeout) as e:
raise IOError(e)
if result.status_code != requests.codes.ok:
raise IOError("HTTP Error: " + str(result.status_code))
zip_file = zipfile.ZipFile(BytesIO(result.content))
files = zip_file.namelist()
cl_json_unzipped = zip_file.read(files[0])
cl_data = json.loads(cl_json_unzipped)
error_count = 0
for call, call_data in cl_data.iteritems():
try:
data = {}
if "firstqso" in call_data:
if call_data["firstqso"] != None:
data["firstqso"] = datetime.strptime(call_data["firstqso"], '%Y-%m-%d %H:%M:%S')
if "lastqso" in call_data:
if call_data["lastqso"] != None:
data["lastqso"] = datetime.strptime(call_data["lastqso"], '%Y-%m-%d %H:%M:%S')
if "last-lotw" in call_data:
if call_data["last-lotw"] != None:
data["last-lotw"] = datetime.strptime(call_data["last-lotw"], '%Y-%m-%d %H:%M:%S')
if "lastupload" in call_data:
if call_data["lastupload"] != None:
data["lastupload"] = datetime.strptime(call_data["lastupload"], '%Y-%m-%d %H:%M:%S')
if "locator" in call_data:
if call_data["locator"] != None:
data["locator"] = call_data["locator"]
if "oqrs" in call_data:
if call_data["oqrs"] != None:
data["oqrs"] = call_data["oqrs"]
clublog[call] = data
except TypeError: #some date fields contain null instead of a valid datetime string - we ignore them
print("Ignoring invalid type in data:", call, call_data)
pass
except ValueError: #some date fiels are invalid. we ignore them for the moment
print("Ignoring invalid data:", call, call_data)
pass
return clublog
def get_eqsl_users(**kwargs):
"""Download the latest official list of EQSL.cc users. The list of users can be found here_.
"""Download the latest official list of `EQSL.cc`__ users. The list of users can be found here_.
Args:
url (str, optional): Download URL
@ -85,7 +177,7 @@ def get_eqsl_users(**kwargs):
>>> except ValueError as e:
>>> print e
'DH1TW' is not in list
.. _here: http://www.eqsl.cc/QSLCard/DownloadedFiles/AGMemberlist.txt
"""

3
pyhamtools/version.py
View file

@ -1,4 +1,3 @@
#VERSION = (0, 5, 0, 'dev')
VERSION = (0, 5, 6)
VERSION = (0, 6, 0)
__release__ = ''.join(['-.'[type(x) == int]+str(x) for x in VERSION])[1:]
__version__ = '.'.join((str(VERSION[0]), str(VERSION[1])))

12
readthedocs-pip-requirements.txt
View file

@ -1,7 +1,7 @@
sphinxcontrib-napoleon>=0.2.7
requests>=2.2.1
pytz>=2014.2
pyephem>=3.7.5.3
redis>=2.10.2
beautifulsoup4>=4.3.2
sphinxcontrib-napoleon>=0.6.1
requests>=2.18.4
pytz>=2017.3
pyephem>=3.7.6.0
redis>=2.10.6
beautifulsoup4>=4.6.0

8
test/conftest.py
View file

@ -66,12 +66,18 @@ def fixCountryFile(request):
Lib = LookupLib("countryfile")
return(Lib)
@pytest.fixture(scope="module", params=["clublogapi", "clublogxml", "countryfile"])
@pytest.fixture(scope="module", params=["clublogxml", "countryfile"])
def fix_callinfo(request, fixApiKey):
lib = LookupLib(request.param, fixApiKey)
callinfo = Callinfo(lib)
return(callinfo)
# @pytest.fixture(scope="module", params=["clublogapi", "clublogxml", "countryfile"])
# def fix_callinfo(request, fixApiKey):
# lib = LookupLib(request.param, fixApiKey)
# callinfo = Callinfo(lib)
# return(callinfo)
@pytest.fixture(scope="module")
def fix_redis():
import redis

BIN
test/fixtures/clublog-users.json.zip vendored Normal file
View file

Binary file not shown.

5
test/fixtures/generate_lotw_data.py vendored Normal file
View file

@ -0,0 +1,5 @@
from pyhamtools.qsl import get_lotw_users
lotw = get_lotw_users()
print lotw
# pipe output into file

34
test/test_callinfo.py
View file

@ -13,7 +13,7 @@ response_prefix_DH_clublog = {
'adif': 230,
'continent': 'EU',
'latitude': 51.0,
'longitude': -10.0,
'longitude': 10.0,
'cqz': 14,
}
@ -22,14 +22,14 @@ response_prefix_DH_countryfile = {
'adif': 230,
'continent': 'EU',
'latitude': 51.0,
'longitude': -10.0,
'longitude': 10.0,
'cqz': 14,
'ituz': 28
}
response_prefix_C6A_clublog = {
'country': 'BAHAMAS',
'longitude': 76.0,
'longitude': -76.0,
'cqz': 8,
'adif': 60,
'latitude': 24.25,
@ -38,7 +38,7 @@ response_prefix_C6A_clublog = {
response_prefix_C6A_countryfile = {
'country': 'Bahamas',
'longitude': 76.0,
'longitude': -76.0,
'cqz': 8,
'adif': 60,
'latitude': 24.25,
@ -53,7 +53,7 @@ response_prefix_VK9NDX_countryfile = {
u'cqz': 32,
u'ituz': 60,
u'latitude': -29.03,
u'longitude': -167.93
u'longitude': 167.93
}
response_prefix_VK9DNX_clublog = {
@ -62,7 +62,7 @@ response_prefix_VK9DNX_clublog = {
u'country': u'NORFOLK ISLAND',
u'cqz': 32,
u'latitude': -29.0,
u'longitude': -168.0
u'longitude': 168.0
}
response_prefix_VK9DWX_clublog = {
@ -71,7 +71,7 @@ response_prefix_VK9DWX_clublog = {
u'country': u'WILLIS ISLAND',
u'cqz': 30,
u'latitude': -16.2,
u'longitude': -150.0
u'longitude': 150.0
}
response_prefix_VK9DLX_clublog = {
@ -80,7 +80,7 @@ response_prefix_VK9DLX_clublog = {
u'country': u'LORD HOWE ISLAND',
u'cqz': 30,
u'latitude': -31.6,
u'longitude': -159.1
u'longitude': 159.1
}
response_prefix_VK9DLX_countryfile = {
@ -90,7 +90,7 @@ response_prefix_VK9DLX_countryfile = {
u'cqz': 30,
u'ituz': 60,
u'latitude': -31.55,
u'longitude': -159.08
u'longitude': 159.08
}
response_prefix_VK9GMW_clublog = {
@ -99,7 +99,7 @@ response_prefix_VK9GMW_clublog = {
u'country': u'MELLISH REEF',
u'cqz': 30,
u'latitude': -17.6,
u'longitude': -155.8
u'longitude': 155.8
}
response_callsign_exceptions_7N1PRD_0_clublog = {
@ -108,7 +108,7 @@ response_callsign_exceptions_7N1PRD_0_clublog = {
u'country': u'JAPAN',
u'cqz': 25,
u'latitude': 35.7,
u'longitude': -139.8
u'longitude': 139.8
}
response_callsign_exceptions_SV8GXQ_P_QRP_clublog = {
@ -117,7 +117,7 @@ response_callsign_exceptions_SV8GXQ_P_QRP_clublog = {
u'country': u'GREECE',
u'cqz': 20,
u'latitude': 38.0,
u'longitude': -23.7
u'longitude': 23.7
}
response_Exception_VP8STI_with_start_and_stop_date = {
@ -125,7 +125,7 @@ response_Exception_VP8STI_with_start_and_stop_date = {
'country': u'SOUTH SANDWICH ISLANDS',
'continent': u'SA',
'latitude': -59.45,
'longitude': 27.4,
'longitude': -27.4,
'cqz': 13,
}
@ -135,7 +135,7 @@ response_Exception_VK9XO_with_start_date = {
'country': 'CHRISTMAS ISLAND',
'continent': 'OC',
'latitude': -10.50,
'longitude': -105.70,
'longitude': 105.70,
'cqz': 29
}
@ -144,13 +144,13 @@ response_zone_exception_dp0gvn = {
'adif': 13,
'cqz': 38,
'latitude': -65.0,
'longitude': 64.0,
'longitude': -64.0,
'continent': 'AN'
}
response_lat_long_dh1tw = {
const.LATITUDE: 51.0,
const.LONGITUDE: -10.0
const.LONGITUDE: 10.0
}
response_maritime_mobile = {
@ -177,7 +177,7 @@ response_callsign_exceptions_7QAA_clublog = {
u'country': u'MALAWI',
u'cqz': 37,
u'latitude': -14.9,
u'longitude': -34.4
u'longitude': 34.4
}

28
test/test_clublog.py Normal file
View file

@ -0,0 +1,28 @@
import os
import datetime
import pytest
from pyhamtools.qsl import get_clublog_users
class Test_clublog_methods:
def test_check_content_with_mocked_http_server(self, httpserver):
httpserver.serve_content(
open('./fixtures/clublog-users.json.zip').read())
data = get_clublog_users(url=httpserver.url)
assert len(data) == 139081
def test_download_lotw_list_and_check_types(self):
data = get_clublog_users()
assert isinstance(data, dict)
for key, value in data.iteritems():
assert isinstance(key, unicode)
assert isinstance(value, dict)
def test_with_invalid_url(self):
with pytest.raises(IOError):
get_clublog_users(url="https://FAKE.csv")

20
test/test_lookuplib_clublogxml.py
View file

@ -18,7 +18,7 @@ response_Entity_230 = {
'country': u'FEDERAL REPUBLIC OF GERMANY',
'continent': u'EU',
'latitude': 51.0,
'longitude': -10.0,
'longitude': 10.0,
'cqz': 14,
'prefix' : u'DL',
'deleted' : False,
@ -29,7 +29,7 @@ response_Exception_KC6MM_1990 = {
'country': u'PALAU',
'continent': u'OC',
'latitude': 9.50,
'longitude': -138.20,
'longitude': 138.20,
'cqz': 27,
}
@ -38,7 +38,7 @@ response_Exception_KC6MM_1992 = {
'country': u'PALAU',
'continent': u'OC',
'latitude': 9.50,
'longitude': -138.20,
'longitude': 138.20,
'cqz': 27,
}
@ -47,7 +47,7 @@ response_Exception_VK9XX_with_end_date = {
'country': u'CHRISTMAS ISLAND',
'continent': u'OC',
'latitude': -10.50,
'longitude': -105.70,
'longitude': 105.70,
'cqz': 29,
}
@ -56,7 +56,7 @@ response_Exception_VK9XO_with_start_date = {
'country': u'CHRISTMAS ISLAND',
'continent': u'OC',
'latitude': -10.50,
'longitude': -105.70,
'longitude': 105.70,
'cqz': 29,
}
@ -65,7 +65,7 @@ response_Exception_AX9NYG = {
'country': u'COCOS (KEELING) ISLAND',
'continent': u'OC',
'latitude': -12.20,
'longitude': -96.80,
'longitude': 96.80,
'cqz': 29,
}
@ -74,7 +74,7 @@ response_Prefix_DH = {
'adif' : 230,
'continent': u'EU',
'latitude': 51.0,
'longitude': -10.0,
'longitude': 10.0,
'cqz': 14,
}
@ -83,7 +83,7 @@ response_Prefix_VK9_until_1975 = {
'adif' : 198,
'continent': u'OC',
'latitude': -9.40,
'longitude': -147.10,
'longitude': 147.10,
'cqz': 28,
}
@ -92,7 +92,7 @@ response_Prefix_VK9_starting_1976 = {
'adif' : 189,
'continent': u'OC',
'latitude': -29.00,
'longitude': -168.00,
'longitude': 168.00,
'cqz': 32,
}
@ -101,7 +101,7 @@ response_Prefix_ZD5_1964_to_1971 = {
'adif' : 468,
'continent': u'AF',
'latitude': -26.30,
'longitude': -31.10,
'longitude': 31.10,
'cqz': 38,
}

4
test/test_lookuplib_countryfile.py
View file

@ -13,7 +13,7 @@ response_Prefix_DH = {
'country': 'Fed. Rep. of Germany',
'continent': 'EU',
'latitude': 51.0,
'longitude': -10.0,
'longitude': 10.0,
'cqz': 14,
'ituz' : 28
}
@ -24,7 +24,7 @@ response_Exception_3D2RI = {
'country': 'Rotuma Island',
'continent': 'OC',
'latitude': -12.48,
'longitude': -177.08,
'longitude': 177.08,
'cqz': 32,
'ituz' : 56
}

16
test/test_lookuplib_qrz.py
View file

@ -38,11 +38,9 @@ response_XX1XX = {
}
response_XX2XX = {
u'addr1': u'1234 Main Road.',
u'addr2': u'Las Vegasvillee',
u'adif': 446,
u'bio': u'218',
u'biodate': datetime(2017, 6, 16, 15, 47, 47, tzinfo=UTC),
u'bio': u'349',
u'biodate': datetime(2017, 9, 5, 22, 28, 42, tzinfo=UTC),
u'born': 1932,
u'callsign': u'XX2XX',
u'ccode': 1230,
@ -52,16 +50,16 @@ response_XX2XX = {
u'email': 'dummy2@qrz.com',
u'eqsl': False,
u'fname': u'Gooberd',
u'geoloc': u'user',
u'geoloc': u'grid',
u'image': u'https://s3.amazonaws.com/files.qrz.com/x/xx2xx/oval_bumper_sticker4.png',
u'imageinfo': u'285:500:44218',
u'iota': u'NA-022',
u'ituz': 5,
u'land': u'Morocco',
u'latitude': 52.195073,
u'latitude': 52.1875,
u'license_class': u'A',
u'locator': u'JO02be',
u'longitude': 0.124962,
u'longitude': 0.125,
u'lotw': False,
u'moddate': datetime(2017, 6, 16, 19, 22, 21, tzinfo=UTC),
u'mqsl': False,
@ -77,8 +75,8 @@ response_XX3XX = {
u'addr2': u'Shady Circle Roads',
u'adif': 79,
u'aliases': [u'XX3XX/W7'],
u'bio': u'1940',
u'biodate': datetime(2015, 8, 13, 23, 14, 38, tzinfo=UTC),
u'bio': u'16',
u'biodate': datetime(2018, 1, 24, 20, 55, 27, tzinfo=UTC),
u'born': 2010,
u'callsign': u'XX3XX',
u'ccode': 130,

2
test/test_lookuplib_redis.py
View file

@ -18,7 +18,7 @@ response_Exception_VP8STI_with_start_and_stop_date = {
'country': u'SOUTH SANDWICH ISLANDS',
'continent': u'SA',
'latitude': -59.45,
'longitude': 27.4,
'longitude': -27.4,
'cqz': 13,
}