diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index fd3203a..00b15a8 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -1,6 +1,19 @@ Changelog --------- +PyHamTools 0.6.0 +================ + +23. January 2018 + + * BREAKING CHANGE: Longitude is now provided with the correct sign for all + lookup libraries. The AD1C cty format used by Countryfile and ClublogAPI + provide the longitude with the wrong sign. This is now covered and internally + corrected. East = positive longitude, West = negative longitude. + * Added a function to download the Clublog user list and the associated activity dates + * updated requirements for libraries used by pyhamtools + + PyHamTools 0.5.6 ================ @@ -46,6 +59,7 @@ PyHamTools 0.5.2 14. April 2015 * catching another bug related to QRZ.com sessions + PyHamTools 0.5.1 diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 58d3867..17ea32c 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -74,7 +74,7 @@ Now we can query the information conveniently through our Callinfo object: 'adif': 230, 'continent': 'EU', 'latitude': 51.0, - 'longitude': -10.0, + 'longitude': 10.0, 'cqz': 14, 'ituz': 28 } diff --git a/pyhamtools/lookuplib.py b/pyhamtools/lookuplib.py index d56bcef..0a511fb 100644 --- a/pyhamtools/lookuplib.py +++ b/pyhamtools/lookuplib.py @@ -177,7 +177,7 @@ class LookupLib(object): u'cqz': 32, u'ituz': 56, u'latitude': -12.48, - u'longitude': -177.08 + u'longitude': 177.08 } @@ -255,7 +255,7 @@ class LookupLib(object): { 'deleted': False, 'country': u'TURKMENISTAN', - 'longitude': -58.4, + 'longitude': 58.4, 'cqz': 17, 'prefix': u'EZ', 'latitude': 38.0, @@ -338,7 +338,7 @@ class LookupLib(object): >>> print my_lookuplib.lookup_callsign("VK9XO", timestamp) { 'country': u'CHRISTMAS ISLAND', - 'longitude': -105.7, + 'longitude': 105.7, 'cqz': 29, 'adif': 35, 'latitude': -10.5, @@ -500,7 +500,7 @@ class LookupLib(object): { 'adif': 230, 'country': u'Fed. Rep. of Germany', - 'longitude': -10.0, + 'longitude': 10.0, 'cqz': 14, 'ituz': 28, 'latitude': 51.0, @@ -691,7 +691,7 @@ class LookupLib(object): for item in jsonLookup: if item == "Name": lookup[const.COUNTRY] = jsonLookup["Name"] elif item == "DXCC": lookup[const.ADIF] = int(jsonLookup["DXCC"]) - elif item == "Lon": lookup[const.LONGITUDE] = float(jsonLookup["Lon"]) + elif item == "Lon": lookup[const.LONGITUDE] = float(jsonLookup["Lon"])*(-1) elif item == "Lat": lookup[const.LATITUDE] = float(jsonLookup["Lat"]) elif item == "CQZ": lookup[const.CQZ] = int(jsonLookup["CQZ"]) elif item == "Continent": lookup[const.CONTINENT] = jsonLookup["Continent"] @@ -1172,7 +1172,7 @@ class LookupLib(object): elif item.tag == "cont": entity[const.CONTINENT] = unicode(item.text) elif item.tag == "long": - entity[const.LONGITUDE] = float(item.text)*(-1) + entity[const.LONGITUDE] = float(item.text) elif item.tag == "lat": entity[const.LATITUDE] = float(item.text) elif item.tag == "start": @@ -1220,7 +1220,7 @@ class LookupLib(object): elif item.tag == "cont": call_exception[const.CONTINENT] = unicode(item.text) elif item.tag == "long": - call_exception[const.LONGITUDE] = float(item.text)*(-1) + call_exception[const.LONGITUDE] = float(item.text) elif item.tag == "lat": call_exception[const.LATITUDE] = float(item.text) elif item.tag == "start": @@ -1261,7 +1261,7 @@ class LookupLib(object): elif item.tag == "cont": prefix[const.CONTINENT] = unicode(item.text) elif item.tag == "long": - prefix[const.LONGITUDE] = float(item.text)*(-1) + prefix[const.LONGITUDE] = float(item.text) elif item.tag == "lat": prefix[const.LATITUDE] = float(item.text) elif item.tag == "start": @@ -1383,7 +1383,7 @@ class LookupLib(object): entry[const.ITUZ] = int(cty_list[item]["ITUZone"]) entry[const.CONTINENT] = unicode(cty_list[item]["Continent"]) entry[const.LATITUDE] = float(cty_list[item]["Latitude"]) - entry[const.LONGITUDE] = float(cty_list[item]["Longitude"]) + entry[const.LONGITUDE] = float(cty_list[item]["Longitude"])*(-1) if cty_list[item]["ExactCallsign"]: if call in exceptions_index.keys(): @@ -1433,7 +1433,7 @@ class LookupLib(object): else: err_str = "HTTP Status Code: " + str(response.status_code) + " HTTP Response: " + str(response.text) self._logger.error(err_str) - if response.text.strip() == error1 or response.text.strip() == error2: + if error1 in response.text.strip() or error2 in response.text.strip(): raise APIKeyMissingError else: raise LookupError(err_str) diff --git a/pyhamtools/qsl.py b/pyhamtools/qsl.py index 33e5494..4550b1f 100644 --- a/pyhamtools/qsl.py +++ b/pyhamtools/qsl.py @@ -3,6 +3,9 @@ import re import requests import redis +import zipfile +import json +from io import BytesIO from requests.exceptions import ConnectionError, HTTPError, Timeout def get_lotw_users(**kwargs): @@ -16,6 +19,7 @@ def get_lotw_users(**kwargs): Raises: IOError: When network is unavailable, file can't be downloaded or processed + ValueError: Raised when data from file can't be read Example: @@ -63,8 +67,96 @@ def get_lotw_users(**kwargs): return lotw +def get_clublog_users(**kwargs): + """Download the latest offical list of `Clublog`__ users. + + Args: + url (str, optional): Download URL + + Returns: + dict: Dictionary containing (if data available) the fields: + firstqso, lastqso, last-lotw, lastupload (datetime), + locator (string) and oqrs (boolean) + + Raises: + IOError: When network is unavailable, file can't be downloaded or processed + + Example: + The following example downloads the Clublog user list and returns a dictionary with the data of HC2/AL1O: + + >>> from pyhamtools.qsl import get_clublog_users + >>> clublog = get_lotw_users() + >>> clublog['HC2/AL1O'] + {'firstqso': datetime.datetime(2012, 1, 1, 19, 59, 27), + 'last-lotw': datetime.datetime(2013, 5, 9, 1, 56, 23), + 'lastqso': datetime.datetime(2013, 5, 5, 6, 39, 3), + 'lastupload': datetime.datetime(2013, 5, 8, 15, 0, 6), + 'oqrs': True} + + .. _CLUBLOG: https://secure.clublog.org + __ CLUBLOG_ + + """ + + url = "" + + clublog = {} + + try: + url = kwargs['url'] + except KeyError: + url = "https://secure.clublog.org/clublog-users.json.zip" + + try: + result = requests.get(url) + except (ConnectionError, HTTPError, Timeout) as e: + raise IOError(e) + + + if result.status_code != requests.codes.ok: + raise IOError("HTTP Error: " + str(result.status_code)) + + zip_file = zipfile.ZipFile(BytesIO(result.content)) + files = zip_file.namelist() + cl_json_unzipped = zip_file.read(files[0]) + + cl_data = json.loads(cl_json_unzipped) + + error_count = 0 + + for call, call_data in cl_data.iteritems(): + try: + data = {} + if "firstqso" in call_data: + if call_data["firstqso"] != None: + data["firstqso"] = datetime.strptime(call_data["firstqso"], '%Y-%m-%d %H:%M:%S') + if "lastqso" in call_data: + if call_data["lastqso"] != None: + data["lastqso"] = datetime.strptime(call_data["lastqso"], '%Y-%m-%d %H:%M:%S') + if "last-lotw" in call_data: + if call_data["last-lotw"] != None: + data["last-lotw"] = datetime.strptime(call_data["last-lotw"], '%Y-%m-%d %H:%M:%S') + if "lastupload" in call_data: + if call_data["lastupload"] != None: + data["lastupload"] = datetime.strptime(call_data["lastupload"], '%Y-%m-%d %H:%M:%S') + if "locator" in call_data: + if call_data["locator"] != None: + data["locator"] = call_data["locator"] + if "oqrs" in call_data: + if call_data["oqrs"] != None: + data["oqrs"] = call_data["oqrs"] + clublog[call] = data + except TypeError: #some date fields contain null instead of a valid datetime string - we ignore them + print("Ignoring invalid type in data:", call, call_data) + pass + except ValueError: #some date fiels are invalid. we ignore them for the moment + print("Ignoring invalid data:", call, call_data) + pass + + return clublog + def get_eqsl_users(**kwargs): - """Download the latest official list of EQSL.cc users. The list of users can be found here_. + """Download the latest official list of `EQSL.cc`__ users. The list of users can be found here_. Args: url (str, optional): Download URL @@ -85,7 +177,7 @@ def get_eqsl_users(**kwargs): >>> except ValueError as e: >>> print e 'DH1TW' is not in list - + .. _here: http://www.eqsl.cc/QSLCard/DownloadedFiles/AGMemberlist.txt """ diff --git a/pyhamtools/version.py b/pyhamtools/version.py index 45817b9..d9f139c 100644 --- a/pyhamtools/version.py +++ b/pyhamtools/version.py @@ -1,4 +1,3 @@ -#VERSION = (0, 5, 0, 'dev') -VERSION = (0, 5, 6) +VERSION = (0, 6, 0) __release__ = ''.join(['-.'[type(x) == int]+str(x) for x in VERSION])[1:] __version__ = '.'.join((str(VERSION[0]), str(VERSION[1]))) diff --git a/readthedocs-pip-requirements.txt b/readthedocs-pip-requirements.txt index 6dd0038..b267d25 100644 --- a/readthedocs-pip-requirements.txt +++ b/readthedocs-pip-requirements.txt @@ -1,7 +1,7 @@ -sphinxcontrib-napoleon>=0.2.7 -requests>=2.2.1 -pytz>=2014.2 -pyephem>=3.7.5.3 -redis>=2.10.2 -beautifulsoup4>=4.3.2 +sphinxcontrib-napoleon>=0.6.1 +requests>=2.18.4 +pytz>=2017.3 +pyephem>=3.7.6.0 +redis>=2.10.6 +beautifulsoup4>=4.6.0 diff --git a/test/conftest.py b/test/conftest.py index fd27ede..999bcb8 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -66,12 +66,18 @@ def fixCountryFile(request): Lib = LookupLib("countryfile") return(Lib) -@pytest.fixture(scope="module", params=["clublogapi", "clublogxml", "countryfile"]) +@pytest.fixture(scope="module", params=["clublogxml", "countryfile"]) def fix_callinfo(request, fixApiKey): lib = LookupLib(request.param, fixApiKey) callinfo = Callinfo(lib) return(callinfo) +# @pytest.fixture(scope="module", params=["clublogapi", "clublogxml", "countryfile"]) +# def fix_callinfo(request, fixApiKey): +# lib = LookupLib(request.param, fixApiKey) +# callinfo = Callinfo(lib) +# return(callinfo) + @pytest.fixture(scope="module") def fix_redis(): import redis diff --git a/test/fixtures/clublog-users.json.zip b/test/fixtures/clublog-users.json.zip new file mode 100644 index 0000000..bcd0278 Binary files /dev/null and b/test/fixtures/clublog-users.json.zip differ diff --git a/test/fixtures/generate_lotw_data.py b/test/fixtures/generate_lotw_data.py new file mode 100644 index 0000000..afa3ec8 --- /dev/null +++ b/test/fixtures/generate_lotw_data.py @@ -0,0 +1,5 @@ +from pyhamtools.qsl import get_lotw_users + +lotw = get_lotw_users() +print lotw +# pipe output into file diff --git a/test/test_callinfo.py b/test/test_callinfo.py index a3d044d..ca25674 100644 --- a/test/test_callinfo.py +++ b/test/test_callinfo.py @@ -13,7 +13,7 @@ response_prefix_DH_clublog = { 'adif': 230, 'continent': 'EU', 'latitude': 51.0, - 'longitude': -10.0, + 'longitude': 10.0, 'cqz': 14, } @@ -22,14 +22,14 @@ response_prefix_DH_countryfile = { 'adif': 230, 'continent': 'EU', 'latitude': 51.0, - 'longitude': -10.0, + 'longitude': 10.0, 'cqz': 14, 'ituz': 28 } response_prefix_C6A_clublog = { 'country': 'BAHAMAS', - 'longitude': 76.0, + 'longitude': -76.0, 'cqz': 8, 'adif': 60, 'latitude': 24.25, @@ -38,7 +38,7 @@ response_prefix_C6A_clublog = { response_prefix_C6A_countryfile = { 'country': 'Bahamas', - 'longitude': 76.0, + 'longitude': -76.0, 'cqz': 8, 'adif': 60, 'latitude': 24.25, @@ -53,7 +53,7 @@ response_prefix_VK9NDX_countryfile = { u'cqz': 32, u'ituz': 60, u'latitude': -29.03, - u'longitude': -167.93 + u'longitude': 167.93 } response_prefix_VK9DNX_clublog = { @@ -62,7 +62,7 @@ response_prefix_VK9DNX_clublog = { u'country': u'NORFOLK ISLAND', u'cqz': 32, u'latitude': -29.0, - u'longitude': -168.0 + u'longitude': 168.0 } response_prefix_VK9DWX_clublog = { @@ -71,7 +71,7 @@ response_prefix_VK9DWX_clublog = { u'country': u'WILLIS ISLAND', u'cqz': 30, u'latitude': -16.2, - u'longitude': -150.0 + u'longitude': 150.0 } response_prefix_VK9DLX_clublog = { @@ -80,7 +80,7 @@ response_prefix_VK9DLX_clublog = { u'country': u'LORD HOWE ISLAND', u'cqz': 30, u'latitude': -31.6, - u'longitude': -159.1 + u'longitude': 159.1 } response_prefix_VK9DLX_countryfile = { @@ -90,7 +90,7 @@ response_prefix_VK9DLX_countryfile = { u'cqz': 30, u'ituz': 60, u'latitude': -31.55, - u'longitude': -159.08 + u'longitude': 159.08 } response_prefix_VK9GMW_clublog = { @@ -99,7 +99,7 @@ response_prefix_VK9GMW_clublog = { u'country': u'MELLISH REEF', u'cqz': 30, u'latitude': -17.6, - u'longitude': -155.8 + u'longitude': 155.8 } response_callsign_exceptions_7N1PRD_0_clublog = { @@ -108,7 +108,7 @@ response_callsign_exceptions_7N1PRD_0_clublog = { u'country': u'JAPAN', u'cqz': 25, u'latitude': 35.7, - u'longitude': -139.8 + u'longitude': 139.8 } response_callsign_exceptions_SV8GXQ_P_QRP_clublog = { @@ -117,7 +117,7 @@ response_callsign_exceptions_SV8GXQ_P_QRP_clublog = { u'country': u'GREECE', u'cqz': 20, u'latitude': 38.0, - u'longitude': -23.7 + u'longitude': 23.7 } response_Exception_VP8STI_with_start_and_stop_date = { @@ -125,7 +125,7 @@ response_Exception_VP8STI_with_start_and_stop_date = { 'country': u'SOUTH SANDWICH ISLANDS', 'continent': u'SA', 'latitude': -59.45, - 'longitude': 27.4, + 'longitude': -27.4, 'cqz': 13, } @@ -135,7 +135,7 @@ response_Exception_VK9XO_with_start_date = { 'country': 'CHRISTMAS ISLAND', 'continent': 'OC', 'latitude': -10.50, - 'longitude': -105.70, + 'longitude': 105.70, 'cqz': 29 } @@ -144,13 +144,13 @@ response_zone_exception_dp0gvn = { 'adif': 13, 'cqz': 38, 'latitude': -65.0, - 'longitude': 64.0, + 'longitude': -64.0, 'continent': 'AN' } response_lat_long_dh1tw = { const.LATITUDE: 51.0, - const.LONGITUDE: -10.0 + const.LONGITUDE: 10.0 } response_maritime_mobile = { @@ -177,7 +177,7 @@ response_callsign_exceptions_7QAA_clublog = { u'country': u'MALAWI', u'cqz': 37, u'latitude': -14.9, - u'longitude': -34.4 + u'longitude': 34.4 } diff --git a/test/test_clublog.py b/test/test_clublog.py new file mode 100644 index 0000000..d9cb783 --- /dev/null +++ b/test/test_clublog.py @@ -0,0 +1,28 @@ +import os +import datetime + +import pytest + +from pyhamtools.qsl import get_clublog_users + + +class Test_clublog_methods: + + def test_check_content_with_mocked_http_server(self, httpserver): + httpserver.serve_content( + open('./fixtures/clublog-users.json.zip').read()) + + data = get_clublog_users(url=httpserver.url) + assert len(data) == 139081 + + def test_download_lotw_list_and_check_types(self): + + data = get_clublog_users() + assert isinstance(data, dict) + for key, value in data.iteritems(): + assert isinstance(key, unicode) + assert isinstance(value, dict) + + def test_with_invalid_url(self): + with pytest.raises(IOError): + get_clublog_users(url="https://FAKE.csv") diff --git a/test/test_lookuplib_clublogxml.py b/test/test_lookuplib_clublogxml.py index 8fefefd..ee865ab 100644 --- a/test/test_lookuplib_clublogxml.py +++ b/test/test_lookuplib_clublogxml.py @@ -18,7 +18,7 @@ response_Entity_230 = { 'country': u'FEDERAL REPUBLIC OF GERMANY', 'continent': u'EU', 'latitude': 51.0, - 'longitude': -10.0, + 'longitude': 10.0, 'cqz': 14, 'prefix' : u'DL', 'deleted' : False, @@ -29,7 +29,7 @@ response_Exception_KC6MM_1990 = { 'country': u'PALAU', 'continent': u'OC', 'latitude': 9.50, - 'longitude': -138.20, + 'longitude': 138.20, 'cqz': 27, } @@ -38,7 +38,7 @@ response_Exception_KC6MM_1992 = { 'country': u'PALAU', 'continent': u'OC', 'latitude': 9.50, - 'longitude': -138.20, + 'longitude': 138.20, 'cqz': 27, } @@ -47,7 +47,7 @@ response_Exception_VK9XX_with_end_date = { 'country': u'CHRISTMAS ISLAND', 'continent': u'OC', 'latitude': -10.50, - 'longitude': -105.70, + 'longitude': 105.70, 'cqz': 29, } @@ -56,7 +56,7 @@ response_Exception_VK9XO_with_start_date = { 'country': u'CHRISTMAS ISLAND', 'continent': u'OC', 'latitude': -10.50, - 'longitude': -105.70, + 'longitude': 105.70, 'cqz': 29, } @@ -65,7 +65,7 @@ response_Exception_AX9NYG = { 'country': u'COCOS (KEELING) ISLAND', 'continent': u'OC', 'latitude': -12.20, - 'longitude': -96.80, + 'longitude': 96.80, 'cqz': 29, } @@ -74,7 +74,7 @@ response_Prefix_DH = { 'adif' : 230, 'continent': u'EU', 'latitude': 51.0, - 'longitude': -10.0, + 'longitude': 10.0, 'cqz': 14, } @@ -83,7 +83,7 @@ response_Prefix_VK9_until_1975 = { 'adif' : 198, 'continent': u'OC', 'latitude': -9.40, - 'longitude': -147.10, + 'longitude': 147.10, 'cqz': 28, } @@ -92,7 +92,7 @@ response_Prefix_VK9_starting_1976 = { 'adif' : 189, 'continent': u'OC', 'latitude': -29.00, - 'longitude': -168.00, + 'longitude': 168.00, 'cqz': 32, } @@ -101,7 +101,7 @@ response_Prefix_ZD5_1964_to_1971 = { 'adif' : 468, 'continent': u'AF', 'latitude': -26.30, - 'longitude': -31.10, + 'longitude': 31.10, 'cqz': 38, } diff --git a/test/test_lookuplib_countryfile.py b/test/test_lookuplib_countryfile.py index 10a5dad..85f9e3e 100644 --- a/test/test_lookuplib_countryfile.py +++ b/test/test_lookuplib_countryfile.py @@ -13,7 +13,7 @@ response_Prefix_DH = { 'country': 'Fed. Rep. of Germany', 'continent': 'EU', 'latitude': 51.0, - 'longitude': -10.0, + 'longitude': 10.0, 'cqz': 14, 'ituz' : 28 } @@ -24,7 +24,7 @@ response_Exception_3D2RI = { 'country': 'Rotuma Island', 'continent': 'OC', 'latitude': -12.48, - 'longitude': -177.08, + 'longitude': 177.08, 'cqz': 32, 'ituz' : 56 } diff --git a/test/test_lookuplib_qrz.py b/test/test_lookuplib_qrz.py index b6c3058..e35bc23 100644 --- a/test/test_lookuplib_qrz.py +++ b/test/test_lookuplib_qrz.py @@ -38,11 +38,9 @@ response_XX1XX = { } response_XX2XX = { - u'addr1': u'1234 Main Road.', - u'addr2': u'Las Vegasvillee', u'adif': 446, - u'bio': u'218', - u'biodate': datetime(2017, 6, 16, 15, 47, 47, tzinfo=UTC), + u'bio': u'349', + u'biodate': datetime(2017, 9, 5, 22, 28, 42, tzinfo=UTC), u'born': 1932, u'callsign': u'XX2XX', u'ccode': 1230, @@ -52,16 +50,16 @@ response_XX2XX = { u'email': 'dummy2@qrz.com', u'eqsl': False, u'fname': u'Gooberd', - u'geoloc': u'user', + u'geoloc': u'grid', u'image': u'https://s3.amazonaws.com/files.qrz.com/x/xx2xx/oval_bumper_sticker4.png', u'imageinfo': u'285:500:44218', u'iota': u'NA-022', u'ituz': 5, u'land': u'Morocco', - u'latitude': 52.195073, + u'latitude': 52.1875, u'license_class': u'A', u'locator': u'JO02be', - u'longitude': 0.124962, + u'longitude': 0.125, u'lotw': False, u'moddate': datetime(2017, 6, 16, 19, 22, 21, tzinfo=UTC), u'mqsl': False, @@ -77,8 +75,8 @@ response_XX3XX = { u'addr2': u'Shady Circle Roads', u'adif': 79, u'aliases': [u'XX3XX/W7'], - u'bio': u'1940', - u'biodate': datetime(2015, 8, 13, 23, 14, 38, tzinfo=UTC), + u'bio': u'16', + u'biodate': datetime(2018, 1, 24, 20, 55, 27, tzinfo=UTC), u'born': 2010, u'callsign': u'XX3XX', u'ccode': 130, diff --git a/test/test_lookuplib_redis.py b/test/test_lookuplib_redis.py index 7ccdeb0..bf28ec8 100644 --- a/test/test_lookuplib_redis.py +++ b/test/test_lookuplib_redis.py @@ -18,7 +18,7 @@ response_Exception_VP8STI_with_start_and_stop_date = { 'country': u'SOUTH SANDWICH ISLANDS', 'continent': u'SA', 'latitude': -59.45, - 'longitude': 27.4, + 'longitude': -27.4, 'cqz': 13, }