corrected negative longitude problem and added clublog user lookup

docs/source/changelog.rst

@@ -1,6 +1,19 @@
 Changelog
 ---------
 
+PyHamTools 0.6.0
+================
+
+23. January 2018
+
+ * BREAKING CHANGE: Longitude is now provided with the correct sign for all
+   lookup libraries. The AD1C cty format used by Countryfile and ClublogAPI
+   provide the longitude with the wrong sign. This is now covered and internally
+   corrected. East = positive longitude, West = negative longitude.
+ * Added a function to download the Clublog user list and the associated activity dates 
+ * updated requirements for libraries used by pyhamtools
+
+
 PyHamTools 0.5.6
 ================
 
@@ -46,6 +59,7 @@ PyHamTools 0.5.2
 14. April 2015
 
  * catching another bug related to QRZ.com sessions
+ 
 
 
 PyHamTools 0.5.1

docs/source/examples.rst

@@ -74,7 +74,7 @@ Now we can query the information conveniently through our Callinfo object:
        'adif': 230,
        'continent': 'EU',
        'latitude': 51.0,
-       'longitude': -10.0,
+       'longitude': 10.0,
        'cqz': 14,
        'ituz': 28
    }

pyhamtools/lookuplib.py

@@ -177,7 +177,7 @@ class LookupLib(object):
              u'cqz': 32,
              u'ituz': 56,
              u'latitude': -12.48,
-             u'longitude': -177.08
+             u'longitude': 177.08
            }
 
 
@@ -255,7 +255,7 @@ class LookupLib(object):
            {
             'deleted': False,
             'country': u'TURKMENISTAN',
-            'longitude': -58.4,
+            'longitude': 58.4,
             'cqz': 17,
             'prefix': u'EZ',
             'latitude': 38.0,
@@ -338,7 +338,7 @@ class LookupLib(object):
            >>> print my_lookuplib.lookup_callsign("VK9XO", timestamp)
            {
             'country': u'CHRISTMAS ISLAND',
-            'longitude': -105.7,
+            'longitude': 105.7,
             'cqz': 29,
             'adif': 35,
             'latitude': -10.5,
@@ -500,7 +500,7 @@ class LookupLib(object):
            {
             'adif': 230,
             'country': u'Fed. Rep. of Germany',
-            'longitude': -10.0,
+            'longitude': 10.0,
             'cqz': 14,
             'ituz': 28,
             'latitude': 51.0,
@@ -691,7 +691,7 @@ class LookupLib(object):
         for item in jsonLookup:
             if item == "Name": lookup[const.COUNTRY] = jsonLookup["Name"]
             elif item == "DXCC": lookup[const.ADIF] = int(jsonLookup["DXCC"])
-            elif item == "Lon": lookup[const.LONGITUDE] = float(jsonLookup["Lon"])
+            elif item == "Lon": lookup[const.LONGITUDE] = float(jsonLookup["Lon"])*(-1)
             elif item == "Lat": lookup[const.LATITUDE] = float(jsonLookup["Lat"])
             elif item == "CQZ": lookup[const.CQZ] = int(jsonLookup["CQZ"])
             elif item == "Continent": lookup[const.CONTINENT] = jsonLookup["Continent"]
@@ -1172,7 +1172,7 @@ class LookupLib(object):
                         elif item.tag == "cont":
                             entity[const.CONTINENT] = unicode(item.text)
                         elif item.tag == "long":
-                            entity[const.LONGITUDE] = float(item.text)*(-1)
+                            entity[const.LONGITUDE] = float(item.text)
                         elif item.tag == "lat":
                             entity[const.LATITUDE] = float(item.text)
                         elif item.tag == "start":
@@ -1220,7 +1220,7 @@ class LookupLib(object):
                     elif item.tag == "cont":
                         call_exception[const.CONTINENT] = unicode(item.text)
                     elif item.tag == "long":
-                        call_exception[const.LONGITUDE] = float(item.text)*(-1)
+                        call_exception[const.LONGITUDE] = float(item.text)
                     elif item.tag == "lat":
                         call_exception[const.LATITUDE] = float(item.text)
                     elif item.tag == "start":
@@ -1261,7 +1261,7 @@ class LookupLib(object):
                     elif item.tag == "cont":
                         prefix[const.CONTINENT] = unicode(item.text)
                     elif item.tag == "long":
-                        prefix[const.LONGITUDE] = float(item.text)*(-1)
+                        prefix[const.LONGITUDE] = float(item.text)
                     elif item.tag == "lat":
                         prefix[const.LATITUDE] = float(item.text)
                     elif item.tag == "start":
@@ -1383,7 +1383,7 @@ class LookupLib(object):
             entry[const.ITUZ] = int(cty_list[item]["ITUZone"])
             entry[const.CONTINENT] = unicode(cty_list[item]["Continent"])
             entry[const.LATITUDE] = float(cty_list[item]["Latitude"])
-            entry[const.LONGITUDE] = float(cty_list[item]["Longitude"])
+            entry[const.LONGITUDE] = float(cty_list[item]["Longitude"])*(-1)
 
             if cty_list[item]["ExactCallsign"]:
                 if call in exceptions_index.keys():
@@ -1433,7 +1433,7 @@ class LookupLib(object):
         else:
             err_str = "HTTP Status Code: " + str(response.status_code) + " HTTP Response: " + str(response.text)
             self._logger.error(err_str)
-            if response.text.strip() == error1 or response.text.strip() == error2:
+            if error1 in response.text.strip() or error2 in response.text.strip():
                 raise APIKeyMissingError
             else:
                 raise LookupError(err_str)

pyhamtools/qsl.py

@@ -3,6 +3,9 @@ import re
 
 import requests
 import redis
+import zipfile
+import json
+from io import BytesIO
 from requests.exceptions import ConnectionError, HTTPError, Timeout
 
 def get_lotw_users(**kwargs):
@@ -16,6 +19,7 @@ def get_lotw_users(**kwargs):
 
         Raises:
             IOError: When network is unavailable, file can't be downloaded or processed
+            
             ValueError: Raised when data from file can't be read
 
         Example:
@@ -63,8 +67,96 @@ def get_lotw_users(**kwargs):
 
     return lotw
 
+def get_clublog_users(**kwargs):
+    """Download the latest offical list of `Clublog`__ users.
+
+        Args:
+            url (str, optional): Download URL
+
+        Returns:
+            dict: Dictionary containing (if data available) the fields:
+                firstqso, lastqso, last-lotw, lastupload (datetime), 
+                locator (string) and oqrs (boolean)
+
+        Raises:
+            IOError: When network is unavailable, file can't be downloaded or processed
+
+        Example:
+           The following example downloads the Clublog user list and returns a dictionary with the data of HC2/AL1O:
+
+           >>> from pyhamtools.qsl import get_clublog_users
+           >>> clublog = get_lotw_users()
+           >>> clublog['HC2/AL1O']
+           {'firstqso': datetime.datetime(2012, 1, 1, 19, 59, 27),
+            'last-lotw': datetime.datetime(2013, 5, 9, 1, 56, 23),
+            'lastqso': datetime.datetime(2013, 5, 5, 6, 39, 3),
+            'lastupload': datetime.datetime(2013, 5, 8, 15, 0, 6),
+            'oqrs': True}
+
+    .. _CLUBLOG: https://secure.clublog.org
+    __ CLUBLOG_ 
+
+    """
+    
+    url = ""
+    
+    clublog = {}
+    
+    try: 
+        url = kwargs['url']
+    except KeyError:
+        url = "https://secure.clublog.org/clublog-users.json.zip"
+    
+    try: 
+        result = requests.get(url)
+    except (ConnectionError, HTTPError, Timeout) as e:
+        raise IOError(e)
+
+    
+    if result.status_code != requests.codes.ok:
+        raise IOError("HTTP Error: " + str(result.status_code))
+
+    zip_file = zipfile.ZipFile(BytesIO(result.content))
+    files = zip_file.namelist()
+    cl_json_unzipped = zip_file.read(files[0])
+
+    cl_data = json.loads(cl_json_unzipped)
+
+    error_count = 0
+
+    for call, call_data in cl_data.iteritems():
+        try:
+            data = {}
+            if "firstqso" in call_data:
+                if call_data["firstqso"] != None:
+                    data["firstqso"] = datetime.strptime(call_data["firstqso"], '%Y-%m-%d %H:%M:%S') 
+            if "lastqso" in call_data:
+                if call_data["lastqso"] != None:
+                    data["lastqso"] = datetime.strptime(call_data["lastqso"], '%Y-%m-%d %H:%M:%S')
+            if "last-lotw" in call_data:
+                if call_data["last-lotw"] != None:
+                    data["last-lotw"] = datetime.strptime(call_data["last-lotw"], '%Y-%m-%d %H:%M:%S')
+            if "lastupload" in call_data:
+                if call_data["lastupload"] != None:
+                    data["lastupload"] = datetime.strptime(call_data["lastupload"], '%Y-%m-%d %H:%M:%S')
+            if "locator" in call_data:
+                if call_data["locator"] != None:
+                    data["locator"] = call_data["locator"]
+            if "oqrs" in call_data:
+                if call_data["oqrs"] != None:
+                    data["oqrs"] = call_data["oqrs"]
+            clublog[call] = data
+        except TypeError: #some date fields contain null instead of a valid datetime string - we ignore them
+            print("Ignoring invalid type in data:", call, call_data)
+            pass
+        except ValueError: #some date fiels are invalid. we ignore them for the moment
+            print("Ignoring invalid data:", call, call_data)
+            pass
+
+    return clublog
+
 def get_eqsl_users(**kwargs):
-    """Download the latest official list of EQSL.cc users. The list of users can be found here_.
+    """Download the latest official list of `EQSL.cc`__ users. The list of users can be found here_.
 
         Args:
             url (str, optional): Download URL
@@ -85,7 +177,7 @@ def get_eqsl_users(**kwargs):
            >>> except ValueError as e:
            >>>    print e
            'DH1TW' is not in list
-                
+    
     .. _here: http://www.eqsl.cc/QSLCard/DownloadedFiles/AGMemberlist.txt
 
     """    

pyhamtools/version.py

@@ -1,4 +1,3 @@
-#VERSION = (0, 5, 0, 'dev')
+VERSION = (0, 6, 0)
-VERSION = (0, 5, 6)
 __release__ = ''.join(['-.'[type(x) == int]+str(x) for x in VERSION])[1:]
 __version__ = '.'.join((str(VERSION[0]), str(VERSION[1])))

readthedocs-pip-requirements.txt

@@ -1,7 +1,7 @@
-sphinxcontrib-napoleon>=0.2.7
+sphinxcontrib-napoleon>=0.6.1
-requests>=2.2.1
+requests>=2.18.4
-pytz>=2014.2
+pytz>=2017.3
-pyephem>=3.7.5.3
+pyephem>=3.7.6.0
-redis>=2.10.2
+redis>=2.10.6
-beautifulsoup4>=4.3.2
+beautifulsoup4>=4.6.0
 

test/conftest.py

@@ -66,12 +66,18 @@ def fixCountryFile(request):
     Lib = LookupLib("countryfile")
     return(Lib)
 
-@pytest.fixture(scope="module", params=["clublogapi", "clublogxml", "countryfile"])
+@pytest.fixture(scope="module", params=["clublogxml", "countryfile"])
 def fix_callinfo(request, fixApiKey):
     lib = LookupLib(request.param, fixApiKey)
     callinfo = Callinfo(lib)
     return(callinfo)
 
+# @pytest.fixture(scope="module", params=["clublogapi", "clublogxml", "countryfile"])
+# def fix_callinfo(request, fixApiKey):
+#     lib = LookupLib(request.param, fixApiKey)
+#     callinfo = Callinfo(lib)
+#     return(callinfo)
+
 @pytest.fixture(scope="module")
 def fix_redis():
     import redis

test/fixtures/generate_lotw_data.py

@@ -0,0 +1,5 @@
+from pyhamtools.qsl import get_lotw_users
+
+lotw = get_lotw_users()
+print lotw
+# pipe output into file

test/test_callinfo.py

@@ -13,7 +13,7 @@ response_prefix_DH_clublog = {
     'adif': 230,
     'continent': 'EU',
     'latitude': 51.0,
-    'longitude': -10.0,
+    'longitude': 10.0,
     'cqz': 14,
 }
 
@@ -22,14 +22,14 @@ response_prefix_DH_countryfile = {
     'adif': 230,
     'continent': 'EU',
     'latitude': 51.0,
-    'longitude': -10.0,
+    'longitude': 10.0,
     'cqz': 14,
     'ituz': 28
 }
 
 response_prefix_C6A_clublog = {
     'country': 'BAHAMAS',
-    'longitude': 76.0,
+    'longitude': -76.0,
     'cqz': 8,
     'adif': 60,
     'latitude': 24.25,
@@ -38,7 +38,7 @@ response_prefix_C6A_clublog = {
 
 response_prefix_C6A_countryfile = {
     'country': 'Bahamas',
-    'longitude': 76.0,
+    'longitude': -76.0,
     'cqz': 8,
     'adif': 60,
     'latitude': 24.25,
@@ -53,7 +53,7 @@ response_prefix_VK9NDX_countryfile = {
     u'cqz': 32,
     u'ituz': 60,
     u'latitude': -29.03,
-    u'longitude': -167.93
+    u'longitude': 167.93
 }
 
 response_prefix_VK9DNX_clublog = {
@@ -62,7 +62,7 @@ response_prefix_VK9DNX_clublog = {
     u'country': u'NORFOLK ISLAND',
     u'cqz': 32,
     u'latitude': -29.0,
-    u'longitude': -168.0
+    u'longitude': 168.0
 }
 
 response_prefix_VK9DWX_clublog = {
@@ -71,7 +71,7 @@ response_prefix_VK9DWX_clublog = {
     u'country': u'WILLIS ISLAND',
     u'cqz': 30,
     u'latitude': -16.2,
-    u'longitude': -150.0
+    u'longitude': 150.0
 }
 
 response_prefix_VK9DLX_clublog = {
@@ -80,7 +80,7 @@ response_prefix_VK9DLX_clublog = {
     u'country': u'LORD HOWE ISLAND',
     u'cqz': 30,
     u'latitude': -31.6,
-    u'longitude': -159.1
+    u'longitude': 159.1
 }
 
 response_prefix_VK9DLX_countryfile = {
@@ -90,7 +90,7 @@ response_prefix_VK9DLX_countryfile = {
      u'cqz': 30,
      u'ituz': 60,
      u'latitude': -31.55,
-     u'longitude': -159.08
+     u'longitude': 159.08
 }
 
 response_prefix_VK9GMW_clublog = {
@@ -99,7 +99,7 @@ response_prefix_VK9GMW_clublog = {
     u'country': u'MELLISH REEF',
     u'cqz': 30,
     u'latitude': -17.6,
-    u'longitude': -155.8
+    u'longitude': 155.8
 }
 
 response_callsign_exceptions_7N1PRD_0_clublog = {
@@ -108,7 +108,7 @@ response_callsign_exceptions_7N1PRD_0_clublog = {
     u'country': u'JAPAN',
     u'cqz': 25,
     u'latitude': 35.7,
-    u'longitude': -139.8
+    u'longitude': 139.8
 }
 
 response_callsign_exceptions_SV8GXQ_P_QRP_clublog = {
@@ -117,7 +117,7 @@ response_callsign_exceptions_SV8GXQ_P_QRP_clublog = {
     u'country': u'GREECE',
     u'cqz': 20,
     u'latitude': 38.0,
-    u'longitude': -23.7
+    u'longitude': 23.7
 }
 
 response_Exception_VP8STI_with_start_and_stop_date = {
@@ -125,7 +125,7 @@ response_Exception_VP8STI_with_start_and_stop_date = {
            'country': u'SOUTH SANDWICH ISLANDS',
            'continent': u'SA',
            'latitude': -59.45,
-           'longitude': 27.4,
+           'longitude': -27.4,
            'cqz': 13,
         }
 
@@ -135,7 +135,7 @@ response_Exception_VK9XO_with_start_date = {
            'country': 'CHRISTMAS ISLAND',
            'continent': 'OC',
            'latitude': -10.50,
-           'longitude': -105.70,
+           'longitude': 105.70,
            'cqz': 29
         }
 
@@ -144,13 +144,13 @@ response_zone_exception_dp0gvn = {
     'adif': 13,
     'cqz': 38,
     'latitude': -65.0,
-    'longitude': 64.0,
+    'longitude': -64.0,
     'continent': 'AN'
 }
 
 response_lat_long_dh1tw = {
     const.LATITUDE: 51.0,
-    const.LONGITUDE: -10.0
+    const.LONGITUDE: 10.0
 }
 
 response_maritime_mobile = {
@@ -177,7 +177,7 @@ response_callsign_exceptions_7QAA_clublog = {
     u'country': u'MALAWI',
     u'cqz': 37,
     u'latitude': -14.9,
-    u'longitude': -34.4
+    u'longitude': 34.4
 }
 
 

test/test_clublog.py

@@ -0,0 +1,28 @@
+import os
+import datetime
+
+import pytest
+
+from pyhamtools.qsl import get_clublog_users
+
+
+class Test_clublog_methods:
+
+    def test_check_content_with_mocked_http_server(self, httpserver):
+        httpserver.serve_content(
+            open('./fixtures/clublog-users.json.zip').read())
+
+        data = get_clublog_users(url=httpserver.url)
+        assert len(data) == 139081
+
+    def test_download_lotw_list_and_check_types(self):
+
+        data = get_clublog_users()
+        assert isinstance(data, dict)
+        for key, value in data.iteritems():
+            assert isinstance(key, unicode)
+            assert isinstance(value, dict)
+
+    def test_with_invalid_url(self):
+        with pytest.raises(IOError):
+            get_clublog_users(url="https://FAKE.csv")

test/test_lookuplib_clublogxml.py

@@ -18,7 +18,7 @@ response_Entity_230 = {
           'country': u'FEDERAL REPUBLIC OF GERMANY',
           'continent': u'EU',
           'latitude': 51.0,
-          'longitude': -10.0,
+          'longitude': 10.0,
           'cqz': 14,
           'prefix' : u'DL',
           'deleted' : False,
@@ -29,7 +29,7 @@ response_Exception_KC6MM_1990 = {
            'country': u'PALAU',
            'continent': u'OC',
            'latitude': 9.50,
-           'longitude': -138.20,
+           'longitude': 138.20,
            'cqz': 27,
         }
 
@@ -38,7 +38,7 @@ response_Exception_KC6MM_1992 = {
            'country': u'PALAU',
            'continent': u'OC',
            'latitude': 9.50,
-           'longitude': -138.20,
+           'longitude': 138.20,
            'cqz': 27,
         }
 
@@ -47,7 +47,7 @@ response_Exception_VK9XX_with_end_date = {
            'country': u'CHRISTMAS ISLAND',
            'continent': u'OC',
            'latitude': -10.50,
-           'longitude': -105.70,
+           'longitude': 105.70,
            'cqz': 29,
         }
 
@@ -56,7 +56,7 @@ response_Exception_VK9XO_with_start_date = {
            'country': u'CHRISTMAS ISLAND',
            'continent': u'OC',
            'latitude': -10.50,
-           'longitude': -105.70,
+           'longitude': 105.70,
            'cqz': 29,
         }
 
@@ -65,7 +65,7 @@ response_Exception_AX9NYG = {
            'country': u'COCOS (KEELING) ISLAND',
            'continent': u'OC',
            'latitude': -12.20,
-           'longitude': -96.80,
+           'longitude': 96.80,
            'cqz': 29,
         }
 
@@ -74,7 +74,7 @@ response_Prefix_DH = {
     'adif' : 230,
     'continent': u'EU',
     'latitude': 51.0,
-    'longitude': -10.0,
+    'longitude': 10.0,
     'cqz': 14,
 }
 
@@ -83,7 +83,7 @@ response_Prefix_VK9_until_1975 = {
     'adif' : 198,
     'continent': u'OC',
     'latitude': -9.40,
-    'longitude': -147.10,
+    'longitude': 147.10,
     'cqz': 28,
 }
 
@@ -92,7 +92,7 @@ response_Prefix_VK9_starting_1976 = {
     'adif' : 189,
     'continent': u'OC',
     'latitude': -29.00,
-    'longitude': -168.00,
+    'longitude': 168.00,
     'cqz': 32,
 }
 
@@ -101,7 +101,7 @@ response_Prefix_ZD5_1964_to_1971 = {
     'adif' : 468,
     'continent': u'AF',
     'latitude': -26.30,
-    'longitude': -31.10,
+    'longitude': 31.10,
     'cqz': 38,
 }
 

test/test_lookuplib_countryfile.py

@@ -13,7 +13,7 @@ response_Prefix_DH = {
           'country': 'Fed. Rep. of Germany', 
           'continent': 'EU', 
           'latitude': 51.0, 
-          'longitude': -10.0, 
+          'longitude': 10.0, 
           'cqz': 14,
           'ituz' : 28
         }
@@ -24,7 +24,7 @@ response_Exception_3D2RI = {
           'country': 'Rotuma Island', 
           'continent': 'OC', 
           'latitude': -12.48, 
-          'longitude': -177.08, 
+          'longitude': 177.08, 
           'cqz': 32,
           'ituz' : 56
         }

test/test_lookuplib_qrz.py

@@ -38,11 +38,9 @@ response_XX1XX = {
  }
 
 response_XX2XX = {
-    u'addr1': u'1234 Main Road.',
-    u'addr2': u'Las Vegasvillee',
     u'adif': 446,
-    u'bio': u'218',
+    u'bio': u'349',
-    u'biodate': datetime(2017, 6, 16, 15, 47, 47, tzinfo=UTC),
+    u'biodate': datetime(2017, 9, 5, 22, 28, 42, tzinfo=UTC),
     u'born': 1932,
     u'callsign': u'XX2XX',
     u'ccode': 1230,
@@ -52,16 +50,16 @@ response_XX2XX = {
     u'email': 'dummy2@qrz.com',
     u'eqsl': False,
     u'fname': u'Gooberd',
-    u'geoloc': u'user',
+    u'geoloc': u'grid',
     u'image': u'https://s3.amazonaws.com/files.qrz.com/x/xx2xx/oval_bumper_sticker4.png',
     u'imageinfo': u'285:500:44218',
     u'iota': u'NA-022',
     u'ituz': 5,
     u'land': u'Morocco',
-    u'latitude': 52.195073,
+    u'latitude': 52.1875,
     u'license_class': u'A',
     u'locator': u'JO02be',
-    u'longitude': 0.124962,
+    u'longitude': 0.125,
     u'lotw': False,
     u'moddate': datetime(2017, 6, 16, 19, 22, 21, tzinfo=UTC),
     u'mqsl': False,
@@ -77,8 +75,8 @@ response_XX3XX = {
      u'addr2': u'Shady Circle Roads',
      u'adif': 79,
      u'aliases': [u'XX3XX/W7'],
-     u'bio': u'1940',
+     u'bio': u'16',
-     u'biodate': datetime(2015, 8, 13, 23, 14, 38, tzinfo=UTC),
+     u'biodate': datetime(2018, 1, 24, 20, 55, 27, tzinfo=UTC),
      u'born': 2010,
      u'callsign': u'XX3XX',
      u'ccode': 130,

test/test_lookuplib_redis.py

@@ -18,7 +18,7 @@ response_Exception_VP8STI_with_start_and_stop_date = {
            'country': u'SOUTH SANDWICH ISLANDS',
            'continent': u'SA',
            'latitude': -59.45,
-           'longitude': 27.4,
+           'longitude': -27.4,
            'cqz': 13,
         }