merging v0.8.0 branch

- finally using libxml2 parser in beautifulsoup/lxml
- minor bug in parsing the CCC field (qrz.com)
- fixed VK9XX fixture (lat/long)
- added support for python 3.10 and 3.11, pypy3.7-3.9
- removed support for python 3.4
- fixed escapings in regular expressions
- replaced exefile from past.buildins with custom function

.appveyor.yml

@@ -1,51 +0,0 @@
-build: false
-image: "Visual Studio 2019"
-
-environment:
-  matrix:
-    - PYTHON: "C:\\Python27"
-      PYTHON_VERSION: "2.7.8"
-      PYTHON_ARCH: "32"
-
-    - PYTHON: "C:\\Python35"
-      PYTHON_VERSION: "3.5.4"
-      PYTHON_ARCH: "32"
-
-    - PYTHON: "C:\\Python36"
-      PYTHON_VERSION: "3.6.4"
-      PYTHON_ARCH: "32"
-
-    - PYTHON: "C:\\Python37"
-      PYTHON_VERSION: "3.7.5"
-      PYTHON_ARCH: "32"
-
-    - PYTHON: "C:\\Python38"
-      PYTHON_VERSION: "3.8.15"
-      PYTHON_ARCH: "32"
-
-    - PYTHON: "C:\\Python39"
-      PYTHON_VERSION: "3.9.15"
-      PYTHON_ARCH: "64"
-
-    - PYTHON: "C:\\Python310"
-      PYTHON_VERSION: "3.10.7"
-      PYTHON_ARCH: "64"
-
-    - PYTHON: "C:\\Python311"
-      PYTHON_VERSION: "3.11.0"
-      PYTHON_ARCH: "64"
-
-init:
-  - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
-  - "%PYTHON%/python.exe -m pip install --upgrade pip"
-
-install:
-  - nuget install redis-64 -excludeversion
-  - redis-64\tools\redis-server.exe --service-install
-  - redis-64\tools\redis-server.exe --service-start
-  - "%PYTHON%/python.exe -m pip install -e ."
-  - "%PYTHON%/python.exe -m pip install -r requirements-docs.txt"
-  - "%PYTHON%/python.exe -m pip install -r requirements-pytest.txt"
-
-test_script:
-  - "%PYTHON%/Scripts/pytest"

.github/workflows/test.yml

@@ -7,21 +7,27 @@ jobs:
     runs-on: "ubuntu-latest"
     name: "Ubuntu latest - Python ${{ matrix.python-version }}"
     env:
-      USING_COVERAGE: '3.10'
+      USING_COVERAGE: '3.11'
 
     strategy:
       matrix:
-        python-version: ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy2"]
+        python-version: ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy2.7", "pypy3.7", "pypy3.8", "pypy3.9"]
         redis-version: [6]
 
     steps:
-      - uses: "actions/checkout@v2"
+      - uses: "actions/checkout@v3"
-      - uses: "actions/setup-python@v2"
+      - uses: "actions/setup-python@v4"
         with:
           python-version: "${{ matrix.python-version }}"
+          cache: "pip"
+          cache-dependency-path: |
+            **/setup.py
+            **/requirements*.txt
+
       - name: "Install dependencies"
         run: |
           set -xe
+          sudo apt-get install -y libxml2-dev libxslt-dev
           python -VV
           python -m pip install --upgrade pip setuptools wheel codecov
           python -m pip install -e .
@@ -39,14 +45,13 @@ jobs:
           QRZ_USERNAME: ${{ secrets.QRZ_USERNAME }}
           QRZ_PWD: ${{ secrets.QRZ_PWD }}
           PYTHON_VERSION: ${{ matrix.python-version }}
-        # delay the execution randomly by 1-20sec to reduce the
+        # delay the execution randomly by a couple of seconds to reduce the amount 
-        # amount of concurrent API calls on Clublog and QRZ.com
+        # of concurrent API calls on Clublog and QRZ.com when all CI jobs execute simultaneously
-        # when all CI jobs execute simultaneously
         run: |
-          sleep $[ ( $RANDOM % 20 )  + 1 ]s 
+          sleep $[ ( $RANDOM % 10 )  + 1 ]s 
           pytest --cov=./
           if [[ $PYTHON_VERSION == 3.11 ]]; then codecov; fi
-          if [[ $PYTHON_VERSION == 3.11 ]]; then cd docs && make html; fi
+          cd docs && make html
 
   # publish_package:
   #   runs-on: "ubuntu-latest"
@@ -59,41 +64,52 @@ jobs:
   #         user: __token__
   #         password: ${{ secrets.PYPI_API_TOKEN }}
 
-  # test_windows:
+  test_windows:
-  #   runs-on: "windows-latest"
+    runs-on: "windows-latest"
-  #   name: "Windows latest - Python ${{ matrix.python-version }}"
+    name: "Windows latest - Python ${{ matrix.python-version }}"
-  #   env:
-  #     USING_COVERAGE: '3.9'
 
-  #   strategy:
+    strategy:
-  #     matrix:
+      matrix:
-  #       python-version: ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9"]
+        # lxml support for windows/python3.11 still missing (Dec 2022)
-  #       redis-version: ["6.2"]
+        # https://github.com/lxml/lxml/pull/360
+        # python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
 
-  #   steps:
+    steps:
-  #     - uses: "actions/checkout@v2"
+      - uses: "actions/checkout@v3"
-  #     - uses: "actions/setup-python@v2"
+      - uses: "actions/setup-python@v4"
-  #       with:
+        with:
-  #         python-version: "${{ matrix.python-version }}"
+          python-version: "${{ matrix.python-version }}"
-  #     - name: "Install dependencies"
+          cache: "pip"
-  #       run: |
+          cache-dependency-path: |
-  #         python -VV
+                setup.py
-  #         python -m pip install --upgrade pip setuptools wheel codecov
+                requirements*.txt
-  #         python -m pip install -e .
+      - name: "Install dependencies"
-  #         python -m pip install -r requirements-pytest.txt
+        run: |
-  #         python -m pip install -r requirements-docs.txt
+          python -VV
-  #     - name: Setup redis
+          python -m pip install --upgrade pip setuptools wheel codecov
-  #       uses: shogo82148/actions-setup-redis@v1
+          python -m pip install -e .
-  #       with:
+          python -m pip install -r requirements-pytest.txt
-  #         redis-version: ${{ matrix.redis-version }}
+          python -m pip install -r requirements-docs.txt
-  #     - name: "Run tests for ${{ matrix.python-version }}"
+      - name: Setup redis
-  #       env: 
+        # We have to download and install a non-official redis windows port
-  #         CLUBLOG_APIKEY: ${{ secrets.CLUBLOG_APIKEY }}
+        # since there is no official redis version for windows.
-  #         QRZ_USERNAME: ${{ secrets.QRZ_USERNAME }}
+        # 5.0 is good enough for our purposes. After installing the msi, 
-  #         QRZ_PWD: ${{ secrets.QRZ_PWD }}
+        # redis will startup as a service. 
-  #         PYTHON_VERSION: ${{ matrix.python-version }}
+        # There are no github-actions supporting redis on windows.
-  #       # delay the execution randomly by 1-20sec to reduce the
+        # Github Actions Container services are also not available for windows.
-  #       # amount of concurrent API calls on Clublog and QRZ.com
+        run: |
-  #       # when all CI jobs execute simultaneously
+          C:\msys64\usr\bin\wget.exe https://github.com/tporadowski/redis/releases/download/v5.0.14.1/Redis-x64-5.0.14.1.msi
-  #       run: |
+          msiexec /quiet /i Redis-x64-5.0.14.1.msi
-  #         pytest
+      - name: "Run tests for ${{ matrix.python-version }}"
+        env: 
+          CLUBLOG_APIKEY: ${{ secrets.CLUBLOG_APIKEY }}
+          QRZ_USERNAME: ${{ secrets.QRZ_USERNAME }}
+          QRZ_PWD: ${{ secrets.QRZ_PWD }}
+          PYTHON_VERSION: ${{ matrix.python-version }}
+        # delay the execution randomly by 1-20sec to reduce the
+        # amount of concurrent API calls on Clublog and QRZ.com
+        # when all CI jobs execute simultaneously
+        run: |
+          start-sleep -Seconds (1..10 | get-random)
+          pytest

README.md

@@ -1,7 +1,6 @@
 # pyhamtools
 
 ![Build Status](https://github.com/dh1tw/pyhamtools/actions/workflows/test.yml/badge.svg)
-![Build status](https://ci.appveyor.com/api/projects/status/8rfgr7x6w1arixrh/branch/master?svg=true)
 [![codecov](https://codecov.io/gh/dh1tw/pyhamtools/branch/master/graph/badge.svg)](https://codecov.io/gh/dh1tw/pyhamtools)
 [![PyPI version](https://badge.fury.io/py/pyhamtools.svg)](https://badge.fury.io/py/pyhamtools)
 
@@ -32,18 +31,22 @@ This Library is used in production at the [DXHeat.com DX Cluster](https://dxheat
 
 ## Compatibility
 
-Pyhamtools is since version 0.6.0 compatible with > Python 2.7 and > python 3.3.
+Pyhamtools is compatible with Python 2.7 and Python >=3.6.
 We check compatibility on OSX, Windows, and Linux with the following Python
 versions:
 
-* Python 2.7
+* Python 2.7 (will be deprecated in 2023)
-* Python 3.4 (will be deprecated in 2022)
+* Python 3.5 (has been deprecated in 2022)
-* Python 3.5 (will be deprecated in 2022)
+* Python 3.6 (will be deprecated in 2023)
-* Python 3.6
 * Python 3.7
 * Python 3.8
 * Python 3.9
+* Python 3.10
+* Python 3.11
 * [pypy2](https://pypy.org/) (Python 2)
+* [pypy3.7](https://pypy.org/)
+* [pypy3.8](https://pypy.org/)
+* [pypy3.9](https://pypy.org/)
 
 ## Documentation
 
@@ -55,8 +58,20 @@ Check out the full documentation including the changelog at:
 Pyhamtools is published under the permissive [MIT License](http://choosealicense.com/licenses/mit/). You can find a good comparison of
 Open Source Software licenses, including the MIT license at [choosealicense.com](http://choosealicense.com/licenses/)
 
+## Dependencies
+
+Starting with version 0.8.0, `libxml2-dev` and `libxslt-dev` are required dependencies.
+
 ## Installation
 
+Install the dependencies (e.g. on Debian/Ubuntu):
+
+```bash
+
+$ sudo apt-get install libxml2-dev libxslt-dev
+
+```
+
 The easiest way to install pyhamtools is through the packet manager `pip`:
 
 ```bash
@@ -65,6 +80,14 @@ $ pip install pyhamtools
 
 ```
 
+Christoph, @df7cb is kindly maintaining a Debian package as an alternative way to install pyhamtools:
+
+```bash
+
+$ sudo apt-get install pyhamtools
+
+```
+
 ## Example: How to use pyhamtools
 
 ``` python

docs/source/changelog.rst

@@ -1,6 +1,21 @@
 Changelog
 ---------
 
+PyHamtools 0.8.0
+================
+
+05. December 2022
+
+* Finally switched to XML parser in BeautifulSoup for qrz.com (requires libxml2-dev and libxslt-dev packages!)
+* Fixed minor bug in parsing the CCC field of qrz.com XML messages
+* Fixed VK9XX test fixture (Latitude & Longitude)
+* Added support for CPython 3.10 and 3.11
+* Added support for PyPy 3.7, 3.8, 3.9
+* Dropped support for Python 3.4
+* Fixed regular expression escapings which were marked as deprecated (since Python 3.6)
+* Replaced legacy execfile function in test package to remove the deprecation warning about 'imp'
+
+
 PyHamtools 0.7.10
 ================
 

docs/source/conf.py

@@ -35,7 +35,7 @@ sys.path.insert(0,"/Users/user/projects/pyhamtools/pyhamtools")
 # ones.
 extensions = [
     'sphinx.ext.autodoc',
-    'sphinxcontrib.napoleon',
+    'sphinx.ext.napoleon',
 ]
 
 # Add any paths that contain templates here, relative to this directory.

pyhamtools/callinfo.py

@@ -70,7 +70,7 @@ class Callinfo(object):
         """
 
         callsign = callsign.upper()
-        homecall = re.search('[\d]{0,1}[A-Z]{1,2}\d([A-Z]{1,4}|\d{3,3}|\d{1,3}[A-Z])[A-Z]{0,5}', callsign)
+        homecall = re.search('[\\d]{0,1}[A-Z]{1,2}\\d([A-Z]{1,4}|\\d{3,3}|\\d{1,3}[A-Z])[A-Z]{0,5}', callsign)
         if homecall:
             homecall = homecall.group(0)
             return homecall
@@ -126,10 +126,10 @@ class Callinfo(object):
         if timestamp is None:
             timestamp = datetime.utcnow().replace(tzinfo=UTC)
 
-        if re.search('[/A-Z0-9\-]{3,15}', entire_callsign):  # make sure the call has at least 3 characters
+        if re.search('[/A-Z0-9\\-]{3,15}', entire_callsign):  # make sure the call has at least 3 characters
 
-            if re.search('\-\d{1,3}$', entire_callsign):  # cut off any -10 / -02 appendixes
+            if re.search('\\-\\d{1,3}$', entire_callsign):  # cut off any -10 / -02 appendixes
-                callsign = re.sub('\-\d{1,3}$', '', entire_callsign)
+                callsign = re.sub('\\-\\d{1,3}$', '', entire_callsign)
 
             if re.search('/[A-Z0-9]{1,4}/[A-Z0-9]{1,4}$', callsign):
                 callsign = re.sub('/[A-Z0-9]{1,4}$', '', callsign)  # cut off 2. appendix DH1TW/HC2/P -> DH1TW/HC2
@@ -192,11 +192,11 @@ class Callinfo(object):
                     data[const.BEACON] = True
                     return data
 
-                elif re.search('\d$', appendix):
+                elif re.search('\\d$', appendix):
-                    area_nr = re.search('\d$', appendix).group(0)
+                    area_nr = re.search('\\d$', appendix).group(0)
-                    callsign = re.sub('/\d$', '', callsign) #remove /number
+                    callsign = re.sub('/\\d$', '', callsign) #remove /number
-                    if len(re.findall(r'\d+', callsign)) == 1: #call has just on digit e.g. DH1TW
+                    if len(re.findall(r'\\d+', callsign)) == 1: #call has just on digit e.g. DH1TW
-                        callsign = re.sub('[\d]+', area_nr, callsign)
+                        callsign = re.sub('[\\d]+', area_nr, callsign)
                     else: # call has several digits e.g. 7N4AAL
                         pass # no (two) digit prefix countries known where appendix would change entity
                     return self._iterate_prefix(callsign, timestamp)
@@ -205,7 +205,7 @@ class Callinfo(object):
                     return self._iterate_prefix(callsign, timestamp)
 
             # regular callsigns, without prefix or appendix
-            elif re.match('^[\d]{0,1}[A-Z]{1,2}\d([A-Z]{1,4}|\d{3,3}|\d{1,3}[A-Z])[A-Z]{0,5}$', callsign):
+            elif re.match('^[\\d]{0,1}[A-Z]{1,2}\\d([A-Z]{1,4}|\\d{3,3}|\\d{1,3}[A-Z])[A-Z]{0,5}$', callsign):
                 return self._iterate_prefix(callsign, timestamp)
 
             # callsigns with prefixes (xxx/callsign)
@@ -215,7 +215,7 @@ class Callinfo(object):
                 #make sure that the remaining part is actually a callsign (avoid: OZ/JO81)
                 rest = re.search('/[A-Z0-9]+', entire_callsign)
                 rest = re.sub('/', '', rest.group(0))
-                if re.match('^[\d]{0,1}[A-Z]{1,2}\d([A-Z]{1,4}|\d{3,3}|\d{1,3}[A-Z])[A-Z]{0,5}$', rest):
+                if re.match('^[\\d]{0,1}[A-Z]{1,2}\\d([A-Z]{1,4}|\\d{3,3}|\\d{1,3}[A-Z])[A-Z]{0,5}$', rest):
                     return self._iterate_prefix(pfx)
 
         if entire_callsign in callsign_exceptions:

pyhamtools/version.py

@@ -1,3 +1,3 @@
-VERSION = (0, 7, 11)
+VERSION = (0, 8, 0)
 __release__ = ''.join(['-.'[type(x) == int]+str(x) for x in VERSION])[1:]
 __version__ = '.'.join((str(VERSION[0]), str(VERSION[1])))

requirements-docs.txt

@@ -1,2 +1 @@
-sphinx>=1.8.5
+sphinx>=1.8.5
-sphinxcontrib-napoleon>=0.7

requirements-pytest.txt

@@ -1,5 +1,5 @@
-pytest>=7.0.0; python_version>='3.6'
+pytest>=7.0.0; python_version>='3.7'
-pytest==4.6.11; python_version<='3.5' and python_version>='2.7'
+pytest==4.6.11; python_version<='3.6' and python_version>='2.7'
 pytest-blockage>=0.2.2
 pytest-localserver>=0.5
 pytest-cov>=2.12

test/execfile.py

@@ -0,0 +1,17 @@
+# In Python3 the function 'execfile' has been deprecated. The alternative is 'exec'.
+# While the package 'past.builtins' provide a python2 / python3 compatible version of 'execfile', 
+# the import of 'past.builtins' keeps on throwing a deprecation warning about 'imp'.
+# Therefore the version of 'execfile' from 'past/builtins' has been replaced by this alternative
+# version, found on: https://stackoverflow.com/a/41658338/2292376
+
+# When support of Python2 is finally dropped, this function can be removed
+
+def execfile(filepath, globals=None, locals=None):
+    if globals is None:
+        globals = {}
+    globals.update({
+        "__file__": filepath,
+        "__name__": "__main__",
+    })
+    with open(filepath, 'rb') as file:
+        exec(compile(file.read(), filepath, 'exec'), globals, locals)

test/test_eqsl.py

@@ -1,4 +1,4 @@
-from past.builtins import execfile
+from .execfile import execfile
 import os
 import sys
 import datetime

test/test_lookuplib_clublogxml.py

@@ -46,8 +46,8 @@ response_Exception_VK9XX_with_end_date = {
            'adif': 35,
            'country': u'CHRISTMAS ISLAND',
            'continent': u'OC',
-           'latitude': -10.44,
+           'latitude': -10.52,
-           'longitude': 105.71,
+           'longitude': 105.54,
            'cqz': 29,
         }
 

test/test_lotw.py

@@ -2,10 +2,21 @@ import os
 import sys
 import datetime
 
-from past.builtins import execfile
+from .execfile import execfile
 from future.utils import iteritems
 import pytest
 
+def execfile(filepath, globals=None, locals=None):
+    if globals is None:
+        globals = {}
+    globals.update({
+        "__file__": filepath,
+        "__name__": "__main__",
+    })
+    with open(filepath, 'rb') as file:
+        exec(compile(file.read(), filepath, 'exec'), globals, locals)
+
+
 from pyhamtools.qsl import get_lotw_users
 
 if sys.version_info.major == 3: