Merge 1c28865f3d into d006e2e587

.github/workflows/test_basic_client.yml

@@ -0,0 +1,27 @@
+name: "Test: Basic Client"
+
+on:
+  workflow_dispatch:
+  push:
+
+jobs:
+  run-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Run tests
+        run: |
+          cd py-kms; timeout 30 python3 pykms_Server.py -F STDOUT -s ./pykms_database.db &
+          sleep 5
+          python3 pykms_Client.py -F STDOUT # fresh client
+          python3 pykms_Client.py -F STDOUT -c 174f5409-0624-4ce3-b209-adde1091956b # (maybe) existing client
+          python3 pykms_Client.py -F STDOUT -c 174f5409-0624-4ce3-b209-adde1091956b # now-for-sure existing client

.github/workflows/test_image_build.yml

@@ -1,4 +1,4 @@
-name: Test-Build Docker Image
+name: "Test: Build Docker Image"
 
 on:
   workflow_dispatch:

docker/docker-py3-kms-minimal/Dockerfile

@@ -9,7 +9,7 @@ ENV LCID=1033
 ENV CLIENT_COUNT=26
 ENV ACTIVATION_INTERVAL=120
 ENV RENEWAL_INTERVAL=10080
-ENV HWID RANDOM
+ENV HWID=RANDOM
 ENV LOGLEVEL=INFO
 ENV LOGFILE=STDOUT
 ENV LOGSIZE=""

docs/Contributing.md

@@ -13,3 +13,14 @@ Awesome! But before you write or modify the existing source code, please note th
 ```
 - Wrap lines only if really long (it does not matter 79 chars return)
 - For the rest a bit as it comes with a look at [PEP8](https://www.python.org/dev/peps/pep-0008/) :)
+
+Test your changes, please. For example, run the server via:
+```bash
+python3 pykms_Server.py -F STDOUT -s ./pykms_database.db
+```
+Then trigger (multiple) client requests and check the output for errors via:
+```bash
+python3 pykms_Client.py -F STDOUT # fresh client
+python3 pykms_Client.py -F STDOUT -c 174f5409-0624-4ce3-b209-adde1091956b # (maybe) existing client
+python3 pykms_Client.py -F STDOUT -c 174f5409-0624-4ce3-b209-adde1091956b # now-for-sure existing client
+```

py-kms/pykms_Base.py

@@ -193,16 +193,17 @@ could be detected as not genuine !{end}" %currentClientCount)
                 infoDict = {
                         "machineName" : kmsRequest.getMachineName(),
                         "clientMachineId" : str(clientMachineId),
-                        "appId" : appName,
+                        "applicationId" : appName,
                         "skuId" : skuName,
                         "licenseStatus" : kmsRequest.getLicenseStatus(),
-                        "requestTime" : int(time.time()),
+                        "lastRequestIP" : self.srv_config['raddr'][0], # (ip, port)
+                        "lastRequestTime" : int(time.time()),
                         "kmsEpid" : None
                 }
 
                 loggersrv.info("Machine Name: %s" % infoDict["machineName"])
                 loggersrv.info("Client Machine ID: %s" % infoDict["clientMachineId"])
-                loggersrv.info("Application ID: %s" % infoDict["appId"])
+                loggersrv.info("Application ID: %s" % infoDict["applicationId"])
                 loggersrv.info("SKU ID: %s" % infoDict["skuId"])
                 loggersrv.info("License Status: %s" % infoDict["licenseStatus"])
                 loggersrv.info("Request Time: %s" % local_dt.strftime('%Y-%m-%d %H:%M:%S %Z (UTC%z)'))
@@ -211,7 +212,7 @@ could be detected as not genuine !{end}" %currentClientCount)
                         loggersrv.mininfo("", extra = {'host': str(self.srv_config['raddr']),
                                                        'status' : infoDict["licenseStatus"],
                                                        'product' : infoDict["skuId"]})
-                # Create database.
+                # Send change to database.
                 if self.srv_config['sqlite']:
                         sql_update(self.srv_config['sqlite'], infoDict)
 

py-kms/pykms_Client.py

@@ -170,7 +170,7 @@ def client_update():
         for appitem in appitems:
                 kmsitems = appitem['KmsItems']
                 for kmsitem in kmsitems:                                
-                        name = re.sub('\(.*\)', '', kmsitem['DisplayName']) # Remove bracets
+                        name = re.sub(r'\(.*\)', '', kmsitem['DisplayName']) # Remove brackets
                         name = name.replace('2015', '') # Remove specific years
                         name = name.replace(' ', '') # Ignore whitespaces
                         name = name.replace('/11', '', 1) # Cut out Windows 11, as it is basically Windows 10
@@ -328,7 +328,7 @@ def createKmsRequestBase():
         requestDict['clientMachineId'] = UUID(uuid.UUID(clt_config['cmid']).bytes_le if (clt_config['cmid'] is not None) else uuid.uuid4().bytes_le)
         requestDict['previousClientMachineId'] = '\0' * 16 # I'm pretty sure this is supposed to be a null UUID.
         requestDict['requiredClientCount'] = clt_config['RequiredClientCount']
-        requestDict['requestTime'] = dt_to_filetime(datetime.datetime.utcnow())
+        requestDict['requestTime'] = dt_to_filetime(datetime.datetime.now(datetime.timezone.utc))
         requestDict['machineName'] = (clt_config['machine'] if (clt_config['machine'] is not None) else
                                       ''.join(random.choice(string.ascii_letters + string.digits) for i in range(random.randint(2,63)))).encode('utf-16le')
         requestDict['mnPad'] = '\0'.encode('utf-16le') * (63 - len(requestDict['machineName'].decode('utf-16le')))

py-kms/pykms_Server.py

@@ -12,9 +12,10 @@ import threading
 import socketserver
 import queue as Queue
 import selectors
+import traceback
 from time import monotonic as time
 
-import pykms_RpcBind, pykms_RpcRequest
+import pykms_RpcBind, pykms_RpcRequest, pykms_Sql
 from pykms_RpcBase import rpcBase
 from pykms_Dcerpc import MSRPCHeader
 from pykms_Misc import check_setup, check_lcid, check_other
@@ -22,7 +23,6 @@ from pykms_Misc import KmsParser, KmsParserException, KmsParserHelp
 from pykms_Misc import kms_parser_get, kms_parser_check_optionals, kms_parser_check_positionals, kms_parser_check_connect
 from pykms_Format import enco, deco, pretty_printer, justify
 from pykms_Connect import MultipleListener
-from pykms_Sql import sql_initialize
 
 srv_version             = "py-kms_2020-10-01"
 __license__             = "The Unlicense"
@@ -124,7 +124,8 @@ class KeyServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
                                put_text = "{reverse}{red}{bold}Server connection timed out. Exiting...{end}")
 
         def handle_error(self, request, client_address):
-                pass
+                pretty_printer(log_obj = loggersrv.error,
+                                   put_text = "{reverse}{red}{bold}Exception happened during processing of request from %s:\n%s{end}" % (str(client_address), traceback.format_exc()))
 
 
 class server_thread(threading.Thread):
@@ -379,12 +380,9 @@ def server_check():
                                 put_text = "{reverse}{yellow}{bold}You specified a folder instead of a database file! This behavior is not officially supported anymore, please change your start parameters soon.{end}")
                         srv_config['sqlite'] = os.path.join(srv_config['sqlite'], 'pykms_database.db')
 
-                try:
-                        import sqlite3
-                        sql_initialize(srv_config['sqlite'])
-                except ImportError:
-                        pretty_printer(log_obj = loggersrv.warning,
-                                put_text = "{reverse}{yellow}{bold}Module 'sqlite3' not installed, database support disabled.{end}")
+                if pykms_Sql.available:
+                        pykms_Sql.sql_initialize(srv_config['sqlite'])
+                else:
                         srv_config['sqlite'] = False
 
         # Check other specific server options.

py-kms/pykms_Sql.py

@@ -1,121 +1,131 @@
 #!/usr/bin/env python3
 
-import datetime
+from datetime import datetime
 import os
 import logging
 
-# sqlite3 is optional.
-try:
-        import sqlite3
-except ImportError:
-        pass
-
-from pykms_Format import pretty_printer
-
 #--------------------------------------------------------------------------------------------------------------------------------------------------------
 
 loggersrv = logging.getLogger('logsrv')
+_column_names = ('clientMachineId', 'machineName', 'applicationId', 'skuId', 'licenseStatus', 'lastRequestTime', 'kmsEpid', 'requestCount', 'lastRequestIP')
+
+# sqlite3 is optional.
+available = False
+try:
+        import sqlite3
+        available = True
+except ImportError:
+        pass
 
 def sql_initialize(dbName):
+        if available is False:
+                loggersrv.info("'sqlite3' module not found! SQLite database support cannot be enabled.")
+                return
+        loggersrv.debug(f'SQLite database support enabled. Database file: "{dbName}"')
         if not os.path.isfile(dbName):
-                # Initialize the database.
+                # Initialize the database
                 loggersrv.debug(f'Initializing database file "{dbName}"...')
-                con = None
-                try:
-                        con = sqlite3.connect(dbName)
+                with sqlite3.connect(dbName) as con:
                         cur = con.cursor()
-                        cur.execute("CREATE TABLE clients(clientMachineId TEXT , machineName TEXT, applicationId TEXT, skuId TEXT, licenseStatus TEXT, lastRequestTime INTEGER, kmsEpid TEXT, requestCount INTEGER, PRIMARY KEY(clientMachineId, applicationId))")
+                        cur.execute("CREATE TABLE clients(clientMachineId TEXT, machineName TEXT, applicationId TEXT, skuId TEXT, licenseStatus TEXT, lastRequestTime INTEGER, kmsEpid TEXT, requestCount INTEGER, PRIMARY KEY(clientMachineId, applicationId))")
+
+        if os.path.isfile(dbName):
+                # Update database
+                with sqlite3.connect(dbName) as con:
+                        cur = con.cursor()
+                        # Create simple "metadata" table if not exists.
+                        cur.execute("CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT);")
+                        # Get the current schema version
+                        cur.execute("SELECT value FROM metadata WHERE key='schema_version';")
+                        row = cur.fetchone()
+                        if row is None:
+                                current_version = 0
+                        else:
+                                current_version = int(row[0])
+                        loggersrv.debug(f'Current database schema version: {current_version}')
+                        # Apply necessary migrations
+                        if current_version < 1:
+                                # v1: Add "lastRequestIP" column to "clients" table.
+                                loggersrv.info("Upgrading database schema to version 1...")
+                                cur.execute("ALTER TABLE clients ADD COLUMN lastRequestIP TEXT;")
+                                cur.execute("INSERT OR REPLACE INTO metadata (key, value) VALUES ('schema_version', '1');")
+                                loggersrv.info("Database schema updated to version 1.")
 
-                except sqlite3.Error as e:
-                        pretty_printer(log_obj = loggersrv.error, to_exit = True, put_text = "{reverse}{red}{bold}Sqlite Error: %s. Exiting...{end}" %str(e))
-                finally:
-                        if con:
-                                con.commit()
-                                con.close()
 
 def sql_get_all(dbName):
+        if available is False:
+                return
         if not os.path.isfile(dbName):
                 return None
         with sqlite3.connect(dbName) as con:
+                con.row_factory = sqlite3.Row
                 cur = con.cursor()
-                cur.execute("SELECT * FROM clients")
+                cur.execute(f"SELECT {', '.join(_column_names)} FROM clients")
                 clients = []
                 for row in cur.fetchall():
-                        clients.append({
-                                'clientMachineId': row[0],
-                                'machineName': row[1],
-                                'applicationId': row[2],
-                                'skuId': row[3],
-                                'licenseStatus': row[4],
-                                'lastRequestTime': datetime.datetime.fromtimestamp(row[5]).isoformat(),
-                                'kmsEpid': row[6],
-                                'requestCount': row[7]
-                        })
+                        loggersrv.debug(f"Row: {row}")
+                        obj = {}
+                        for col_name in _column_names:
+                                if col_name == "lastRequestTime":
+                                        obj[col_name] = datetime.fromtimestamp(row['lastRequestTime']).isoformat()
+                                else:
+                                        obj[col_name] = row[col_name]
+                        loggersrv.debug(f"Obj: {obj}")
+                        clients.append(obj)
                 return clients
 
 def sql_update(dbName, infoDict):
-        con = None
-        try:
-                con = sqlite3.connect(dbName)
-                cur = con.cursor()
-                cur.execute("SELECT * FROM clients WHERE clientMachineId=:clientMachineId AND applicationId=:appId;", infoDict)
-                try:
-                        data = cur.fetchone()
-                        if not data:
-                                # Insert row.
-                                cur.execute("INSERT INTO clients (clientMachineId, machineName, applicationId, \
-skuId, licenseStatus, lastRequestTime, requestCount) VALUES (:clientMachineId, :machineName, :appId, :skuId, :licenseStatus, :requestTime, 1);", infoDict)
-                        else:
-                                # Update data.
-                                if data[1] != infoDict["machineName"]:
-                                        cur.execute("UPDATE clients SET machineName=:machineName WHERE \
-clientMachineId=:clientMachineId AND applicationId=:appId;", infoDict)
-                                if data[2] != infoDict["appId"]:
-                                        cur.execute("UPDATE clients SET applicationId=:appId WHERE \
-clientMachineId=:clientMachineId AND applicationId=:appId;", infoDict)
-                                if data[3] != infoDict["skuId"]:
-                                        cur.execute("UPDATE clients SET skuId=:skuId WHERE \
-clientMachineId=:clientMachineId AND applicationId=:appId;", infoDict)
-                                if data[4] != infoDict["licenseStatus"]:
-                                        cur.execute("UPDATE clients SET licenseStatus=:licenseStatus WHERE \
-clientMachineId=:clientMachineId AND applicationId=:appId;", infoDict)
-                                if data[5] != infoDict["requestTime"]:
-                                        cur.execute("UPDATE clients SET lastRequestTime=:requestTime WHERE \
-clientMachineId=:clientMachineId AND applicationId=:appId;", infoDict)
-                                # Increment requestCount
-                                cur.execute("UPDATE clients SET requestCount=requestCount+1 WHERE \
-clientMachineId=:clientMachineId AND applicationId=:appId;", infoDict)
+        if available is False:
+                return
 
-                except sqlite3.Error as e:
-                        pretty_printer(log_obj = loggersrv.error, to_exit = True,
-                                       put_text = "{reverse}{red}{bold}Sqlite Error: %s. Exiting...{end}" %str(e))
-        except sqlite3.Error as e:
-                pretty_printer(log_obj = loggersrv.error, to_exit = True,
-                               put_text = "{reverse}{red}{bold}Sqlite Error: %s. Exiting...{end}" %str(e))
-        finally:
-                if con:
-                    con.commit()
-                    con.close()
+        # make sure all column names are present
+        for col_name in _column_names:
+                if col_name in ["requestCount", "kmsEpid"]:
+                        continue
+                if col_name not in infoDict:
+                        raise ValueError(f"infoDict is missing required column: {col_name}")
+
+        with sqlite3.connect(dbName) as con:
+                con.row_factory = sqlite3.Row
+                cur = con.cursor()
+                cur.execute(f"SELECT {', '.join(_column_names)} FROM clients WHERE clientMachineId=:clientMachineId AND applicationId=:applicationId;", infoDict)
+                data = cur.fetchone()
+                if not data:
+                        # Insert new row with all given info
+                        infoDict["kmsEpid"] = ""  # Default empty value
+                        infoDict["requestCount"] = 1
+                        cur.execute(f"""INSERT INTO clients ({', '.join(_column_names)})
+                                VALUES ({', '.join(':' + col for col in _column_names)});""", infoDict)
+
+                else:
+                        # Update only changed columns
+                        common_postfix = "WHERE clientMachineId=:clientMachineId AND applicationId=:applicationId"
+                        def update_column_if_changed(column_name, new_value):
+                                assert "clientMachineId" in infoDict and "applicationId" in infoDict, "infoDict must contain 'clientMachineId' and 'applicationId'"
+                                if column_name not in _column_names:
+                                        raise ValueError(f"Unknown column name: {column_name}")
+                                if data[column_name] != new_value:
+                                        query = f"UPDATE clients SET {column_name}=:value {common_postfix}"
+                                        cur.execute(query, {"value": new_value, "clientMachineId": infoDict['clientMachineId'], "applicationId": infoDict['applicationId']})
+
+                        # Dynamically check and maybe update all columns
+                        for column_name in _column_names:
+                                if column_name in ["clientMachineId", "applicationId", "requestCount"]:
+                                        continue  # Skip these columns
+                                if column_name == "kmsEpid":
+                                        # this one can only be updated by the special function
+                                        continue
+                                update_column_if_changed(column_name, infoDict[column_name])
+
+                        # Finally increment requestCount
+                        cur.execute(f"UPDATE clients SET requestCount=requestCount+1 {common_postfix}", infoDict)
 
 def sql_update_epid(dbName, kmsRequest, response, appName):
-        cmid = str(kmsRequest['clientMachineId'].get())
-        con = None
-        try:
-                con = sqlite3.connect(dbName)
-                cur = con.cursor()
-                cur.execute("SELECT * FROM clients WHERE clientMachineId=? AND applicationId=?;", (cmid, appName))
-                try:
-                        data = cur.fetchone()
-                        cur.execute("UPDATE clients SET kmsEpid=? WHERE \
-clientMachineId=? AND applicationId=?;", (str(response["kmsEpid"].decode('utf-16le')), cmid, appName))
+        if available is False:
+                return
 
-                except sqlite3.Error as e:
-                        pretty_printer(log_obj = loggersrv.error, to_exit = True,
-                                       put_text = "{reverse}{red}{bold}Sqlite Error: %s. Exiting...{end}" %str(e))
-        except sqlite3.Error as e:
-                pretty_printer(log_obj = loggersrv.error, to_exit = True,
-                               put_text = "{reverse}{red}{bold}Sqlite Error: %s. Exiting...{end}" %str(e))
-        finally:
-                if con:
-                        con.commit()
-                        con.close()
+        cmid = str(kmsRequest['clientMachineId'].get())
+        with sqlite3.connect(dbName) as con:
+                cur = con.cursor()
+                cur.execute("UPDATE clients SET kmsEpid=? WHERE clientMachineId=? AND applicationId=?;",
+                        (str(response["kmsEpid"].decode('utf-16le')), cmid, appName))

py-kms/templates/clients.html

@@ -57,6 +57,7 @@ th {
             <th>Application ID</th>
             <th><abbr title="Stock Keeping Unit">SKU</abbr> ID</th>
             <th>License Status</th>
+            <th>Last Address</th>
             <th>Last Seen</th>
             <th>KMS <abbr title="Enhanced Privacy ID">EPID</abbr></th>
             <th>Seen Count</th>
@@ -65,7 +66,9 @@ th {
     <tbody>
         {% for client in clients %}
         <tr>
-            <th><pre class="clientMachineId">{{ client.clientMachineId }}</pre></th>
+            <th>
+                <pre class="clientMachineId">{{ client.clientMachineId }}</pre>
+            </th>
             <td class="machineName">
                 {% if client.machineName | length > 16 %}
                 <abbr title="{{ client.machineName }}">{{ client.machineName | truncate(16, True, '...') }}</abbr>
@@ -76,6 +79,7 @@ th {
             <td>{{ client.applicationId }}</td>
             <td>{{ client.skuId }}</td>
             <td>{{ client.licenseStatus }}</td>
+            <td>{{ client.lastRequestIP or "N/A" }}</td>
             <td class="convert_timestamp">{{ client.lastRequestTime }}</td>
             <td>
                 {% if client.kmsEpid | length > 16 %}
@@ -95,9 +99,10 @@ th {
         <p>Whoops?</p>
     </div>
     <div class="message-body">
-        This page seems to be empty, because no clients are available. Try to use the server with a compartible client to add it to the database.
+        This page seems to be empty, because no clients are available. Try to use the server with a compatible client
+        to add it to the database.
     </div>
 </article>
 {% endif %}
 {% endif %}
-{% endblock %}
+{% endblock %}