Split initialization of SystemRescue into parts done before and in parallel to networking (#304)

Configuring the nofirewall option by the sysrescue-initialize script must be done before the ip6?tables service.
Configuring the firewall should be finished before beginning to initialize the network.

But the rest of sysrescue-initialize should still be done in parallel to networking
being set up for a fast boot sequence. Also some services in sysrescue-initialize require
networking being online, for example load_srm with a remote url.

The proper way to accommodate all these needs is to split sysrescue:
sysrescue-initialize-prenet: must be finished before the network-pre.target
sysrescue-initialize-whilenet: started after sysrescue-initialize-prenet, can run in parallel
                               to networking being set up.

ChangeLog

@@ -8,6 +8,7 @@ https://gitlab.com/systemrescue/systemrescue-sources/-/issues/278
 ------------------------------------------------------------------------------
 9.05 (YYYY-MM-DD):
 ------------------------------------------------------------------------------
+* Split initialization of SystemRescue into parts done before and in parallel to networking (#304)
 * Add a new style for configuring autorun scripts ("autorun.exec") (#287)
 * Change the default for ar_nowait to true: don't wait at the end of autorun by default anymore
 * Deprecate storing autorun scripts in the root of the boot disk (#252)

airootfs/etc/systemd/scripts/sysrescue-initialize-prenet

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+#
+# initialize SystemRescue, do the parts that must be finished before networking is started
+# Keep as short as possible, move non-critical parts to -whilenet to not block the boot unnecessarily
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import subprocess
+import json
+import glob
+import os
+import sys
+import re
+import tempfile
+import functools
+import configparser
+
+# flush stdout buffer after each print call: immediately show the user what is going on
+print = functools.partial(print, flush=True)
+
+def strtobool (val):
+    """Convert a string representation of truth to true (1) or false (0).
+
+    True values are 'y', 'yes', 't', 'true', 'on', '1', '1.0'; false values
+    are 'n', 'no', 'f', 'false', 'off', '0', '0.0'.  Raises ValueError if
+    'val' is anything else.
+    
+    Function adapted from Pythons distutils.util.py because it will be deprecated soon
+    Copyright (c) Python Software Foundation; All Rights Reserved
+    """
+    val = str(val).lower()
+    if val in ('y', 'yes', 't', 'true', 'on', '1', '1.0'):
+        return True
+    elif val in ('n', 'no', 'f', 'false', 'off', '0', '0.0'):
+        return False
+    else:
+        raise ValueError("invalid truth value %r" % (val,))
+
+# ==============================================================================
+# Initialization
+# ==============================================================================
+print(f"====> Script {sys.argv[0]} starting ...")
+errcnt = 0
+
+# ==============================================================================
+# Read the effective configuration file
+# ==============================================================================
+print(f"====> Read the effective configuration file ...")
+effectivecfg = "/run/archiso/config/sysrescue-effective-config.json"
+if os.path.exists(effectivecfg) == False:
+    print (f"Failed to find effective configuration file in {effectivecfg}")
+    sys.exit(1)
+
+with open(effectivecfg) as file:
+    config = json.load(file)
+
+# ==============================================================================
+# Sanitize config, initialize variables
+# Make sysrescue-initialize work safely without them being defined or have a wrong type
+# Also show the effective configuration
+# ==============================================================================
+print(f"====> Showing the effective global configuration (except clear passwords) ...")
+
+def read_cfg_value(scope, name, defaultval, printval):
+    if not scope in config:
+        val = defaultval
+    elif name in config[scope]:
+        chkval = config[scope][name]
+        try:
+            if isinstance(chkval, list) or isinstance(chkval, dict):
+                raise TypeError(f"must be a {type(defaultval)}, not a {type(chkval)}")
+            elif isinstance(defaultval, bool) and not isinstance(chkval, bool):
+                val = strtobool(chkval)
+            else:
+                val = type(defaultval)(chkval)
+        except (TypeError, ValueError) as e:
+            if printval:
+                print(f"config['{scope}']['{name}'] with {chkval} is not the same type as defaultval: {e}")
+            else:
+                print(f"config['{scope}']['{name}'] is not the same type as defaultval: {e}")
+            val = defaultval
+    else:
+        val = defaultval
+
+    if printval:
+        print(f"config['{scope}']['{name}']={val}")
+    
+    return val
+
+nofirewall = read_cfg_value('global','nofirewall', False, True)
+
+# ==============================================================================
+# Apply the effective configuration
+# ==============================================================================
+print(f"====> Applying pre-network configuration ...")
+
+# Disable the firewall
+if nofirewall == True:
+    # The firewall service(s) must be in the Before-section of sysrescue-initialize-prenet.service
+    p = subprocess.run(["systemctl", "disable", "--now", "iptables.service", "ip6tables.service"], text=True)
+    if p.returncode == 0:
+        print (f"Have disabled the firewall successfully")
+    else:
+        print (f"Failed to disable the firewall")
+        errcnt+=1
+
+# ==============================================================================
+# customize sysctl
+# Should be pre-network to allow tweaking network-specific sysctls
+# ==============================================================================
+
+if 'sysconfig' in config and 'sysctl' in config['sysconfig'] and \
+  config['sysconfig']['sysctl'] and isinstance(config['sysconfig']['sysctl'], dict):
+    print(f"====> Customizing sysctl options ...")
+    sysctllines = ""
+    for key, value in config['sysconfig']['sysctl'].items():
+        sysctllines+=f"{key} = {value}\n"
+
+    # pipe config into sysctl
+    p = subprocess.run(["sysctl", "--load=-"], text=True, input=sysctllines)
+    if p.returncode != 0:
+        print (f"Some or all sysctl options couldn't be set")
+        errcnt+=1
+
+# ==============================================================================
+# End of the script
+# ==============================================================================
+print(f"====> Script {sys.argv[0]} completed with {errcnt} errors ...")
+sys.exit(errcnt)

airootfs/etc/systemd/scripts/sysrescue-initialize-whilenet

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
-
+#
+# initialize SystemRescue, do the parts that can be done in parallel to networking being set up
+#
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import subprocess
@@ -109,7 +111,6 @@ setkmap = read_cfg_value('global','setkmap', "", True)
 rootshell = read_cfg_value('global','rootshell', "", True)
 rootpass = read_cfg_value('global','rootpass', "", False)
 rootcryptpass = read_cfg_value('global','rootcryptpass', "", False)
-nofirewall = read_cfg_value('global','nofirewall', False, True)
 noautologin = read_cfg_value('global','noautologin', False, True)
 dostartx = read_cfg_value('global','dostartx', False, True)
 dovnc = read_cfg_value('global','dovnc', False, True)
@@ -160,16 +161,6 @@ if rootcryptpass != "":
         print (f"Failed to change the root password")
         errcnt+=1
 
-# Disable the firewall
-if nofirewall == True:
-    # The firewall service(s) must be in the Before-section of sysrescue-initialize.service
-    p = subprocess.run(["systemctl", "disable", "--now", "iptables.service", "ip6tables.service"], text=True)
-    if p.returncode == 0:
-        print (f"Have disabled the firewall successfully")
-    else:
-        print (f"Failed to disable the firewall")
-        errcnt+=1
-
 # Auto-start the graphical environment (tty1 only)
 if dostartx == True:
     str = '[[ ! $DISPLAY ]] && [[ ! $SSH_TTY ]] && [[ $XDG_VTNR == 1 ]] && startx'
@@ -321,25 +312,9 @@ if 'sysconfig' in config and 'ca-trust' in config['sysconfig'] and config['sysco
     with open(firefox_policy_path, "w", encoding='utf-8') as polfile:
         json.dump(ff_policy, polfile, ensure_ascii=False, indent=2)
 
-# ==============================================================================
-# customize sysctl
-# ==============================================================================
-
-if 'sysconfig' in config and 'sysctl' in config['sysconfig'] and \
-  config['sysconfig']['sysctl'] and isinstance(config['sysconfig']['sysctl'], dict):
-    print(f"====> Customizing sysctl options ...")
-    sysctllines = ""
-    for key, value in config['sysconfig']['sysctl'].items():
-        sysctllines+=f"{key} = {value}\n"
-
-    # pipe config into sysctl
-    p = subprocess.run(["sysctl", "--load=-"], text=True, input=sysctllines)
-    if p.returncode != 0:
-        print (f"Some or all sysctl options couldn't be set")
-        errcnt+=1
-
 # ==============================================================================
 # late-load a SystemRescueModule (SRM)
+# load-srm contains code that waits for the networking being up if necessary
 # ==============================================================================
 
 if late_load_srm != "":

airootfs/etc/systemd/system/sysrescue-autorun.service

@@ -1,6 +1,6 @@
 [Unit]
 Description=SystemRescue Autorun
-After=network.target network-online.target sysrescue-initialize.service
+After=network.target network-online.target sysrescue-initialize-whilenet.service
 Before=getty-pre.target
 Wants=getty-pre.target network-online.target
 

airootfs/etc/systemd/system/sysrescue-initialize-prenet.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=SystemRescue Initialization, before networking
+Before=network-pre.target iptables.service ip6tables.service sysrescue-initialize-whilenet.service
+Wants=network-pre.target sysrescue-initialize-whilenet.service
+
+[Service]
+Type=oneshot
+ExecStart=/etc/systemd/scripts/sysrescue-initialize-prenet
+RemainAfterExit=true
+StandardOutput=journal+console
+
+[Install]
+WantedBy=multi-user.target

airootfs/etc/systemd/system/sysrescue-initialize-whilenet.service

@@ -1,11 +1,12 @@
 [Unit]
-Description=SystemRescue Initialization
+Description=SystemRescue Initialization, parallel to networking
 Before=getty-pre.target
 Wants=getty-pre.target
+After=sysrescue-initialize-prenet.service
 
 [Service]
 Type=oneshot
-ExecStart=/etc/systemd/scripts/sysrescue-initialize.py
+ExecStart=/etc/systemd/scripts/sysrescue-initialize-whilenet
 RemainAfterExit=true
 StandardOutput=journal+console
 

airootfs/root/customize_airootfs.sh

@@ -39,7 +39,8 @@ systemctl enable iptables.service
 systemctl enable ip6tables.service
 systemctl enable choose-mirror.service
 systemctl enable sshd.service
-systemctl enable sysrescue-initialize.service
+systemctl enable sysrescue-initialize-prenet.service
+systemctl enable sysrescue-initialize-whilenet.service
 systemctl enable sysrescue-autorun.service
 systemctl enable qemu-guest-agent.service
 systemctl enable var-lib-pacman\\x2drolling-local.mount