diff --git a/ChangeLog b/ChangeLog index d47f2f9..8170cd7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,13 @@ SystemRescue ChangeLog ====================== +------------------------------------------------------------------------------ +9.03 (YYYY-MM-DD): +------------------------------------------------------------------------------- +* Change how the "sysrescuecfg" boot command line option is implemented: + Merge given config files after reading the ones in the sysrescue.d dir +* Change YAML config file loading logic to fully merge multiple files (#254) + ------------------------------------------------------------------------------- 9.02 (2022-04-09): ------------------------------------------------------------------------------- diff --git a/airootfs/etc/systemd/scripts/sysrescue-autorun b/airootfs/etc/systemd/scripts/sysrescue-autorun index ab465ff..09fdf54 100755 --- a/airootfs/etc/systemd/scripts/sysrescue-autorun +++ b/airootfs/etc/systemd/scripts/sysrescue-autorun @@ -124,7 +124,17 @@ def search_autoruns(dirname, suffixes, copyfilefct): found+=1 return found +def read_cfg_value(name, defaultval=None): + if name in config: + val = config[name] + else: + val = defaultval + print(f"config['{name}']={val}") + return val + def main(): + global config + errcnt=0 # in case no autorun executed logging.basicConfig(filename='/var/log/sysrescue-autorun.log', format='%(asctime)s %(message)s', level=logging.DEBUG) writemsg('Initializing autorun ...') @@ -135,39 +145,47 @@ def main(): sys.exit(1) with open(effectivecfg) as file: fullcfg = json.load(file) - config = fullcfg['autorun'] - #print(json.dumps(config, indent=4)) + if 'autorun' in fullcfg: + config = fullcfg['autorun'] + else: + config = { } # ---- parse legacy options passed on the boot command line for curopt in open("/proc/cmdline","r").read().split(): if re.match('^autoruns=', curopt): # "autoruns" is the legacy name for 'ar_suffixes' config['ar_suffixes'] = curopt.replace('autoruns=','') - # ---- show the effective configuration + # ---- load the config into local variables, allow the keys not to exist + # ---- also print the effective configuration logging.info(f"Showing the effective autorun configuration ...") - for key, val in config.items(): - logging.info(f"config['{key}']={val}") + ar_suffixes = read_cfg_value('ar_suffixes', "") + ar_disable = read_cfg_value('ar_disable', False) + ar_attempts = read_cfg_value('ar_attempts', 1) + ar_source = read_cfg_value('ar_source', "") + ar_ignorefail = read_cfg_value('ar_ignorefail', False) + ar_nodel = read_cfg_value('ar_nodel', False) + ar_nowait = read_cfg_value('ar_nowait', False) # ---- determine the effective script files suffixes - if config['ar_suffixes'] in (None, 'no', ''): + if ar_suffixes in (None, 'no', ''): suffixes=[''] else: - suffixes=[''] + str(config['ar_suffixes']).split(',') + suffixes=[''] + str(ar_suffixes).split(',') logging.info(f"suffixes={suffixes}") # ---- exit here is there is nothing to do - if config['ar_disable'] == True: + if ar_disable == True: writemsg(f"Autorun has been disabled using ar_disable, exiting now") sys.exit(0) # ---- parse autorun sources ---- - if re.match('^https?://', config['ar_source']): - while config['ar_attempts'] > 0 and not autorunfiles: + if re.match('^https?://', ar_source): + while ar_attempts > 0 and not autorunfiles: time.sleep(1) - config['ar_attempts'] -= 1 - search_autoruns(config['ar_source'], suffixes, copyfilefct_http) - elif re.match('^/dev/', config['ar_source']): # mount a partition/device - mnt1=('mount',config['ar_source'],autorunmnt) + ar_attempts -= 1 + search_autoruns(ar_source, suffixes, copyfilefct_http) + elif re.match('^/dev/', ar_source): # mount a partition/device + mnt1=('mount',ar_source,autorunmnt) mnt2=('umount',autorunmnt) p = subprocess.Popen(mnt1) p.wait() @@ -176,8 +194,8 @@ def main(): sys.exit(1) search_autoruns(autorunmnt, suffixes, copyfilefct_basic) subprocess.Popen(mnt2) - elif re.match('^nfs://', config['ar_source']): # mount an nfs share - source=config['ar_source'].replace('nfs://','') + elif re.match('^nfs://', ar_source): # mount an nfs share + source=ar_source.replace('nfs://','') mnt1=('mount','-t','nfs','-o','nolock',source,autorunmnt) mnt2=('umount',autorunmnt) p = subprocess.Popen(mnt1) @@ -187,8 +205,8 @@ def main(): sys.exit(1) search_autoruns(autorunmnt, suffixes, copyfilefct_basic) subprocess.Popen(mnt2) - elif re.match('^smb://', config['ar_source']): # mount a samba share - source=config['ar_source'].replace('smb://','') + elif re.match('^smb://', ar_source): # mount a samba share + source=ar_source.replace('smb://','') mnt1=('mount','-t','cifs','//%s'%source,autorunmnt) mnt2=('umount',autorunmnt) p = subprocess.Popen(mnt1) @@ -260,20 +278,20 @@ def main(): writemsg (f'Execution of {filebase} returned {returncode}') if returncode != 0: errcnt += 1 - if config['ar_ignorefail'] == False: + if ar_ignorefail == False: writemsg (f'Now aborting autorun as {filebase} has failed') break; # Stop on the first script failure # ---- delete the copies of the scripts ---- - if config['ar_nodel'] == False: + if ar_nodel == False: for curfile in autorunfiles: writemsg (f'removing {curfile}') os.unlink(curfile) # ---- wait a keypress feature ----- if os.path.isfile('/etc/ar_nowait'): - config['ar_nowait'] = True - if (config['ar_nowait'] == False) and (len(autorunfiles) > 0): + ar_nowait = True + if (ar_nowait != True) and (len(autorunfiles) > 0): writemsg(f'Autorun scripts completed with {errcnt} errors, press to continue') sys.stdin.read(1) diff --git a/airootfs/etc/systemd/scripts/sysrescue-initialize.py b/airootfs/etc/systemd/scripts/sysrescue-initialize.py index aa51139..b6ed0ec 100755 --- a/airootfs/etc/systemd/scripts/sysrescue-initialize.py +++ b/airootfs/etc/systemd/scripts/sysrescue-initialize.py @@ -28,17 +28,35 @@ with open(effectivecfg) as file: config = json.load(file) # ============================================================================== -# Show the effective configuration +# Sanitize config, initialize variables +# Make sysrescue-initialize work safely without them being defined +# Also show the effective configuration # ============================================================================== print(f"====> Showing the effective global configuration (except clear passwords) ...") -print(f"config['global']['setkmap']='{config['global']['setkmap']}'") -print(f"config['global']['rootshell']='{config['global']['rootshell']}'") -print(f"config['global']['rootcryptpass']='{config['global']['rootcryptpass']}'") -print(f"config['global']['nofirewall']={config['global']['nofirewall']}") -print(f"config['global']['dostartx']={config['global']['dostartx']}") -print(f"config['global']['noautologin']={config['global']['noautologin']}") -print(f"config['global']['dovnc']={config['global']['dovnc']}") -print(f"config['global']['late_load_srm']={config['global']['late_load_srm']}") + +def read_cfg_value(scope, name, printval): + if not scope in config: + val = None + elif name in config[scope]: + val = config[scope][name] + else: + val = None + + if printval: + print(f"config['{scope}']['{name}']={val}") + + return val + +setkmap = read_cfg_value('global','setkmap', True) +rootshell = read_cfg_value('global','rootshell', True) +rootpass = read_cfg_value('global','rootpass', False) +rootcryptpass = read_cfg_value('global','rootcryptpass', False) +nofirewall = read_cfg_value('global','nofirewall', True) +noautologin = read_cfg_value('global','noautologin', True) +dostartx = read_cfg_value('global','dostartx', True) +dovnc = read_cfg_value('global','dovnc', True) +vncpass = read_cfg_value('global','vncpass', False) +late_load_srm = read_cfg_value('global','late_load_srm', True) # ============================================================================== # Apply the effective configuration @@ -46,7 +64,6 @@ print(f"config['global']['late_load_srm']={config['global']['late_load_srm']}") print(f"====> Applying configuration ...") # Configure keyboard layout if requested in the configuration -setkmap = config['global']['setkmap'] if (setkmap != None) and (setkmap != ""): p = subprocess.run(["localectl", "set-keymap", setkmap], text=True) if p.returncode == 0: @@ -56,7 +73,6 @@ if (setkmap != None) and (setkmap != ""): errcnt+=1 # Configure root login shell if requested in the configuration -rootshell = config['global']['rootshell'] if (rootshell != None) and (rootshell != ""): p = subprocess.run(["chsh", "--shell", rootshell, "root"], text=True) if p.returncode == 0: @@ -66,7 +82,6 @@ if (rootshell != None) and (rootshell != ""): errcnt+=1 # Set the system root password from a clear password -rootpass = config['global']['rootpass'] if (rootpass != None) and (rootpass != ""): p = subprocess.run(["chpasswd", "--crypt-method", "SHA512"], text=True, input=f"root:{rootpass}") if p.returncode == 0: @@ -78,7 +93,6 @@ if (rootpass != None) and (rootpass != ""): # Set the system root password from an encrypted password # A password can be encrypted using a one-line python3 command such as: # python3 -c 'import crypt; print(crypt.crypt("MyPassWord123", crypt.mksalt(crypt.METHOD_SHA512)))' -rootcryptpass = config['global']['rootcryptpass'] if (rootcryptpass != None) and (rootcryptpass != ""): p = subprocess.run(["chpasswd", "--encrypted"], text=True, input=f"root:{rootcryptpass}") if p.returncode == 0: @@ -88,7 +102,7 @@ if (rootcryptpass != None) and (rootcryptpass != ""): errcnt+=1 # Disable the firewall -if config['global']['nofirewall'] == True: +if nofirewall == True: # The firewall service(s) must be in the Before-section of sysrescue-initialize.service p = subprocess.run(["systemctl", "disable", "--now", "iptables.service", "ip6tables.service"], text=True) if p.returncode == 0: @@ -98,7 +112,7 @@ if config['global']['nofirewall'] == True: errcnt+=1 # Auto-start the graphical environment (tty1 only) -if config['global']['dostartx'] == True: +if dostartx == True: str = '[[ ! $DISPLAY ]] && [[ ! $SSH_TTY ]] && [[ $XDG_VTNR == 1 ]] && startx' if (os.path.exists("/root/.bash_profile") == False) or (open("/root/.bash_profile", 'r').read().find(str) == -1): file1 = open("/root/.bash_profile", "a") @@ -109,7 +123,7 @@ if config['global']['dostartx'] == True: file2.close() # Require authenticated console access -if config['global']['noautologin'] == True: +if noautologin == True: p = subprocess.run(["systemctl", "revert", "getty@.service", "serial-getty@.service"], text=True) if p.returncode == 0: print (f"Have enabled authenticated console access successfully") @@ -118,7 +132,6 @@ if config['global']['noautologin'] == True: errcnt+=1 # Set the VNC password from a clear password -vncpass = config['global']['vncpass'] if (vncpass != None) and (vncpass != ""): os.makedirs("/root/.vnc", exist_ok = True) p = subprocess.run(["x11vnc", "-storepasswd", vncpass, "/root/.vnc/passwd"], text=True) @@ -129,7 +142,7 @@ if (vncpass != None) and (vncpass != ""): errcnt+=1 # Auto-start x11vnc with the graphical environment -if config['global']['dovnc'] == True: +if dovnc == True: print (f"Enabling VNC Server in /root/.xprofile ...") file = open("/root/.xprofile", "w") file.write("""[ -f ~/.vnc/passwd ] && pwopt="-usepw" || pwopt="-nopw"\n""") @@ -141,7 +154,7 @@ if config['global']['dovnc'] == True: # ============================================================================== ca_anchor_path = "/etc/ca-certificates/trust-source/anchors/" -if config['sysconfig']['ca-trust']: +if 'sysconfig' in config and 'ca-trust' in config['sysconfig'] and config['sysconfig']['ca-trust']: print(f"====> Adding trusted CA certificates ...") for name, cert in sorted(config['sysconfig']['ca-trust'].items()): @@ -156,7 +169,6 @@ if config['sysconfig']['ca-trust']: # late-load a SystemRescueModule (SRM) # ============================================================================== -late_load_srm = config['global']['late_load_srm'] if (late_load_srm != None) and (late_load_srm != ""): print(f"====> Late-loading SystemRescueModule (SRM) ...") p = subprocess.run(["/usr/share/sysrescue/bin/load-srm", late_load_srm], text=True) diff --git a/airootfs/usr/bin/sysrescue-configuration.lua b/airootfs/usr/bin/sysrescue-configuration.lua index 22ae7ad..6cbe9c7 100755 --- a/airootfs/usr/bin/sysrescue-configuration.lua +++ b/airootfs/usr/bin/sysrescue-configuration.lua @@ -105,20 +105,51 @@ function search_cmdline_option(optname, multiple) end -- Process a block of yaml configuration and override the current configuration with new values -function process_yaml_config(curconfig) - if (curconfig == nil) or (type(curconfig) ~= "table") then - io.stderr:write(string.format("This is not valid yaml, it will be ignored\n")) +function process_yaml_config(config_content) + if (config_content == nil) then + io.stderr:write(string.format("Error downloading or empty file received\n")) return false end - for scope, entries in pairs(config) do - for key, val in pairs(entries) do - if (curconfig[scope] ~= nil) and (curconfig[scope][key] ~= nil) then - print("- Overriding config['"..scope.."']['"..key.."'] with the value from the yaml file") - config[scope][key] = curconfig[scope][key] + if pcall(function() curconfig = yaml.load(config_content) end) then + if (curconfig == nil) or (type(curconfig) ~= "table") then + io.stderr:write(string.format("This is not valid yaml (=no table), it will be ignored\n")) + return false + end + merge_config_table(config, curconfig, "config") + return true + else + io.stderr:write(string.format("Failed parsing yaml, it will be ignored\n")) + return false + end +end + +-- Recursive merge of a config table +-- config_table: references the current level within the global config +-- new_table: the current level within the new yaml we want to merge right now +-- leveltext: textual representation of the current level used for messages, split by "|" +function merge_config_table(config_table, new_table, leveltext) + for key, value in pairs(new_table) do + -- loop through the current level of the new config + if (config_table[key] == nil) then + -- a key just existing in the new config, not in current config -> copy it + print("- Merging "..leveltext.."|"..key.." into the config") + config_table[key] = value + else + -- key of the new config also exisiting in the current config: check value type + if (type(value) == "nil" or (type(value) == "string" and value == "")) then + -- remove an existing table entry with an empty value + print("- Removing "..leveltext.."|"..key) + config_table[key] = nil + elseif (type(value) == "table" and type(config_table[key]) == "table") then + -- old and new values are tables: recurse + merge_config_table(config_table[key], value, leveltext.."|"..key) + else + -- overwrite the old value + print("- Overriding "..leveltext.."|"..key.." with the value from the yaml file") + config_table[key] = value end end end - return true end -- Download a file over http/https and return the contents of the file or nil if it fails @@ -153,55 +184,24 @@ end errcnt = 0 -- ============================================================================== --- Define the default configuration +-- We start with an empty global config +-- the default config is usually in the first yaml file parsed (100-defaults.yaml) -- ============================================================================== -print ("====> Define the default configuration ...") -config = { - ["global"] = { - ['copytoram'] = false, - ['checksum'] = false, - ['loadsrm'] = false, - ['late_load_srm'] = "", - ['dostartx'] = false, - ['dovnc'] = false, - ['noautologin'] = false, - ['nofirewall'] = false, - ['rootshell'] = "", - ['rootpass'] = "", - ['rootcryptpass'] = "", - ['setkmap'] = "", - ['vncpass'] = "", - }, - ["autorun"] = { - ['ar_disable'] = false, - ['ar_nowait'] = false, - ['ar_nodel'] = false, - ['ar_ignorefail'] = false, - ['ar_attempts'] = 1, - ['ar_source'] = "", - ['ar_suffixes'] = "0,1,2,3,4,5,6,7,8,9,A,B,C,D,E,F", - }, - ["sysconfig"] = { - ["ca-trust"] = {}, - }, -} +config = { } -- ============================================================================== --- Override the configuration with values from yaml files +-- Merge one yaml file after the other in lexicographic order -- ============================================================================== -print ("====> Overriding the default configuration with values from yaml files ...") +print ("====> Merging configuration with values from yaml files ...") confdirs = {"/run/archiso/bootmnt/sysrescue.d", "/run/archiso/copytoram/sysrescue.d"} -conffiles = search_cmdline_option("sysrescuecfg", true) -- Process local yaml configuration files for _, curdir in ipairs(confdirs) do if lfs.attributes(curdir, "mode") == "directory" then print("Searching for yaml configuration files in "..curdir.." ...") - for _, curfile in ipairs(list_config_files(curdir, conffiles)) do + for _, curfile in ipairs(list_config_files(curdir, {})) do print(string.format("Processing local yaml configuration file: %s ...", curfile)) - local curconfig = yaml.loadpath(curfile) - --print("++++++++++++++\n"..yaml.dump(curconfig).."++++++++++++++\n") - if process_yaml_config(curconfig) == false then + if process_yaml_config(read_file_contents(curfile)) == false then errcnt = errcnt + 1 end end @@ -210,35 +210,82 @@ for _, curdir in ipairs(confdirs) do end end --- Process remote yaml configuration files +-- Process explicitly configured configuration files +-- these are parsed afterwards and in the order given, so they have precedence +conffiles = search_cmdline_option("sysrescuecfg", true) print("Searching for remote yaml configuration files ...") for _, curfile in ipairs(conffiles) do if string.match(curfile, "^https?://") then print(string.format("Processing remote yaml configuration file: %s ...", curfile)) local contents = download_file(curfile) - if (contents == nil) or (process_yaml_config(yaml.load(contents)) == false) then + if process_yaml_config(contents) == false then errcnt = errcnt + 1 end + elseif string.match(curfile, "^/") then + -- we have a local file with absolute path + print(string.format("Processing local yaml configuration file: %s ...",curfile)) + if process_yaml_config(read_file_contents(curfile)) == false then + errcnt = errcnt + 1 + end + else + -- we have a local file with relative path, prefix the one existing config dir + -- this will apply the config again, but later than before, giving it higher priority + for _, curdir in ipairs(confdirs) do + if lfs.attributes(curdir, "mode") == "directory" then + print(string.format("Processing local yaml configuration file: %s ...",curdir.."/"..curfile)) + if process_yaml_config(read_file_contents(curdir.."/"..curfile)) == false then + errcnt = errcnt + 1 + end + -- just try the explicitly configured filename with one dir prefix + break + end + end end end -- ============================================================================== -- Override the configuration with values passed on the boot command line +-- +-- NOTE: boot command line options are only for legacy compatibility and +-- very common options. Consider carfully before adding new boot +-- command line options. New features should by default just be +-- configured through the yaml config. -- ============================================================================== + +cmdline_options = { + ['copytoram'] = "global", + ['checksum'] = "global", + ['loadsrm'] = "global", + ['dostartx'] = "global", + ['dovnc'] = "global", + ['noautologin'] = "global", + ['nofirewall'] = "global", + ['rootshell'] = "global", + ['rootpass'] = "global", + ['rootcryptpass'] = "global", + ['setkmap'] = "global", + ['vncpass'] = "global", + ['ar_disable'] = "autorun", + ['ar_nowait'] = "autorun", + ['ar_nodel'] = "autorun", + ['ar_ignorefail'] = "autorun", + ['ar_attempts'] = "autorun", + ['ar_source'] = "autorun", + ['ar_suffixes'] = "autorun" +} + print ("====> Overriding the configuration with options passed on the boot command line ...") -for _, scope in ipairs({"global", "autorun"}) do - for key,val in pairs(config[scope]) do - optresult = search_cmdline_option(key, false) - if optresult == true then - print("- Option '"..key.."' has been enabled on the boot command line") - config[scope][key] = optresult - elseif optresult == false then - print("- Option '"..key.."' has been disabled on the boot command line") - config[scope][key] = optresult - elseif optresult ~= nil then - print("- Option '"..key.."' has been defined as '"..optresult.."' on the boot command line") - config[scope][key] = optresult - end +for option, scope in pairs(cmdline_options) do + optresult = search_cmdline_option(option, false) + if optresult == true then + print("- Option '"..option.."' has been enabled on the boot command line") + config[scope][option] = optresult + elseif optresult == false then + print("- Option '"..option.."' has been disabled on the boot command line") + config[scope][option] = optresult + elseif optresult ~= nil then + print("- Option '"..option.."' has been defined as '"..optresult.."' on the boot command line") + config[scope][option] = optresult end end diff --git a/sysrescue.d/100-defaults.yaml b/sysrescue.d/100-defaults.yaml index 05e4ef2..2934d13 100644 --- a/sysrescue.d/100-defaults.yaml +++ b/sysrescue.d/100-defaults.yaml @@ -4,6 +4,8 @@ global: checksum: false loadsrm: false dostartx: false + dovnc: false + noautologin: false nofirewall: false autorun: