Merge branch 'yaml-merge' into 'main'

change yaml config loading logic to full merge

See merge request systemrescue/systemrescue-sources!192
This commit is contained in:
Francois Dupoux 2022-05-09 11:26:35 +00:00
commit ada815f07a
5 changed files with 190 additions and 104 deletions

View file

@ -1,6 +1,13 @@
SystemRescue ChangeLog
======================
------------------------------------------------------------------------------
9.03 (YYYY-MM-DD):
-------------------------------------------------------------------------------
* Change how the "sysrescuecfg" boot command line option is implemented:
Merge given config files after reading the ones in the sysrescue.d dir
* Change YAML config file loading logic to fully merge multiple files (#254)
-------------------------------------------------------------------------------
9.02 (2022-04-09):
-------------------------------------------------------------------------------

View file

@ -124,7 +124,17 @@ def search_autoruns(dirname, suffixes, copyfilefct):
found+=1
return found
def read_cfg_value(name, defaultval=None):
if name in config:
val = config[name]
else:
val = defaultval
print(f"config['{name}']={val}")
return val
def main():
global config
errcnt=0 # in case no autorun executed
logging.basicConfig(filename='/var/log/sysrescue-autorun.log', format='%(asctime)s %(message)s', level=logging.DEBUG)
writemsg('Initializing autorun ...')
@ -135,39 +145,47 @@ def main():
sys.exit(1)
with open(effectivecfg) as file:
fullcfg = json.load(file)
config = fullcfg['autorun']
#print(json.dumps(config, indent=4))
if 'autorun' in fullcfg:
config = fullcfg['autorun']
else:
config = { }
# ---- parse legacy options passed on the boot command line
for curopt in open("/proc/cmdline","r").read().split():
if re.match('^autoruns=', curopt): # "autoruns" is the legacy name for 'ar_suffixes'
config['ar_suffixes'] = curopt.replace('autoruns=','')
# ---- show the effective configuration
# ---- load the config into local variables, allow the keys not to exist
# ---- also print the effective configuration
logging.info(f"Showing the effective autorun configuration ...")
for key, val in config.items():
logging.info(f"config['{key}']={val}")
ar_suffixes = read_cfg_value('ar_suffixes', "")
ar_disable = read_cfg_value('ar_disable', False)
ar_attempts = read_cfg_value('ar_attempts', 1)
ar_source = read_cfg_value('ar_source', "")
ar_ignorefail = read_cfg_value('ar_ignorefail', False)
ar_nodel = read_cfg_value('ar_nodel', False)
ar_nowait = read_cfg_value('ar_nowait', False)
# ---- determine the effective script files suffixes
if config['ar_suffixes'] in (None, 'no', ''):
if ar_suffixes in (None, 'no', ''):
suffixes=['']
else:
suffixes=[''] + str(config['ar_suffixes']).split(',')
suffixes=[''] + str(ar_suffixes).split(',')
logging.info(f"suffixes={suffixes}")
# ---- exit here is there is nothing to do
if config['ar_disable'] == True:
if ar_disable == True:
writemsg(f"Autorun has been disabled using ar_disable, exiting now")
sys.exit(0)
# ---- parse autorun sources ----
if re.match('^https?://', config['ar_source']):
while config['ar_attempts'] > 0 and not autorunfiles:
if re.match('^https?://', ar_source):
while ar_attempts > 0 and not autorunfiles:
time.sleep(1)
config['ar_attempts'] -= 1
search_autoruns(config['ar_source'], suffixes, copyfilefct_http)
elif re.match('^/dev/', config['ar_source']): # mount a partition/device
mnt1=('mount',config['ar_source'],autorunmnt)
ar_attempts -= 1
search_autoruns(ar_source, suffixes, copyfilefct_http)
elif re.match('^/dev/', ar_source): # mount a partition/device
mnt1=('mount',ar_source,autorunmnt)
mnt2=('umount',autorunmnt)
p = subprocess.Popen(mnt1)
p.wait()
@ -176,8 +194,8 @@ def main():
sys.exit(1)
search_autoruns(autorunmnt, suffixes, copyfilefct_basic)
subprocess.Popen(mnt2)
elif re.match('^nfs://', config['ar_source']): # mount an nfs share
source=config['ar_source'].replace('nfs://','')
elif re.match('^nfs://', ar_source): # mount an nfs share
source=ar_source.replace('nfs://','')
mnt1=('mount','-t','nfs','-o','nolock',source,autorunmnt)
mnt2=('umount',autorunmnt)
p = subprocess.Popen(mnt1)
@ -187,8 +205,8 @@ def main():
sys.exit(1)
search_autoruns(autorunmnt, suffixes, copyfilefct_basic)
subprocess.Popen(mnt2)
elif re.match('^smb://', config['ar_source']): # mount a samba share
source=config['ar_source'].replace('smb://','')
elif re.match('^smb://', ar_source): # mount a samba share
source=ar_source.replace('smb://','')
mnt1=('mount','-t','cifs','//%s'%source,autorunmnt)
mnt2=('umount',autorunmnt)
p = subprocess.Popen(mnt1)
@ -260,20 +278,20 @@ def main():
writemsg (f'Execution of {filebase} returned {returncode}')
if returncode != 0:
errcnt += 1
if config['ar_ignorefail'] == False:
if ar_ignorefail == False:
writemsg (f'Now aborting autorun as {filebase} has failed')
break; # Stop on the first script failure
# ---- delete the copies of the scripts ----
if config['ar_nodel'] == False:
if ar_nodel == False:
for curfile in autorunfiles:
writemsg (f'removing {curfile}')
os.unlink(curfile)
# ---- wait a keypress feature -----
if os.path.isfile('/etc/ar_nowait'):
config['ar_nowait'] = True
if (config['ar_nowait'] == False) and (len(autorunfiles) > 0):
ar_nowait = True
if (ar_nowait != True) and (len(autorunfiles) > 0):
writemsg(f'Autorun scripts completed with {errcnt} errors, press <Enter> to continue')
sys.stdin.read(1)

View file

@ -28,17 +28,35 @@ with open(effectivecfg) as file:
config = json.load(file)
# ==============================================================================
# Show the effective configuration
# Sanitize config, initialize variables
# Make sysrescue-initialize work safely without them being defined
# Also show the effective configuration
# ==============================================================================
print(f"====> Showing the effective global configuration (except clear passwords) ...")
print(f"config['global']['setkmap']='{config['global']['setkmap']}'")
print(f"config['global']['rootshell']='{config['global']['rootshell']}'")
print(f"config['global']['rootcryptpass']='{config['global']['rootcryptpass']}'")
print(f"config['global']['nofirewall']={config['global']['nofirewall']}")
print(f"config['global']['dostartx']={config['global']['dostartx']}")
print(f"config['global']['noautologin']={config['global']['noautologin']}")
print(f"config['global']['dovnc']={config['global']['dovnc']}")
print(f"config['global']['late_load_srm']={config['global']['late_load_srm']}")
def read_cfg_value(scope, name, printval):
if not scope in config:
val = None
elif name in config[scope]:
val = config[scope][name]
else:
val = None
if printval:
print(f"config['{scope}']['{name}']={val}")
return val
setkmap = read_cfg_value('global','setkmap', True)
rootshell = read_cfg_value('global','rootshell', True)
rootpass = read_cfg_value('global','rootpass', False)
rootcryptpass = read_cfg_value('global','rootcryptpass', False)
nofirewall = read_cfg_value('global','nofirewall', True)
noautologin = read_cfg_value('global','noautologin', True)
dostartx = read_cfg_value('global','dostartx', True)
dovnc = read_cfg_value('global','dovnc', True)
vncpass = read_cfg_value('global','vncpass', False)
late_load_srm = read_cfg_value('global','late_load_srm', True)
# ==============================================================================
# Apply the effective configuration
@ -46,7 +64,6 @@ print(f"config['global']['late_load_srm']={config['global']['late_load_srm']}")
print(f"====> Applying configuration ...")
# Configure keyboard layout if requested in the configuration
setkmap = config['global']['setkmap']
if (setkmap != None) and (setkmap != ""):
p = subprocess.run(["localectl", "set-keymap", setkmap], text=True)
if p.returncode == 0:
@ -56,7 +73,6 @@ if (setkmap != None) and (setkmap != ""):
errcnt+=1
# Configure root login shell if requested in the configuration
rootshell = config['global']['rootshell']
if (rootshell != None) and (rootshell != ""):
p = subprocess.run(["chsh", "--shell", rootshell, "root"], text=True)
if p.returncode == 0:
@ -66,7 +82,6 @@ if (rootshell != None) and (rootshell != ""):
errcnt+=1
# Set the system root password from a clear password
rootpass = config['global']['rootpass']
if (rootpass != None) and (rootpass != ""):
p = subprocess.run(["chpasswd", "--crypt-method", "SHA512"], text=True, input=f"root:{rootpass}")
if p.returncode == 0:
@ -78,7 +93,6 @@ if (rootpass != None) and (rootpass != ""):
# Set the system root password from an encrypted password
# A password can be encrypted using a one-line python3 command such as:
# python3 -c 'import crypt; print(crypt.crypt("MyPassWord123", crypt.mksalt(crypt.METHOD_SHA512)))'
rootcryptpass = config['global']['rootcryptpass']
if (rootcryptpass != None) and (rootcryptpass != ""):
p = subprocess.run(["chpasswd", "--encrypted"], text=True, input=f"root:{rootcryptpass}")
if p.returncode == 0:
@ -88,7 +102,7 @@ if (rootcryptpass != None) and (rootcryptpass != ""):
errcnt+=1
# Disable the firewall
if config['global']['nofirewall'] == True:
if nofirewall == True:
# The firewall service(s) must be in the Before-section of sysrescue-initialize.service
p = subprocess.run(["systemctl", "disable", "--now", "iptables.service", "ip6tables.service"], text=True)
if p.returncode == 0:
@ -98,7 +112,7 @@ if config['global']['nofirewall'] == True:
errcnt+=1
# Auto-start the graphical environment (tty1 only)
if config['global']['dostartx'] == True:
if dostartx == True:
str = '[[ ! $DISPLAY ]] && [[ ! $SSH_TTY ]] && [[ $XDG_VTNR == 1 ]] && startx'
if (os.path.exists("/root/.bash_profile") == False) or (open("/root/.bash_profile", 'r').read().find(str) == -1):
file1 = open("/root/.bash_profile", "a")
@ -109,7 +123,7 @@ if config['global']['dostartx'] == True:
file2.close()
# Require authenticated console access
if config['global']['noautologin'] == True:
if noautologin == True:
p = subprocess.run(["systemctl", "revert", "getty@.service", "serial-getty@.service"], text=True)
if p.returncode == 0:
print (f"Have enabled authenticated console access successfully")
@ -118,7 +132,6 @@ if config['global']['noautologin'] == True:
errcnt+=1
# Set the VNC password from a clear password
vncpass = config['global']['vncpass']
if (vncpass != None) and (vncpass != ""):
os.makedirs("/root/.vnc", exist_ok = True)
p = subprocess.run(["x11vnc", "-storepasswd", vncpass, "/root/.vnc/passwd"], text=True)
@ -129,7 +142,7 @@ if (vncpass != None) and (vncpass != ""):
errcnt+=1
# Auto-start x11vnc with the graphical environment
if config['global']['dovnc'] == True:
if dovnc == True:
print (f"Enabling VNC Server in /root/.xprofile ...")
file = open("/root/.xprofile", "w")
file.write("""[ -f ~/.vnc/passwd ] && pwopt="-usepw" || pwopt="-nopw"\n""")
@ -141,7 +154,7 @@ if config['global']['dovnc'] == True:
# ==============================================================================
ca_anchor_path = "/etc/ca-certificates/trust-source/anchors/"
if config['sysconfig']['ca-trust']:
if 'sysconfig' in config and 'ca-trust' in config['sysconfig'] and config['sysconfig']['ca-trust']:
print(f"====> Adding trusted CA certificates ...")
for name, cert in sorted(config['sysconfig']['ca-trust'].items()):
@ -156,7 +169,6 @@ if config['sysconfig']['ca-trust']:
# late-load a SystemRescueModule (SRM)
# ==============================================================================
late_load_srm = config['global']['late_load_srm']
if (late_load_srm != None) and (late_load_srm != ""):
print(f"====> Late-loading SystemRescueModule (SRM) ...")
p = subprocess.run(["/usr/share/sysrescue/bin/load-srm", late_load_srm], text=True)

View file

@ -105,20 +105,51 @@ function search_cmdline_option(optname, multiple)
end
-- Process a block of yaml configuration and override the current configuration with new values
function process_yaml_config(curconfig)
if (curconfig == nil) or (type(curconfig) ~= "table") then
io.stderr:write(string.format("This is not valid yaml, it will be ignored\n"))
function process_yaml_config(config_content)
if (config_content == nil) then
io.stderr:write(string.format("Error downloading or empty file received\n"))
return false
end
for scope, entries in pairs(config) do
for key, val in pairs(entries) do
if (curconfig[scope] ~= nil) and (curconfig[scope][key] ~= nil) then
print("- Overriding config['"..scope.."']['"..key.."'] with the value from the yaml file")
config[scope][key] = curconfig[scope][key]
if pcall(function() curconfig = yaml.load(config_content) end) then
if (curconfig == nil) or (type(curconfig) ~= "table") then
io.stderr:write(string.format("This is not valid yaml (=no table), it will be ignored\n"))
return false
end
merge_config_table(config, curconfig, "config")
return true
else
io.stderr:write(string.format("Failed parsing yaml, it will be ignored\n"))
return false
end
end
-- Recursive merge of a config table
-- config_table: references the current level within the global config
-- new_table: the current level within the new yaml we want to merge right now
-- leveltext: textual representation of the current level used for messages, split by "|"
function merge_config_table(config_table, new_table, leveltext)
for key, value in pairs(new_table) do
-- loop through the current level of the new config
if (config_table[key] == nil) then
-- a key just existing in the new config, not in current config -> copy it
print("- Merging "..leveltext.."|"..key.." into the config")
config_table[key] = value
else
-- key of the new config also exisiting in the current config: check value type
if (type(value) == "nil" or (type(value) == "string" and value == "")) then
-- remove an existing table entry with an empty value
print("- Removing "..leveltext.."|"..key)
config_table[key] = nil
elseif (type(value) == "table" and type(config_table[key]) == "table") then
-- old and new values are tables: recurse
merge_config_table(config_table[key], value, leveltext.."|"..key)
else
-- overwrite the old value
print("- Overriding "..leveltext.."|"..key.." with the value from the yaml file")
config_table[key] = value
end
end
end
return true
end
-- Download a file over http/https and return the contents of the file or nil if it fails
@ -153,55 +184,24 @@ end
errcnt = 0
-- ==============================================================================
-- Define the default configuration
-- We start with an empty global config
-- the default config is usually in the first yaml file parsed (100-defaults.yaml)
-- ==============================================================================
print ("====> Define the default configuration ...")
config = {
["global"] = {
['copytoram'] = false,
['checksum'] = false,
['loadsrm'] = false,
['late_load_srm'] = "",
['dostartx'] = false,
['dovnc'] = false,
['noautologin'] = false,
['nofirewall'] = false,
['rootshell'] = "",
['rootpass'] = "",
['rootcryptpass'] = "",
['setkmap'] = "",
['vncpass'] = "",
},
["autorun"] = {
['ar_disable'] = false,
['ar_nowait'] = false,
['ar_nodel'] = false,
['ar_ignorefail'] = false,
['ar_attempts'] = 1,
['ar_source'] = "",
['ar_suffixes'] = "0,1,2,3,4,5,6,7,8,9,A,B,C,D,E,F",
},
["sysconfig"] = {
["ca-trust"] = {},
},
}
config = { }
-- ==============================================================================
-- Override the configuration with values from yaml files
-- Merge one yaml file after the other in lexicographic order
-- ==============================================================================
print ("====> Overriding the default configuration with values from yaml files ...")
print ("====> Merging configuration with values from yaml files ...")
confdirs = {"/run/archiso/bootmnt/sysrescue.d", "/run/archiso/copytoram/sysrescue.d"}
conffiles = search_cmdline_option("sysrescuecfg", true)
-- Process local yaml configuration files
for _, curdir in ipairs(confdirs) do
if lfs.attributes(curdir, "mode") == "directory" then
print("Searching for yaml configuration files in "..curdir.." ...")
for _, curfile in ipairs(list_config_files(curdir, conffiles)) do
for _, curfile in ipairs(list_config_files(curdir, {})) do
print(string.format("Processing local yaml configuration file: %s ...", curfile))
local curconfig = yaml.loadpath(curfile)
--print("++++++++++++++\n"..yaml.dump(curconfig).."++++++++++++++\n")
if process_yaml_config(curconfig) == false then
if process_yaml_config(read_file_contents(curfile)) == false then
errcnt = errcnt + 1
end
end
@ -210,35 +210,82 @@ for _, curdir in ipairs(confdirs) do
end
end
-- Process remote yaml configuration files
-- Process explicitly configured configuration files
-- these are parsed afterwards and in the order given, so they have precedence
conffiles = search_cmdline_option("sysrescuecfg", true)
print("Searching for remote yaml configuration files ...")
for _, curfile in ipairs(conffiles) do
if string.match(curfile, "^https?://") then
print(string.format("Processing remote yaml configuration file: %s ...", curfile))
local contents = download_file(curfile)
if (contents == nil) or (process_yaml_config(yaml.load(contents)) == false) then
if process_yaml_config(contents) == false then
errcnt = errcnt + 1
end
elseif string.match(curfile, "^/") then
-- we have a local file with absolute path
print(string.format("Processing local yaml configuration file: %s ...",curfile))
if process_yaml_config(read_file_contents(curfile)) == false then
errcnt = errcnt + 1
end
else
-- we have a local file with relative path, prefix the one existing config dir
-- this will apply the config again, but later than before, giving it higher priority
for _, curdir in ipairs(confdirs) do
if lfs.attributes(curdir, "mode") == "directory" then
print(string.format("Processing local yaml configuration file: %s ...",curdir.."/"..curfile))
if process_yaml_config(read_file_contents(curdir.."/"..curfile)) == false then
errcnt = errcnt + 1
end
-- just try the explicitly configured filename with one dir prefix
break
end
end
end
end
-- ==============================================================================
-- Override the configuration with values passed on the boot command line
--
-- NOTE: boot command line options are only for legacy compatibility and
-- very common options. Consider carfully before adding new boot
-- command line options. New features should by default just be
-- configured through the yaml config.
-- ==============================================================================
cmdline_options = {
['copytoram'] = "global",
['checksum'] = "global",
['loadsrm'] = "global",
['dostartx'] = "global",
['dovnc'] = "global",
['noautologin'] = "global",
['nofirewall'] = "global",
['rootshell'] = "global",
['rootpass'] = "global",
['rootcryptpass'] = "global",
['setkmap'] = "global",
['vncpass'] = "global",
['ar_disable'] = "autorun",
['ar_nowait'] = "autorun",
['ar_nodel'] = "autorun",
['ar_ignorefail'] = "autorun",
['ar_attempts'] = "autorun",
['ar_source'] = "autorun",
['ar_suffixes'] = "autorun"
}
print ("====> Overriding the configuration with options passed on the boot command line ...")
for _, scope in ipairs({"global", "autorun"}) do
for key,val in pairs(config[scope]) do
optresult = search_cmdline_option(key, false)
if optresult == true then
print("- Option '"..key.."' has been enabled on the boot command line")
config[scope][key] = optresult
elseif optresult == false then
print("- Option '"..key.."' has been disabled on the boot command line")
config[scope][key] = optresult
elseif optresult ~= nil then
print("- Option '"..key.."' has been defined as '"..optresult.."' on the boot command line")
config[scope][key] = optresult
end
for option, scope in pairs(cmdline_options) do
optresult = search_cmdline_option(option, false)
if optresult == true then
print("- Option '"..option.."' has been enabled on the boot command line")
config[scope][option] = optresult
elseif optresult == false then
print("- Option '"..option.."' has been disabled on the boot command line")
config[scope][option] = optresult
elseif optresult ~= nil then
print("- Option '"..option.."' has been defined as '"..optresult.."' on the boot command line")
config[scope][option] = optresult
end
end

View file

@ -4,6 +4,8 @@ global:
checksum: false
loadsrm: false
dostartx: false
dovnc: false
noautologin: false
nofirewall: false
autorun: