diff --git a/server.py b/server.py
index 80cc2f18..cfb21a6e 100644
--- a/server.py
+++ b/server.py
@@ -236,14 +236,14 @@ if __name__ == "__main__":
         settings_file = Path(shared.args.settings)
     elif Path('user_data/settings.yaml').exists():
         settings_file = Path('user_data/settings.yaml')
-    elif Path('user_data/settings.json').exists():
-        settings_file = Path('user_data/settings.json')
 
     if settings_file is not None:
         logger.info(f"Loading settings from \"{settings_file}\"")
-        file_contents = open(settings_file, 'r', encoding='utf-8').read()
-        new_settings = json.loads(file_contents) if settings_file.suffix == "json" else yaml.safe_load(file_contents)
-        shared.settings.update(new_settings)
+        with open(settings_file, 'r', encoding='utf-8') as f:
+            new_settings = yaml.safe_load(f.read())
+
+        if new_settings:
+            shared.settings.update(new_settings)
 
     # Fallback settings for models
     shared.model_config['.*'] = get_fallback_settings()