RSX: workaround for color/depth aliasing heurestic edge case failures (#18644)

The color/depth alias collapse heuristic at get_framebuffer_layout()
picks depth whenever depth_test_enabled or stencil_test_enabled is set.
But test means read, not write - so deferred renderers that run a
Z-prepass and then a G-buffer pass with depth-test ON, depth-write OFF,
color-write ON get classified as a depth pass even though they're
writing color. The color writes get silently dropped.

In Starhawk this killed every lit surface - characters, skybox, anything
that goes through the deferred lighting path rendered black or
invisible. Terrain, particles, and emissive geometry kept working
because they don't go through that same pipeline.

Fix is to check writes instead of tests:

    if (zeta_write_enabled && !color_write_enabled)
        keep_as_depth();  // Z-prepass, shadow gen
    else
        keep_as_color();  // G-buffer / lit pass

For the both-writes case I went with color since losing color is much
more obvious visually than losing depth (the engine's Z-prepass usually
still has the depth around).

Tested with Starhawk [BCUS98181] in menu and gameplay - before: missing
characters and sky. After: matches PS3 reference. Logged every aliasing
event during a couple minutes of gameplay and they were all the G-buffer
pattern (depth-test ON, depth-write OFF, color-write ON), all handled
right. The depth-keep branch wasn't actually exercised in Starhawk, but
it's there for games that do use a Z-prepass to an aliased buffer.

Fixes #11877.

rpcs3/Emu/RSX/RSXThread.cpp

@@ -1586,19 +1586,29 @@ namespace rsx
 
 				m_graphics_state.set(rsx::rtt_config_contested);
 
-				// TODO: Research clearing both depth AND color
-				// TODO: If context is creation_draw, deal with possibility of a lost buffer clear
-				if (depth_test_enabled || stencil_test_enabled || (!layout.color_write_enabled[index] && layout.zeta_write_enabled))
-				{
-					// Use address for depth data
-					layout.color_addresses[index] = 0;
-					continue;
-				}
-				else
+				if (g_cfg.video.fb_aliasing_bias == framebuffer_aliasing_bias::prefer_color
+					&& layout.color_write_enabled[index]
+					&& !layout.zeta_write_enabled)
 				{
 					// Use address for color data
 					layout.zeta_address = 0;
 				}
+				else
+				{
+					// TODO: Research clearing both depth AND color
+					// TODO: If context is creation_draw, deal with possibility of a lost buffer clear
+					if (depth_test_enabled || stencil_test_enabled || (!layout.color_write_enabled[index] && layout.zeta_write_enabled))
+					{
+						// Use address for depth data
+						layout.color_addresses[index] = 0;
+						continue;
+					}
+					else
+					{
+						// Use address for color data
+						layout.zeta_address = 0;
+					}
+				}
 			}
 
 			ensure(layout.color_addresses[index]);

rpcs3/Emu/system_config.h

@@ -142,6 +142,7 @@ struct cfg_root : cfg::node
 		cfg::_bool stretch_to_display_area{ this, "Stretch To Display Area", false, true };
 		cfg::_bool force_high_precision_z_buffer{ this, "Force High Precision Z buffer" };
 		cfg::_bool strict_rendering_mode{ this, "Strict Rendering Mode" };
+		cfg::_enum<framebuffer_aliasing_bias> fb_aliasing_bias{ this, "Framebuffer Aliasing Heuristic Bias", framebuffer_aliasing_bias::_auto, true };
 		cfg::_bool disable_zcull_queries{ this, "Disable ZCull Occlusion Queries", false, true };
 		cfg::_bool disable_video_output{ this, "Disable Video Output", false, true };
 		cfg::_bool disable_vertex_cache{ this, "Disable Vertex Cache", false };

rpcs3/Emu/system_config_types.cpp

@@ -87,6 +87,22 @@ void fmt_class_string<msaa_level>::format(std::string& out, u64 arg)
 	});
 }
 
+template <>
+void fmt_class_string<framebuffer_aliasing_bias>::format(std::string& out, u64 arg)
+{
+	format_enum(out, arg, [](framebuffer_aliasing_bias value)
+	{
+		switch (value)
+		{
+		case framebuffer_aliasing_bias::_auto: return "Auto";
+		case framebuffer_aliasing_bias::prefer_color: return "Prefer Color";
+		case framebuffer_aliasing_bias::prefer_depth: return "Prefer Depth";
+		}
+
+		return unknown;
+	});
+}
+
 template <>
 void fmt_class_string<keyboard_handler>::format(std::string& out, u64 arg)
 {

rpcs3/Emu/system_config_types.h

@@ -226,6 +226,13 @@ enum class msaa_level
 	_auto
 };
 
+enum class framebuffer_aliasing_bias
+{
+	_auto,
+	prefer_color,
+	prefer_depth,
+};
+
 enum class detail_level
 {
 	none,

rpcs3/rpcs3qt/emu_settings.cpp

@@ -1113,6 +1113,14 @@ QString emu_settings::GetLocalizedSetting(const QString& original, emu_settings_
 		case msaa_level::_auto: return tr("Auto", "MSAA");
 		}
 		break;
+	case emu_settings_type::FramebufferAliasingBias:
+		switch (static_cast<framebuffer_aliasing_bias>(index))
+		{
+		case framebuffer_aliasing_bias::_auto: return tr("Auto", "Framebuffer Aliasing Heuristic Bias");
+		case framebuffer_aliasing_bias::prefer_color: return tr("Prefer Color", "Framebuffer Aliasing Heuristic Bias");
+		case framebuffer_aliasing_bias::prefer_depth: return tr("Prefer Depth", "Framebuffer Aliasing Heuristic Bias");
+		}
+		break;
 	case emu_settings_type::ShaderPrecisionQuality:
 		switch (static_cast<gpu_preset_level>(index))
 		{

rpcs3/rpcs3qt/emu_settings_type.cpp

@@ -79,6 +79,7 @@ const std::map<emu_settings_type, cfg_location> settings_location =
 	{ emu_settings_type::StretchToDisplayArea,       get_cfg_location(local_cfg.video.stretch_to_display_area) },
 	{ emu_settings_type::ForceHighpZ,                get_cfg_location(local_cfg.video.force_high_precision_z_buffer) },
 	{ emu_settings_type::StrictRenderingMode,        get_cfg_location(local_cfg.video.strict_rendering_mode) },
+	{ emu_settings_type::FramebufferAliasingBias,    get_cfg_location(local_cfg.video.fb_aliasing_bias) },
 	{ emu_settings_type::DisableVertexCache,         get_cfg_location(local_cfg.video.disable_vertex_cache) },
 	{ emu_settings_type::DisableOcclusionQueries,    get_cfg_location(local_cfg.video.disable_zcull_queries) },
 	{ emu_settings_type::DisableVideoOutput,         get_cfg_location(local_cfg.video.disable_video_output) },

rpcs3/rpcs3qt/emu_settings_type.h

@@ -74,6 +74,7 @@ enum class emu_settings_type
 	VulkanAdapter,
 	ForceHighpZ,
 	StrictRenderingMode,
+	FramebufferAliasingBias,
 	DisableVertexCache,
 	DisableOcclusionQueries,
 	DisableVideoOutput,

rpcs3/rpcs3qt/settings_dialog.cpp

@@ -2417,6 +2417,9 @@ settings_dialog::settings_dialog(std::shared_ptr<gui_settings> gui_settings, std
 	m_emu_settings->EnhanceCheckBox(ui->disableHwOcclusionQueries, emu_settings_type::DisableOcclusionQueries);
 	SubscribeTooltip(ui->disableHwOcclusionQueries, tooltips.settings.disable_occlusion_queries);
 
+	m_emu_settings->EnhanceComboBox(ui->fbAliasingBias, emu_settings_type::FramebufferAliasingBias);
+	SubscribeTooltip(ui->fbAliasingBias, tooltips.settings.fb_aliasing_bias);
+
 	m_emu_settings->EnhanceCheckBox(ui->disableVideoOutput, emu_settings_type::DisableVideoOutput);
 	SubscribeTooltip(ui->disableVideoOutput, tooltips.settings.disable_video_output);
 

rpcs3/rpcs3qt/settings_dialog.ui

@@ -4683,6 +4683,18 @@
                </layout>
               </widget>
              </item>
+             <item>
+              <widget class="QGroupBox" name="gb_fbAliasingBias">
+               <property name="title">
+                <string>Framebuffer Aliasing Heuristic Bias</string>
+               </property>
+               <layout class="QVBoxLayout" name="gb_fbAliasingBias_layout">
+                <item>
+                 <widget class="QComboBox" name="fbAliasingBias"/>
+                </item>
+               </layout>
+              </widget>
+             </item>
              <item>
               <widget class="QGroupBox" name="gb_log_levels">
                <property name="title">

rpcs3/rpcs3qt/tooltips.h

@@ -118,6 +118,7 @@ public:
 		const QString debug_overlay_mouse          = tr("Provides a graphical overlay with mouse input values.\nThis is only shown if the other debug overlays are disabled.\nIf unsure, don't use this option.");
 		const QString log_shader_programs          = tr("Dump game shaders to file. Only useful to developers.\nIf unsure, don't use this option.");
 		const QString disable_occlusion_queries    = tr("Disables running occlusion queries. Minor to moderate performance boost.\nMight introduce issues with broken occlusion e.g missing geometry and extreme pop-in.");
+		const QString fb_aliasing_bias             = tr("Controls how RPCS3 resolves render targets where color and depth alias the same memory. Auto is recommended for most games.\n· Auto is the existing behavior, biased toward depth.\n· Prefer Color keeps the color binding when color-write is enabled and depth-write is not. Fixes missing geometry in some deferred renderers (e.g. Starhawk) at the cost of skipping depth test for that draw.\n· Prefer Depth is the same as Auto for now.");
 		const QString disable_video_output         = tr("Disables all video output and PS3 graphical rendering.\nIts only use case is to evaluate performance on CELL for development.");
 		const QString force_cpu_blit_emulation     = tr("Forces emulation of all blit and image manipulation operations on the CPU.\nRequires 'Write Color Buffers' option to also be enabled in most cases to avoid missing graphics.\nSignificantly degrades performance but is more accurate in some cases.\nThis setting overrides the 'GPU texture scaling' option.");
 		const QString disable_vulkan_mem_allocator = tr("Disables the custom Vulkan memory allocator and reverts to direct calls to VkAllocateMemory/VkFreeMemory.");