diff --git a/rpcs3/Emu/RSX/GL/GLDraw.cpp b/rpcs3/Emu/RSX/GL/GLDraw.cpp
index c564adfa20..e7d1338868 100644
--- a/rpcs3/Emu/RSX/GL/GLDraw.cpp
+++ b/rpcs3/Emu/RSX/GL/GLDraw.cpp
@@ -353,7 +353,9 @@ void GLGSRender::load_texture_env()
 
 		if (sampler_state->format_ex.texel_remap_control &&
 			sampler_state->image_handle &&
-			sampler_state->upload_context == rsx::texture_upload_context::shader_read) [[ unlikely ]]
+			sampler_state->upload_context == rsx::texture_upload_context::shader_read &&
+			(current_fp_metadata.bx2_texture_reads_mask & (1u << i)) == 0 &&
+			g_cfg.video.use_hardware_texel_remapping) [[ unlikely ]]
 		{
 			// Check if we need to override the view format
 			const auto gl_format = sampler_state->image_handle->view_format();
diff --git a/rpcs3/Emu/RSX/VK/VKDraw.cpp b/rpcs3/Emu/RSX/VK/VKDraw.cpp
index 6bec02ab7c..8f14d7e8c7 100644
--- a/rpcs3/Emu/RSX/VK/VKDraw.cpp
+++ b/rpcs3/Emu/RSX/VK/VKDraw.cpp
@@ -345,7 +345,9 @@ void VKGSRender::load_texture_env()
 
 		if (sampler_state->format_ex.texel_remap_control &&
 			sampler_state->image_handle &&
-			sampler_state->upload_context == rsx::texture_upload_context::shader_read) [[ unlikely ]]
+			sampler_state->upload_context == rsx::texture_upload_context::shader_read &&
+			(current_fp_metadata.bx2_texture_reads_mask & (1u << i)) == 0 &&
+			g_cfg.video.use_hardware_texel_remapping) [[ unlikely ]]
 		{
 			// Check if we need to override the view format
 			const auto vk_format = sampler_state->image_handle->format();
diff --git a/rpcs3/Emu/RSX/VK/VKTextureCache.cpp b/rpcs3/Emu/RSX/VK/VKTextureCache.cpp
index 2427cd8da3..0c3df5cd9c 100644
--- a/rpcs3/Emu/RSX/VK/VKTextureCache.cpp
+++ b/rpcs3/Emu/RSX/VK/VKTextureCache.cpp
@@ -1153,7 +1153,8 @@ namespace vk
 		const bool upload_async = rsx::get_current_renderer()->get_backend_config().supports_asynchronous_compute;
 		rsx::flags32_t create_flags = 0;
 
-		if (context == rsx::texture_upload_context::shader_read)
+		if (context == rsx::texture_upload_context::shader_read &&
+			g_cfg.video.use_hardware_texel_remapping)
 		{
 			create_flags |= texture_create_flags::mutable_format;
 		}
diff --git a/rpcs3/Emu/system_config.h b/rpcs3/Emu/system_config.h
index aaa889ce1f..c9e763b5af 100644
--- a/rpcs3/Emu/system_config.h
+++ b/rpcs3/Emu/system_config.h
@@ -176,6 +176,7 @@ struct cfg_root : cfg::node
 		cfg::_bool disable_async_host_memory_manager{ this, "Disable Asynchronous Memory Manager", false, true };
 		cfg::_enum<output_scaling_mode> output_scaling{ this, "Output Scaling Mode", output_scaling_mode::bilinear, true };
 		cfg::_bool record_with_overlays{ this, "Record With Overlays", true, true };
+		cfg::_bool use_hardware_texel_remapping{ this, "Use Hardware Texel Remapping", true, false };
 
 		struct node_vk : cfg::node
 		{