diff --git a/Common/GPU/Vulkan/VulkanRenderManager.cpp b/Common/GPU/Vulkan/VulkanRenderManager.cpp
index e28efceba72a..2a2dd0b1e5eb 100644
--- a/Common/GPU/Vulkan/VulkanRenderManager.cpp
+++ b/Common/GPU/Vulkan/VulkanRenderManager.cpp
@@ -273,14 +273,21 @@ class CreateMultiPipelinesTask : public Task {
 
 	// Use during shutdown to make sure there aren't any leftover tasks sitting queued.
 	// Could probably be done more elegantly. Like waiting for all tasks of a type, or saving pointers to them, or something...
-	static void WaitForAll();
+	// Returns the maximum value of tasks in flight seen during the wait.
+	static int WaitForAll();
 	static std::atomic<int> tasksInFlight_;
 };
 
-void CreateMultiPipelinesTask::WaitForAll() {
-	while (tasksInFlight_.load() > 0) {
+int CreateMultiPipelinesTask::WaitForAll() {
+	int inFlight = 0;
+	int maxInFlight = 0;
+	while ((inFlight = tasksInFlight_.load()) > 0) {
+		if (inFlight > maxInFlight) {
+			maxInFlight = inFlight;
+		}
 		sleep_ms(2, "create-multi-pipelines-wait");
 	}
+	return maxInFlight;
 }
 
 std::atomic<int> CreateMultiPipelinesTask::tasksInFlight_;
@@ -410,7 +417,7 @@ void VulkanRenderManager::StopThreads() {
 		presentWaitThread_.join();
 	}
 
-	INFO_LOG(Log::G3D, "Vulkan compiler thread joined. Now wait for any straggling compile tasks.");
+	INFO_LOG(Log::G3D, "Vulkan compiler thread joined. Now wait for any straggling compile tasks. runCompileThread_ = %d", (int)runCompileThread_);
 	CreateMultiPipelinesTask::WaitForAll();
 
 	{
@@ -775,6 +782,10 @@ void VulkanRenderManager::ReportBadStateForDraw() {
 	ERROR_LOG_REPORT_ONCE(baddraw, Log::G3D, "Can't draw: %s%s. Step count: %d", cause1, cause2, (int)steps_.size());
 }
 
+int VulkanRenderManager::WaitForPipelines() {
+	return CreateMultiPipelinesTask::WaitForAll();
+}
+
 VKRGraphicsPipeline *VulkanRenderManager::CreateGraphicsPipeline(VKRGraphicsPipelineDesc *desc, PipelineFlags pipelineFlags, uint32_t variantBitmask, VkSampleCountFlagBits sampleCount, bool cacheLoad, const char *tag) {
 	if (!desc->vertexShader || !desc->fragmentShader) {
 		ERROR_LOG(Log::G3D, "Can't create graphics pipeline with missing vs/ps: %p %p", desc->vertexShader, desc->fragmentShader);
diff --git a/Common/GPU/Vulkan/VulkanRenderManager.h b/Common/GPU/Vulkan/VulkanRenderManager.h
index ee6f0435614b..bfe0f3bf02cb 100644
--- a/Common/GPU/Vulkan/VulkanRenderManager.h
+++ b/Common/GPU/Vulkan/VulkanRenderManager.h
@@ -283,6 +283,8 @@ class VulkanRenderManager {
 
 	void ReportBadStateForDraw();
 
+	int WaitForPipelines();
+
 	void NudgeCompilerThread() {
 		compileQueueMutex_.lock();
 		compileCond_.notify_one();
diff --git a/Common/Render/TextureAtlas.cpp b/Common/Render/TextureAtlas.cpp
index f41cbc1402a1..24eb22f63c0e 100644
--- a/Common/Render/TextureAtlas.cpp
+++ b/Common/Render/TextureAtlas.cpp
@@ -61,8 +61,10 @@ bool Atlas::Load(const uint8_t *data, size_t data_size) {
 		return false;
 	}
 
-	images = reader.ReadMultipleAlloc<AtlasImage>(num_images, header.version >= 1);
+	delete[] images;
+	delete[] fonts;
 
+	images = reader.ReadMultipleAlloc<AtlasImage>(num_images, header.version >= 1);
 	fonts = new AtlasFont[num_fonts];
 	for (int i = 0; i < num_fonts; i++) {
 		AtlasFontHeader font_header = reader.Read<AtlasFontHeader>();
diff --git a/Core/CoreParameter.h b/Core/CoreParameter.h
index d51b3c727c0c..43d5549a58ba 100644
--- a/Core/CoreParameter.h
+++ b/Core/CoreParameter.h
@@ -23,7 +23,7 @@
 #include "Core/Compatibility.h"
 #include "Core/Loaders.h"
 
-enum GPUCore {
+enum GPUCore : int {
 	GPUCORE_GLES,
 	GPUCORE_SOFTWARE,
 	GPUCORE_DIRECTX9,
diff --git a/Core/System.cpp b/Core/System.cpp
index 88e1a9922879..d785e0835f0e 100644
--- a/Core/System.cpp
+++ b/Core/System.cpp
@@ -575,6 +575,21 @@ bool PSP_InitStart(const CoreParameter &coreParam) {
 			return;
 		}
 
+		// Initialize the GPU as far as we can here (do things like load cache files).
+
+		if (!gpu) {  // should be!
+			INFO_LOG(Log::Loader, "Starting graphics...");
+			Draw::DrawContext *draw = g_CoreParameter.graphicsContext ? g_CoreParameter.graphicsContext->GetDrawContext() : nullptr;
+			// This set the `gpu` global.
+			GPUCore gpuCore = PSP_CoreParameter().gpuCore;
+			bool success = GPU_Init(gpuCore, g_CoreParameter.graphicsContext, draw);
+			if (!success) {
+				*error_string = "Unable to initialize rendering engine.";
+				CPU_Shutdown(false);
+				g_bootState = BootState::Failed;
+			}
+		}
+
 		g_bootState = BootState::Complete;
 	});
 
@@ -604,27 +619,11 @@ BootState PSP_InitUpdate(std::string *error_string) {
 		return BootState::Failed;
 	}
 
-	// Ok, async boot completed, let's finish up things on the main thread.
-	if (!gpu) {  // should be!
-		INFO_LOG(Log::Loader, "Starting graphics...");
-		Draw::DrawContext *draw = g_CoreParameter.graphicsContext ? g_CoreParameter.graphicsContext->GetDrawContext() : nullptr;
-		// This set the `gpu` global.
-		bool success = GPU_Init(g_CoreParameter.graphicsContext, draw);
-		if (!success) {
-			*error_string = "Unable to initialize rendering engine.";
-			PSP_Shutdown(false);
-			g_bootState = BootState::Off;
-			return BootState::Failed;
-		}
-	}
-
-	// TODO: This should all be checked during GPU_Init.
-	if (!GPU_IsStarted()) {
-		*error_string = "Unable to initialize rendering engine.";
-		PSP_Shutdown(false);
-		g_bootState = BootState::Off;
-		Core_NotifyLifecycle(CoreLifecycle::START_COMPLETE);
-		return BootState::Failed;
+	// Ok, async part of the boot completed, let's finish up things on the main thread.
+	if (gpu) {
+		gpu->FinishInitOnMainThread();
+	} else {
+		_dbg_assert_(gpu);
 	}
 
 	Core_NotifyLifecycle(CoreLifecycle::START_COMPLETE);
diff --git a/GPU/Common/FramebufferManagerCommon.cpp b/GPU/Common/FramebufferManagerCommon.cpp
index e45e7ab8bfad..350f6d9fd609 100644
--- a/GPU/Common/FramebufferManagerCommon.cpp
+++ b/GPU/Common/FramebufferManagerCommon.cpp
@@ -2783,7 +2783,7 @@ void FramebufferManagerCommon::NotifyBlockTransferAfter(u32 dstBasePtr, int dstS
 
 		// A few games use this INSTEAD of actually drawing the video image to the screen, they just blast it to
 		// the backbuffer. Detect this and have the framebuffermanager draw the pixels.
-		if (!useBufferedRendering_ && currentRenderVfb_ != dstRect.vfb) {
+		if ((!useBufferedRendering_ && currentRenderVfb_ != dstRect.vfb) || dstRect.vfb == nullptr) {
 			return;
 		}
 
diff --git a/GPU/D3D11/GPU_D3D11.cpp b/GPU/D3D11/GPU_D3D11.cpp
index 496b51c2d0db..d59b24850add 100644
--- a/GPU/D3D11/GPU_D3D11.cpp
+++ b/GPU/D3D11/GPU_D3D11.cpp
@@ -39,8 +39,6 @@ GPU_D3D11::GPU_D3D11(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 	context_ = (ID3D11DeviceContext *)draw->GetNativeObject(Draw::NativeObject::CONTEXT);
 	D3D_FEATURE_LEVEL featureLevel = (D3D_FEATURE_LEVEL)draw->GetNativeObject(Draw::NativeObject::FEATURE_LEVEL);
 
-	stockD3D11.Create(device_);
-
 	shaderManagerD3D11_ = new ShaderManagerD3D11(draw, device_, context_, featureLevel);
 	framebufferManagerD3D11_ = new FramebufferManagerD3D11(draw);
 	framebufferManager_ = framebufferManagerD3D11_;
@@ -77,6 +75,11 @@ GPU_D3D11::GPU_D3D11(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 	textureCache_->NotifyConfigChanged();
 }
 
+void GPU_D3D11::FinishInitOnMainThread() {
+	textureCacheD3D11_->InitDeviceObjects();
+	stockD3D11.Create(device_);
+}
+
 GPU_D3D11::~GPU_D3D11() {
 	stockD3D11.Destroy();
 }
diff --git a/GPU/D3D11/GPU_D3D11.h b/GPU/D3D11/GPU_D3D11.h
index e1d7c5115d79..007607f883a2 100644
--- a/GPU/D3D11/GPU_D3D11.h
+++ b/GPU/D3D11/GPU_D3D11.h
@@ -34,6 +34,8 @@ class GPU_D3D11 : public GPUCommonHW {
 	GPU_D3D11(GraphicsContext *gfxCtx, Draw::DrawContext *draw);
 	~GPU_D3D11();
 
+	void FinishInitOnMainThread() override;
+
 	u32 CheckGPUFeatures() const override;
 
 	void GetStats(char *buffer, size_t bufsize) override;
diff --git a/GPU/D3D11/TextureCacheD3D11.cpp b/GPU/D3D11/TextureCacheD3D11.cpp
index 902841bbaa57..c62ed66a6f8c 100644
--- a/GPU/D3D11/TextureCacheD3D11.cpp
+++ b/GPU/D3D11/TextureCacheD3D11.cpp
@@ -42,8 +42,6 @@ struct DepthPushConstants {
 	float pad[2];
 };
 
-#define INVALID_TEX (ID3D11ShaderResourceView *)(-1LL)
-
 static const D3D11_INPUT_ELEMENT_DESC g_QuadVertexElements[] = {
 	{ "POSITION", 0, DXGI_FORMAT_R32G32B32_FLOAT, 0, 0, },
 	{ "TEXCOORD", 0, DXGI_FORMAT_R32G32_FLOAT, 0, 12,},
@@ -135,21 +133,21 @@ TextureCacheD3D11::TextureCacheD3D11(Draw::DrawContext *draw, Draw2D *draw2D)
 	device_ = (ID3D11Device *)draw->GetNativeObject(Draw::NativeObject::DEVICE);
 	context_ = (ID3D11DeviceContext *)draw->GetNativeObject(Draw::NativeObject::CONTEXT);
 
-	lastBoundTexture = INVALID_TEX;
-
-	D3D11_BUFFER_DESC desc{ sizeof(DepthPushConstants), D3D11_USAGE_DYNAMIC, D3D11_BIND_CONSTANT_BUFFER, D3D11_CPU_ACCESS_WRITE };
-	HRESULT hr = device_->CreateBuffer(&desc, nullptr, &depalConstants_);
-	_dbg_assert_(SUCCEEDED(hr));
-
-	HRESULT result = 0;
+	lastBoundTexture_ = D3D11_INVALID_TEX;
 
-	nextTexture_ = nullptr;
+	InitDeviceObjects();
 }
 
 TextureCacheD3D11::~TextureCacheD3D11() {
 	Clear(true);
 }
 
+void TextureCacheD3D11::InitDeviceObjects() {
+	D3D11_BUFFER_DESC desc{ sizeof(DepthPushConstants), D3D11_USAGE_DYNAMIC, D3D11_BIND_CONSTANT_BUFFER, D3D11_CPU_ACCESS_WRITE };
+	HRESULT hr = device_->CreateBuffer(&desc, nullptr, &depalConstants_);
+	_dbg_assert_(SUCCEEDED(hr));
+}
+
 void TextureCacheD3D11::SetFramebufferManager(FramebufferManagerD3D11 *fbManager) {
 	framebufferManager_ = fbManager;
 }
@@ -168,7 +166,7 @@ void TextureCacheD3D11::ReleaseTexture(TexCacheEntry *entry, bool delete_them) {
 }
 
 void TextureCacheD3D11::ForgetLastTexture() {
-	lastBoundTexture = INVALID_TEX;
+	lastBoundTexture_ = D3D11_INVALID_TEX;
 
 	ID3D11ShaderResourceView *nullTex[4]{};
 	context_->PSSetShaderResources(0, 4, nullTex);
@@ -217,9 +215,9 @@ void TextureCacheD3D11::BindTexture(TexCacheEntry *entry) {
 		return;
 	}
 	ID3D11ShaderResourceView *textureView = DxView(entry);
-	if (textureView != lastBoundTexture) {
+	if (textureView != lastBoundTexture_) {
 		context_->PSSetShaderResources(0, 1, &textureView);
-		lastBoundTexture = textureView;
+		lastBoundTexture_ = textureView;
 	}
 	int maxLevel = (entry->status & TexCacheEntry::STATUS_NO_MIPS) ? 0 : entry->maxLevel;
 	SamplerCacheKey samplerKey = GetSamplingParams(maxLevel, entry);
diff --git a/GPU/D3D11/TextureCacheD3D11.h b/GPU/D3D11/TextureCacheD3D11.h
index e0d6b6fa7bef..124cdd705cd2 100644
--- a/GPU/D3D11/TextureCacheD3D11.h
+++ b/GPU/D3D11/TextureCacheD3D11.h
@@ -42,6 +42,8 @@ class SamplerCacheD3D11 {
 	std::map<SamplerCacheKey, Microsoft::WRL::ComPtr<ID3D11SamplerState>> cache_;
 };
 
+#define D3D11_INVALID_TEX (ID3D11ShaderResourceView *)(-1LL)
+
 class TextureCacheD3D11 : public TextureCacheCommon {
 public:
 	TextureCacheD3D11(Draw::DrawContext *draw, Draw2D *draw2D);
@@ -56,6 +58,8 @@ class TextureCacheD3D11 : public TextureCacheCommon {
 	void DeviceLost() override { draw_ = nullptr; }
 	void DeviceRestore(Draw::DrawContext *draw) override { draw_ = draw; }
 
+	void InitDeviceObjects();
+
 protected:
 	void BindTexture(TexCacheEntry *entry) override;
 	void Unbind() override;
@@ -82,7 +86,7 @@ class TextureCacheD3D11 : public TextureCacheCommon {
 
 	SamplerCacheD3D11 samplerCache_;
 
-	ID3D11ShaderResourceView *lastBoundTexture;
+	ID3D11ShaderResourceView *lastBoundTexture_ = D3D11_INVALID_TEX;
 	Microsoft::WRL::ComPtr<ID3D11Buffer> depalConstants_;
 };
 
diff --git a/GPU/Directx9/GPU_DX9.cpp b/GPU/Directx9/GPU_DX9.cpp
index 2adf0c62222f..092af920ee0f 100644
--- a/GPU/Directx9/GPU_DX9.cpp
+++ b/GPU/Directx9/GPU_DX9.cpp
@@ -75,7 +75,6 @@ GPU_DX9::GPU_DX9(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 
 	// Some of our defaults are different from hw defaults, let's assert them.
 	// We restore each frame anyway, but here is convenient for tests.
-	dxstate.Restore();
 	textureCache_->NotifyConfigChanged();
 
 	if (g_Config.bHardwareTessellation) {
@@ -87,6 +86,11 @@ GPU_DX9::GPU_DX9(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 	}
 }
 
+void GPU_DX9::FinishInitOnMainThread() {
+	textureCacheDX9_->InitDeviceObjects();
+	dxstate.Restore();
+}
+
 u32 GPU_DX9::CheckGPUFeatures() const {
 	u32 features = GPUCommonHW::CheckGPUFeatures();
 	features |= GPU_USE_16BIT_FORMATS;
diff --git a/GPU/Directx9/GPU_DX9.h b/GPU/Directx9/GPU_DX9.h
index 5b675ae481a7..e199ed17b73a 100644
--- a/GPU/Directx9/GPU_DX9.h
+++ b/GPU/Directx9/GPU_DX9.h
@@ -34,6 +34,7 @@ class FramebufferManagerDX9;
 class GPU_DX9 : public GPUCommonHW {
 public:
 	GPU_DX9(GraphicsContext *gfxCtx, Draw::DrawContext *draw);
+	void FinishInitOnMainThread() override;
 
 	u32 CheckGPUFeatures() const override;
 
diff --git a/GPU/Directx9/TextureCacheDX9.cpp b/GPU/Directx9/TextureCacheDX9.cpp
index 69a2412546e5..d263a7e0579d 100644
--- a/GPU/Directx9/TextureCacheDX9.cpp
+++ b/GPU/Directx9/TextureCacheDX9.cpp
@@ -73,6 +73,13 @@ TextureCacheDX9::TextureCacheDX9(Draw::DrawContext *draw, Draw2D *draw2D)
 	lastBoundTexture = INVALID_TEX;
 	device_ = (LPDIRECT3DDEVICE9)draw->GetNativeObject(Draw::NativeObject::DEVICE);
 	deviceEx_ = (LPDIRECT3DDEVICE9EX)draw->GetNativeObject(Draw::NativeObject::DEVICE_EX);
+}
+
+TextureCacheDX9::~TextureCacheDX9() {
+	Clear(true);
+}
+
+void TextureCacheDX9::InitDeviceObjects() {
 	D3DCAPS9 pCaps;
 	ZeroMemory(&pCaps, sizeof(pCaps));
 	HRESULT result = 0;
@@ -92,10 +99,6 @@ TextureCacheDX9::TextureCacheDX9(Draw::DrawContext *draw, Draw2D *draw2D)
 	device_->CreateVertexDeclaration(g_FramebufferVertexElements, &pFramebufferVertexDecl);
 }
 
-TextureCacheDX9::~TextureCacheDX9() {
-	Clear(true);
-}
-
 void TextureCacheDX9::SetFramebufferManager(FramebufferManagerDX9 *fbManager) {
 	framebufferManager_ = fbManager;
 }
diff --git a/GPU/Directx9/TextureCacheDX9.h b/GPU/Directx9/TextureCacheDX9.h
index 31471fadff18..109e4d48eb94 100644
--- a/GPU/Directx9/TextureCacheDX9.h
+++ b/GPU/Directx9/TextureCacheDX9.h
@@ -46,6 +46,7 @@ class TextureCacheDX9 : public TextureCacheCommon {
 	void DeviceLost() override { draw_ = nullptr; }
 	void DeviceRestore(Draw::DrawContext *draw) override { draw_ = draw; }
 
+	void InitDeviceObjects();
 protected:
 	void BindTexture(TexCacheEntry *entry) override;
 	void Unbind() override;
@@ -73,7 +74,7 @@ class TextureCacheDX9 : public TextureCacheCommon {
 	IDirect3DBaseTexture9 *lastBoundTexture = nullptr;
 	float maxAnisotropyLevel;
 
-	FramebufferManagerDX9 *framebufferManagerDX9_;
+	FramebufferManagerDX9 *framebufferManagerDX9_ = nullptr;
 };
 
 static D3DFORMAT getClutDestFormat(GEPaletteFormat format);
diff --git a/GPU/GPU.cpp b/GPU/GPU.cpp
index 7dfc759ebe4f..3a31d1269f8b 100644
--- a/GPU/GPU.cpp
+++ b/GPU/GPU.cpp
@@ -43,12 +43,6 @@ GPUStatistics gpuStats;
 GPUCommon *gpu;
 GPUDebugInterface *gpuDebug;
 
-template <typename T>
-static void SetGPU(T *obj) {
-	gpu = obj;
-	gpuDebug = obj;
-}
-
 #ifdef USE_CRT_DBG
 #undef new
 #endif
@@ -59,70 +53,78 @@ bool GPU_IsStarted() {
 	return false;
 }
 
-bool GPU_Init(GraphicsContext *ctx, Draw::DrawContext *draw) {
-	const auto &gpuCore = PSP_CoreParameter().gpuCore;
-	_assert_(draw || gpuCore == GPUCORE_SOFTWARE);
+static GPUCommon *CreateGPUCore(GPUCore gpuCore, GraphicsContext *ctx, Draw::DrawContext *draw) {
 #if PPSSPP_PLATFORM(UWP)
+	// TODO: Can probably remove this special case.
 	if (gpuCore == GPUCORE_SOFTWARE) {
-		SetGPU(new SoftGPU(ctx, draw));
+		return new SoftGPU(ctx, draw);
 	} else {
-		SetGPU(new GPU_D3D11(ctx, draw));
+		return new GPU_D3D11(ctx, draw);
 	}
-	return true;
 #else
 	switch (gpuCore) {
 	case GPUCORE_GLES:
 		// Disable GLES on ARM Windows (but leave it enabled on other ARM platforms).
 #if PPSSPP_API(ANY_GL)
-		SetGPU(new GPU_GLES(ctx, draw));
-		break;
+		return new GPU_GLES(ctx, draw);
 #else
-		return false;
+		return nullptr;
 #endif
 	case GPUCORE_SOFTWARE:
-		SetGPU(new SoftGPU(ctx, draw));
-		break;
+		return new SoftGPU(ctx, draw);
 	case GPUCORE_DIRECTX9:
 #if PPSSPP_API(D3D9)
-		SetGPU(new GPU_DX9(ctx, draw));
-		break;
+		return new GPU_DX9(ctx, draw);
 #else
-		return false;
+		return nullptr;
 #endif
 	case GPUCORE_DIRECTX11:
 #if PPSSPP_API(D3D11)
-		SetGPU(new GPU_D3D11(ctx, draw));
-		break;
+		return new GPU_D3D11(ctx, draw);
 #else
-		return false;
+		return nullptr;
 #endif
 #if !PPSSPP_PLATFORM(SWITCH)
 	case GPUCORE_VULKAN:
 		if (!ctx) {
+			// Can this happen?
 			ERROR_LOG(Log::G3D, "Unable to init Vulkan GPU backend, no context");
-			break;
+			return nullptr;
 		}
-		SetGPU(new GPU_Vulkan(ctx, draw));
-		break;
+		return new GPU_Vulkan(ctx, draw);
 #endif
 	default:
-		break;
+		return nullptr;
 	}
+#endif
+}
 
-	if (gpu && !gpu->IsStarted())
-		SetGPU<SoftGPU>(nullptr);
+bool GPU_Init(GPUCore gpuCore, GraphicsContext *ctx, Draw::DrawContext *draw) {
+	_assert_(draw || gpuCore == GPUCORE_SOFTWARE);
+	GPUCommon *createdGPU = CreateGPUCore(gpuCore, ctx, draw);
+
+	// This can happen on some memory allocation failure, but can probably just be ignored in practice.
+	if (createdGPU && !createdGPU->IsStarted()) {
+		delete createdGPU;
+		createdGPU = nullptr;
+	}
+
+	if (createdGPU) {
+		gpu = createdGPU;
+		gpuDebug = createdGPU;
+	}
 
 	return gpu != nullptr;
-#endif
 }
+
 #ifdef USE_CRT_DBG
 #define new DBG_NEW
 #endif
 
 void GPU_Shutdown() {
-
 	delete gpu;
 	gpu = nullptr;
+	gpuDebug = nullptr;
 }
 
 const char *RasterChannelToString(RasterChannel channel) {
diff --git a/GPU/GPU.h b/GPU/GPU.h
index 442586f24fd0..50957e80026b 100644
--- a/GPU/GPU.h
+++ b/GPU/GPU.h
@@ -21,6 +21,8 @@
 #include <cstring>
 #include <cstdint>
 
+enum GPUCore : int;
+
 class GPUCommon;
 class GPUDebugInterface;
 class GraphicsContext;
@@ -181,7 +183,7 @@ namespace Draw {
 	class DrawContext;
 }
 
-bool GPU_Init(GraphicsContext *ctx, Draw::DrawContext *draw);
+bool GPU_Init(GPUCore gpuCore, GraphicsContext *ctx, Draw::DrawContext *draw);
 bool GPU_IsStarted();
 void GPU_Shutdown();
 
diff --git a/GPU/GPUCommon.h b/GPU/GPUCommon.h
index 2b4827ac2121..15bdf63d89f9 100644
--- a/GPU/GPUCommon.h
+++ b/GPU/GPUCommon.h
@@ -201,12 +201,21 @@ inline bool IsTrianglePrim(GEPrimitiveType prim) {
 
 class GPUCommon : public GPUDebugInterface {
 public:
+	// The constructor might run on the loader thread.
 	GPUCommon(GraphicsContext *gfxCtx, Draw::DrawContext *draw);
+
+	// FinishInitOnMainThread runs on the main thread, of course.
+	virtual void FinishInitOnMainThread() {}
+
 	virtual ~GPUCommon() {}
 
 	Draw::DrawContext *GetDrawContext() {
 		return draw_;
 	}
+
+	virtual void DeviceLost() = 0;
+	virtual void DeviceRestore(Draw::DrawContext *draw) = 0;
+
 	virtual u32 CheckGPUFeatures() const = 0;
 
 	virtual void UpdateCmdInfo() = 0;
@@ -257,9 +266,6 @@ class GPUCommon : public GPUDebugInterface {
 
 	virtual void ReapplyGfxState();
 
-	virtual void DeviceLost() = 0;
-	virtual void DeviceRestore(Draw::DrawContext *draw) = 0;
-
 	// Returns true if we should split the call across GE execution.
 	// For example, a debugger is active.
 	bool ShouldSplitOverGe() const;
diff --git a/GPU/Vulkan/DrawEngineVulkan.h b/GPU/Vulkan/DrawEngineVulkan.h
index f46ddddb146d..505f73718ffd 100644
--- a/GPU/Vulkan/DrawEngineVulkan.h
+++ b/GPU/Vulkan/DrawEngineVulkan.h
@@ -229,5 +229,5 @@ class DrawEngineVulkan : public DrawEngineCommon {
 	FBOTexState fboTexBindState_ = FBO_TEX_NONE;
 
 	// Hardware tessellation
-	TessellationDataTransferVulkan *tessDataTransferVulkan;
+	TessellationDataTransferVulkan *tessDataTransferVulkan = nullptr;
 };
diff --git a/GPU/Vulkan/GPU_Vulkan.cpp b/GPU/Vulkan/GPU_Vulkan.cpp
index f3e5a0b2dfb2..896a0be79621 100644
--- a/GPU/Vulkan/GPU_Vulkan.cpp
+++ b/GPU/Vulkan/GPU_Vulkan.cpp
@@ -21,6 +21,7 @@
 #include "Common/Profiler/Profiler.h"
 
 #include "Common/Log.h"
+#include "Common/TimeUtil.h"
 #include "Common/File/FileUtil.h"
 #include "Common/GraphicsContext.h"
 
@@ -42,7 +43,6 @@
 GPU_Vulkan::GPU_Vulkan(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 	: GPUCommonHW(gfxCtx, draw), drawEngine_(draw) {
 	gstate_c.SetUseFlags(CheckGPUFeatures());
-	drawEngine_.InitDeviceObjects();
 
 	VulkanContext *vulkan = (VulkanContext *)gfxCtx->GetAPIContext();
 
@@ -66,13 +66,10 @@ GPU_Vulkan::GPU_Vulkan(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 	framebufferManagerVulkan_->SetTextureCache(textureCacheVulkan_);
 	framebufferManagerVulkan_->SetDrawEngine(&drawEngine_);
 	framebufferManagerVulkan_->SetShaderManager(shaderManagerVulkan_);
-	framebufferManagerVulkan_->Init(msaaLevel_);
 	textureCacheVulkan_->SetFramebufferManager(framebufferManagerVulkan_);
 	textureCacheVulkan_->SetShaderManager(shaderManagerVulkan_);
 	textureCacheVulkan_->SetDrawEngine(&drawEngine_);
 
-	InitDeviceObjects();
-
 	// Sanity check gstate
 	if ((int *)&gstate.transferstart - (int *)&gstate != 0xEA) {
 		ERROR_LOG(Log::G3D, "gstate has drifted out of sync!");
@@ -82,6 +79,8 @@ GPU_Vulkan::GPU_Vulkan(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 
 	textureCache_->NotifyConfigChanged();
 
+	drawEngine_.InitDeviceObjects();  // Creates important things like the pipeline layout. Required for loading the disk cache.
+
 	// Load shader cache.
 	std::string discID = g_paramSFO.GetDiscID();
 	if (discID.size()) {
@@ -89,6 +88,13 @@ GPU_Vulkan::GPU_Vulkan(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 		shaderCachePath_ = GetSysDirectory(DIRECTORY_APP_CACHE) / (discID + ".vkshadercache");
 		LoadCache(shaderCachePath_);
 	}
+
+	InitDeviceObjects();
+}
+
+void GPU_Vulkan::FinishInitOnMainThread() {
+	// This can end up stopping/starting the vulkan render manager.
+	framebufferManagerVulkan_->Init(msaaLevel_);
 }
 
 void GPU_Vulkan::LoadCache(const Path &filename) {
@@ -126,6 +132,14 @@ void GPU_Vulkan::LoadCache(const Path &filename) {
 	}
 	fclose(f);
 
+	// Now, since we're on the loader thread, we can just block here until all pipelines are actually created.
+	// This makes it so that the on-screen spinner keeps spinning until we are done.
+	double start = time_now_d();
+	VulkanRenderManager *rm = (VulkanRenderManager *)draw_->GetNativeObject(Draw::NativeObject::RENDER_MANAGER);
+	int maxTasksSeen = rm->WaitForPipelines();
+	double seconds = time_now_d() - start;
+	INFO_LOG(Log::G3D, "Waited %0.1fms for at least %d pipeline tasks to finish compiling.", maxTasksSeen, seconds * 1000.0);
+
 	if (!result) {
 		WARN_LOG(Log::G3D, "Incompatible Vulkan pipeline cache - rebuilding.");
 		// Bad cache file for this GPU/Driver/etc. Delete it.
diff --git a/GPU/Vulkan/GPU_Vulkan.h b/GPU/Vulkan/GPU_Vulkan.h
index 3d943de1cf86..7e21011ef124 100644
--- a/GPU/Vulkan/GPU_Vulkan.h
+++ b/GPU/Vulkan/GPU_Vulkan.h
@@ -38,6 +38,8 @@ class GPU_Vulkan : public GPUCommonHW {
 	GPU_Vulkan(GraphicsContext *gfxCtx, Draw::DrawContext *draw);
 	~GPU_Vulkan();
 
+	void FinishInitOnMainThread() override;
+
 	// This gets called on startup and when we get back from settings.
 	u32 CheckGPUFeatures() const override;
 
diff --git a/GPU/Vulkan/PipelineManagerVulkan.h b/GPU/Vulkan/PipelineManagerVulkan.h
index 4ee6e40a2b30..b1b83664e07a 100644
--- a/GPU/Vulkan/PipelineManagerVulkan.h
+++ b/GPU/Vulkan/PipelineManagerVulkan.h
@@ -95,6 +95,7 @@ class PipelineManagerVulkan {
 	void DeviceRestore(VulkanContext *vulkan);
 
 	void InvalidateMSAAPipelines();
+	void BlockUntilReady();
 
 	std::string DebugGetObjectString(const std::string &id, DebugShaderType type, DebugShaderStringType stringType, ShaderManagerVulkan *shaderManager);
 	std::vector<std::string> DebugGetObjectIDs(DebugShaderType type) const;
diff --git a/UI/EmuScreen.cpp b/UI/EmuScreen.cpp
index 04b0393fd835..b7bc33b27057 100644
--- a/UI/EmuScreen.cpp
+++ b/UI/EmuScreen.cpp
@@ -1208,7 +1208,7 @@ void EmuScreen::CreateViews() {
 	root_->Add(buttons);
 
 	resumeButton_ = buttons->Add(new Button(dev->T("Resume")));
-	resumeButton_->OnClick.Add([this](UI::EventParams &) {
+	resumeButton_->OnClick.Add([](UI::EventParams &) {
 		if (coreState == CoreState::CORE_RUNTIME_ERROR) {
 			// Force it!
 			Memory::MemFault_IgnoreLastCrash();
diff --git a/UI/EmuScreen.h b/UI/EmuScreen.h
index b0a2a8758700..ae84e3708d5f 100644
--- a/UI/EmuScreen.h
+++ b/UI/EmuScreen.h
@@ -151,9 +151,10 @@ class EmuScreen : public UIScreen {
 
 	ImGuiContext *ctx_ = nullptr;
 
-	// TODO: Ugly!
 	bool frameStep_ = false;
+#ifndef MOBILE_DEVICE
 	bool startDumping_ = false;
+#endif
 };
 
 bool MustRunBehind();
diff --git a/UI/NativeApp.cpp b/UI/NativeApp.cpp
index 7e4e908aa9b4..969e57278632 100644
--- a/UI/NativeApp.cpp
+++ b/UI/NativeApp.cpp
@@ -1282,7 +1282,6 @@ bool NativeIsAtTopLevel() {
 	Screen *currentScreen = g_screenManager->topScreen();
 	if (currentScreen) {
 		bool top = currentScreen->isTopLevel();
-		INFO_LOG(Log::System, "Screen toplevel: %i", (int)top);
 		return currentScreen->isTopLevel();
 	} else {
 		ERROR_LOG(Log::System, "No current screen");