Skip to content

Commit 1054fda

Browse files
flightlessmangoflightlessmango
committed
server: don't use multiple GPUs for now
1 parent 1ba7684 commit 1054fda

6 files changed

Lines changed: 65 additions & 97 deletions

File tree

‎ipc/client.cpp‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,6 @@ void destroy_client_res(clientRes* r) {
1919
destroy_vk_images(r->device, buf.source.image_res);
2020
buf.dmabuf.gbm = {};
2121

22-
if (buf.sync.semaphore) {
23-
vkDestroySemaphore(r->device, buf.sync.semaphore, nullptr);
24-
buf.sync.semaphore = VK_NULL_HANDLE;
25-
}
26-
2722
if (buf.sync.fence) {
2823
vkDestroyFence(r->device, buf.sync.fence, nullptr);
2924
buf.sync.fence = VK_NULL_HANDLE;
@@ -229,7 +224,7 @@ int Client::on_connect(sd_bus_message* m, void* userdata, sd_bus_error* ret_erro
229224
}
230225
self->pEngineName = engine;
231226

232-
if (!self->resources->vk) self->resources->vk = self->server->vk(self->renderMinor);
227+
if (!self->resources->vk) self->resources->vk = self->server->vk();
233228

234229
return 0;
235230
}
@@ -597,8 +592,8 @@ int Client::spdlog_msg(sd_bus_message* m, void* userdata, sd_bus_error*) {
597592
}
598593

599594
void Client::frame_ready(uint32_t idx) {
600-
auto sema = resources->buffer[idx].sync.semaphore;
601-
auto fd = unique_fd::adopt(resources->vk->get_semaphore_fd(sema));
595+
auto fence = resources->buffer[idx].sync.fence;
596+
auto fd = unique_fd::adopt(resources->vk->get_fence_fd(fence));
602597
auto weak = self_weak;
603598
post([weak, idx, fd = std::move(fd)]() {
604599
auto self = weak.lock();

‎render/imgui_ctx.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,6 @@ bool ImGuiCtx::draw(std::shared_ptr<clientRes>& r, slot_t &buf) {
397397
}
398398

399399
void ImGuiCtx::record_cmd(slot_t& buf, uint32_t w, uint32_t h) {
400-
vkWaitForFences(vk->device, 1, &buf.sync.fence, VK_FALSE, 100'000'000);
401400
vkResetFences(vk->device, 1, &buf.sync.fence);
402401
vkResetCommandBuffer(buf.sync.cmd, 0);
403402
VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};

‎render/shared.h‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -100,15 +100,8 @@ struct source_t {
100100
};
101101

102102
struct sync_t {
103-
VkSemaphore semaphore = VK_NULL_HANDLE;
104-
unique_fd semaphore_fd;
105-
VkSemaphore consumer_semaphore = VK_NULL_HANDLE;
106-
unique_fd consumer_semaphore_fd;
107-
uint64_t semaphore_last = 1;
108-
uint64_t consumer_last = 0;
109103
VkCommandBuffer cmd = VK_NULL_HANDLE;
110104
VkFence fence = VK_NULL_HANDLE;
111-
bool inited = false;
112105
};
113106

114107
struct slot_t {
@@ -122,7 +115,6 @@ struct clientRes {
122115
VkDevice device;
123116

124117
std::vector<slot_t> buffer;
125-
std::vector<VkFence> in_flight;
126118
VkCommandPool cmd_pool = VK_NULL_HANDLE;
127119
std::mutex m;
128120
std::mutex table_m;

‎render/vulkan_ctx.cpp‎

Lines changed: 50 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -27,17 +27,13 @@ vkb::PhysicalDevice VkCtx::pick_device(vkb::Instance instance) {
2727
selector.add_required_extension(VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME);
2828
selector.add_required_extension(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME);
2929

30-
if (use_dmabuf) {
31-
selector.add_required_extension(VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME);
32-
// Some GPUs will not support this and so we can't use dmabuf
33-
// TODO add opaque fd path to vulkan layer for this case
34-
selector.add_required_extension(VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME);
35-
}
30+
selector.add_required_extension(VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME);
31+
selector.add_required_extension(VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME);
3632

3733
auto devicesRet = selector.select_devices();
3834
if (!devicesRet) {
3935
SPDLOG_ERROR("Selector failed\n");
40-
return vkb::PhysicalDevice();
36+
return {};
4137
}
4238

4339
for (const auto& dev : devicesRet.value()) {
@@ -49,17 +45,13 @@ vkb::PhysicalDevice VkCtx::pick_device(vkb::Instance instance) {
4945
if (!drm.hasRender)
5046
continue;
5147

52-
if (drm.renderMinor != renderMinor)
53-
continue;
54-
5548
return dev;
5649
}
5750

58-
use_dmabuf = false;
59-
return vkb::PhysicalDevice();
51+
SPDLOG_ERROR("No Vulkan device with render node + dmabuf/modifier support\n");
52+
return {};
6053
}
6154

62-
6355
void VkCtx::init(bool enableValidation = true) {
6456
vkb::InstanceBuilder ib;
6557
ib.set_app_name("mangohud-server")
@@ -81,8 +73,8 @@ void VkCtx::init(bool enableValidation = true) {
8173
vkb_device = pick_device(vkbInstance_);
8274

8375
if (!vkb_device.physical_device) {
84-
SPDLOG_ERROR("can't find GPU {}, bailing", renderMinor);
85-
exit(1);
76+
SPDLOG_ERROR("can't find GPU, bailing");
77+
std::abort();
8678
}
8779

8880
vkb::DeviceBuilder db{vkb_device};
@@ -96,6 +88,7 @@ void VkCtx::init(bool enableValidation = true) {
9688

9789
pfn_vkGetMemoryFdPropertiesKHR = (PFN_vkGetMemoryFdPropertiesKHR)vkGetDeviceProcAddr(device, "vkGetMemoryFdPropertiesKHR");
9890
pfn_vkGetSemaphoreFdKHR = (PFN_vkGetSemaphoreFdKHR)vkGetDeviceProcAddr(device, "vkGetSemaphoreFdKHR");
91+
pfn_vkGetFenceFdKHR = (PFN_vkGetFenceFdKHR)vkGetDeviceProcAddr(device, "vkGetFenceFdKHR");
9992
pfn_vkImportSemaphoreFdKHR = (PFN_vkImportSemaphoreFdKHR)vkGetDeviceProcAddr(device, "vkImportSemaphoreFdKHR");
10093
pfn_vkSetDebugUtilsObjectNameEXT = (PFN_vkSetDebugUtilsObjectNameEXT)vkGetDeviceProcAddr(device, "vkSetDebugUtilsObjectNameEXT");
10194
}
@@ -187,7 +180,6 @@ void VkCtx::init_client(clientRes* r, size_t buffer_size) {
187180
return;
188181

189182
r->device = device;
190-
r->server_render_minor = renderMinor;
191183
if (r->buffer.size() < buffer_size)
192184
r->buffer.resize(buffer_size);
193185

@@ -218,24 +210,13 @@ void VkCtx::init_client(clientRes* r, size_t buffer_size) {
218210

219211
// TODO rename this to init sync or something
220212
void VkCtx::create_sync(slot_t* s) {
221-
VkExportSemaphoreCreateInfo exportInfo{
222-
.sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO,
223-
.pNext = nullptr,
224-
.handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
225-
};
226-
227-
VkSemaphoreCreateInfo sci{
228-
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
229-
.pNext = &exportInfo,
230-
.flags = 0,
231-
};
213+
VkExportFenceCreateInfo export_info{};
214+
export_info.sType = VK_STRUCTURE_TYPE_EXPORT_FENCE_CREATE_INFO;
215+
export_info.handleTypes = VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
232216

233-
VkResult r = vkCreateSemaphore(device, &sci, nullptr, &s->sync.semaphore);
234-
if (r != VK_SUCCESS)
235-
SPDLOG_ERROR("vkCreateSemaphore {}", string_VkResult(r));
236-
237-
238-
VkFenceCreateInfo fci{ VK_STRUCTURE_TYPE_FENCE_CREATE_INFO };
217+
VkFenceCreateInfo fci{};
218+
fci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
219+
fci.pNext = &export_info;
239220
fci.flags = VK_FENCE_CREATE_SIGNALED_BIT;
240221
vkCreateFence(device, &fci, nullptr, &s->sync.fence);
241222
}
@@ -338,15 +319,6 @@ bool VkCtx::create_gbm_buffer(clientRes* r, dmabuf_t* buf) {
338319

339320
const uint64_t linear = DRM_FORMAT_MOD_LINEAR;
340321
buf->gbm.bo = gbm_bo_create_with_modifiers(gbm_dev, r->w, r->h, buf->gbm.fourcc, &linear, 1);
341-
if (!buf->gbm.bo) {
342-
buf->gbm.bo = gbm_bo_create(gbm_dev, r->w, r->h, buf->gbm.fourcc, GBM_BO_USE_RENDERING);
343-
if (!buf->gbm.bo) {
344-
fprintf(stderr, "gbm_bo_create_with_modifiers failed\n");
345-
throw;
346-
return false;
347-
}
348-
}
349-
350322
buf->gbm.fd = unique_fd::adopt(gbm_bo_get_fd(buf->gbm.bo));
351323
if (!buf->gbm.fd) {
352324
fprintf(stderr, "Failed to get gbm fd\n");
@@ -355,6 +327,11 @@ bool VkCtx::create_gbm_buffer(clientRes* r, dmabuf_t* buf) {
355327
}
356328

357329
buf->gbm.modifier = gbm_bo_get_modifier(buf->gbm.bo);
330+
if (buf->gbm.modifier != DRM_FORMAT_MOD_LINEAR) {
331+
SPDLOG_ERROR("Expected linear modifier, got 0x{:016x}", buf->gbm.modifier);
332+
return false;
333+
}
334+
358335
buf->gbm.stride = gbm_bo_get_stride_for_plane(buf->gbm.bo, 0);
359336
buf->gbm.offset = gbm_bo_get_offset(buf->gbm.bo, 0);
360337
buf->gbm.plane_size = (uint64_t)buf->gbm.stride * (uint64_t)r->h;
@@ -468,9 +445,9 @@ bool VkCtx::allocate_memory(VkImage image, VkDeviceMemory& memory, clientRes* r,
468445

469446
VkExportMemoryAllocateInfo exportInfo{ VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO };
470447
VkImportMemoryFdInfoKHR importInfo{ VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR };
471-
unique_fd import_fd;
448+
int import_fd = -1;
472449
if (fd >= 0)
473-
import_fd = unique_fd::dup(fd);
450+
import_fd = dup(fd);
474451

475452
if (import_dmabuf) {
476453
if (!import_fd) {
@@ -583,9 +560,6 @@ bool VkCtx::submit(std::shared_ptr<clientRes>& r, int idx) {
583560
submit.commandBufferCount = 1;
584561
submit.pCommandBuffers = &buf.sync.cmd;
585562

586-
submit.signalSemaphoreCount = 1;
587-
submit.pSignalSemaphores = &buf.sync.semaphore;
588-
589563
{
590564
std::scoped_lock lock(m);
591565
vkQueueSubmit(graphicsQueue, 1, &submit, buf.sync.fence);
@@ -594,24 +568,41 @@ bool VkCtx::submit(std::shared_ptr<clientRes>& r, int idx) {
594568
return true;
595569
}
596570

597-
int VkCtx::get_semaphore_fd(VkSemaphore sema) {
598-
VkSemaphoreGetFdInfoKHR fdinfo{
599-
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR,
600-
.pNext = nullptr,
601-
.semaphore = sema,
602-
.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT_KHR,
603-
};
571+
// int VkCtx::get_semaphore_fd(VkSemaphore sema) {
572+
// VkSemaphoreGetFdInfoKHR fdinfo{
573+
// .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR,
574+
// .pNext = nullptr,
575+
// .semaphore = sema,
576+
// .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT_KHR,
577+
// };
578+
579+
// int out_fd = -1;
580+
// // This produces a sync fd that will trigger a false positive double close
581+
// // in valgrind etc. Be warned so you don't spend 3 days like I did
582+
// // tracking it down.
583+
// VkResult r = pfn_vkGetSemaphoreFdKHR(device, &fdinfo, &out_fd);
584+
// if (r != VK_SUCCESS)
585+
// SPDLOG_ERROR("vkGetSemaphoreFdKHR {}", string_VkResult(r));
586+
587+
588+
// return out_fd;
589+
// }
590+
591+
int VkCtx::get_fence_fd(VkFence fence) {
592+
VkFenceGetFdInfoKHR get_fd{};
593+
get_fd.sType = VK_STRUCTURE_TYPE_FENCE_GET_FD_INFO_KHR;
594+
get_fd.fence = fence;
595+
get_fd.handleType = VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
596+
int fd;
604597

605-
int out_fd = -1;
606598
// This produces a sync fd that will trigger a false positive double close
607599
// in valgrind etc. Be warned so you don't spend 3 days like I did
608600
// tracking it down.
609-
VkResult r = pfn_vkGetSemaphoreFdKHR(device, &fdinfo, &out_fd);
601+
VkResult r = pfn_vkGetFenceFdKHR(device, &get_fd, &fd);
610602
if (r != VK_SUCCESS)
611-
SPDLOG_ERROR("vkGetSemaphoreFdKHR {}", string_VkResult(r));
612-
603+
SPDLOG_ERROR("vkGetFenceFdKHR {}", string_VkResult(r));
613604

614-
return out_fd;
605+
return fd;
615606
}
616607

617608
void VkCtx::transition_image(VkCommandBuffer cmd, VkImage image, VkImageLayout oldLayout, VkImageLayout newLayout) {

‎render/vulkan_ctx.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,10 @@ class VkCtx {
1717
VkQueue graphicsQueue = VK_NULL_HANDLE;
1818
uint32_t graphicsQueueFamilyIndex = UINT32_MAX;
1919
VkFormat fmt = VK_FORMAT_B8G8R8A8_SRGB;
20-
int64_t renderMinor;
2120
std::mutex m;
2221
PFN_vkImportSemaphoreFdKHR pfn_vkImportSemaphoreFdKHR = nullptr;
2322

24-
VkCtx(int64_t renderMinor) : renderMinor(renderMinor) {
25-
SPDLOG_DEBUG("VkCtx init with renderMinor: {}", renderMinor);
23+
VkCtx() {
2624
init(true);
2725
imgui = std::make_shared<ImGuiCtx>(this, 4);
2826
};
@@ -34,13 +32,15 @@ class VkCtx {
3432
void create_cmd(clientRes* r, sync_t* s);
3533
void wait_on_semaphores(std::shared_ptr<Client> client);
3634
void sync_wait(std::shared_ptr<Client> client);
37-
int get_semaphore_fd(VkSemaphore sema);
35+
// int get_semaphore_fd(VkSemaphore sema);
36+
int get_fence_fd(VkFence fence);
3837

3938
~VkCtx();
4039
private:
4140
VkDebugUtilsMessengerEXT debugMessenger = VK_NULL_HANDLE;
4241
PFN_vkGetMemoryFdPropertiesKHR pfn_vkGetMemoryFdPropertiesKHR = nullptr;
4342
PFN_vkGetSemaphoreFdKHR pfn_vkGetSemaphoreFdKHR = nullptr;
43+
PFN_vkGetFenceFdKHR pfn_vkGetFenceFdKHR = nullptr;
4444
PFN_vkSetDebugUtilsObjectNameEXT pfn_vkSetDebugUtilsObjectNameEXT = nullptr;
4545

4646
unique_fd phys_fd_;

‎server/server.h‎

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "config.h"
77
#include <spdlog/spdlog.h>
88
#include <spdlog/sinks/stdout_color_sinks.h>
9+
#include <spdlog/cfg/env.h>
910

1011
class MangoHudServer {
1112
public:
@@ -19,31 +20,21 @@ class MangoHudServer {
1920
spdlog::set_default_logger(logger);
2021
// TODO Don't force debug
2122
spdlog::set_level(spdlog::level::debug);
22-
23+
spdlog::cfg::load_env_levels();
2324
config = std::make_shared<Config>();
2425
ipc = std::make_unique<IPCServer>(this);
2526
metrics = std::make_unique<Metrics>(*ipc, config);
2627
loop();
2728
}
2829

29-
std::shared_ptr<VkCtx> vk(uint32_t id) {
30+
std::shared_ptr<VkCtx> vk() {
3031
std::lock_guard<std::mutex> lock(vk_ctx_m);
3132

32-
for (auto it = vk_contexts.begin(); it != vk_contexts.end(); ) {
33-
if (it->expired()) {
34-
it = vk_contexts.erase(it);
35-
continue;
36-
}
37-
38-
auto ctx = it->lock();
39-
if (ctx && ctx->renderMinor == id)
40-
return ctx;
41-
42-
++it;
43-
}
33+
if (auto ctx = vk_ctx.lock())
34+
return ctx;
4435

45-
auto ctx = std::make_shared<VkCtx>(id);
46-
vk_contexts.push_back(ctx);
36+
auto ctx = std::make_shared<VkCtx>();
37+
vk_ctx = ctx;
4738
return ctx;
4839
}
4940

@@ -55,7 +46,7 @@ class MangoHudServer {
5546
std::unique_ptr<IPCServer> ipc;
5647
std::unique_ptr<Metrics> metrics;
5748
std::shared_ptr<spdlog::logger> logger;
58-
std::vector<std::weak_ptr<VkCtx>> vk_contexts;
49+
std::weak_ptr<VkCtx> vk_ctx;
5950
std::mutex vk_ctx_m;
6051
std::atomic<bool> stop {false};
6152

0 commit comments

Comments
 (0)