RenderDoc在Windows上编译

在Mac上编译需要Xcode 12.2

rendererdoc是开源的代码，今天调试一个32位的程序，发现始终无法capture frames

尝试了好几个版本都不行，最关键的问题处在API：None，既然无法得知原因，就直接看代码吧

下载后的rendererdoc代码windows直接编译通过了，因为我看文档说64位的renderdoc可以调试32/64的program，所以直接选择

运行后直接报这个错：

追溯源代码发现：

wchar_t renderdocPath[MAX_PATH] = {0};
// renderdocpath的值是由renderdoc.dll的路径决定的
GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), &renderdocPath[0], MAX_PATH - 1);

// 如果是32位的program，则使用32位的renderdoc，反之使用64位
#if ENABLE(RDOC_X64)
//...
#else
// ...
#endif
wchar_t *paramsAlloc = new wchar_t[2048];
// paramsAlloc的值来自renderdocPath的值
_snwprintf_s(
    paramsAlloc, 2047, 2047,
    L"\"%ls\" capaltbit --pid=%u --capfile=\"%ls\" --debuglog=\"%ls\" --capopts=\"%hs\"",
    renderdocPath, pid, wcapturefile.c_str(), wdebugLogfile.c_str(), optstr.c_str());
// 变量来自paramsAlloc
wchar_t *commandLine = paramsAlloc;
// 这里的retValue=0导致的，原因是commandLine指向了一个不存在的renderrdoc.exe文件，导致程序报错
BOOL retValue = CreateProcessW(NULL, commandLine, &pSec, &tSec, false,
                               CREATE_NEW_CONSOLE | CREATE_SUSPENDED, NULL, NULL, &si, &pi);
if (!retValue)
{
  RDResult result;

  // 提示无法运行32位的rendererCmd捕获32位的程序
  SET_ERROR_RESULT(
      result, ResultCode::InternalError,
      "Can't run 32-bit renderdoccmd to capture 32-bit program."
      "If this is a locally built RenderDoc you must build both 32-bit and 64-bit versions.");
  CloseHandle(hProcess);
  return {result, 0};
}

development环境好像又严格的32/64区分，运行程序时32位的，只能乖乖选择

运行后正常了，下一步就是看API：None时怎么会是

renderdoc\qrenderdoc\Windows\Dialogs\LiveCapture.cpp

extern "C" RENDERDOC_API ITargetControl *RENDERDOC_CC RENDERDOC_CreateTargetControl(
    const rdcstr &URL, uint32_t ident, const rdcstr &clientName, bool forceConnection)
{
  rdcstr host = "localhost";
  if(!URL.empty())
    host = URL;

  rdcstr deviceID = host;
  uint16_t port = ident & 0xffff;

  IDeviceProtocolHandler *protocol = RenderDoc::Inst().GetDeviceProtocol(deviceID);

  if(protocol)
  {
    deviceID = protocol->GetDeviceID(deviceID);
    host = protocol->RemapHostname(deviceID);
    if(host.empty())
      return NULL;

    port = protocol->RemapPort(deviceID, port);
  }
  else
  {
    int32_t idx = deviceID.indexOf(':');
    if(idx > 0)
    {
      host = deviceID.substr(0, idx);
      port = atoi(deviceID.substr(idx + 1).c_str()) & 0xffff;
    }
  }

  if(port == 0)
    return NULL;
  // 通过socket建立链接
  Network::Socket *sock = Network::CreateClientSocket(host, port, 750);

  if(sock == NULL)
    return NULL;

  TargetControl *remote = new TargetControl(sock, clientName, forceConnection != 0);

  if(remote->Connected())
    return remote;

  delete remote;
  return NULL;
}

void LiveCapture::connectionThreadEntry()
{
  // 创建了链接
  ITargetControl *conn = RENDERDOC_CreateTargetControl(m_Hostname, m_RemoteIdent, GetSystemUsername(), true);
  // 从链接池中获取数据
  while (conn && conn->Connected())
  {
    TargetControlMessage msg = conn->ReceiveMessage();
    if (msg.type == TargetControlMessageType::RegisterAPI) // 消息类型
    {
      QString api = msg.apiUse.name;
      bool presenting = msg.apiUse.presenting;
      bool supported = msg.apiUse.supported;
      GUIInvoke::call(this, [this, api, presenting, supported]()
                      {
        m_APIs[api] = APIStatus(presenting, supported);

        if(presenting && supported)
        {
          ui->triggerImmediateCapture->setEnabled(true);
          ui->triggerDelayedCapture->setEnabled(true);
          ui->queueCap->setEnabled(true);
        }

        updateAPIStatus(); });
    }
  }
}
// 更新API状态
void LiveCapture::updateAPIStatus()
{

  QString apiStatus;

  bool nonpresenting = false;

  // add any fully working APIs first in the list.
  for (QString api : m_APIs.keys())
  {
    if (m_APIs[api].supported && m_APIs[api].presenting)
      apiStatus += lit(", <b>%1 (Active)</b>").arg(api);
  }

  // then add any problem APIs
  for (QString api : m_APIs.keys())
  {
    if (!m_APIs[api].supported)
    {
      apiStatus += tr(", %1 (Unsupported)").arg(api);
    }
    else if (!m_APIs[api].presenting)
    {
      apiStatus += tr(", %1 (Not Presenting)").arg(api);
      nonpresenting = true;
    }
  }
  ui->apiStatus->setText(apiStatus);
}

从以上代码看出来，UI界面时在监听net数据，接下来排查方向就是寻找TargetControlMessageType::RegisterAPI的发送源头 renderdoc\renderdoc\core\target_control.cpp