查了一下之後發現，問題是在 llamacpp 的 responses api compatibility layer。

codex 的 view_image 工具執行完後，下一輪會送類似這種內容：

{
  "type": "function_call_output",
  "call_id": "call_xxx",
  "output": [
    {
      "type": "input_image",
      "image_url": "data:image/png;base64,..."
    }
  ]
}

但 llamacpp 目前在處理 function_call_output.output 時，只接受：

{
  "type": "input_text"
}

llamacpp 本身其實早就支援圖片，一般 responses api 的 {"type": "input_image", "image_url": "..."} 本來就會被轉成 chat completions 內部使用的 {"type": "image_url", "image_url": {"url": "..."}}。我現在加的 patch 大致就是把 function_call_output 的 input_image 轉成 tool message 的 image_url。

目前我已經把這個修正提到 llamacpp upstream PR，現在正在等 CI / maintainer review。之前也有人回報過相同問題，只是 issue 後來因為 stale 被關掉了，所以這次被無視的可能性還是比較高的。我把 patch 貼在這裡，有需要的朋友自行 copy 到 llamacpp 的 root 目錄去執行一下，然後重新編譯就能支援 codex 的視覺能力了。

import re
import sys
from pathlib import Path

# source base 目錄
BASE = Path(__file__).resolve().parent
# marker
PATCH_COMMENT = "[codex-patch]"
# 元 message
ORIGINAL_THROW = "Output of tool call should be 'Input text'"
# patch 後
PATCH_THROW = "Output of tool call should be 'Input text' or 'Input image'"

PAT = re.compile(
  r'([ \t]+)if \(!chatcmpl_output\.contains\("type"\) \|\| '
  r'chatcmpl_output\.at\("type"\) != "input_text"\) \{\s*'
  r'throw std::invalid_argument\("Output of tool call should be \'Input text\'"\);\s*'
  r'\}\s*'
  r'[ \t]*chatcmpl_output\["type"\] = "text";'
)

NEW_TEMPLATE = (
  "@@0@@const std::string out_type = json_value(chatcmpl_output, \"type\", std::string());\n"
  "@@0@@if (out_type == \"input_text\") {\n"
  "@@1@@chatcmpl_output[\"type\"] = \"text\";\n"
  "@@0@@} else if (out_type == \"input_image\") {\n"
  "@@1@@// %s allow images in tool call output (multimodal tool results)\n"
  "@@1@@if (!chatcmpl_output.contains(\"image_url\")) {\n"
  "@@2@@throw std::invalid_argument(\"'image_url' is required for 'input_image' tool output\");\n"
  "@@1@@}\n"
  "@@1@@chatcmpl_output = json {\n"
  "@@2@@{\"image_url\", json {\n"
  "@@3@@{\"url\", chatcmpl_output.at(\"image_url\")}\n"
  "@@2@@}},\n"
  "@@2@@{\"type\", \"image_url\"},\n"
  "@@1@@};\n"
  "@@0@@} else {\n"
  "@@1@@throw std::invalid_argument(\"Output of tool call should be 'Input text' or 'Input image'\");\n"
  "@@0@@}"
) % PATCH_COMMENT

def build_replacement(indent: str, eol: str) -> str:
  i0 = indent
  i1 = indent + "    "
  i2 = indent + "        "
  i3 = indent + "            "
  block = (NEW_TEMPLATE
            .replace("@@0@@", i0)
            .replace("@@1@@", i1)
            .replace("@@2@@", i2)
            .replace("@@3@@", i3))
  if eol != "\n":
    block = block.replace("\n", eol)
  return block

def main():
  target = Path(BASE)
  src = target / "tools" / "server" / "server-chat.cpp"
  if not src.is_file():
    print("server-chat.cpp not found at:", target)
    return 1

  raw = src.read_bytes()
  text = raw.decode("utf-8")

  # marker check
  if PATCH_THROW in text or PATCH_COMMENT in text:
    print("SKIP: already patched ->", target)
    return 0

  # source check
  if not PAT.search(text):
    if ORIGINAL_THROW in text:
      print("STOP: file left UNTOUCHED. inspect manually:")
      print("      ->", src)
      return 2
    print("SKIP: the restrictive block is absent. nothing to patch.")
    print("      ->", target)
    return 0

  EOL = "\r\n" in text and "\r\n" or "\n"
  newtext, n = PAT.subn(lambda m: build_replacement(m.group(1), EOL), text, count=1)

  if n != 1:
    print("STOP: expected exactly one match, found", n)
    return 2

  # 備份元檔
  bak = src.with_name(src.name + ".codex.bak")
  if not bak.exists():
    bak.write_bytes(raw)

  src.write_bytes(newtext.encode("utf-8"))
  print("PATCHED :", src)
  print("Backup  :", bak)
  print("Next    : rebuild llama-server, then test a mtmd tool result.")
  return 0

if __name__ == "__main__":
  sys.exit(main())

一個相性問題， codex 預設使用的是 websocket，llamacpp只支援 sse 並不支援 websocket，所以codex 在接 llama-server 時需要手動設定成使用 sse 模式。

monitoring / tracing 問題，codex 自帶的 OTEL 跟 langfuse 的要求完全不匹配，普通的 thin-proxy 很難做到完全支援 langfuse 的 api protocol，需要用到社群的 extension。