diff --git a/hw/display/Makefile.objs b/hw/display/Makefile.objs
index 3c7c75b94d..11321e466b 100644
--- a/hw/display/Makefile.objs
+++ b/hw/display/Makefile.objs
@@ -20,6 +20,8 @@ common-obj-$(CONFIG_MILKYMIST) += milkymist-vgafb.o
 common-obj-$(CONFIG_ZAURUS) += tc6393xb.o
 
 common-obj-$(CONFIG_MILKYMIST_TMU2) += milkymist-tmu2.o
+milkymist-tmu2.o-cflags := $(X11_CFLAGS)
+milkymist-tmu2.o-libs := $(X11_LIBS)
 
 obj-$(CONFIG_OMAP) += omap_dss.o
 obj-$(CONFIG_OMAP) += omap_lcdc.o
diff --git a/ui/Makefile.objs b/ui/Makefile.objs
index cc784346cb..00f6976c30 100644
--- a/ui/Makefile.objs
+++ b/ui/Makefile.objs
@@ -15,10 +15,6 @@ common-obj-$(CONFIG_COCOA) += cocoa.o
 common-obj-$(CONFIG_VNC) += $(vnc-obj-y)
 common-obj-$(call lnot,$(CONFIG_VNC)) += vnc-stubs.o
 
-common-obj-$(CONFIG_X11) += x_keymap.o
-x_keymap.o-cflags := $(X11_CFLAGS)
-x_keymap.o-libs := $(X11_LIBS)
-
 # ui-sdl module
 common-obj-$(CONFIG_SDL) += sdl.mo
 ifeq ($(CONFIG_SDLABI),1.2)
@@ -46,6 +42,13 @@ gtk.mo-objs += gtk-gl-area.o
 endif
 endif
 
+ifeq ($(CONFIG_X11),y)
+sdl.mo-objs += x_keymap.o
+gtk.mo-objs += x_keymap.o
+x_keymap.o-cflags := $(X11_CFLAGS)
+x_keymap.o-libs := $(X11_LIBS)
+endif
+
 common-obj-$(CONFIG_CURSES) += curses.mo
 curses.mo-objs := curses.o
 curses.mo-cflags := $(CURSES_CFLAGS)