exec: Resolve subpages in one step except for IOTLB fills

Except for the case of setting the IOTLB entry in TCG mode, we can avoid
the subpage dispatching handlers and do the resolution directly on
address_space_lookup_region. An IOTLB entry describes a full page, not
only the region that the first access to a sub-divided page may return.

This patch therefore introduces a special translation function,
address_space_translate_for_iotlb, that avoids the subpage resolutions.
In contrast, callers of the existing address_space_translate service
will now always receive the terminal memory region section. This will be
important for breaking the BQL and for enabling unaligned memory region.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

cputlb.c

@@ -256,8 +256,8 @@ void tlb_set_page(CPUArchState *env, target_ulong vaddr,
     }
 
     sz = size;
-    section = address_space_translate(&address_space_memory, paddr, &xlat, &sz,
-                                      false);
+    section = address_space_translate_for_iotlb(&address_space_memory, paddr,
+                                                &xlat, &sz);
     assert(sz >= TARGET_PAGE_SIZE);
 
 #if defined(DEBUG_TLB)

exec.c

@@ -97,6 +97,13 @@ struct AddressSpaceDispatch {
     MemoryListener listener;
 };
 
+#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
+typedef struct subpage_t {
+    MemoryRegion iomem;
+    hwaddr base;
+    uint16_t sub_section[TARGET_PAGE_SIZE];
+} subpage_t;
+
 static MemoryRegionSection *phys_sections;
 static unsigned phys_sections_nb, phys_sections_nb_alloc;
 static uint16_t phys_section_unassigned;
@@ -220,19 +227,28 @@ bool memory_region_is_unassigned(MemoryRegion *mr)
 }
 
 static MemoryRegionSection *address_space_lookup_region(AddressSpace *as,
-                                                        hwaddr addr)
+                                                        hwaddr addr,
+                                                        bool resolve_subpage)
 {
-    return phys_page_find(as->dispatch, addr >> TARGET_PAGE_BITS);
+    MemoryRegionSection *section;
+    subpage_t *subpage;
+
+    section = phys_page_find(as->dispatch, addr >> TARGET_PAGE_BITS);
+    if (resolve_subpage && section->mr->subpage) {
+        subpage = container_of(section->mr, subpage_t, iomem);
+        section = &phys_sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
+    }
+    return section;
 }
 
-MemoryRegionSection *address_space_translate(AddressSpace *as, hwaddr addr,
-                                             hwaddr *xlat, hwaddr *plen,
-                                             bool is_write)
+static MemoryRegionSection *
+address_space_translate_internal(AddressSpace *as, hwaddr addr, hwaddr *xlat,
+                                 hwaddr *plen, bool resolve_subpage)
 {
     MemoryRegionSection *section;
     Int128 diff;
 
-    section = address_space_lookup_region(as, addr);
+    section = address_space_lookup_region(as, addr, resolve_subpage);
     /* Compute offset within MemoryRegionSection */
     addr -= section->offset_within_address_space;
 
@@ -243,6 +259,20 @@ MemoryRegionSection *address_space_translate(AddressSpace *as, hwaddr addr,
     *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
     return section;
 }
+
+MemoryRegionSection *address_space_translate(AddressSpace *as, hwaddr addr,
+                                             hwaddr *xlat, hwaddr *plen,
+                                             bool is_write)
+{
+    return address_space_translate_internal(as, addr, xlat, plen, true);
+}
+
+MemoryRegionSection *
+address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
+                                  hwaddr *plen)
+{
+    return address_space_translate_internal(as, addr, xlat, plen, false);
+}
 #endif
 
 void cpu_exec_init_all(void)
@@ -697,13 +727,6 @@ hwaddr memory_region_section_get_iotlb(CPUArchState *env,
 
 #if !defined(CONFIG_USER_ONLY)
 
-#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
-typedef struct subpage_t {
-    MemoryRegion iomem;
-    hwaddr base;
-    uint16_t sub_section[TARGET_PAGE_SIZE];
-} subpage_t;
-
 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                              uint16_t section);
 static subpage_t *subpage_init(hwaddr base);

include/exec/cputlb.h

@@ -32,6 +32,10 @@ extern int tlb_flush_count;
 
 /* exec.c */
 void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr);
+
+MemoryRegionSection *
+address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
+                                  hwaddr *plen);
 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
                                        MemoryRegionSection *section,
                                        target_ulong vaddr,