Various testing and misc fixes:

- header cleanups for plugins
   - support wider watchpoints
   - tweaks for unreliable and broken CI
   - docker image fixes and verion bumps
   - linux-user guest_base fixes
   - remove flex/bison from various test images
 -----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCgAdFiEEZoWumedRZ7yvyN81+9DbCVqeKkQFAl7eZFMACgkQ+9DbCVqe
 KkQ5IAf/f7icxRp2EK5+Tt1g2nP9/h1T5Vg7Lu76dPm+i8a4f1D/pKyimGMiMYbc
 bGSQn+4rRohocdAS8rTkPfpBYauFvBxFMUCZThseeW4xYM2YcasvPAvsnQlauNy+
 NrIkg6Ptaw7OSG6ecNJ+0qDiNvOzGz0l4+r+JvhF9LjfPmKlOhipnUY3huzreq2c
 4nvlWPxNVx/Bc+bIVPO3F/zMwwyvZkcthCYZij7cBKN3y8LiP31khsR1GbVu7qxy
 G1hl9j+2oJwzY9RpJBr8eVOrcn5OogIiXH22fBOn1SKAaxbg8m2quNy1iNGZuKNa
 A7I+otZJtHbErWscNAdmtDPKUgepeg==
 =LZKS
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/stsquad/tags/pull-testing-and-misc-080620-1' into staging

Various testing and misc fixes:

  - header cleanups for plugins
  - support wider watchpoints
  - tweaks for unreliable and broken CI
  - docker image fixes and verion bumps
  - linux-user guest_base fixes
  - remove flex/bison from various test images

# gpg: Signature made Mon 08 Jun 2020 17:16:19 BST
# gpg:                using RSA key 6685AE99E75167BCAFC8DF35FBD0DB095A9E2A44
# gpg: Good signature from "Alex Bennée (Master Work Key) <alex.bennee@linaro.org>" [full]
# Primary key fingerprint: 6685 AE99 E751 67BC AFC8  DF35 FBD0 DB09 5A9E 2A44

* remotes/stsquad/tags/pull-testing-and-misc-080620-1:
  scripts/coverity-scan: Remove flex/bison packages
  cirrus-ci: Remove flex/bison packages
  tests/vm: Remove flex/bison packages
  tests/docker: Remove flex/bison packages
  linux-user: detect overflow of MAP_FIXED mmap
  tests/tcg: add simple commpage test case
  linux-user: deal with address wrap for ARM_COMMPAGE on 32 bit
  linux-user: provide fallback pgd_find_hole for bare chroots
  hw/virtio/vhost: re-factor vhost-section and allow DIRTY_MEMORY_CODE
  docker: update Ubuntu to 20.04
  tests/docker: fix pre-requisite for debian-tricore-cross
  .shippable: temporaily disable some cross builds
  .travis.yml: allow failure for unreliable hosts
  exec: flush the whole TLB if a watchpoint crosses a page boundary
  tests/plugin: correctly honour io_count
  scripts/clean-includes: Mark 'qemu/qemu-plugin.h' as special header
  qemu-plugin.h: add missing include <stddef.h> to define size_t

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
master
Peter Maydell 2020-06-09 21:30:00 +01:00
commit 9e7f1469b9
28 changed files with 194 additions and 59 deletions

View File

@ -7,7 +7,7 @@ freebsd_12_task:
cpu: 8
memory: 8G
install_script: ASSUME_ALWAYS_YES=yes pkg bootstrap -f ; pkg install -y
bash bison curl cyrus-sasl git glib gmake gnutls gsed
bash curl cyrus-sasl git glib gmake gnutls gsed
nettle perl5 pixman pkgconf png usbredir
script:
- mkdir build

View File

@ -5,8 +5,8 @@ env:
global:
- LC_ALL=C
matrix:
- IMAGE=debian-amd64
TARGET_LIST=x86_64-softmmu,x86_64-linux-user
# - IMAGE=debian-amd64
# TARGET_LIST=x86_64-softmmu,x86_64-linux-user
- IMAGE=debian-win32-cross
TARGET_LIST=arm-softmmu,i386-softmmu,lm32-softmmu
- IMAGE=debian-win64-cross
@ -19,10 +19,10 @@ env:
TARGET_LIST=aarch64-softmmu,aarch64-linux-user
- IMAGE=debian-s390x-cross
TARGET_LIST=s390x-softmmu,s390x-linux-user
- IMAGE=debian-mips-cross
TARGET_LIST=mips-softmmu,mipsel-linux-user
- IMAGE=debian-mips64el-cross
TARGET_LIST=mips64el-softmmu,mips64el-linux-user
# - IMAGE=debian-mips-cross
# TARGET_LIST=mips-softmmu,mipsel-linux-user
# - IMAGE=debian-mips64el-cross
# TARGET_LIST=mips64el-softmmu,mips64el-linux-user
- IMAGE=debian-ppc64el-cross
TARGET_LIST=ppc64-softmmu,ppc64-linux-user,ppc64abi32-linux-user
build:

View File

@ -429,6 +429,7 @@ jobs:
env:
- TEST_CMD="make check check-tcg V=1"
- CONFIG="--disable-containers --target-list=${MAIN_SOFTMMU_TARGETS}"
- UNRELIABLE=true
- name: "[ppc64] GCC check-tcg"
arch: ppc64le
@ -493,6 +494,7 @@ jobs:
env:
- TEST_CMD="make check check-tcg V=1"
- CONFIG="--disable-containers --target-list=${MAIN_SOFTMMU_TARGETS},s390x-linux-user"
- UNRELIABLE=true
script:
- ( cd ${SRC_DIR} ; git submodule update --init roms/SLOF )
- BUILD_RC=0 && make -j${JOBS} || BUILD_RC=$?
@ -535,6 +537,7 @@ jobs:
- TEST_CMD="make check-unit"
- CONFIG="--disable-containers --disable-tcg --enable-kvm
--disable-tools --host-cc=clang --cxx=clang++"
- UNRELIABLE=true
# Release builds
# The make-release script expect a QEMU version, so our tag must start with a 'v'.
@ -556,3 +559,5 @@ jobs:
- mkdir -p release-build && cd release-build
- ../configure ${BASE_CONFIG} ${CONFIG} || { cat config.log && exit 1; }
- make install
allow_failures:
- env: UNRELIABLE=true

8
exec.c
View File

@ -1038,6 +1038,7 @@ int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
int flags, CPUWatchpoint **watchpoint)
{
CPUWatchpoint *wp;
vaddr in_page;
/* forbid ranges which are empty or run off the end of the address space */
if (len == 0 || (addr + len - 1) < addr) {
@ -1058,7 +1059,12 @@ int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
QTAILQ_INSERT_TAIL(&cpu->watchpoints, wp, entry);
}
tlb_flush_page(cpu, addr);
in_page = -(addr | TARGET_PAGE_MASK);
if (len <= in_page) {
tlb_flush_page(cpu, addr);
} else {
tlb_flush(cpu);
}
if (watchpoint)
*watchpoint = wp;

View File

@ -5,7 +5,8 @@ vhost_commit(bool started, bool changed) "Started: %d Changed: %d"
vhost_region_add_section(const char *name, uint64_t gpa, uint64_t size, uint64_t host) "%s: 0x%"PRIx64"+0x%"PRIx64" @ 0x%"PRIx64
vhost_region_add_section_merge(const char *name, uint64_t new_size, uint64_t gpa, uint64_t owr) "%s: size: 0x%"PRIx64 " gpa: 0x%"PRIx64 " owr: 0x%"PRIx64
vhost_region_add_section_aligned(const char *name, uint64_t gpa, uint64_t size, uint64_t host) "%s: 0x%"PRIx64"+0x%"PRIx64" @ 0x%"PRIx64
vhost_section(const char *name, int r) "%s:%d"
vhost_section(const char *name) "%s"
vhost_reject_section(const char *name, int d) "%s:%d"
vhost_iotlb_miss(void *dev, int step) "%p step %d"
# vhost-user.c

View File

@ -27,6 +27,7 @@
#include "migration/blocker.h"
#include "migration/qemu-file-types.h"
#include "sysemu/dma.h"
#include "sysemu/tcg.h"
#include "trace.h"
/* enabled until disconnected backend stabilizes */
@ -403,26 +404,50 @@ static int vhost_verify_ring_mappings(struct vhost_dev *dev,
return r;
}
/*
* vhost_section: identify sections needed for vhost access
*
* We only care about RAM sections here (where virtqueue and guest
* internals accessed by virtio might live). If we find one we still
* allow the backend to potentially filter it out of our list.
*/
static bool vhost_section(struct vhost_dev *dev, MemoryRegionSection *section)
{
bool result;
bool log_dirty = memory_region_get_dirty_log_mask(section->mr) &
~(1 << DIRTY_MEMORY_MIGRATION);
result = memory_region_is_ram(section->mr) &&
!memory_region_is_rom(section->mr);
MemoryRegion *mr = section->mr;
/* Vhost doesn't handle any block which is doing dirty-tracking other
* than migration; this typically fires on VGA areas.
*/
result &= !log_dirty;
if (memory_region_is_ram(mr) && !memory_region_is_rom(mr)) {
uint8_t dirty_mask = memory_region_get_dirty_log_mask(mr);
uint8_t handled_dirty;
if (result && dev->vhost_ops->vhost_backend_mem_section_filter) {
result &=
dev->vhost_ops->vhost_backend_mem_section_filter(dev, section);
/*
* Kernel based vhost doesn't handle any block which is doing
* dirty-tracking other than migration for which it has
* specific logging support. However for TCG the kernel never
* gets involved anyway so we can also ignore it's
* self-modiying code detection flags. However a vhost-user
* client could still confuse a TCG guest if it re-writes
* executable memory that has already been translated.
*/
handled_dirty = (1 << DIRTY_MEMORY_MIGRATION) |
(1 << DIRTY_MEMORY_CODE);
if (dirty_mask & ~handled_dirty) {
trace_vhost_reject_section(mr->name, 1);
return false;
}
if (dev->vhost_ops->vhost_backend_mem_section_filter &&
!dev->vhost_ops->vhost_backend_mem_section_filter(dev, section)) {
trace_vhost_reject_section(mr->name, 2);
return false;
}
trace_vhost_section(mr->name);
return true;
} else {
trace_vhost_reject_section(mr->name, 3);
return false;
}
trace_vhost_section(section->mr->name, result);
return result;
}
static void vhost_begin(MemoryListener *listener)

View File

@ -12,6 +12,7 @@
#include <inttypes.h>
#include <stdbool.h>
#include <stddef.h>
/*
* For best performance, build the plugin with -fvisibility=hidden so that

View File

@ -389,7 +389,7 @@ static bool init_guest_commpage(void)
{
void *want = g2h(ARM_COMMPAGE & -qemu_host_page_size);
void *addr = mmap(want, qemu_host_page_size, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0);
if (addr == MAP_FAILED) {
perror("Allocating guest commpage");
@ -2101,9 +2101,54 @@ static void pgb_have_guest_base(const char *image_name, abi_ulong guest_loaddr,
}
}
/**
* pgd_find_hole_fallback: potential mmap address
* @guest_size: size of available space
* @brk: location of break
* @align: memory alignment
*
* This is a fallback method for finding a hole in the host address
* space if we don't have the benefit of being able to access
* /proc/self/map. It can potentially take a very long time as we can
* only dumbly iterate up the host address space seeing if the
* allocation would work.
*/
static uintptr_t pgd_find_hole_fallback(uintptr_t guest_size, uintptr_t brk,
long align, uintptr_t offset)
{
uintptr_t base;
/* Start (aligned) at the bottom and work our way up */
base = ROUND_UP(mmap_min_addr, align);
while (true) {
uintptr_t align_start, end;
align_start = ROUND_UP(base, align);
end = align_start + guest_size + offset;
/* if brk is anywhere in the range give ourselves some room to grow. */
if (align_start <= brk && brk < end) {
base = brk + (16 * MiB);
continue;
} else if (align_start + guest_size < align_start) {
/* we have run out of space */
return -1;
} else {
int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE | MAP_FIXED;
void * mmap_start = mmap((void *) align_start, guest_size,
PROT_NONE, flags, -1, 0);
if (mmap_start != MAP_FAILED) {
munmap((void *) align_start, guest_size);
return (uintptr_t) mmap_start + offset;
}
base += qemu_host_page_size;
}
}
}
/* Return value for guest_base, or -1 if no hole found. */
static uintptr_t pgb_find_hole(uintptr_t guest_loaddr, uintptr_t guest_size,
long align)
long align, uintptr_t offset)
{
GSList *maps, *iter;
uintptr_t this_start, this_end, next_start, brk;
@ -2116,6 +2161,10 @@ static uintptr_t pgb_find_hole(uintptr_t guest_loaddr, uintptr_t guest_size,
/* Read brk after we've read the maps, which will malloc. */
brk = (uintptr_t)sbrk(0);
if (!maps) {
return pgd_find_hole_fallback(guest_size, brk, align, offset);
}
/* The first hole is before the first map entry. */
this_start = mmap_min_addr;
@ -2125,7 +2174,7 @@ static uintptr_t pgb_find_hole(uintptr_t guest_loaddr, uintptr_t guest_size,
this_end = ((MapInfo *)iter->data)->start;
next_start = ((MapInfo *)iter->data)->end;
align_start = ROUND_UP(this_start, align);
align_start = ROUND_UP(this_start + offset, align);
/* Skip holes that are too small. */
if (align_start >= this_end) {
@ -2175,6 +2224,7 @@ static void pgb_static(const char *image_name, abi_ulong orig_loaddr,
{
uintptr_t loaddr = orig_loaddr;
uintptr_t hiaddr = orig_hiaddr;
uintptr_t offset = 0;
uintptr_t addr;
if (hiaddr != orig_hiaddr) {
@ -2188,18 +2238,19 @@ static void pgb_static(const char *image_name, abi_ulong orig_loaddr,
if (ARM_COMMPAGE) {
/*
* Extend the allocation to include the commpage.
* For a 64-bit host, this is just 4GiB; for a 32-bit host,
* the address arithmetic will wrap around, but the difference
* will produce the correct allocation size.
* For a 64-bit host, this is just 4GiB; for a 32-bit host we
* need to ensure there is space bellow the guest_base so we
* can map the commpage in the place needed when the address
* arithmetic wraps around.
*/
if (sizeof(uintptr_t) == 8 || loaddr >= 0x80000000u) {
hiaddr = (uintptr_t)4 << 30;
hiaddr = (uintptr_t) 4 << 30;
} else {
loaddr = ARM_COMMPAGE & -align;
offset = -(ARM_COMMPAGE & -align);
}
}
addr = pgb_find_hole(loaddr, hiaddr - loaddr, align);
addr = pgb_find_hole(loaddr, hiaddr - loaddr, align, offset);
if (addr == -1) {
/*
* If ARM_COMMPAGE, there *might* be a non-consecutive allocation
@ -2234,7 +2285,7 @@ static void pgb_dynamic(const char *image_name, long align)
* just above that, and maximises the positive guest addresses.
*/
commpage = ARM_COMMPAGE & -align;
addr = pgb_find_hole(commpage, -commpage, align);
addr = pgb_find_hole(commpage, -commpage, align, 0);
assert(addr != -1);
guest_base = addr;
}

View File

@ -467,7 +467,7 @@ abi_long target_mmap(abi_ulong start, abi_ulong len, int prot,
* It can fail only on 64-bit host with 32-bit target.
* On any other target/host host mmap() handles this error correctly.
*/
if (!guest_range_valid(start, len)) {
if (end < start || !guest_range_valid(start, len)) {
errno = ENOMEM;
goto fail;
}

View File

@ -123,6 +123,7 @@ for f in "$@"; do
;;
*include/qemu/osdep.h | \
*include/qemu/compiler.h | \
*include/qemu/qemu-plugin.h | \
*include/glib-compat.h | \
*include/sysemu/os-posix.h | \
*include/sysemu/os-win32.h | \

View File

@ -19,7 +19,6 @@ FROM fedora:30
ENV PACKAGES \
alsa-lib-devel \
bc \
bison \
brlapi-devel \
bzip2 \
bzip2-devel \
@ -30,7 +29,6 @@ ENV PACKAGES \
dbus-daemon \
device-mapper-multipath-devel \
findutils \
flex \
gcc \
gcc-c++ \
gettext \

View File

@ -130,7 +130,7 @@ docker-image-debian-sparc64-cross: docker-image-debian10
docker-image-travis: NOUSER=1
# Specialist build images, sometimes very limited tools
docker-image-tricore-cross: docker-image-debian9
docker-image-debian-tricore-cross: docker-image-debian9
docker-image-debian-arm64-test-cross: docker-image-debian11
# These images may be good enough for building tests but not for test builds

View File

@ -5,13 +5,11 @@ RUN yum -y update
# Please keep this list sorted alphabetically
ENV PACKAGES \
bison \
bzip2 \
bzip2-devel \
ccache \
csnappy-devel \
dbus-daemon \
flex \
gcc-c++ \
gcc \
gettext \

View File

@ -3,11 +3,9 @@ FROM centos:8.1.1911
RUN dnf -y update
ENV PACKAGES \
SDL-devel \
bison \
bzip2 \
bzip2-devel \
dbus-daemon \
flex \
gcc \
gcc-c++ \
gettext \

View File

@ -11,11 +11,9 @@ RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \
DEBIAN_FRONTEND=noninteractive eatmydata \
apt-get install -y --no-install-recommends \
bison \
build-essential \
ca-certificates \
curl \
flex \
gettext \
git \
python3-minimal

View File

@ -18,12 +18,10 @@ RUN apt update && \
DEBIAN_FRONTEND=noninteractive eatmydata \
apt install -y --no-install-recommends \
bc \
bison \
build-essential \
ca-certificates \
clang \
dbus \
flex \
gdb-multiarch \
gettext \
git \

View File

@ -18,11 +18,9 @@ RUN apt update && \
DEBIAN_FRONTEND=noninteractive eatmydata \
apt install -y --no-install-recommends \
bc \
bison \
build-essential \
ca-certificates \
clang \
flex \
gdb-multiarch \
gettext \
git \

View File

@ -3,7 +3,6 @@ FROM fedora:30
# Please keep this list sorted alphabetically
ENV PACKAGES \
bc \
bison \
brlapi-devel \
bzip2 \
bzip2-devel \
@ -13,7 +12,6 @@ ENV PACKAGES \
dbus-daemon \
device-mapper-multipath-devel \
findutils \
flex \
gcc \
gcc-c++ \
gettext \

View File

@ -9,8 +9,8 @@
# system won't pick up that it has changed.
#
FROM ubuntu:19.04
ENV PACKAGES flex bison \
FROM ubuntu:20.04
ENV PACKAGES \
ccache \
clang \
dbus \

View File

@ -1,5 +1,5 @@
FROM ubuntu:18.04
ENV PACKAGES flex bison \
ENV PACKAGES \
ccache \
clang \
gcc \

View File

@ -28,7 +28,7 @@ static void plugin_exit(qemu_plugin_id_t id, void *p)
g_string_printf(out, "mem accesses: %" PRIu64 "\n", mem_count);
if (do_haddr) {
g_string_append_printf(out, "io accesses: %" PRIu64 "\n", mem_count);
g_string_append_printf(out, "io accesses: %" PRIu64 "\n", io_count);
}
qemu_plugin_outs(out->str);
}

View File

@ -68,6 +68,8 @@ run-semiconsole-arm: semiconsole-arm
run-plugin-semiconsole-arm-with-%:
$(call skip-test, $<, "MANUAL ONLY")
ARM_TESTS += commpage
TESTS += $(ARM_TESTS)
# On ARM Linux only supports 4k pages

61
tests/tcg/arm/commpage.c Normal file
View File

@ -0,0 +1,61 @@
/*
* Verify the COMMPAGE emulation
*
* The ARM commpage is a set of user space helper functions provided
* by the kernel in an effort to ease portability of user space code
* between different CPUs with potentially different capabilities. It
* is a 32 bit invention and similar to the vdso segment in many ways.
*
* The ABI is documented in the Linux kernel:
* Documentation/arm/kernel_userspace_helpers.rst
*
* Copyright (c) 2020 Linaro Ltd
*
* SPDX-License-Identifier: GPL-2.0-or-later
*/
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#define ARM_COMMPAGE (0xffff0f00u)
#define ARM_KUSER_VERSION (*(int32_t *)(ARM_COMMPAGE + 0xfc))
typedef void * (get_tls_fn)(void);
#define ARM_KUSER_GET_TLS (*(get_tls_fn *)(ARM_COMMPAGE + 0xe0))
typedef int (cmpxchg_fn)(int oldval, int newval, volatile int *ptr);
#define ARM_KUSER_CMPXCHG (*(cmpxchg_fn *)(ARM_COMMPAGE + 0xc0))
typedef void (dmb_fn)(void);
#define ARM_KUSER_DMB (*(dmb_fn *)(ARM_COMMPAGE + 0xa0))
typedef int (cmpxchg64_fn)(const int64_t *oldval,
const int64_t *newval,
volatile int64_t *ptr);
#define ARM_KUSER_CMPXCHG64 (*(cmpxchg64_fn *)(ARM_COMMPAGE + 0x60))
#define fail_unless(x) \
do { \
if (!(x)) { \
fprintf(stderr, "FAILED at %s:%d\n", __FILE__, __LINE__); \
exit(EXIT_FAILURE); \
} \
} while (0)
int main(int argc, char *argv[argc])
{
void *kuser_tls;
int val = 1;
const int64_t oldval = 1, newval = 2;
int64_t val64 = 1;
fail_unless(ARM_KUSER_VERSION == 0x5);
kuser_tls = ARM_KUSER_GET_TLS();
printf("TLS = %p\n", kuser_tls);
fail_unless(kuser_tls != 0);
fail_unless(ARM_KUSER_CMPXCHG(1, 2, &val) == 0);
printf("val = %d\n", val);
/* this is a crash test, not checking an actual barrier occurs */
ARM_KUSER_DMB();
fail_unless(ARM_KUSER_CMPXCHG64(&oldval, &newval, &val64) == 0);
printf("val64 = %lld\n", val64);
return 0;
}

View File

@ -32,7 +32,6 @@ class FedoraVM(basevm.BaseVM):
pkgs = [
# tools
'git-core',
'flex', 'bison',
'gcc', 'binutils', 'make',
# perl

View File

@ -38,7 +38,6 @@ class FreeBSDVM(basevm.BaseVM):
"bash",
"gmake",
"gsed",
"flex", "bison",
# libs: crypto
"gnutls",

View File

@ -36,7 +36,6 @@ class NetBSDVM(basevm.BaseVM):
"bash",
"gmake",
"gsed",
"flex", "bison",
# libs: crypto
"gnutls",

View File

@ -35,7 +35,6 @@ class OpenBSDVM(basevm.BaseVM):
"bash",
"gmake",
"gsed",
"bison",
# libs: usb
"libusb1",

View File

@ -52,7 +52,7 @@ class UbuntuX86VM(basevm.BaseVM):
self.ssh_root_check("sed -ie s/^#\ deb-src/deb-src/g /etc/apt/sources.list")
self.ssh_root_check("apt-get update")
self.ssh_root_check("apt-get build-dep -y qemu")
self.ssh_root_check("apt-get install -y libfdt-dev flex bison language-pack-en")
self.ssh_root_check("apt-get install -y libfdt-dev language-pack-en")
self.ssh_root("poweroff")
self.wait()
os.rename(img_tmp, img)