From 5dfa88f7162f390463b227940e84a23af5407744 Mon Sep 17 00:00:00 2001 From: Max Filippov Date: Mon, 17 Sep 2018 11:13:14 -0700 Subject: [PATCH] linux-user: do setrlimit selectively setrlimit guest calls that affect memory resources (RLIMIT_{AS,DATA,STACK}) may interfere with QEMU internal memory management. They may result in QEMU lockup because mprotect call in page_unprotect would fail with ENOMEM error code, causing infinite loop of SIGSEGV. E.g. it happens when running libstdc++ testsuite for xtensa target on x86_64 host. Don't call host setrlimit for memory-related resources. Reviewed-by: Peter Maydell Signed-off-by: Max Filippov Message-Id: <20180917181314.22551-1-jcmvbkbc@gmail.com> [lv: rebase on master] Signed-off-by: Laurent Vivier --- linux-user/syscall.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/linux-user/syscall.c b/linux-user/syscall.c index 019af632df..ae3c0dfef7 100644 --- a/linux-user/syscall.c +++ b/linux-user/syscall.c @@ -7879,7 +7879,21 @@ static abi_long do_syscall1(void *cpu_env, int num, abi_long arg1, rlim.rlim_cur = target_to_host_rlim(target_rlim->rlim_cur); rlim.rlim_max = target_to_host_rlim(target_rlim->rlim_max); unlock_user_struct(target_rlim, arg2, 0); - return get_errno(setrlimit(resource, &rlim)); + /* + * If we just passed through resource limit settings for memory then + * they would also apply to QEMU's own allocations, and QEMU will + * crash or hang or die if its allocations fail. Ideally we would + * track the guest allocations in QEMU and apply the limits ourselves. + * For now, just tell the guest the call succeeded but don't actually + * limit anything. + */ + if (resource != RLIMIT_AS && + resource != RLIMIT_DATA && + resource != RLIMIT_STACK) { + return get_errno(setrlimit(resource, &rlim)); + } else { + return 0; + } } #endif #ifdef TARGET_NR_getrlimit