From ffb4fbf90a2f63c9cb33e4bb9f854c79bf04ca4a Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Mon, 2 Nov 2020 16:52:18 +0000 Subject: [PATCH] tests/qtest/npcm7xx_rng-test: Disable randomness tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The randomness tests in the NPCM7xx RNG test fail intermittently but fairly frequently. On my machine running the test in a loop: while QTEST_QEMU_BINARY=./qemu-system-aarch64 ./tests/qtest/npcm7xx_rng-test; do true; done will fail in less than a minute with an error like: ERROR:../../tests/qtest/npcm7xx_rng-test.c:256:test_first_byte_runs: assertion failed (calc_runs_p(buf.l, sizeof(buf) * BITS_PER_BYTE) > 0.01): (0.00286205989 > 0.01) (Failures have been observed on all 4 of the randomness tests, not just first_byte_runs.) It's not clear why these tests are failing like this, but intermittent failures make CI and merge testing awkward, so disable running them unless a developer specifically sets QEMU_TEST_FLAKY_RNG_TESTS when running the test suite, until we work out the cause. Signed-off-by: Peter Maydell Reviewed-by: Philippe Mathieu-Daudé Message-id: 20201102152454.8287-1-peter.maydell@linaro.org Reviewed-by: Havard Skinnemoen --- tests/qtest/npcm7xx_rng-test.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/qtest/npcm7xx_rng-test.c b/tests/qtest/npcm7xx_rng-test.c index da6e639bf6..e7cde85fbb 100644 --- a/tests/qtest/npcm7xx_rng-test.c +++ b/tests/qtest/npcm7xx_rng-test.c @@ -265,10 +265,16 @@ int main(int argc, char **argv) qtest_add_func("npcm7xx_rng/enable_disable", test_enable_disable); qtest_add_func("npcm7xx_rng/rosel", test_rosel); - qtest_add_func("npcm7xx_rng/continuous/monobit", test_continuous_monobit); - qtest_add_func("npcm7xx_rng/continuous/runs", test_continuous_runs); - qtest_add_func("npcm7xx_rng/first_byte/monobit", test_first_byte_monobit); - qtest_add_func("npcm7xx_rng/first_byte/runs", test_first_byte_runs); + /* + * These tests fail intermittently; only run them on explicit + * request until we figure out why. + */ + if (getenv("QEMU_TEST_FLAKY_RNG_TESTS")) { + qtest_add_func("npcm7xx_rng/continuous/monobit", test_continuous_monobit); + qtest_add_func("npcm7xx_rng/continuous/runs", test_continuous_runs); + qtest_add_func("npcm7xx_rng/first_byte/monobit", test_first_byte_monobit); + qtest_add_func("npcm7xx_rng/first_byte/runs", test_first_byte_runs); + } qtest_start("-machine npcm750-evb"); ret = g_test_run();