mirror of https://github.com/proxmox/mirror_qemu
tests/docker: add --registry support to tooling
This allows us to point the tools towards a registry from which they can grab pre-built layers instead of doing everything from scratch each time. To enable this we need to be using the DOCKER_BUILDKIT engine. Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Message-Id: <20200701135652.1366-25-alex.bennee@linaro.org>master
parent
8a8a50a957
commit
e6f1306b10
|
@ -13,6 +13,7 @@ DOCKER_IMAGES := $(sort $(notdir $(basename $(wildcard $(DOCKER_FILES_DIR)/*.doc
|
||||||
DOCKER_TARGETS := $(patsubst %,docker-image-%,$(DOCKER_IMAGES))
|
DOCKER_TARGETS := $(patsubst %,docker-image-%,$(DOCKER_IMAGES))
|
||||||
# Use a global constant ccache directory to speed up repetitive builds
|
# Use a global constant ccache directory to speed up repetitive builds
|
||||||
DOCKER_CCACHE_DIR := $$HOME/.cache/qemu-docker-ccache
|
DOCKER_CCACHE_DIR := $$HOME/.cache/qemu-docker-ccache
|
||||||
|
DOCKER_REGISTRY := $(if $(REGISTRY),$(REGISTRY),registry.gitlab.com/qemu-project/qemu)
|
||||||
|
|
||||||
DOCKER_TESTS := $(notdir $(shell \
|
DOCKER_TESTS := $(notdir $(shell \
|
||||||
find $(SRC_PATH)/tests/docker/ -name 'test-*' -type f))
|
find $(SRC_PATH)/tests/docker/ -name 'test-*' -type f))
|
||||||
|
@ -56,7 +57,9 @@ else
|
||||||
docker-image-%: $(DOCKER_FILES_DIR)/%.docker
|
docker-image-%: $(DOCKER_FILES_DIR)/%.docker
|
||||||
$(call quiet-command,\
|
$(call quiet-command,\
|
||||||
$(DOCKER_SCRIPT) build -t qemu/$* -f $< \
|
$(DOCKER_SCRIPT) build -t qemu/$* -f $< \
|
||||||
$(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \
|
$(if $V,,--quiet) \
|
||||||
|
$(if $(NOCACHE),--no-cache, \
|
||||||
|
$(if $(DOCKER_REGISTRY),--registry $(DOCKER_REGISTRY))) \
|
||||||
$(if $(NOUSER),,--add-current-user) \
|
$(if $(NOUSER),,--add-current-user) \
|
||||||
$(if $(EXTRA_FILES),--extra-files $(EXTRA_FILES))\
|
$(if $(EXTRA_FILES),--extra-files $(EXTRA_FILES))\
|
||||||
$(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\
|
$(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\
|
||||||
|
@ -213,6 +216,7 @@ endif
|
||||||
@echo ' Include extra files in image.'
|
@echo ' Include extra files in image.'
|
||||||
@echo ' ENGINE=auto/docker/podman'
|
@echo ' ENGINE=auto/docker/podman'
|
||||||
@echo ' Specify which container engine to run.'
|
@echo ' Specify which container engine to run.'
|
||||||
|
@echo ' REGISTRY=url Cache builds from registry (default:$(DOCKER_REGISTRY))'
|
||||||
|
|
||||||
# This rule if for directly running against an arbitrary docker target.
|
# This rule if for directly running against an arbitrary docker target.
|
||||||
# It is called by the expanded docker targets (e.g. make
|
# It is called by the expanded docker targets (e.g. make
|
||||||
|
|
|
@ -221,6 +221,13 @@ class Docker(object):
|
||||||
""" Running Docker commands """
|
""" Running Docker commands """
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self._command = _guess_engine_command()
|
self._command = _guess_engine_command()
|
||||||
|
|
||||||
|
if "docker" in self._command and "TRAVIS" not in os.environ:
|
||||||
|
os.environ["DOCKER_BUILDKIT"] = "1"
|
||||||
|
self._buildkit = True
|
||||||
|
else:
|
||||||
|
self._buildkit = False
|
||||||
|
|
||||||
self._instance = None
|
self._instance = None
|
||||||
atexit.register(self._kill_instances)
|
atexit.register(self._kill_instances)
|
||||||
signal.signal(signal.SIGTERM, self._kill_instances)
|
signal.signal(signal.SIGTERM, self._kill_instances)
|
||||||
|
@ -289,10 +296,24 @@ class Docker(object):
|
||||||
return labels.get("com.qemu.dockerfile-checksum", "")
|
return labels.get("com.qemu.dockerfile-checksum", "")
|
||||||
|
|
||||||
def build_image(self, tag, docker_dir, dockerfile,
|
def build_image(self, tag, docker_dir, dockerfile,
|
||||||
quiet=True, user=False, argv=None, extra_files_cksum=[]):
|
quiet=True, user=False, argv=None, registry=None,
|
||||||
|
extra_files_cksum=[]):
|
||||||
if argv is None:
|
if argv is None:
|
||||||
argv = []
|
argv = []
|
||||||
|
|
||||||
|
# pre-calculate the docker checksum before any
|
||||||
|
# substitutions we make for caching
|
||||||
|
checksum = _text_checksum(_dockerfile_preprocess(dockerfile))
|
||||||
|
|
||||||
|
if registry is not None:
|
||||||
|
dockerfile = dockerfile.replace("FROM qemu/",
|
||||||
|
"FROM %s/qemu/" %
|
||||||
|
(registry))
|
||||||
|
# see if we can fetch a cache copy, may fail...
|
||||||
|
pull_args = ["pull", "%s/%s" % (registry, tag)]
|
||||||
|
self._do(pull_args, quiet=quiet)
|
||||||
|
|
||||||
|
|
||||||
tmp_df = tempfile.NamedTemporaryFile(mode="w+t",
|
tmp_df = tempfile.NamedTemporaryFile(mode="w+t",
|
||||||
encoding='utf-8',
|
encoding='utf-8',
|
||||||
dir=docker_dir, suffix=".docker")
|
dir=docker_dir, suffix=".docker")
|
||||||
|
@ -306,15 +327,23 @@ class Docker(object):
|
||||||
(uname, uid, uname))
|
(uname, uid, uname))
|
||||||
|
|
||||||
tmp_df.write("\n")
|
tmp_df.write("\n")
|
||||||
tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
|
tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % (checksum))
|
||||||
_text_checksum(_dockerfile_preprocess(dockerfile)))
|
|
||||||
for f, c in extra_files_cksum:
|
for f, c in extra_files_cksum:
|
||||||
tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c))
|
tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c))
|
||||||
|
|
||||||
tmp_df.flush()
|
tmp_df.flush()
|
||||||
|
|
||||||
self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv +
|
build_args = ["build", "-t", tag, "-f", tmp_df.name]
|
||||||
[docker_dir],
|
if self._buildkit:
|
||||||
|
build_args += ["--build-arg", "BUILDKIT_INLINE_CACHE=1"]
|
||||||
|
|
||||||
|
if registry is not None:
|
||||||
|
cache = "%s/%s" % (registry, tag)
|
||||||
|
build_args += ["--cache-from", cache]
|
||||||
|
build_args += argv
|
||||||
|
build_args += [docker_dir]
|
||||||
|
|
||||||
|
self._do_check(build_args,
|
||||||
quiet=quiet)
|
quiet=quiet)
|
||||||
|
|
||||||
def update_image(self, tag, tarball, quiet=True):
|
def update_image(self, tag, tarball, quiet=True):
|
||||||
|
@ -403,6 +432,8 @@ class BuildCommand(SubCommand):
|
||||||
parser.add_argument("--add-current-user", "-u", dest="user",
|
parser.add_argument("--add-current-user", "-u", dest="user",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
help="Add the current user to image's passwd")
|
help="Add the current user to image's passwd")
|
||||||
|
parser.add_argument("--registry", "-r",
|
||||||
|
help="cache from docker registry")
|
||||||
parser.add_argument("-t", dest="tag",
|
parser.add_argument("-t", dest="tag",
|
||||||
help="Image Tag")
|
help="Image Tag")
|
||||||
parser.add_argument("-f", dest="dockerfile",
|
parser.add_argument("-f", dest="dockerfile",
|
||||||
|
@ -458,7 +489,8 @@ class BuildCommand(SubCommand):
|
||||||
for k, v in os.environ.items()
|
for k, v in os.environ.items()
|
||||||
if k.lower() in FILTERED_ENV_NAMES]
|
if k.lower() in FILTERED_ENV_NAMES]
|
||||||
dkr.build_image(tag, docker_dir, dockerfile,
|
dkr.build_image(tag, docker_dir, dockerfile,
|
||||||
quiet=args.quiet, user=args.user, argv=argv,
|
quiet=args.quiet, user=args.user,
|
||||||
|
argv=argv, registry=args.registry,
|
||||||
extra_files_cksum=cksum)
|
extra_files_cksum=cksum)
|
||||||
|
|
||||||
rmtree(docker_dir)
|
rmtree(docker_dir)
|
||||||
|
|
Loading…
Reference in New Issue