From b38a60cfc5b5dca25a674ebe5887737525aa353f Mon Sep 17 00:00:00 2001 From: Magnus Granberg Date: Wed, 15 Feb 2023 22:30:32 +0100 Subject: Use command shell to build docker images Signed-off-by: Magnus Granberg --- buildbot_gentoo_ci/config/builders.py | 24 +++--- buildbot_gentoo_ci/config/buildfactorys.py | 6 +- buildbot_gentoo_ci/config/schedulers.py | 22 ++--- buildbot_gentoo_ci/config/workers.py | 12 ++- buildbot_gentoo_ci/db/model.py | 1 + buildbot_gentoo_ci/db/workers.py | 3 +- buildbot_gentoo_ci/steps/nodes.py | 121 ++++++++++++++++++++++++---- docker/GentooBuildbotWorkerBuild.Dockerfile | 31 ++++--- master.cfg | 14 +++- sql/gentoo_ci_schema.sql | 14 +++- 10 files changed, 186 insertions(+), 62 deletions(-) diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py index edcd0c1..b3ddb16 100644 --- a/buildbot_gentoo_ci/config/builders.py +++ b/buildbot_gentoo_ci/config/builders.py @@ -13,7 +13,7 @@ def getWorkersLocal(worker_type, workers): worker_list.append(worker['name']) return worker_list -def getWorkersDocker(worker_type, workers): +def getWorkers(worker_type, workers): worker_list = [] for worker in workers: if worker['type'] == worker_type and worker['enable'] is True: @@ -44,11 +44,11 @@ def CanWorkerUpdateV(builder, wfb, request): print('Find a worker: NO') return False -def gentoo_builders(worker_data_local, worker_data_docker): +def gentoo_builders(worker_data): b = [] b.append(util.BuilderConfig( name='update_db_check', - workername=getWorkersLocal('local', worker_data_local)[0], + workername=getWorkersLocal('local', worker_data['local'])[0], workerbuilddir='builds', collapseRequests=False, factory=buildfactorys.update_db_check() @@ -56,7 +56,7 @@ def gentoo_builders(worker_data_local, worker_data_docker): ) b.append(util.BuilderConfig( name='update_repo_check', - workername=getWorkersLocal('local', worker_data_local)[1], + workername=getWorkersLocal('local', worker_data['local'])[1], workerbuilddir='builds', collapseRequests=True, factory=buildfactorys.update_repo_check() @@ -67,7 +67,7 @@ def gentoo_builders(worker_data_local, worker_data_docker): # (builders.UpdateRepos step) b.append(util.BuilderConfig( name='update_cpv_data', - workernames=getWorkersDocker('log', worker_data_docker)[0], + workernames=getWorkers('log', worker_data['docker'])[0], workerbuilddir='builds', collapseRequests=False, factory=buildfactorys.update_db_cpv() @@ -76,7 +76,7 @@ def gentoo_builders(worker_data_local, worker_data_docker): # Use multiplay workers b.append(util.BuilderConfig( name='update_v_data', - workername=getWorkersDocker('log', worker_data_docker)[0], + workername=getWorkers('log', worker_data['docker'])[0], workerbuilddir='builds', collapseRequests=False, canStartBuild=CanWorkerUpdateV, @@ -86,7 +86,7 @@ def gentoo_builders(worker_data_local, worker_data_docker): # Use multiplay workers b.append(util.BuilderConfig( name='build_request_data', - workernames=getWorkersLocal('local', worker_data_local), + workernames=getWorkersLocal('local', worker_data['local']), collapseRequests=False, factory=buildfactorys.build_request_check() ) @@ -94,7 +94,7 @@ def gentoo_builders(worker_data_local, worker_data_docker): # Use multiplay workers b.append(util.BuilderConfig( name='run_build_request', - workernames=getWorkersDocker('build', worker_data_docker), + workernames=getWorkers('build', worker_data['docker']), canStartBuild=CanWorkerBuildProject, collapseRequests=False, factory=buildfactorys.run_build_request() @@ -103,19 +103,19 @@ def gentoo_builders(worker_data_local, worker_data_docker): # Use multiplay workers b.append(util.BuilderConfig( name='parse_build_log', - workernames=getWorkersDocker('log', worker_data_docker)[1:], + workernames=getWorkers('log', worker_data['docker'])[1:], collapseRequests=False, factory=buildfactorys.parse_build_log() ) ) # For node workers b.append(util.BuilderConfig( - name='run_build_stage4_request', - workernames=getWorkersLocal('local', worker_data_local), + name='run_build_images_request', + workernames=getWorkers('node', worker_data['node']), #FIXME: support more the one node #canStartBuild=CanWorkerBuildProject, collapseRequests=False, - factory=buildfactorys.run_build_stage4_request() + factory=buildfactorys.run_build_images_request() ) ) return b diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py index d07a143..efccc93 100644 --- a/buildbot_gentoo_ci/config/buildfactorys.py +++ b/buildbot_gentoo_ci/config/buildfactorys.py @@ -193,10 +193,10 @@ def parse_build_log(): #f.addStep(logs.SetIrcInfo()) return f -def run_build_stage4_request(): +def run_build_images_request(): f = util.BuildFactory() # set needed Propertys f.addStep(nodes.SetupPropertys()) - # set the needed steps for making the stage4 - f.addStep(nodes.SetupStage4Steps()) + # set the needed steps for making the image + f.addStep(nodes.SetupBuildTypeAndSteps()) return f diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py index 60f1857..7833233 100644 --- a/buildbot_gentoo_ci/config/schedulers.py +++ b/buildbot_gentoo_ci/config/schedulers.py @@ -66,23 +66,17 @@ def gentoo_schedulers(): builderNames = builderUpdateDbNames, change_filter=util.ChangeFilter(branch_fn=bb_branch_fn, category_fn=bb_category_fn), ) - create_stage4 = schedulers.ForceScheduler( - name="create_stage4", - buttonName="Create stage4", - label="My nice create stage4 form", - builderNames=['run_build_stage4_request'], + create_images = schedulers.ForceScheduler( + name="create_images", + buttonName="Create images", + label="My nice create images form", + builderNames=['run_build_images_request'], # A completely customized property list. The name of the # property is the name of the parameter properties=[ util.StringParameter(name="project_uuid", label="Project uuid", default="e89c2c1a-46e0-4ded-81dd-c51afeb7fcfd", size=36), - util.StringParameter(name="worker", - label="New worker uuid", - default="a89c2c1a-46e0-4ded-81dd-c51afeb7fcfd", size=36), - util.StringParameter(name="type", - label="type of worker", - default="docker", size=10), ]) update_cpv_data = schedulers.Triggerable(name="update_cpv_data", builderNames=["update_cpv_data"]) @@ -96,10 +90,10 @@ def gentoo_schedulers(): builderNames=["run_build_request"]) parse_build_log = schedulers.Triggerable(name="parse_build_log", builderNames=["parse_build_log"]) - run_build_stage4_request = schedulers.Triggerable(name="run_build_stage4_request", - builderNames=["run_build_stage4_request"]) + run_build_images_request = schedulers.Triggerable(name="run_build_images_request", + builderNames=["run_build_images_request"]) s = [] - s.append(create_stage4) + s.append(create_images) s.append(scheduler_update_db) s.append(update_repo_check) s.append(update_cpv_data) diff --git a/buildbot_gentoo_ci/config/workers.py b/buildbot_gentoo_ci/config/workers.py index c29502e..7d13f46 100644 --- a/buildbot_gentoo_ci/config/workers.py +++ b/buildbot_gentoo_ci/config/workers.py @@ -54,11 +54,15 @@ def GetBuildDockerImage(props, docker_worker): print(image_data) return image_data['tag'] -def gentoo_workers(worker_data_local, worker_data_docker): +def gentoo_workers(worker_data): w = [] - for local_worker in worker_data_local: + for local_worker in worker_data['local']: if local_worker['enable']: w.append(worker.LocalWorker(local_worker['name'])) + # node workers + for node_worker in worker_data['node']: + if node_worker['enable']: + w.append(worker.Worker(node_worker['uuid'], node_worker['password'])) # docker workers docker_hostconfig = {} # For use of sandbox stuff @@ -67,8 +71,8 @@ def gentoo_workers(worker_data_local, worker_data_docker): # libseccomp overhead # https://github.com/seccomp/libseccomp/issues/153 docker_hostconfig['security_opt'] = ['seccomp=unconfined'] - print(worker_data_docker) - for docker_worker in worker_data_docker: + print(worker_data['docker']) + for docker_worker in worker_data['docker']: #FIXME: get settings from db if docker_worker['type'] == 'build': w.append(worker.DockerLatentWorker(docker_worker['uuid'], diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py index a4b5935..4373ebb 100644 --- a/buildbot_gentoo_ci/db/model.py +++ b/buildbot_gentoo_ci/db/model.py @@ -354,6 +354,7 @@ class Model(base.DBConnectorComponent): sa.Column('tag', sa.String(255), nullable=False), sa.Column('dockerfile', sa.String(255), nullable=False), sa.Column('bootstrap_tag', sa.String(255), nullable=False), + sa.Column('type', sa.Enum('docker',), nullable=False), ) workers_configs = sautils.Table( diff --git a/buildbot_gentoo_ci/db/workers.py b/buildbot_gentoo_ci/db/workers.py index d09d88c..16bc4a1 100644 --- a/buildbot_gentoo_ci/db/workers.py +++ b/buildbot_gentoo_ci/db/workers.py @@ -161,7 +161,8 @@ class WorkersConnectorComponent(base.DBConnectorComponent): name=row.name, tag=row.tag, dockerfile=row.dockerfile, - bootstrap_tag=row.bootstrap_tag + bootstrap_tag=row.bootstrap_tag, + type=row.type ) def _row2dict_worker_config(self, conn, row): return dict( diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py index 902f7a8..f92727b 100644 --- a/buildbot_gentoo_ci/steps/nodes.py +++ b/buildbot_gentoo_ci/steps/nodes.py @@ -25,7 +25,7 @@ from buildbot_gentoo_ci.steps import portage as portage_steps from buildbot_gentoo_ci.steps import repos as repos_steps class SetupPropertys(BuildStep): - name = 'Setup propertys for stage4 image' + name = 'Setup propertys for images build' description = 'Running' descriptionSuffix = None haltOnFailure = True @@ -40,14 +40,112 @@ class SetupPropertys(BuildStep): self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] project_data = yield self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_uuid')) self.setProperty('project_data', project_data, 'project_data') - #FIXME: set it in db node config - self.workerbase = yield os.path.join('/', 'srv', 'gentoo', 'stage4') - self.workerdest = yield os.path.join(self.workerbase, self.getProperty('project_uuid')) + #FIXME: set it in node config + self.workerbase = yield os.path.join('/', 'srv', 'gentoo') + #self.workerdest = yield os.path.join(self.workerbase, self.getProperty('project_uuid')) self.setProperty('workerbase', self.workerbase, 'workerbase') - self.setProperty('workerdest', self.workerdest, 'workerdest') - self.setProperty('stage3', 'image', 'stage3') - # we only support docker for now - self.setProperty('type', 'docker', 'type') + #self.setProperty('workerdest', self.workerdest, 'workerdest') + node_info = {} + workers = yield self.gentooci.db.projects.getWorkersByProjectUuid(project_data['uuid']) + print(workers) + for worker in workers: + worker_data = yield self.gentooci.db.workers.getWorkerByUuid(worker['worker_uuid']) + print(worker_data) + node_data = yield self.gentooci.db.workers.getNodeByWorkerUuid(worker['worker_uuid']) + print(node_data) + if node_info == {} or node_info['uuid'] != node_data['node_uuid']: + node_info = yield self.gentooci.db.workers.getNodeByUuid(node_data['node_uuid']) + image_info = yield self.gentooci.db.workers.getImageById(worker_data['image_id']) + print(node_info) + print(image_info) + self.setProperty('node_data', node_info, 'node_data') + self.setProperty('image_data', image_info, 'image_data') + return SUCCESS + +class SetupBuildStepsForDocker(BuildStep): + name = 'Setup build steps for building docker images' + description = 'Running' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @defer.inlineCallbacks + def run(self): + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] + node_data = self.getProperty("node_data") + image_data = self.getProperty("image_data") + dockerfile = self.getProperty("image_data")['dockerfile'] + '.Dockerfile' + dockerfile_path = os.path.join(self.master.basedir, 'docker') + #FIXME: Set build timeout in config + self.build_timeout = 6600 + aftersteps_list = [] + aftersteps_list.append(steps.FileDownload( + flunkOnFailure=True, + name='Upload Dockerfile', + mastersrc = os.path.join(dockerfile_path, dockerfile), + workerdest = dockerfile + )) + # build the gentoo docker buildbot-worker image + # set the build args + self.buildargs = {} + # set PROJECTUUID + self.buildargs['PROJECTUUID'] = self.getProperty("project_uuid") + # set ACCEPT_KEYWORDS + keyword_data = yield self.gentooci.db.keywords.getKeywordById(self.getProperty('project_data')['keyword_id']) + if self.getProperty('project_data')['status'] == 'unstable': + self.buildargs['KEYWORDS'] = '~' + keyword_data['name'] + else: + self.buildargs['KEYWORDS'] = keyword_data['name'] + # set ARCH + self.buildargs['ARCH'] = keyword_data['name'] + # set MAKEOPTS + # set USE + # set CFLAGS + # set BINHOSTURL + self.buildargs['BINHOSTURL'] = node_data['bin_host_url'] + # set bootstrap image + self.buildargs['BOOTSTRAPTAG'] = image_data['bootstrap_tag'] + # build the gentoo docker buildbot-worker image + aftersteps_list.append(steps.ShellCommand( + flunkOnFailure=True, + name='Build docker buildbot-worker image', + command=['docker', + 'buildx', + 'build', + '--file', dockerfile, + '--build-arg', 'PROJECTUUID='+ self.buildargs['PROJECTUUID'], + '--build-arg', 'KEYWORDS=' + self.buildargs['KEYWORDS'], + '--build-arg', 'ARCH=' + self.buildargs['ARCH'], + '--build-arg', 'BINHOSTURL=' + self.buildargs['BINHOSTURL'], + '--build-arg', 'BOOTSTRAPTAG=' + self.buildargs['BOOTSTRAPTAG'], + '--tag', image_data['tag'], + '.'], + timeout=self.build_timeout + )) + if aftersteps_list != []: + yield self.build.addStepsAfterCurrentStep(aftersteps_list) + return SUCCESS + +class SetupBuildTypeAndSteps(BuildStep): + name = 'Check build type for building images' + description = 'Running' + descriptionSuffix = None + haltOnFailure = True + flunkOnFailure = True + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @defer.inlineCallbacks + def run(self): + aftersteps_list = [] + if self.getProperty("image_data")['type'] == 'docker': + aftersteps_list.append(SetupBuildStepsForDocker()) + if aftersteps_list != []: + yield self.build.addStepsAfterCurrentStep(aftersteps_list) return SUCCESS class SetupStage4Steps(BuildStep): @@ -204,13 +302,6 @@ class SetupStage4Steps(BuildStep): command=['docker', 'import', 'stage4-' + self.getProperty('project_uuid') + '.tar', 'stage4-' + self.getProperty('project_uuid') + ':latest'], workdir=self.getProperty("workerbase") )) - # gentoo docker buildbot-worker image - aftersteps_list.append(steps.ShellCommand( - flunkOnFailure=True, - name='Build docker buildbot-worker image', - command=['docker', 'buildx', 'build', '--file', '../docker/GentooBuildbotWorker.Docker', '--build-arg', 'PROJECTUUID=' + self.getProperty('project_uuid'), '--tag', 'bb-worker-' + self.getProperty('project_uuid') + ':latest', '.'], - workdir=self.getProperty("workerbase") - )) if aftersteps_list != []: yield self.build.addStepsAfterCurrentStep(aftersteps_list) return SUCCESS diff --git a/docker/GentooBuildbotWorkerBuild.Dockerfile b/docker/GentooBuildbotWorkerBuild.Dockerfile index bd357dc..23fc3f0 100644 --- a/docker/GentooBuildbotWorkerBuild.Dockerfile +++ b/docker/GentooBuildbotWorkerBuild.Dockerfile @@ -1,16 +1,19 @@ -# This Dockerfile creates a gentoo stage4 container image with buildbot worker installed. +# This Dockerfile creates a gentoo stage4 container image. By default it +# creates a stage4-amd64 image. It utilizes a multi-stage build and requires +# docker-17.05.0 or later. -ARG TAG=latest +ARG BOOTSTRAPTAG=gentoo/stage3:latest FROM gentoo/portage:latest as portage -FROM gentoo/stage3:$TAG +FROM ${BOOTSTRAPTAG} ARG KEYWORDS ARG PROJECTUUID ARG BINHOSTURL ARG ARCH -ARG TAG=latest +ARG BOOTSTRAPTAG=gentoo/stage3:latest +ARG DOCKERFILE ARG NODE # copy the entire portage volume in @@ -25,13 +28,16 @@ RUN emerge -qv dev-vcs/git # Setup portage for the rest of the build RUN echo -e "[binhost]\npriority = 9999\nsync-uri = ${BINHOSTURL}/${PROJECTUUID}/packages/\n" | cat >> /etc/portage/binrepos.conf\ - && echo 'EMERGE_DEFAULT_OPTS="--buildpkg=y --rebuild-if-new-rev=y --rebuilt-binaries=y --usepkg=y --binpkg-respect-use=n --binpkg-changed-deps=y --nospinner --color=n --ask=n"' | cat >> /etc/portage/make.conf\ + && echo 'EMERGE_DEFAULT_OPTS="--rebuild-if-new-rev=y --rebuilt-binaries=y --usepkg=y --nospinner --color=n --ask=n"' | cat >> /etc/portage/make.conf\ && echo 'FEATURES="${FEATURES} parallel-fetch -merge-sync "' | cat >> /etc/portage/make.conf\ && echo 'FEATURES="${FEATURES} getbinpkg"' | cat >> /etc/portage/make.conf\ && echo 'COMMON_FLAGS="${COMMON_FLAGS} -fno-diagnostics-color -march=native"' | >> /etc/portage/make.conf\ && echo 'USE="X elogind"' | >> /etc/portage/make.conf\ - && echo 'PYTHON_TARGETS="python3_10 python3_9 python3_8 python3_11"' | >> /etc/portage/make.conf\ + # && echo 'PYTHON_TARGETS="python3_10 python3_9 python3_8 python3_11"' | >> /etc/portage/make.conf\ && echo 'ACCEPT_LICENSE="*"'\ + #&& echo 'dev-vcs/git -webdev -gnome-keyring' | cat >> /etc/portage/package.use/git\ + #&& echo 'dev-util/buildbot-worker' | cat >> /etc/portage/package.accept_keywords/buildbot\ + #&& echo 'dev-libs/glib' | cat >> /etc/portage/package.mask/git\ && ( sed -i -e 's/#rc_sys=""/rc_sys="docker"/g' /etc/rc.conf 2>/dev/null || true ) \ && echo 'UTC' > /etc/timezone @@ -59,10 +65,15 @@ RUN echo 'app-text/ansifilter' | cat >> /var/lib/portage/world\ && echo 'app-editors/nano' | cat >> /var/lib/portage/world # Run emerge update and install buildbot-worker +# update perl RUN emerge -1qv perl\ - && perl-cleaner --all\ - && emerge -qv --update --deep --newuse @world\ - && emerge -qv gentoo-kernel-bin\ + && perl-cleaner --all +# update openssl +RUN emerge -1qv openssl python +# update system +RUN emerge -uDNqv --deep @world +# install kernel and buildbot-worker +RUN emerge -qv gentoo-kernel-bin\ && emerge -qv buildbot-worker # Rm the gentoo repo @@ -73,7 +84,7 @@ WORKDIR /var/lib/buildbot_worker RUN wget https://raw.githubusercontent.com/buildbot/buildbot/master/worker/docker/buildbot.tac\ && ( sed -i -e 's/umask = None/umask = 0o22/g' buildbot.tac 2>/dev/null || true ) \ && mkdir info\ - && echo "${PROJECTUUID} ${ARCH} stage3:${TAG} ${NODE}" | cat > info/host + && echo "${PROJECTUUID} ${ARCH} ${BOOTSTRAPTAG} ${DOCKERFILE} ${NODE}" | cat > info/host ENTRYPOINT ["/usr/bin/buildbot-worker"] CMD ["start", "--nodaemon"] #CMD ["/bin/bash"] diff --git a/master.cfg b/master.cfg index cc81d10..e8a8b88 100644 --- a/master.cfg +++ b/master.cfg @@ -3,6 +3,7 @@ from buildbot.plugins import util, secrets from buildbot_gentoo_ci.config import schedulers, workers, builders, service, change_source, reporters +#FIXME: Should have uuid worker_data_local = [ {'name' : 'updatedb_1', 'type' : 'local', 'enable' : True, }, {'name' : 'updatedb_2', 'type' : 'local', 'enable' : True, }, @@ -19,6 +20,15 @@ worker_data_docker = [ {'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfc', 'type' : 'build', 'enable' : True, }, {'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfd', 'type' : 'build', 'enable' : True, }, ] +worker_data_node = [ + {'uuid' : 'c89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'type' : 'node', 'enable' : True, 'password' : util.Secret("Node.c89c2c1a-46e0-4ded-81dd-c51afeb7fcfa")}, +] + +worker_data = { + 'local' : worker_data_local, + 'docker' : worker_data_docker, + 'node' : worker_data_node, +} # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. @@ -36,7 +46,7 @@ c['secretsProviders'].append(secrets.SecretInAFile(dirname="/var/lib/buildmaster # The 'workers' list defines the set of recognized workers. Each element is # a Worker object, specifying a unique worker name and password. The same # worker name and password must be configured on the worker. -c['workers'] = workers.gentoo_workers(worker_data_local, worker_data_docker) +c['workers'] = workers.gentoo_workers(worker_data) # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers @@ -65,7 +75,7 @@ c['schedulers'] = schedulers.gentoo_schedulers() # what steps, and which workers can execute them. Note that any particular build will # only take place on one worker. -c['builders'] = builders.gentoo_builders(worker_data_local, worker_data_docker) +c['builders'] = builders.gentoo_builders(worker_data) ####### BUILDBOT SERVICES diff --git a/sql/gentoo_ci_schema.sql b/sql/gentoo_ci_schema.sql index 5956664..d26e0c8 100644 --- a/sql/gentoo_ci_schema.sql +++ b/sql/gentoo_ci_schema.sql @@ -227,6 +227,17 @@ CREATE TYPE public.workers_type AS ENUM ( ALTER TYPE public.workers_type OWNER TO buildbot; +-- +-- Name: workers_images_type; Type: TYPE; Schema: public; Owner: buildbot +-- + +CREATE TYPE public.workers_images_type AS ENUM ( + 'docker' +); + + +ALTER TYPE public.workers_images_type OWNER TO buildbot; + SET default_tablespace = ''; SET default_table_access_method = heap; @@ -879,7 +890,8 @@ CREATE TABLE public.workers_images ( name character varying, tag character varying, dockerfile character varying, - bootstrap_tag character varying + bootstrap_tag character varying, + type public.workers_images_type ); -- cgit v1.2.3-65-gdbad