path: root/buildbot_gentoo_ci/steps/nodes.py

# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

import os
import re
import json
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry


from portage.versions import catpkgsplit, cpv_getversion
from portage.dep import dep_getcpv, dep_getslot, dep_getrepo

from twisted.internet import defer
from twisted.python import log

from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
from buildbot.process.results import SKIPPED
from buildbot.plugins import steps, util

from buildbot_gentoo_ci.steps import portage as portage_steps
from buildbot_gentoo_ci.steps import repos as repos_steps

class SetupPropertys(BuildStep):
    name = 'Setup propertys for images build'
    description = 'Running'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        # we need project uuid and worker uuid
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        project_data = yield self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_uuid'))
        self.setProperty('project_data', project_data, 'project_data')
        #FIXME: set it in node config
        self.workerbase = yield os.path.join('/', 'srv', 'gentoo')
        #self.workerdest = yield os.path.join(self.workerbase, self.getProperty('project_uuid'))
        self.setProperty('workerbase', self.workerbase, 'workerbase')
        #self.setProperty('workerdest', self.workerdest, 'workerdest')
        node_info = {}
        workers = yield self.gentooci.db.projects.getWorkersByProjectUuid(project_data['uuid'])
        print(workers)
        for worker in workers:
            worker_data = yield self.gentooci.db.workers.getWorkerByUuid(worker['worker_uuid'])
            print(worker_data)
            node_data = yield self.gentooci.db.workers.getNodeByWorkerUuid(worker['worker_uuid'])
            print(node_data)
            if node_info == {} or node_info['uuid'] != node_data['node_uuid']:
                node_info = yield self.gentooci.db.workers.getNodeByUuid(node_data['node_uuid'])
                image_info = yield self.gentooci.db.workers.getImageById(worker_data['image_id'])
                print(node_info)
                print(image_info)
                self.setProperty('node_data', node_info, 'node_data')
                self.setProperty('image_data', image_info, 'image_data')
        return SUCCESS

class SetupBuildStepsForDocker(BuildStep):
    name = 'Setup build steps for building docker images'
    description = 'Running'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        node_data = self.getProperty("node_data")
        image_data = self.getProperty("image_data")
        dockerfile = self.getProperty("image_data")['dockerfile'] + '.Dockerfile'
        dockerfile_path = os.path.join(self.master.basedir, 'docker')
        #FIXME: Set build timeout in config
        self.build_timeout = 6600
        aftersteps_list = []
        aftersteps_list.append(steps.FileDownload(
                        flunkOnFailure=True,
                        name='Upload Dockerfile',
                        mastersrc = os.path.join(dockerfile_path, dockerfile),
                        workerdest = dockerfile
                    ))
        # build the gentoo docker buildbot-worker image
        # set the build args
        self.buildargs = {}
        # set PROJECTUUID
        self.buildargs['PROJECTUUID'] = self.getProperty("project_uuid")
        # set ACCEPT_KEYWORDS
        keyword_data = yield self.gentooci.db.keywords.getKeywordById(self.getProperty('project_data')['keyword_id'])
        if self.getProperty('project_data')['status'] == 'unstable':
            self.buildargs['KEYWORDS'] = '~' + keyword_data['name']
        else:
            self.buildargs['KEYWORDS'] = keyword_data['name']
        # set ARCH
        self.buildargs['ARCH'] = keyword_data['name']
        # set MAKEOPTS
        # set USE
        # set CFLAGS
        # set BINHOSTURL
        self.buildargs['BINHOSTURL'] = node_data['bin_host_url']
        # set bootstrap image
        self.buildargs['BOOTSTRAPTAG'] = image_data['bootstrap_tag']
        # build the gentoo docker buildbot-worker image
        aftersteps_list.append(steps.ShellCommand(
                        flunkOnFailure=True,
                        name='Build docker buildbot-worker image',
                        command=['docker',
                                 'buildx',
                                 'build',
                                 '--file', dockerfile,
                                 '--build-arg', 'PROJECTUUID='+ self.buildargs['PROJECTUUID'],
                                 '--build-arg', 'KEYWORDS=' + self.buildargs['KEYWORDS'],
                                 '--build-arg', 'ARCH=' + self.buildargs['ARCH'],
                                 '--build-arg', 'BINHOSTURL=' + self.buildargs['BINHOSTURL'],
                                 '--build-arg', 'BOOTSTRAPTAG=' + self.buildargs['BOOTSTRAPTAG'],
                                 '--tag', image_data['tag'],
                                 '.'],
                        timeout=self.build_timeout
                    ))
        if aftersteps_list != []:
            yield self.build.addStepsAfterCurrentStep(aftersteps_list)
        return SUCCESS

class SetupBuildTypeAndSteps(BuildStep):
    name = 'Check build type for building images'
    description = 'Running'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        aftersteps_list = []
        if self.getProperty("image_data")['type'] == 'docker':
            aftersteps_list.append(SetupBuildStepsForDocker())
        if aftersteps_list != []:
            yield self.build.addStepsAfterCurrentStep(aftersteps_list)
        return SUCCESS

class SetupStage4Steps(BuildStep):
    name = 'Setup steps for stage4 image'
    description = 'Running'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        self.setProperty('portage_repos_path', self.gentooci.config.project['project']['worker_portage_repos_path'], 'portage_repos_path')
        aftersteps_list = []
        separator = '\n'
        log = yield self.addLog('makeing_stage4')
        if self.getProperty("type") == 'docker':
            print('build this stage4 %s on %s for %s' % (self.getProperty('project_uuid'), self.getProperty('workername'), self.getProperty('project_data')['name']))
            self.descriptionDone = ' '.join(['build this stage4', self.getProperty('project_uuid'), 'on', self.getProperty('workername'), 'for', self.getProperty('project_data')['name']])
            #FIXME: package list should be in the db project
            package_list = ['dev-vcs/git', 'app-text/ansifilter', 'dev-util/pkgcheck', 'dev-lang/rust-bin', 'app-admin/eclean-kernel', 'app-portage/gentoolkit', 'sys-kernel/gentoo-kernel-bin', 'app-editors/nano']
            if 'systemd' or 'openrc' in self.getProperty('project_data')['image']:
                workerdest = yield os.path.join(self.getProperty("workerbase"), self.getProperty('project_uuid'))
                workerdest_etc = yield os.path.join(workerdest, 'etc')
                print(workerdest_etc)
                self.setProperty('rootworkdir', workerdest, 'rootworkdir')
                # create dir
                aftersteps_list.append(steps.ShellCommand(
                        flunkOnFailure=True,
                        name='Create stage4 dir',
                        command=['mkdir', self.getProperty('project_uuid')],
                        workdir=self.getProperty("workerbase")
                        ))
                # download stage3
                aftersteps_list.append(GetSteg3())
                # setup portage
                aftersteps_list.append(repos_steps.UpdateRepos())
                aftersteps_list.append(portage_steps.SetReposConf())
                aftersteps_list.append(portage_steps.SetMakeConf())
                # add localegen
                #FIXME: set that in config
                locale_conf = []
                locale_conf.append('en_US.UTF-8 UTF-8')
                locale_conf.append('en_US ISO-8859-1')
                locale_conf.append('C.UTF8 UTF-8')
                locale_conf_string = separator.join(locale_conf)
                aftersteps_list.append(
                    steps.StringDownload(locale_conf_string + separator,
                                workerdest="locale.gen",
                                workdir=self.getProperty("workerdest") + '/etc'
                                ))
                yield log.addStdout('File: ' + 'locale.gen' + '\n')
                for line in locale_conf:
                    yield log.addStdout(line + '\n')
                aftersteps_list.append(
                    steps.StringDownload('LANG="en_US.utf8"' + separator,
                                workerdest="locale.conf",
                                workdir=self.getProperty("workerdest") + '/etc'
                                ))
                yield log.addStdout('Setting LANG to: ' + 'en_US.utf8' + '\n')
                aftersteps_list.append(SetSystemdNspawnConf())
                # run localgen
                aftersteps_list.append(steps.ShellCommand(
                    flunkOnFailure=True,
                    name='Run locale-gen on the chroot',
                    command=['systemd-nspawn', '-D', self.getProperty('project_uuid'), 'locale-gen'],
                    workdir=self.getProperty("workerbase")
                    ))
                # update timezone
                # add the world file
                package_list_string = separator.join(package_list)
                aftersteps_list.append(
                    steps.StringDownload(package_list_string + separator,
                                workerdest="var/lib/portage/world",
                                workdir=self.getProperty("workerdest")
                                ))
                # update container
                aftersteps_list.append(steps.ShellCommand(
                    flunkOnFailure=True,
                    name='Run update on the chroot',
                    command=['systemd-nspawn', '-D', self.getProperty('project_uuid'), 'emerge', '--update', '--deep', '--newuse', '@world'],
                    workdir=self.getProperty("workerbase")
                    ))
                # install buildbot-worker
                aftersteps_list.append(steps.ShellCommand(
                    flunkOnFailure=True,
                    name='Install buildbot worker on the chroot',
                    command=['systemd-nspawn', '-D', self.getProperty('project_uuid'), 'emerge', 'buildbot-worker'],
                    workdir=self.getProperty("workerbase")
                    ))
                if self.getProperty("type") == 'docker':
                    # copy docker_buildbot.tac to worker dir
                    buildbot_worker_config_file = yield os.path.join(self.master.basedir, 'files', 'docker_buildbot_worker.tac')
                    aftersteps_list.append(steps.FileDownload(
                        flunkOnFailure=True,
                        name='Upload buildbot worker config to the stage4',
                        mastersrc=buildbot_worker_config_file,
                        workerdest='var/lib/buildbot_worker/buildbot.tac',
                        workdir=self.getProperty("workerdest")
                    ))
                # add info to the buildbot worker
                worker_info_list = []
                worker_info_list.append(self.getProperty('project_data')['name'])
                worker_info_list.append(self.getProperty("stage3"))
                #FIXME: worker name of self.getProperty('workername') from node table
                worker_info_list.append('node1')
                print(worker_info_list)
                worker_info = ' '.join(worker_info_list)
                aftersteps_list.append(steps.StringDownload(
                    worker_info + separator,
                    workerdest='var/lib/buildbot_worker/info/host',
                    workdir=self.getProperty("workerdest")
                ))
                #FIXME: add admin info
                # depclean
                aftersteps_list.append(steps.ShellCommand(
                    flunkOnFailure=True,
                    name='Depclean on the chroot',
                    command=['systemd-nspawn', '-D', self.getProperty('project_uuid'), 'emerge', '--depclean'],
                    workdir=self.getProperty("workerbase")
                    ))
                # remove the gentoo repo
                #aftersteps_list.append(steps.ShellCommand(
                #    flunkOnFailure=True,
                #    name='Remove the repo dir',
                #    command=['rm', '-R', self.getProperty('project_uuid') + '/var/db/repos/gentoo'],
                #    workdir=self.getProperty("workerbase")
                #))
                # compress it
                aftersteps_list.append(steps.ShellCommand(
                    flunkOnFailure=True,
                    name='Compress the stage4',
                    command=['tar', '-cf', '../stage4-' + self.getProperty('project_uuid') + '.tar', '.'],
                    workdir=self.getProperty("workerbase") + '/' + self.getProperty('project_uuid')
                ))
                # signing the stage4
                # remove the dir
                aftersteps_list.append(steps.ShellCommand(
                    flunkOnFailure=True,
                    name='Remove the stage4 dir',
                    command=['rm', '-R', self.getProperty('project_uuid')],
                    workdir=self.getProperty("workerbase")
                ))
                # build docker stage4 image and buildbot-worker image
                # FIXME: Use the python docker api
                # FIXME: add date tags
                if self.getProperty("type") == 'docker':
                    aftersteps_list.append(steps.ShellCommand(
                        flunkOnFailure=True,
                        name='Build docker stage4 image',
                        command=['docker', 'import', 'stage4-' + self.getProperty('project_uuid') + '.tar', 'stage4-' + self.getProperty('project_uuid') + ':latest'],
                        workdir=self.getProperty("workerbase")
                    ))
        if aftersteps_list != []:
            yield self.build.addStepsAfterCurrentStep(aftersteps_list)
        return SUCCESS

class GetSteg3(BuildStep):
    name = 'Get the steg3 image'
    description = 'Running'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        compresstype = '.tar.xz'
        downloadbaseurl = 'https://gentoo.osuosl.org/releases'
        #downloadbaseurl = 'https://bouncer.gentoo.org/fetch/root/all/releases'
        aftersteps_list = []
        project_data = self.getProperty('project_data')
        project_keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
        keyword = project_keyword_data['name']
        # if the stage3 end with latest we neeed to get the date
        if project_data['image'].endswith('latest'):
            stage3latest = 'latest' + '-' + project_data['image'].replace('-latest', '') + '.txt'
            downloadurllatest = '/'.join([downloadbaseurl, keyword, 'autobuilds', stage3latest])
            print(downloadurllatest)
            session = requests.Session()
            retry = Retry(connect=3, backoff_factor=0.5)
            adapter = HTTPAdapter(max_retries=retry)
            session.mount('http://', adapter)
            session.mount('https://', adapter)
            r = session.get(downloadurllatest, timeout=5)
            if r.status_code == requests.codes.ok:
                urltext = r.text.split('\n')[2]
                print(urltext)
                date = urltext.split('/')[0]
                image = urltext.split('/')[1].split(' ')[0]
            else:
                r.raise_for_status()
                return FAILURE
        else:
            image = project_data['image'] + compresstype
            date = project_data['image'].split('-')[-1]
        self.descriptionDone = image
        self.setProperty('stage3', image, 'stage3')
        downloadurlstage3 = '/'.join([downloadbaseurl, keyword, 'autobuilds', date, image])
        print(downloadurlstage3)
        aftersteps_list.append(steps.ShellCommand(
                        flunkOnFailure=True,
                        name='Download the stage3 file',
                        command=['wget', '-N', '-nv', downloadurlstage3],
                        workdir=self.getProperty("workerbase")
                        ))
        aftersteps_list.append(steps.ShellCommand(
                        flunkOnFailure=True,
                        name='Download the stage3 DIGESTS file',
                        command=['wget', '-N', '-nv', downloadurlstage3 + '.asc'],
                        workdir=self.getProperty("workerbase")
                        ))
        #FIXME: validate the stage3 file
        aftersteps_list.append(steps.ShellCommand(
                        flunkOnFailure=True,
                        name='Unpack the stage3 file',
                        command=['tar', 'xpf', image, '--xattrs-include=\'*.*\'', '--numeric-owner', '-C',  self.getProperty('project_uuid')],
                        workdir=self.getProperty("workerbase")
                        ))
        if aftersteps_list != []:
            yield self.build.addStepsAfterCurrentStep(aftersteps_list)
        return SUCCESS

class SetSystemdNspawnConf(BuildStep):

    name = 'SetSystemdNspawnConf'
    description = 'Running'
    descriptionDone = 'Ran'
    descriptionSuffix = None
    haltOnFailure = True
    flunkOnFailure = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @defer.inlineCallbacks
    def run(self):
        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
        nspawn_conf_path = '/etc/systemd/nspawn/'
        log = yield self.addLog(self.getProperty('project_uuid') + '.nspawn')
        #FIXME: set it in config
        separator = '\n'
        nspawn_conf = []
        nspawn_conf.append('[Files]')
        nspawn_conf.append('TemporaryFileSystem=/run/lock')
        # db node config portage cache bind
        src_dir = '/srv/gentoo/portage/' + self.getProperty('project_uuid')
        dest_dir = '/var/cache/portage'
        nspawn_conf.append('Bind=' + src_dir + '/distfiles' + ':' + dest_dir + '/distfiles')
        nspawn_conf.append('Bind=' + src_dir + '/packages' + ':' + dest_dir + '/packages')
        nspawn_conf.append('[Exec]')
        nspawn_conf.append('Capability=CAP_NET_ADMIN')
        nspawn_conf.append('[Network]')
        nspawn_conf.append('VirtualEthernet=no')
        nspawn_conf_string = separator.join(nspawn_conf)
        yield self.build.addStepsAfterCurrentStep([
            steps.StringDownload(nspawn_conf_string + separator,
                                workerdest=self.getProperty('project_uuid') + '.nspawn',
                                workdir=nspawn_conf_path)
            ])
        yield log.addStdout('File: ' + self.getProperty('project_uuid') + '.nspawn' + '\n')
        for line in nspawn_conf:
            yield log.addStdout(line + '\n')
        return SUCCESS