# vim: set sw=4 sts=4 et : # Copyright: 2008 Gentoo Foundation # Author(s): Nirbheek Chauhan # License: GPL-3 # # Immortal lh! # import os, shutil, urllib2, atexit import os.path as osp import cPickle as pickle from urllib import urlencode from autotua import fetch, config, sync, chroot, jobuild, crypt def decrypt_if_required(data, crypto): gpg_header = '-----BEGIN PGP MESSAGE-----' if data.split('\n')[0] != gpg_header: return data if not crypto: raise Exception('Encryption selected, but no "crypto"') return crypto.decrypt(data)[0] def talk(url, data=None, crypto=None, encrypt=False): """ Talk to the master server @param url: relative URL to talk to on the master server @type url: string @param data: Data to POST to the server @type data: Anything! @param encrypt: Whether to encrypt the data to the POSTed @type encrypt: bool """ # We not wantz leading '/' if url[0] == '/': url = url[1:] url = urllib2.quote(url) url = '/'.join([config.AUTOTUA_MASTER, url]) if data: # ASCII till I figure out str->bytes data = pickle.dumps(data, 0) if encrypt: if not crypto: raise Exception('Encryption selected, but no "crypto"') data = crypto.encrypt(data) data = urlencode({'data': data}) data = urllib2.urlopen(url, data).read() data = decrypt_if_required(data, crypto) data = pickle.loads(data) return data class Jobs: """Interface to jobs on the master server""" def __init__(self): self.crypto = crypt.Crypto() def import_master_pubkey(self): pubkey = 'autotua_master.asc' url = '/'.join([config.AUTOTUA_MASTER, 'slave_api', pubkey]) pubkey = urllib2.urlopen(url).read() self.crypto.import_pubkey(pubkey) def getlist(self, maintainer=None): """ Get a list of jobs """ jobs = [] url = 'slave_api/' if maintainer: url += '~%s/' % maintainer url += 'jobs/' job_list = talk(url, crypto=self.crypto) for job_data in job_list: jobs.append(Job(job_data)) return jobs def getjob(self, maintainer, job_name): """ Get a job's data """ url = 'slave_api/jobs/~%s/%s' % (maintainer, job_name) job_data = talk(url, crypto=self.crypto) return Job(job_data) def takejob(self, maintainer, job_name): """ Take a specific job for running """ job = self.getjob(maintainer, job_name) url = 'slave_api/slaves/accept/' data = {'maintainer': job.maint, 'name': job.name} ret = talk(url, data, crypto=self.crypto, encrypt=True) if not ret: raise Exception('Unable to register job') print ret return job class Job: """A Job.""" def __init__(self, job_data): self.maint_details = job_data['maintainer'] self.maint = job_data['maintainer']['username'] self.name = job_data['name'] self.stage = fetch.Fetchable(uri=[job_data['stage']]) self.jobdir = osp.join(config.WORKDIR, self.maint, self.name) self.jobtagedir = osp.join(self.jobdir, 'jobtage') self.jobtagerev = job_data['jobtagerev'] self.atoms = job_data['atoms'] self.jobuilds = [] self.chroot = chroot.WorkChroot(self.jobdir, self.stage.filename) self.next_phase = 'fetch' atexit.register(self.tidy) def __repr__(self): return '<%s: %s>' % (self.name, 'Job object') def __str__(self): return '%s object' % self.name # Get jobuild SRC_URI -> $tmpdir/jobfiles # jobfile -(link)-> $chroot_tmpdir/jobfiles def _setup_jobfiles(self): job_src = [] action = 'link' fetcher = fetch.Fetcher(config.JOBFILE_DIR) processor = jobuild.Processor(None, self.chroot) for jbld in self.jobuilds: processor.jobuild = jbld src_uri = processor.get_var('SRC_URI').split() for i in src_uri: job_src.append(fetch.Fetchable(uri=[i])) src = fetcher.tarballdir+'/%s' dest = self.chroot.chrootdir+config.CHAUTOTUA_DIR+'/jobfiles/%s' for fetchable in job_src: fetcher.fetch(fetchable) src = src % fetchable.filename dest = dest % fetchable.filename if action == 'link': try: os.link(src, dest) except OSError: print "Chroot and Jobfiles are on different devices. Falling back to copying..." action = 'copy' if action == 'copy': shutil.copyfile(src, dest) def fetch(self): # Job metadata stuff ## Get stage3 (if required) print 'Fetching stage...' fetcher = fetch.Fetcher(config.STAGE_DIR) fetcher.fetch(self.stage) # Sync jobtage tree print 'Syncing jobtage tree...' sync.Syncer().sync() # Export from local jobtage tree print 'Exporting jobtage tree...' sync.Syncer(uri=config.JOBTAGE_DIR, destdir=self.jobtagedir, rev=self.jobtagerev, scheme="git-export").sync() ## Read config, get portage snapshot if required #self._fetch_portage_snapshot() self.next_phase = 'prepare' def prepare(self): # Chroot setup needs to be done before parsing jobuilds # because all parsing is done inside the chroot print 'Setup the chroot for usage...' self.chroot.setup() # Create jobuild objects for parsing for atom in self.atoms.split(): self.jobuilds.append(jobuild.Jobuild(self.jobtagedir, atom)) print 'Fetch jobuild SRC_URI and hardlink/copy into chroot' self._setup_jobfiles() self.next_phase = 'run' def run(self): processor = jobuild.Processor(None, self.chroot) for jbld in self.jobuilds: processor.jobuild = jbld print 'Running jobuild "%s"' % jbld.atom processor.run_phase('all') self.next_phase = 'tidy' def tidy(self): print 'Tidying up..' self.chroot.tidy() self.next_phase = '' def clean(self): # Tidy up before cleaning self.tidy() shutil.rmtree(self.jobdir) os.removedirs(osp.join(config.WORKDIR, self.maint)) def everything(self): while self.next_phase: exec('self.%s()' % self.next_phase) print 'Everything done.' if __name__ == "__main__": jobs = Jobs() print 'Importing server public key' jobs.import_master_pubkey() print 'Registering sample job "Sample AutotuA job" for running' job = jobs.takejob('test_user', 'Sample AutotuA job') job.next_phase = 'prepare' if os.getuid() == 0: job.everything() else: print 'You need to be root to run job.everything()'