Basic slave-master stateful encrypted interaction

* Slave can "take" jobs from the master now
    - Asymmetrical encryption is used via GPG
    - models.Slave stores the gpg fingerprint in models.GPGFingerprintField
    - Slave imports the master's GPG key (/slave_api/autotua_master.asc)
        * Currently, Slave registration is manual (./manage.py shell)
    - Slave does fancy encrypted pickle talking (autotua.talk()) :)
        * "Take" jobs via autotua.Jobs().takejob(maintainer, job_name)
    - slave/autotua/crypt/__init__.py:
        * Implements the glue with `gpg` (maybe pygnupg later?)
        * Crypto() object. Has encrypt() and decrypt()
            - Also see autotua.decrypt_if_required()
    - GNUPGHOME for the slave is /var/tmp/autotua
        * => Job().fetch() requires root access (userpriv/sandbox later)

* Phases store state to allow pausing/stopping and resuming of jobs
    - Future feature, not really used ATM
    - Job().everything() has prelim support for "resume"

* Various small bug fixes and tweaks
    - Yes, I know I need to make this stuff more atomic :p

4 files changed, 207 insertions, 12 deletions

diff --git a/slave/autotua/__init__.py b/slave/autotua/__init__.py
index 3e23fc3..5f02947 100644
--- a/slave/autotua/__init__.py
+++ b/slave/autotua/__init__.py
@@ -9,25 +9,94 @@
 import os, shutil, urllib2, atexit
 import os.path as osp
 import cPickle as pickle
-from autotua import fetch, config, sync, chroot, jobuild
+from urllib import urlencode
+from autotua import fetch, config, sync, chroot, jobuild, crypt
+
+def decrypt_if_required(data, crypto):
+    gpg_header = '-----BEGIN PGP MESSAGE-----'
+    if data.split('\n')[0] != gpg_header:
+        return data
+    if not crypto:
+        raise Exception('Encryption selected, but no "crypto"')
+    return crypto.decrypt(data)[0]
+
+def talk(url, data=None, crypto=None, encrypt=False):
+    """
+    Talk to the master server
+    @param url: relative URL to talk to on the master server
+    @type url: string
+
+    @param data: Data to POST to the server
+    @type data: Anything!
+
+    @param encrypt: Whether to encrypt the data to the POSTed
+    @type encrypt: bool
+    """
+    # We not wantz leading '/'
+    if url[0] == '/':
+        url = url[1:]
+    url = urllib2.quote(url)
+    url = '/'.join([config.AUTOTUA_MASTER, url])
+    if data:
+        # ASCII till I figure out str->bytes
+        data = pickle.dumps(data, 0)
+        if encrypt:
+            if not crypto:
+                raise Exception('Encryption selected, but no "crypto"')
+            data = crypto.encrypt(data)
+        data = urlencode({'data': data})
+    data = urllib2.urlopen(url, data).read()
+    data = decrypt_if_required(data, crypto)
+    data = pickle.loads(data)
+    return data
 
 class Jobs:
-    """Interface to jobs on the master server that we can do"""
+    """Interface to jobs on the master server"""
 
     def __init__(self):
-        self.pubkey = ''
+        self.crypto = crypt.Crypto()
+
+    def import_master_pubkey(self):
+        pubkey = 'autotua_master.asc'
+        url = '/'.join([config.AUTOTUA_MASTER, 'slave_api', pubkey])
+        pubkey = urllib2.urlopen(url).read()
+        self.crypto.import_pubkey(pubkey)
 
-    def getjobs(self):
+    def getlist(self, maintainer=None):
         """
         Get a list of jobs
-        (skeleton code atm)
         """
         jobs = []
-        job_list = pickle.load(urllib2.urlopen(config.AUTOTUA_MASTER+'/slave_api/jobs'))
+        url = 'slave_api/'
+        if maintainer:
+            url += '~%s/' % maintainer
+        url += 'jobs/' 
+        job_list = talk(url, crypto=self.crypto)
         for job_data in job_list:
             jobs.append(Job(job_data))
         return jobs
 
+    def getjob(self, maintainer, job_name):
+        """
+        Get a job's data
+        """
+        url = 'slave_api/jobs/~%s/%s' % (maintainer, job_name)
+        job_data = talk(url, crypto=self.crypto)
+        return Job(job_data)
+
+    def takejob(self, maintainer, job_name):
+        """
+        Take a specific job for running
+        """
+        job = self.getjob(maintainer, job_name)
+        url = 'slave_api/slaves/accept/'
+        data = {'maintainer': job.maint, 'name': job.name}
+        ret = talk(url, data, crypto=self.crypto, encrypt=True)
+        if not ret:
+            raise Exception('Unable to register job')
+        print ret
+        return job
+
 class Job:
     """A Job."""
 
@@ -42,6 +111,7 @@ class Job:
         self.atoms = job_data['atoms']
         self.jobuilds = []
         self.chroot = chroot.WorkChroot(self.jobdir, self.stage.filename)
+        self.next_phase = 'fetch'
         atexit.register(self.tidy)
 
     def __repr__(self):
@@ -92,6 +162,7 @@ class Job:
                     rev=self.jobtagerev, scheme="git-export").sync()
         ## Read config, get portage snapshot if required
         #self._fetch_portage_snapshot()
+        self.next_phase = 'prepare'
 
     def prepare(self):
         # Chroot setup needs to be done before parsing jobuilds
@@ -103,6 +174,7 @@ class Job:
             self.jobuilds.append(jobuild.Jobuild(self.jobtagedir, atom))
         print 'Fetch jobuild SRC_URI and hardlink/copy into chroot'
         self._setup_jobfiles()
+        self.next_phase = 'run'
 
     def run(self):
         processor = jobuild.Processor(None, self.chroot)
@@ -110,10 +182,12 @@ class Job:
             processor.jobuild = jbld
             print 'Running jobuild "%s"' % jbld.atom
             processor.run_phase('all')
+        self.next_phase = 'tidy'
 
     def tidy(self):
         print 'Tidying up..'
         self.chroot.tidy()
+        self.next_phase = ''
 
     def clean(self):
         # Tidy up before cleaning
@@ -121,12 +195,19 @@ class Job:
         shutil.rmtree(self.jobdir)
         os.removedirs(osp.join(config.WORKDIR, self.maint))
 
+    def everything(self):
+        while self.next_phase:
+            exec('self.%s()' % self.next_phase)
+        print 'Everything done.'
+
 if __name__ == "__main__":
-    job = Jobs().getjobs()[0]
-    job.fetch()
+    jobs = Jobs()
+    print 'Importing server public key'
+    jobs.import_master_pubkey()
+    print 'Registering sample job "Sample AutotuA job" for running'
+    job = jobs.takejob('test_user', 'Sample AutotuA job')
+    job.next_phase = 'prepare'
     if os.getuid() == 0:
-        job.prepare()
-        job.run()
-        job.tidy()
+        job.everything()
     else:
-        print 'You need to be root to run job.prepare(), job.run() and job.tidy()'
+        print 'You need to be root to run job.everything()'
diff --git a/slave/autotua/config.py b/slave/autotua/config.py
index 11fe625..7ab4267 100644
--- a/slave/autotua/config.py
+++ b/slave/autotua/config.py
@@ -18,8 +18,10 @@ IGNORE_PROXY        = False
 
 LOGFILE             = '/var/log/autotua/slave.log'
 TMPDIR              = '/var/tmp/autotua'
+GPGHOME             = '/var/tmp/autotua/.gnupg'
 
 AUTOTUA_MASTER      = ''
+MASTER_PUBKEY       = 'autotua_master.asc'
 JOBTAGE_URI         = 'git://git.overlays.gentoo.org/proj/jobtage.git'
 
 # Bind mounted inside the chroots for use if defined
diff --git a/slave/autotua/crypt/__init__.py b/slave/autotua/crypt/__init__.py
new file mode 100644
index 0000000..bb97d17
--- /dev/null
+++ b/slave/autotua/crypt/__init__.py
@@ -0,0 +1,111 @@
+# vim: set sw=4 sts=4 et :
+# Copyright: 2008 Gentoo Foundation
+# Author(s): Nirbheek Chauhan <nirbheek.chauhan@gmail.com>
+# License: GPL-3
+#
+
+import subprocess, os
+from .. import config
+
+class Crypto(object):
+    """
+    Data Encrypter/Decrypter
+    """
+
+    def __init__(self, gpghome=config.GPGHOME):
+        """
+        @param gpghome: Home directory for GPG
+        @type gpghome: string
+        """
+        self.gpghome = gpghome
+        self.gpgcmd = 'gpg -a --keyid-format long --trust-model always '
+        self.gpgcmd += '--homedir="%s" ' % self.gpghome
+        if not os.path.exists(self.gpghome+'/secring.gpg'):
+            raise Exception('"%s": Invalid GPG homedir' % self.gpghome)
+
+    def _get_fp_from_keyid(self, keyid):
+        gpg_args = '--with-colons --fingerprint --list-keys "%s"' % keyid
+        process = subprocess.Popen(self.gpgcmd+gpg_args, shell=True,
+                                   stdout=subprocess.PIPE)
+        output = process.stdout.readlines()
+        process.wait()
+        for line in output:
+            # Fingerprint line
+            if line.startswith('fpr'):
+                # Fingerprint
+                return line.split(':')[-2]
+
+    def export_pubkey(self, file, which):
+        gpg_args = '--export "%s" > "%s"' % (which, file)
+        print self.gpgcmd+gpg_args
+        subprocess.check_call(self.gpgcmd+gpg_args, shell=True)
+
+    def import_pubkey(self, pubkey):
+        gpg_args = '--import <<<"%s"' % pubkey
+        subprocess.check_call(self.gpgcmd+gpg_args, shell=True)
+
+    def init_gpghome(self, name='Test AutotuA Slave', email='test_slave@test.org',
+                     expire='5y', length='4096'):
+        """
+        Initialize a GnuPG home by generating keys
+        """
+        params = (('Key-Type', 'DSA'),
+                  ('Key-Length', '1024'),
+                  ('Subkey-Type', 'ELG-E'),
+                  ('Subkey-Length', length),
+                  ('Name-Real', name),
+                  ('Name-Email', email),
+                  ('Expire-Date', expire),)
+        # Batch mode
+        gpg_args = '--batch --gen-key <<<"'
+        gpg_args += '%echo Generating keys.. [Worship the gods of randomness :p]'
+        for param in params:
+            gpg_args += '\n%s: %s' % param
+        gpg_args += '"'
+        subprocess.check_call(self.gpgcmd+gpg_args, shell=True)
+        print 'Done.'
+
+    def encrypt(self, data, recipient='Autotua Master'):
+        """
+        @param data: Data to be encrypted
+        @type data: string
+
+        @param recipient: Recipient for the data
+        @type recipient: string
+
+        returns: encrypted_data
+        """
+        gpg_args = '--encrypt --sign --recipient "%s" <<<"%s"' % (recipient, data)
+        process = subprocess.Popen(self.gpgcmd+gpg_args, shell=True,
+                                   stdout=subprocess.PIPE)
+        data = process.stdout.read()
+        process.wait()
+        if process.returncode < 0:
+            raise Exception('Unable to encrypt, something went wrong :(')
+        return data
+
+    def decrypt(self, data):
+        """
+        @param data: Data to be encrypted
+        @type data: string
+
+        returns: (decrypted_data, sender)
+        """
+        gpg_args = '--decrypt <<<"%s"' % data
+        process = subprocess.Popen(self.gpgcmd+gpg_args, shell=True,
+                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        ddata = process.stdout.read()[:-1] # Extra \n at the end :-/
+        # Get the output to stderr
+        gpg_out = process.stderr.readlines()
+        process.wait()
+        if process.returncode < 0 or not ddata:
+            raise Exception('Unable to decrypt, something went wrong')
+        # Get the line with the DSA long key ID
+        for line in gpg_out:
+            if line.find('using DSA key') != -1:
+                # Get the long key ID
+                gpg_out = line.split()[-1]
+                break
+        # Get the fingerprint
+        sender = self._get_fp_from_keyid(gpg_out)
+        return (ddata, sender)
diff --git a/slave/config/slave.cfg b/slave/config/slave.cfg
index dda3226..47216ca 100644
--- a/slave/config/slave.cfg
+++ b/slave/config/slave.cfg
@@ -8,6 +8,7 @@ verbose             = True
 ;ignore_proxy       = False
 ;logfile            = '/var/log/autotua/slave.log'
 ;tmpdir             = '/var/tmp/autotua'
+;gpg_home           = '/var/tmp/autotua/.gnupg'
 # You need to set this if you're running a local AutotuA master
 ;autotua_master     = ''
 ;jobtage_uri        = 'git://git.overlays.gentoo.org/proj/jobtage.git'