Merge remote branches 'aheadley/rpm' and 'aheadley/clean-up'

Author: agrif

worker.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 import builder
 import uploader
 import sys
@@ -12,48 +14,55 @@
 import subprocess
 import urllib
 import urllib2
+import logging
 
 try:
     import gearman
 except ImportError:
     sys.path.append(r"c:\devel\python-gearman")
     import gearman
 
-secret_key = None
-if 'SECRET_KEY' in os.environ:
-    secret_key = os.environ['SECRET_KEY'].strip()
-else:
-    secret_key_path = os.path.split(sys.argv[0])[0]
-    secret_key_path = os.path.join(secret_key_path, 'secret_key.txt')
-    try:
-        with open(secret_key_path) as f:
-            secret_key = f.readline().strip()
-    except:
-        print "You must create the file `%s'" % (secret_key_path,)
-        print "and fill it with the build system password on the top line."
-        print "(or put it in the SECRET_KEY environment variable.)"
-        sys.exit(1)
+logger = logging.getLogger("Worker")
+logger.setLevel(logging.DEBUG)
+logging_handler = logging.StreamHandler()
+logger.addHandler(logging_handler)
+
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+logging_handler.setFormatter(formatter)
+
+try:
+    import worker_config
+except ImportError, e:
+    logger.exception('Error importing worker config file')
+    sys.exit(1)
 
 #upload = uploader.S3Uploader()
 upload = uploader.OverviewerOrgUploader()
-gm_worker = gearman.GearmanWorker(["192.168.1.4:9092", "em32.net:9092"])
+gm_worker = gearman.GearmanWorker(worker_config.gearman_hosts)
 
 def package_hook(platform, repo, checkout, commit, version, url):
-    hook = "http://overviewer.org/hooks/package"
-    data = urllib.urlencode({'platform':platform, 'repo':repo, 'checkout':checkout, 'commit':commit, 'version':version, 'url':url})
-    key = urllib.urlencode({'key':secret_key})
-    
+    hook = worker_config.package_hook_url
+    data = urllib.urlencode({
+        'platform': platform,
+        'repo': repo,
+        'checkout': checkout,
+        'commit': commit,
+        'version': version,
+        'url': url,
+    })
+    key = urllib.urlencode({'key': worker_config.secret_key})
+
     try:
         f = urllib2.urlopen(hook + '?' + key, data)
         f.read()
         f.close()
     except Exception, e:
-        print "could not do POST hook:", e
+        logger.exception('could not do POST hook')
         pass # couldn't contact the server, ack!
 
 def signAndPickle(d):
     data = cPickle.dumps(d)
-    h = hmac.new(secret_key, data, hashlib.sha256)
+    h = hmac.new(worker_config.secret_key, data, hashlib.sha256)
     return h.digest() + data
 
 def uploadLogs(b, result):
@@ -62,49 +71,56 @@ def uploadLogs(b, result):
 
     err_log = "build_logs/%s.stderr.txt" % now
     result['build_log_stderr'] = upload.upload(err_log, b.stderr_log[1])
-    
+
     out_log = "build_logs/%s.stdout.txt" % now
     result['build_log_stdout'] = upload.upload(out_log, b.stdout_log[1])
 
 def build(worker, job):
-    print "got a job!"
+    logger.info('got a job!')
     result = dict(status=None)
 
     if len(job.data) < 33:
-        result['status'] = 'ERROR'
-        print "ERROR:  job data is not valid.  too short"
-        result['msg'] = "ERROR:  job data is not valid.  too short"
+        status = 'ERROR'
+        msg = 'Job data is not valid: too short'
+        result['status'] = status
+        result['msg'] = '%s: %s' % (status, msg)
+        logger.error(msg)
         return signAndPickle(result)
 
     received_h = job.data[0:32]
     # calc hmac of the resulting data
     data = job.data[32:]
-    h = hmac.new(secret_key, data, hashlib.sha256)
+    h = hmac.new(worker_config.secret_key, data, hashlib.sha256)
     if (h.digest() != received_h):
-        result['status'] = 'ERROR'
-        print "ERROR:  job data is not valid.  bad signature"
-        result['msg'] = "ERROR:  job data is not valid.  bad signature"
+        status = 'ERROR'
+        msg = 'Job data is not valid: bad signature'
+        result['status'] = status
+        logger.error(msg)
+        result['msg'] = '%s: %s' % (status, msg)
         return signAndPickle(result)
 
-    print "Data is valid, doing to depickle"
+    logger.info('Data is valid, depickling')
     try:
         depick = cPickle.loads(data)
-
-    except: 
-        result['status'] = 'ERROR'
-        result['msg'] = "ERROR: can't depickle"
-        print "ERROR: can't depickle"
+    except:
+        status = 'ERROR'
+        msg = 'Can\'t depickle'
+        result['status'] = status
+        result['msg'] = '%s: %s' % (status, msg)
+        logger.error(msg)
         return signAndPickle(result)
 
     if type(depick) != dict:
-        result['status'] = 'ERROR'
-        result['msg'] = "ERROR: input must be a dictionary"
-        print "ERROR: input must be a dictionary"
+        status = 'ERROR'
+        msg = 'Input must be a dictionary'
+        result['status'] = status
+        result['msg'] = '%s: %s' % (status, msg)
+        logger.error(msg)
         return signAndPickle(result)
 
 
     defaults = depick
-    print defaults
+    logger.debug(defaults)
 
     platform = job.task.split("_", 1)[1]
     b = builder.Builder.builders[platform](**defaults)
@@ -120,20 +136,21 @@ def build(worker, job):
     except:
         result['status'] = 'ERROR'
         result['msg'] = 'Error in either the clone or the checkout'
-        uploadLogs(b, result) 
+        logger.exception('Error in either the clone or checkout')
+        uploadLogs(b, result)
         return signAndPickle(result)
 
     zipname = b.filename()
-    print "zipname -->%s<--" % zipname
-    
+    logger.debug('zipname: %s' % zipname)
+
     # before we take the time to build, first see if a copy of this
     # already exists on S3:
     if upload.check_exists(zipname):
         result['status'] = 'SUCCESS'
         result['built'] = False
-        print "found a copy already!"
+        logger.info('zipname already exists on upload target')
         result['url'] = upload.get_url(zipname)
-        uploadLogs(b, result) 
+        uploadLogs(b, result)
         return signAndPickle(result)
 
     try:
@@ -143,55 +160,56 @@ def build(worker, job):
 
         b.post_build()
     except:
-        print "something failed"
+        logger.exception('build failed')
         result['status'] = 'ERROR'
         result['msg'] = "ERROR: build failed.  Check the build logs for info"
-        uploadLogs(b, result) 
+        uploadLogs(b, result)
         return signAndPickle(result)
 
 
     archive = b.package()
     worker.send_job_status(job, 3 + num_phases, 4 + num_phases)
-    print "archive: -->%s<--" % archive
-    print "done!"
+    logger.debug('archive: %s' % archive)
+    logger.info('Build done!')
 
     # upload
     try:
         url = upload.upload(zipname, archive)
         result['status'] = 'SUCCESS'
         result['built'] = True
         result['url'] = url
-        uploadLogs(b, result) 
-        
-        package_hook(platform, defaults.get('repo', ''), defaults.get('checkout', 'master'), b.getCommit(), b.getVersion(), url)
-        
+        uploadLogs(b, result)
+
+        package_hook(platform, defaults.get('repo', ''),
+            defaults.get('checkout', 'master'), b.getCommit(),
+            b.getVersion(), url)
+
         return signAndPickle(result)
 
     except:
-        print "failed to upload to S3"
-        traceback.print_exc()
+        logger.exception('Failed to upload to S3')
         result['status'] = 'ERROR'
         result['msg'] = "ERROR: failed to upload to S3"
         return signAndPickle(result)
 
 if __name__ == "__main__":
     if not builder.Builder.builders:
-        print "no supported builders found, exiting..."
+        logger.error('No supported builders found, exiting')
         sys.exit(1)
-    
+
     this_plat = builder.Builder.builders.keys()[0]
+    logger.info('Identifying as %s_worker', this_plat)
     gm_worker.set_client_id("%s_worker" % this_plat)
     for platform in builder.Builder.builders:
+        logger.info('Register builder: %s' % platform)
         gm_worker.register_task("build_%s" % platform, build)
 
     while(1):
-        print "Starting worker for %s" % this_plat
+        logger.info('Starting worker')
         try:
             gm_worker.work()
         except gearman.errors.ServerUnavailable:
-            print "Server disconnected.  Trying again"
-
-    sys.exit(0)
-
-
+            logger.warning('Server disconnected. Trying again')
+            time.sleep(1)
 
+    sys.exit(0)

worker_config.py-example

@@ -0,0 +1,8 @@
+#sssh it's a secret
+secret_key = 'no u'
+
+#list of host:port for gearman hosts to try connecting to
+gearman_hosts = ['192.168.1.4:9092', 'em32.net:9092']
+
+#no idea what this really does
+package_hook_url = 'http://overviewer.org/hooks/package'