Module factory
[hide private]

Source Code for Module factory

  1  #!/usr/bin/env python 
  2  # Officeshots.org - Test your office documents in different applications 
  3  # Copyright (C) 2009 Stichting Lone Wolves 
  4  # Written by Sander Marechal <s.marechal@jejik.com> 
  5  # 
  6  # This program is free software: you can redistribute it and/or modify 
  7  # it under the terms of the GNU Affero General Public License as 
  8  # published by the Free Software Foundation, either version 3 of the 
  9  # License, or (at your option) any later version. 
 10  # 
 11  # This program is distributed in the hope that it will be useful, 
 12  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 13  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 14  # GNU Affero General Public License for more details. 
 15  # 
 16  # You should have received a copy of the GNU Affero General Public License 
 17  # along with this program.  If not, see <http://www.gnu.org/licenses/>. 
 18   
 19  """ 
 20  This is the core Factory class and main method of the Officeshots 
 21  factory. Execute it with -h or --help to see the available options 
 22  """ 
 23   
 24  import os 
 25  import sys 
 26  import time 
 27  import socket 
 28  import logging 
 29  import ConfigParser 
 30   
 31  from optparse import OptionParser 
 32  from backends import BackendException 
 33  from xmlrpclib import ServerProxy, Error, Fault 
 34   
 35  LOGLEVELS = {'debug': logging.DEBUG, 
 36               'info': logging.INFO, 
 37               'warning': logging.WARNING, 
 38               'error': logging.ERROR, 
 39               'critical': logging.CRITICAL} 
 40   
41 -class Factory():
42 """ 43 The core factory class communicates with the Officeshots server and passes 44 requests on to any of the available workers 45 """ 46
47 - def configure(self, options):
48 self.options = options 49 self.config = ConfigParser.RawConfigParser() 50 self.config.read(os.path.abspath(self.options.config_file)) 51 52 # Configure logging 53 if self.options.debug: 54 logging.basicConfig(format = self.config.get('global', 'log_format'), level = logging.DEBUG) 55 else: 56 level = LOGLEVELS.get(self.config.get('global', 'log_level'), logging.NOTSET) 57 try: 58 logging.basicConfig( 59 format = self.config.get('global', 'log_format'), 60 filename = self.config.get('global', 'log_file'), 61 filemode = 'a', 62 level = level 63 ) 64 except IOError, e: 65 print "Logfile IO error. Please make sure that the log_file setting is correct in config.ini" 66 sys.exit(1) 67 68 # Load factory name 69 self.name = self.config.get('global', 'factory_name') 70 71 # Configure the XMLRPC proxy 72 transport_name = self.config.get('global', 'transport') 73 transport = self.load('transports.' + transport_name, 'SSLTransport') 74 if transport is None: 75 print "Transport %s could not be loaded" % transport_name 76 sys.exit(1) 77 78 transport = transport( 79 self.config.get('global', 'tls_key_file'), 80 self.config.get('global', 'tls_certificate_file') 81 ) 82 self.proxy = ServerProxy(self.config.get('global', 'xmlrpc_endpoint'), transport=transport, verbose=self.options.debug) 83 84 # Load all the backends 85 self.backends = [] 86 sections = [s.strip() for s in self.config.get('global', 'backends').split(',')] 87 88 for section in sections: 89 backend_name = self.config.get(section, 'backend') 90 if backend_name is None: 91 continue 92 93 backend = self.load('backends.' + backend_name.lower(), backend_name) 94 backend = backend(self.options, self.config, section) 95 try: 96 backend.initialize() 97 except BackendException, e: 98 logging.warning('Error initializing backend %s for %s: ' + str(e), backend_name, section) 99 continue 100 101 self.backends.append(backend) 102 103 if len(self.backends) == 0: 104 logging.critical('No backends could be loaded') 105 return False 106 107 # Configuration succeeded 108 return True
109
110 - def load(self, package, class_name):
111 """ 112 A convenience function to import class_name from package 113 """ 114 try: 115 module = __import__(package, globals(), locals(), class_name) 116 except ImportError, e: 117 logging.warning('Error importing %s from %s. ' + str(e), class_name, package) 118 return None 119 120 return getattr(module, class_name)
121 122 123
124 - def systemload(self):
125 """ 126 Return the average system load 127 """ 128 try: 129 return max(os.getloadavg()) 130 except (AttributeError, OSError): 131 return None
132
133 - def loop(self):
134 """ 135 A single iteration of the main loop. 136 Return False to terminate the application 137 """ 138 # Keep an eye on system load 139 load = self.systemload() 140 maxload = self.config.getfloat('global', 'load_max') 141 if load > maxload: 142 logging.debug("Systemload %.2f exceeds limit %.2f. Sleeping." % (load, maxload)) 143 time.sleep(60) 144 return True 145 146 # Poll for a job. Sleep for a minute if there's no work 147 try: 148 job = self.proxy.jobs.poll(self.name) 149 except socket.error, ex: 150 logging.warning("Cannot connect to server. Sleeping.") 151 time.sleep(60) 152 return True 153 154 if len(job) == 0: 155 logging.debug('No jobs found. Sleeping.') 156 time.sleep(60) 157 return True 158 159 # We have work. Find a backend to pass it off to 160 for backend in self.backends: 161 if backend.can_process(job): 162 try: 163 (format, document) = backend.process(job) 164 except BackendException, ex: 165 logging.warning(ex) 166 if not ex.recoverable: 167 logging.warning('Removing backend') 168 self.backends.remove(backend) 169 if len(self.backends) == 0: 170 logging.critical('No more active backends.') 171 return False 172 return True 173 174 try: 175 self.proxy.jobs.finish(self.name, job['job'], format, document) 176 except socket.error, ex: 177 logging.warning("Cannot connect to server. Job cannot be finished. Sleeping.") 178 time.sleep(60) 179 return True 180 181 logging.info('Processed job %s', job['job']) 182 return True 183 184 logging.warning('No suitable backend found for job') 185 # TODO: Do something smart about that, like deactivating the related worker on the server 186 return True
187 188
189 - def run(self):
190 """ 191 This is the main execution loop 192 """ 193 logging.info('Starting factory server.') 194 while self.loop(): 195 pass
196 197 198 199 if __name__ == "__main__": 200 parser = OptionParser(usage='Usage: %prog [options]') 201 parser.add_option('-c', '--config-file', action='store', type='string', dest='config_file', 202 default='../conf/config.ini', help='Full path to the configuration file to read.') 203 parser.add_option('-d', '--debug', action='store_true', dest='debug', 204 help='When in debug mode all errors will be written to the console and logging will be set to debug.') 205 (options, args) = parser.parse_args() 206 207 factory = Factory() 208 if factory.configure(options): 209 factory.run() 210