[Erp5-report] r13656 - /erp5/trunk/utils/oood/
nobody at svn.erp5.org
nobody at svn.erp5.org
Mon Mar 26 16:14:49 CEST 2007
Author: bartek
Date: Mon Mar 26 16:14:45 2007
New Revision: 13656
URL: http://svn.erp5.org?rev=13656&view=rev
Log:
Code reformatting.
Some more docstrings.
Updated README.
Modified:
erp5/trunk/utils/oood/README
erp5/trunk/utils/oood/factory.py
erp5/trunk/utils/oood/lib.py
erp5/trunk/utils/oood/mimemapper.py
erp5/trunk/utils/oood/pool.py
erp5/trunk/utils/oood/serw.py
erp5/trunk/utils/oood/testOoodBasicOperations.py
erp5/trunk/utils/oood/worker.py
Modified: erp5/trunk/utils/oood/README
URL: http://svn.erp5.org/erp5/trunk/utils/oood/README?rev=13656&r1=13655&r2=13656&view=diff
==============================================================================
--- erp5/trunk/utils/oood/README (original)
+++ erp5/trunk/utils/oood/README Mon Mar 26 16:14:45 2007
@@ -49,9 +49,13 @@
TESTING:
-Run test_worker.py to test the main tool.
-Run test_server.py to test communication and xmlrpc interface.
-Run test_timeout.py to check if the OOo gets properly restarted.
+Run testOoodBasicOperations.py to check if it works.
+Run testOoodOldFormats.py to check if older OpenOffice and StarOffice
+formats are really supported.
+Run testOoodHighLoad.py to see how it performs. This test takes a long
+time to run, and even longer to set up when run for the first time; but
+it does its best to strangle oood. Consult documentation in source code
+for more information.
# vi:tw=72 ai ft=txt
Modified: erp5/trunk/utils/oood/factory.py
URL: http://svn.erp5.org/erp5/trunk/utils/oood/factory.py?rev=13656&r1=13655&r2=13656&view=diff
==============================================================================
--- erp5/trunk/utils/oood/factory.py (original)
+++ erp5/trunk/utils/oood/factory.py Mon Mar 26 16:14:45 2007
@@ -27,11 +27,15 @@
#
##############################################################################
-import os, sys, unittest
+import os
+import sys
+
import config
import lib
+
sys.path.append(config.uno_path)
import uno
+
import worker
class WorkerFactory(object):
@@ -42,9 +46,9 @@
self.resolver = self.smgrLocal.createInstanceWithContext("com.sun.star.bridge.UnoUrlResolver", self.ctxLocal)
def build(self, pool, i):
- '''
+ """
Create a context, bind it to next instance of OO using calculated port number.
- '''
+ """
instance_port = config.pool_port_range_start + i
ctx = self.resolver.resolve("uno:socket,host=%s,port=%d;urp;StarOffice.ComponentContext" % (config.pool_host, instance_port))
smgr = ctx.ServiceManager
Modified: erp5/trunk/utils/oood/lib.py
URL: http://svn.erp5.org/erp5/trunk/utils/oood/lib.py?rev=13656&r1=13655&r2=13656&view=diff
==============================================================================
--- erp5/trunk/utils/oood/lib.py (original)
+++ erp5/trunk/utils/oood/lib.py Mon Mar 26 16:14:45 2007
@@ -27,7 +27,9 @@
#
##############################################################################
-import time, sys
+import threading
+import time
+import sys
# Ad-hoc patch. Should be more generic
sys.path.append('/etc/oood/')
@@ -54,11 +56,15 @@
if len(s) > max_sv_size:
max_sv_size = len(s)
+loglock = threading.Lock()
def log(msg, severity=0):
+ loglock.acquire()
f = open(config.log_file, 'a')
- s = '%s: %*s: %s\r\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), max_sv_size, sv[severity], msg)
+ thname = threading.currentThread().getName()
+ s = '%s [%s]: %*s: %s\r\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), thname, max_sv_size, sv[severity], msg)
f.write(s)
f.close()
+ loglock.release()
# vim: shiftwidth=2
Modified: erp5/trunk/utils/oood/mimemapper.py
URL: http://svn.erp5.org/erp5/trunk/utils/oood/mimemapper.py?rev=13656&r1=13655&r2=13656&view=diff
==============================================================================
--- erp5/trunk/utils/oood/mimemapper.py (original)
+++ erp5/trunk/utils/oood/mimemapper.py Mon Mar 26 16:14:45 2007
@@ -37,7 +37,7 @@
class MimeMapper:
- types_map={
+ types_map = {
'application/vnd.oasis.opendocument.text':
{'ext':'odt',
'allowed_to':['pdf', 'doc', 'rtf', 'html-writer', 'txt'],
@@ -60,7 +60,7 @@
},
}
- property_map={
+ property_map = {
'pdf':{
'mime_type':('application/pdf',),
'outprops':(PropertyValue("FilterName",0,"writer_pdf_Export",0),PropertyValue("Overwrite",0,True,0)),
@@ -169,39 +169,48 @@
}
def __init__(self):
- # what can we convert from - [{non_OOo_mime_type:{'ext':target_OOo_extensions,'mime_type':target_OOo_mimetype'}},]
- self.convertable={}
+ """
+ what can we convert from - [{non_OOo_mime_type:{'ext':target_OOo_extensions,'mime_type':target_OOo_mimetype'}},]
+ what we can convert to - [{OOo_mime_type:{type:{'ext':target_non_OOo_extension,'outprops':properties'}},]
+ """
+ self.convertable = {}
for k,v in self.types_map.iteritems():
for allowed in v['allowed_from']:
- pm=self.property_map[allowed]
+ pm = self.property_map[allowed]
if pm.get('mime_type') is None:
raise Exception('doc to convert from must have a mimetype')
for mt in pm.get('mime_type'):
- self.convertable[mt]={'ext':v['ext'],'mime_type':k}
- # convertions to - [{OOo_mime_type:{type:{'ext':target_non_OOo_extension,'outprops':properties'}},]
- self.generable={}
+ self.convertable[mt] = {'ext':v['ext'],'mime_type':k}
+ self.generable = {}
for k,v in self.types_map.iteritems():
- pm={}
+ pm = {}
for ext in v['allowed_to']:
- props={'ext':ext,'outprops':self.property_map[ext]['outprops']}
- pm[ext]=props
- self.generable[k]=pm
+ props = {'ext':ext,'outprops':self.property_map[ext]['outprops']}
+ pm[ext] = props
+ self.generable[k] = pm
# for convenience - lists of OOo and non-OOo mimetypes
- self.ooo_types=self.types_map.keys()
- self.non_ooo_types=[v['mime_type'] for v in self.property_map.values() if v['mime_type'] is not None]
+ self.ooo_types = self.types_map.keys()
+ self.non_ooo_types = [v['mime_type'] for v in self.property_map.values() if v['mime_type'] is not None]
self.non_ooo_types = list(reduce(operator.add, self.non_ooo_types))
def getAllowedTo(self,mimetype):
+ """
+ returns a list of tuples made of (type, label)
+ for all formats to which we can convert the given mimetype
+ """
return [(type,self.property_map[type].get('label',type)) for type in self.types_map[mimetype]['allowed_to']]
def getMimeFor(self,format):
+ """
+ return mime type for a given format (format is file name extension)
+ """
try:
return self.property_map[format]['mime_type']
except KeyError:
- raise Exception('no mime type for format '+format)
-
-
-mimemapper=MimeMapper()
+ raise Exception('no mime type for format ' + format)
+
+
+mimemapper = MimeMapper()
if __name__=='__main__':
print '========================================'
Modified: erp5/trunk/utils/oood/pool.py
URL: http://svn.erp5.org/erp5/trunk/utils/oood/pool.py?rev=13656&r1=13655&r2=13656&view=diff
==============================================================================
--- erp5/trunk/utils/oood/pool.py (original)
+++ erp5/trunk/utils/oood/pool.py Mon Mar 26 16:14:45 2007
@@ -28,30 +28,42 @@
##############################################################################
-import lib,threading,os,time,sys
+import os
+import sys
+import threading
+import time
+
import config
-import factory,start
+import factory
+import lib
+import start
class Pool(object):
- '''manages a pool of worker object; number of workers is
- set in config directs requests to the first available
- worker uses semaphore to hold request untill one is
- available '''
-
- total=config.pool_size
- inst={}
- timers={}
+ """
+ manages a pool of worker object; number of workers is
+ set in config directs requests to the first available
+ worker uses semaphore to hold request untill one is
+ available
+ """
+ total = config.pool_size
+ inst = {}
+ timers = {}
def __init__(self):
- '''set up semaphore, make workers'''
- self.sem=threading.Semaphore(self.total)
+ """
+ set up semaphore, make workers
+ """
+ self.sem = threading.Semaphore(self.total)
for i in range(self.total):
self._mkWorker(i)
def acquire(self):
self.sem.acquire()
- def release(self,i):
+ def release(self, i):
+ """
+ release semaphore, try to delete timer thread
+ """
try:
self.timers[i].cancel()
del(self.timers[i])
@@ -60,43 +72,39 @@
self.sem.release()
def getWorker(self):
- '''
- returns first available instance, marks it as busy etc.
- we use index to name temporary files which are per
- worker. IDEA: use additional threads to test OOo instance
- that should be available and timeout if there is no
- response or do sthg else to keep semaphore in sync if OOo
- crashes, restart OOo instances etc.
- '''
- w=None
+ """
+ wait until there is an available worker
+ returns first available instance, marks it as busy etc.
+ we use index to name temporary files which are per
+ worker.
+ This method also starts timer thread which is responsible for killing
+ and restarting OOo instance and rebuilding worker if there was timeout
+ (which usu. means that OOo instance crashed)
+ """
+ w = None
self.acquire() # wait untill something is free
for idx in range(self.total):
candidate = self.inst[idx]
if not candidate.busy and not candidate.dead:
- w=self.inst[idx]
+ w = self.inst[idx]
break
- if w is None: # means semaphore is out of sync with pool
+ if w is None: # means semaphore is out of sync with pool - this signals a serious problem
raise lib.WorkerNotAvailable('sorry')
w.setBusy()
self.timers[idx] = threading.Timer(config.instance_timeout, self.rebuild(idx))
self.timers[idx].start()
- return idx,w
+ return idx, w
def rebuild(self, i):
- '''
- triggered by a timer thread in case worker takes too long
- rebuilds the worker instance
- '''
+ """
+ triggered by a timer thread in case worker takes too long
+ rebuilds the worker instance
+ """
def _rebuild():
"""
- would be best to kill only one instance, but killing pid returned by
- spawn leaves child process running, and killing by process group id
- kills everything because all instances have the same pgid
- (the one that start.py had).
- Worse still, after restart instances have the same pgid as the server,
- so we'd kill the server along with them.
- So, we hack - we do "ps" to find parent process and explicitly kill both
- it is not very portable, but I don't have a better idea...
+ This is run when a timer thread expires, which means that the OOo instance if frozen
+ kills OOod instance, kills worker, starts new instance and worker, replaces the old worker
+ in the pool with the new one
"""
print 'That takes too long !'
lib.log('Instance %s not responding - restart it' % i, 3)
@@ -113,6 +121,13 @@
self.inst[i] = (factory.builder.build(self, i))
def _killWorker(self, i):
+ """
+ this basically means marking worker as "dead"
+ there is no way to really destroy the object; it is removed from the pool so it should be
+ removed by garbage collector
+ but we have to mark it, otherwise it may hang around for a while, complete the job and release semaphore
+ which would cause serious problem
+ """
self.inst[i].dead = True
pool = Pool()
Modified: erp5/trunk/utils/oood/serw.py
URL: http://svn.erp5.org/erp5/trunk/utils/oood/serw.py?rev=13656&r1=13655&r2=13656&view=diff
==============================================================================
--- erp5/trunk/utils/oood/serw.py (original)
+++ erp5/trunk/utils/oood/serw.py Mon Mar 26 16:14:45 2007
@@ -31,120 +31,144 @@
import sys
sys.path.append('/etc/oood/')
+import base64
+import config
+import cStringIO
+import glob
+import os
+import pdb
+import random
+import re
+import socket
+import string
+import sys
+import threading
+import time
+import traceback
+import zipfile
+
from SimpleXMLRPCServer import *
from SocketServer import ThreadingMixIn
-import time,threading,base64,os,pdb,sys,socket,traceback,string, random
-import zipfile, cStringIO, glob, re
+
import lib
-import config
import pool
+
from com.sun.star.uno import RuntimeException as unoRuntimeException
-nulltrans=string.maketrans('','')
-remother=nulltrans.translate(nulltrans,string.letters)
-
-class MySerw(ThreadingMixIn,SimpleXMLRPCServer):
+nulltrans = string.maketrans('', '')
+remother = nulltrans.translate(nulltrans, string.letters)
+
+class MySerw(ThreadingMixIn, SimpleXMLRPCServer):
def server_bind(self):
- self.socket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
+ self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind(self.server_address)
class Procesor(object):
- '''main request handler, registered with the server handler
- functions are auto-generated named run_<funcname>, args:
- filename, data, [metadata, extension]
- (only positional args are supported - this is xmlrpclib limitation)
- '''
-
- public_funcs=('convert','getmetadata','setmetadata','generate','getmetadatatoolong')
+ """
+ main request handler, registered with the server handler
+ functions are auto-generated named run_<funcname>, args:
+ filename, data, [metadata, extension]
+ (only positional args are supported - this is xmlrpclib limitation)
+ """
+
+ public_funcs = ('convert', 'getmetadata', 'setmetadata', 'generate', 'getmetadatatoolong')
def __init__(self):
self._generateFuncs()
def _generateFuncs(self):
for f in self.public_funcs:
- setattr(self,'run_'+f,self._wrap(f))
-
- def _asciify(self,s):
- '''for clearing chars out of filenames'''
- return s.translate(nulltrans,remother)
-
- def _mkName(self,fname):
- '''make a complete file name with path we save the file
- under a derivative of its original name, mostly for
- debugging
- '''
+ setattr(self, 'run_' + f, self._wrap(f))
+
+ def _asciify(self, s):
+ """
+ for clearing chars out of filenames
+ """
+ return s.translate(nulltrans, remother)
+
+ def _mkName(self, fname):
+ """
+ make a complete file name with path we save the file
+ under a derivative of its original name, mostly for
+ debugging
+ """
return os.path.join(config.oood_home, 'tmp', fname)
- def convert(self,w,kw):
+ def convert(self, w, kw):
w.run_convert(kw)
return True
- def getmetadatatoolong(self,w,kw):
+ def getmetadatatoolong(self, w, kw):
w.run_getmetadatatoolong(kw)
return True
- def getmetadata(self,w,kw):
- meta=w.run_getmetadata(kw)
- return meta,None
-
- def setmetadata(self,w,kw):
- res=w.run_setmetadata(kw)
+ def getmetadata(self, w, kw):
+ meta = w.run_getmetadata(kw)
+ return meta, None
+
+ def setmetadata(self, w, kw):
+ res = w.run_setmetadata(kw)
if res:
return True
else:
raise lib.SetMetaFailed()
- def generate(self,w,kw):
- '''generate file in a desired format (provided we already
- have an OOo file)'''
+ def generate(self, w, kw):
+ """
+ generate file in a desired format (provided we already
+ have an OOo file)
+ """
w.run_generate(kw)
return True
- def getAllowedTargets(self,mimetype):
- '''get list of allowed target formats from worker
- different interface (no file), so we don't wrap it'''
- id,w=pool.pool.getWorker()
+ def getAllowedTargetItemList(self, mimetype):
+ """
+ get list of allowed target formats from worker
+ different interface (no file), so we don't wrap it
+ """
+ id, w = pool.pool.getWorker()
try:
- return w.getAllowedTargets(mimetype)
+ return w.getAllowedTargetItemList(mimetype)
finally:
w.setFree()
def _pre(self,kw):
- '''write data to a temp file, get worker instance'''
- id,w=pool.pool.getWorker()
+ """
+ write data to a temp file, get worker instance
+ """
+ id, w = pool.pool.getWorker()
if kw.get('data') is not None and kw['data']!='':
if kw.get('filename') is None:
# we generate a random name mostly because unit tests of ERP5 have a problem with it
- kw['filename']=str(random.random()).split('.')[1]
+ kw['filename'] = str(random.random()).split('.')[1]
# we have to store in a file for OOo to open
- filename='%d_%s' % (id,self._asciify(kw['filename'])) # in case we have two files of the same name
- filename=self._mkName(filename)
- f=open(filename,'w')
- #f2=open(self._mkName('temp64'),'w') # for debugging only
- #f2.write(kw['data'])
- #f2.close()
+ filename = '%d_%s' % (id, self._asciify(kw['filename'])) # in case we have two files of the same name
+ filename = self._mkName(filename)
+ f = open(filename, 'w')
f.write(base64.decodestring(kw['data']))
f.close()
- kw['filename']=filename
- return id,w
-
- def _safeRemove(self,filename):
- '''remove a file or log warning if none exists'''
- any=False
- pth=os.path.join(config.oood_home, 'tmp', os.path.basename(filename))
- for f in glob.glob(pth+'*'):
+ kw['filename'] = filename
+ return id, w
+
+ def _safeRemove(self, filename):
+ """
+ remove a file or log warning if none exists
+ """
+ any = False
+ pth = os.path.join(config.oood_home, 'tmp', os.path.basename(filename))
+ for f in glob.glob(pth + '*'):
lib.log('removing %s' % f)
os.remove(self._mkName(f))
- any=True
+ any = True
if not any:
- lib.log('no file beginning with %s' % filename,1)
-
- def _post(self,w,kw):
- '''
- remove temp files, free worker
- newfname only if there was convertion
- '''
+ lib.log('no file beginning with %s' % filename, 1)
+
+ def _post(self, w, kw):
+ """
+ remove temp files, free worker
+ newfname only if there was convertion
+ """
try:
# because we are in finally, things can be none
# if there was an exception
@@ -155,58 +179,59 @@
finally:
w.setFree()
- def _wrap(self,funcname):
- '''wrap function execute _pre and _post if newfname,
- which means there was a convertion, send data back to the
- user
- '''
- argtpl=('filename','data','meta','extension')
+ def _wrap(self, funcname):
+ """
+ wrap function execute _pre and _post if newfname,
+ which means there was a convertion, send data back to the
+ user
+ """
+ argtpl=('filename', 'data', 'meta', 'extension')
def _run(*args):
def _process(kw):
- print 'running',func.__name__
- res=func(w,kw) # XXX am I sure it is threadsafe?
+ print 'running', func.__name__
+ res = func(w, kw) # XXX am I sure it is threadsafe?
if kw.get('newfilename') is None:
# we return empty data
- kw['data']=''
+ kw['data'] = ''
else:
try: # we can do it only if the argument is a string
- tozip=kw.get('extension').startswith('html') # if html, we pack the output files
+ tozip = kw.get('extension').startswith('html') # if html, we pack the output files
except AttributeError:
- tozip=False
+ tozip = False
if tozip:
#first delete the original
- fname=kw['filename']
- fullname=self._mkName(fname)
+ fname = kw['filename']
+ fullname = self._mkName(fname)
os.remove(fullname)
- arch=cStringIO.StringIO()
- pack=zipfile.ZipFile(arch,'a')
- r=re.compile(kw['extension']+'$')
- for f in glob.glob(fullname+'*'):
- name=os.path.basename(f)
+ arch = cStringIO.StringIO()
+ pack = zipfile.ZipFile(arch, 'a')
+ r = re.compile(kw['extension'] + '$')
+ for f in glob.glob(fullname + '*'):
+ name = os.path.basename(f)
# fix html extension
- name=r.sub('html',name)
- pack.write(f,name)
+ name = r.sub('html', name)
+ pack.write(f, name)
pack.close()
arch.seek(0)
- kw['data']=base64.encodestring(arch.read())
+ kw['data'] = base64.encodestring(arch.read())
arch.close()
- kw['mime']='application/zip' # generation returns mime type
+ kw['mime'] = 'application/zip' # generation returns mime type
else:
- kw['data']=base64.encodestring(open(self._mkName(kw['newfilename'])).read())
+ kw['data'] = base64.encodestring(open(self._mkName(kw['newfilename'])).read())
return kw
# let's get to work
- kw=dict(zip(argtpl,args))
+ kw = dict(zip(argtpl, args))
w = None # initialize worker variable
try:
try:
- id,w=self._pre(kw)
+ id, w = self._pre(kw)
return _process(kw)
except unoRuntimeException:
- print "unoRuntimeException in "+str(id)
- rebuilder=pool.pool.rebuild(id)()
- id,w=self._pre(kw)
+ print "unoRuntimeException in " + str(id)
+ rebuilder = pool.pool.rebuild(id)()
+ id, w = self._pre(kw)
return _process(kw)
- except Exception,e:
+ except Exception, e:
print e
traceback.print_tb(sys.exc_info()[2])
raise e
@@ -214,12 +239,12 @@
try:
if w is not None:
# do cleanup only if worker has been created
- self._post(w,kw)
- except Exception,e:
+ self._post(w, kw)
+ except Exception, e:
print e
traceback.print_tb(sys.exc_info()[2])
raise e
- func=getattr(self,funcname)
+ func = getattr(self, funcname)
return _run
# vim: shiftwidth=2
Modified: erp5/trunk/utils/oood/testOoodBasicOperations.py
URL: http://svn.erp5.org/erp5/trunk/utils/oood/testOoodBasicOperations.py?rev=13656&r1=13655&r2=13656&view=diff
==============================================================================
--- erp5/trunk/utils/oood/testOoodBasicOperations.py (original)
+++ erp5/trunk/utils/oood/testOoodBasicOperations.py Mon Mar 26 16:14:45 2007
@@ -58,7 +58,7 @@
res = sp.run_setmetadata('test.odt', base64.encodestring(data), newmeta)
open('doc/out/test_changed.odt', 'w').write(base64.decodestring(res['data']))
# getting metadata back after the change
- data = open('doc/test_changed.odt').read()
+ data = open('doc/out/test_changed.odt').read()
res = sp.run_getmetadata('test_changed.odt', base64.encodestring(data))
self.assertEqual(res['meta']['title'], 'HELLOOOOOO')
self.assertEqual(res['meta']['reference'], 'blaaaaaah')
@@ -146,4 +146,4 @@
-# vim: filetype = python syntax=python shiftwidth=2
+# vim: filetype=python syntax=python shiftwidth=2
Modified: erp5/trunk/utils/oood/worker.py
URL: http://svn.erp5.org/erp5/trunk/utils/oood/worker.py?rev=13656&r1=13655&r2=13656&view=diff
==============================================================================
--- erp5/trunk/utils/oood/worker.py (original)
+++ erp5/trunk/utils/oood/worker.py Mon Mar 26 16:14:45 2007
@@ -42,134 +42,163 @@
from mimemapper import mimemapper
def caseUp(s):
- '''OOo uses erp5-like naming convention, so we do the same
- except for MIMEType'''
- if s=='MIMEType':return s
+ """
+ OOo uses erp5-like naming convention, so we do the same
+ except for MIMEType
+ """
+ if s == 'MIMEType':return s
return ''.join([st.capitalize() for st in s.split('_')])
def caseDown(s):
try:
- s=s.encode()
+ s = s.encode()
except AttributeError:
pass
- if len(s)<=1:return s.lower()
- if s=='MIMEType':return s
- r=lambda c: c in string.uppercase and '_'+c or c
- s=[s[0]]+map(r,s[1:])
- s=''.join(s).lower()
+ if len(s) <= 1:return s.lower()
+ if s == 'MIMEType':return s
+ r = lambda c: c in string.uppercase and '_' + c or c
+ s=[s[0]] + map(r, s[1:])
+ s = ''.join(s).lower()
return s
class Worker(object):
- '''
- This class encapsulates an OOo instance providing interface
- to functions. Can be accessed directly by functions
- "convert" etc., but should be through wrappers
- "run_convert", which is safe and resets the worker finally.
- '''
+ """
+ This class encapsulates an OOo instance providing interface
+ to functions. Can be accessed directly by functions
+ "convert" etc., but should be through wrappers
+ "run_convert", which is safe and resets the worker finally.
+ """
# for automatic interface generation
- public_funcs=('convert','generate','setmetadata','getmetadata','getmetadatatoolong')
+ public_funcs=('convert', 'generate', 'setmetadata', 'getmetadata', 'getmetadatatoolong')
inProps = PropertyValue( "Hidden" , 0 , True, 0 ),
- metafields=('title','subject','keywords','description','MIMEType')
- userfields=('reference','version','language')
- busy=False
+ metafields=('title', 'subject', 'keywords', 'description', 'MIMEType')
+ userfields=('reference', 'version', 'language')
+ # this attribute says if the worker is doing something, or waiting for request
+ busy = False
+ # this attribute says if the worker is removed from the pool because of a timeout
dead = False
- def __init__(self,idx,desktop,pool=None):
- self.idx=idx
- self.desktop=desktop
- self.pool=pool
- self.cwd=systemPathToFileUrl(config.oood_home)
+ def __init__(self,idx,desktop,pool = None):
+ self.idx = idx
+ self.desktop = desktop
+ self.pool = pool
+ self.cwd = systemPathToFileUrl(config.oood_home)
self._reset()
self._generateFuncs()
def setBusy(self):
- self.busy=True
+ self.busy = True
def setFree(self):
- '''marks self as not busy, releases semaphore in the
- pool object'''
+ """
+ marks self as not busy, releases semaphore in the
+ pool object
+ The worker can be marked as "dead" if it has timed out, been removed from the pool,
+ but hasn't been removed by garbage collector
+ We have to check because otherwise it will release the semaphore and make it out of sync with the pool
+ """
if not self.dead:
- self.busy=False
+ self.busy = False
self.pool.release(self.idx)
- def convert(self,kw):
- '''convertion to OOo, produces metadata and OOo file url
- '''
+ def convert(self, kw):
+ """
+ convertion to OOo, produces metadata and OOo file url
+ """
self._loadFile(kw['filename'])
self._convert()
# if the doc doesn't have title or reference, we use filename for them
# only on ERP5 object (we don't set them on file)
# strip filename of #_ prefix
try:
- name=os.path.basename(kw['filename'])
- strippedfilename=name[name.index('_')+1:]
+ name = os.path.basename(kw['filename'])
+ strippedfilename = name[name.index('_') + 1:]
except ValueError:
- strippedfilename=name
+ strippedfilename = name
if not self.metadata.get('title'):
- self.metadata['title']=strippedfilename
+ self.metadata['title'] = strippedfilename
if not self.metadata.get('reference'):
- self.metadata['reference']=strippedfilename
- kw['mime']=self.metadata.get('MIMEType')
- kw['meta']=self.metadata
- kw['newfilename']=fileUrlToSystemPath(self.destUrl)
-
- def generate(self,kw):
- '''generate file in a requested format, return file url
- and a respective mime type'''
+ self.metadata['reference'] = strippedfilename
+ kw['mime'] = self.metadata.get('MIMEType')
+ kw['meta'] = self.metadata
+ kw['newfilename'] = fileUrlToSystemPath(self.destUrl)
+
+ def generate(self, kw):
+ """
+ generate file in a requested format, return file url
+ and a respective mime type
+ """
self._loadFile(kw['filename'])
self._generate(kw['extension'])
- mime=mimemapper.getMimeFor(kw['extension'])
+ mime = mimemapper.getMimeFor(kw['extension'])
kw['mime'] = mime[0] # it is a tuple; can be multiple but we don't know which exactly was requested
- kw['newfilename']=fileUrlToSystemPath(self.destUrl)
-
- def setmetadata(self,kw):
- '''set metadata on OOo file (fname does not change)'''
+ kw['newfilename'] = fileUrlToSystemPath(self.destUrl)
+
+ def setmetadata(self, kw):
+ """
+ set metadata on OOo file (fname does not change)
+ """
self._loadFile(kw['filename'])
self._setMetadata(kw['meta'])
- kw['newfilename']=kw['filename']
+ kw['newfilename'] = kw['filename']
self._saveFile()
return True
def getmetadatatoolong(self,kw):
"""
- for debugging - to see if timeout works
- """
- self._loadFile(kw['filename'])
- kw['meta']=self._getMetadata()
+ for debugging - to see if timeout works
+ """
+ self._loadFile(kw['filename'])
+ kw['meta'] = self._getMetadata()
time.sleep(config.instance_timeout + 120)
return True
- def getmetadata(self,kw):
- self._loadFile(kw['filename'])
- meta=self._getMetadata()
+ def getmetadata(self, kw):
+ """
+ Get metadata from the OOo file
+ """
+ self._loadFile(kw['filename'])
+ meta = self._getMetadata()
kw['meta']=meta
return meta
def _generateFuncs(self):
- '''generates functions run_<funcname>
- XXX we should prob'ly change naming convention and generate
- automatically based on name pattern'''
+ """
+ generates functions run_<funcname>
+ XXX we should probably change naming convention and generate
+ automatically based on name pattern
+ or use decorators
+ """
for f in self.public_funcs:
- fname='run_'+f
- setattr(self,fname,self._gener(f))
-
- def _gener(self,f):
- def _run(*args,**kwargs):
- '''if anything goes wrong, we reset the worker'''
+ fname = 'run_' + f
+ setattr(self, fname,self._gener(f))
+
+ def _gener(self, f):
+ """
+ Wrapper for API functions
+ runs the function, and then resets the worker
+ no matter what happened
+ """
+ def _run(*args, **kwargs):
+ """
+ if anything goes wrong, we reset the worker
+ """
try:
- res=func(*args,**kwargs)
+ res = func(*args, **kwargs)
finally:
self._reset()
return res
- func=getattr(self,f)
+ func = getattr(self, f)
return _run
def _reset(self):
- '''reset all worker's properties'''
+ """
+ reset all worker's properties
+ """
try:
self.doc.close(True)
except:
@@ -178,9 +207,9 @@
setattr(self,atr,None)
def _loadFile(self,fname):
- self.fileUrl=absolutize(self.cwd,systemPathToFileUrl(fname))
- try:
- self.doc=self.desktop.loadComponentFromURL(self.fileUrl,'_blank',0,self.inProps)
+ self.fileUrl = absolutize(self.cwd,systemPathToFileUrl(fname))
+ try:
+ self.doc = self.desktop.loadComponentFromURL(self.fileUrl,'_blank',0,self.inProps)
except IllegalArgumentException:
raise lib.NotFound(self.fileUrl)
if not self.doc:
@@ -193,105 +222,117 @@
self.doc.close(True)
def _checkMimeType(self,format):
- '''we check mime type - convert OOo and non-OOo files,
- set metadata and generate other formats only on OOo
- format: 0 - non-ooo, 1-ooo, 2-both'''
+ """
+ we check mime type - convert OOo and non-OOo files,
+ set metadata and generate other formats only on OOo
+ format: 0 - non-ooo, 1-ooo, 2-both
+ """
if not self.metadata or not self.metadata.has_key('MIMEType'): # sometimes missing
- self.metadata=self._getMetadata()
- if format==1 and self.metadata['MIMEType'] in mimemapper.ooo_types: return True
- if format==0 and self.metadata['MIMEType'] in mimemapper.non_ooo_types: return True
- if format==2 and self.metadata['MIMEType'] in mimemapper.ooo_types+mimemapper.non_ooo_types: return True
+ self.metadata = self._getMetadata()
+ if format == 1 and self.metadata['MIMEType'] in mimemapper.ooo_types: return True
+ if format == 0 and self.metadata['MIMEType'] in mimemapper.non_ooo_types: return True
+ if format == 2 and self.metadata['MIMEType'] in mimemapper.ooo_types+mimemapper.non_ooo_types: return True
raise lib.IllegalMimeType(self.metadata['MIMEType'])
def _generate(self,format):
- '''
- '''
+ """
+ From the ODF file that was submitted to the worker, now generate an output file
+ in a desired format
+ """
self._checkMimeType(1)
- outProps=mimemapper.generable[self.metadata['MIMEType']].get(format,None)
+ outProps = mimemapper.generable[self.metadata['MIMEType']].get(format, None)
if outProps is None:
raise lib.IllegalFormat(format)
- self.destUrl=self.fileUrl+'.'+outProps['ext'] # we want it very unique
- try:
- self.doc.storeToURL(self.destUrl,outProps['outprops'])
+ self.destUrl = self.fileUrl + '.' + outProps['ext'] # we want it very unique
+ try:
+ self.doc.storeToURL(self.destUrl, outProps['outprops'])
except ErrorCodeIOException:
- raise lib.GenerationFailed(self.fileUrl+' to format '+format)
+ raise lib.GenerationFailed(self.fileUrl + ' to format ' + format)
return self.destUrl
def _getMetadata(self):
- '''extract metadata, assign as dictionary
- no MIME check - we get them also before conversion
- we convert names between two naming conventions
- '''
- self.dinfo=self.doc.getDocumentInfo()
- metadata={}
+ """
+ extract metadata, assign as dictionary
+ no MIME check - we get them also before conversion
+ we convert names between two naming conventions
+ """
+ self.dinfo = self.doc.getDocumentInfo()
+ metadata = {}
for n in self.metafields:
- v=self.dinfo.getPropertyValue(caseUp(n))
+ v = self.dinfo.getPropertyValue(caseUp(n))
if v:
- metadata[n]=v
+ metadata[n] = v
for i in range(self.dinfo.getUserFieldCount()):
- n=self.dinfo.getUserFieldName(i)
- v=self.dinfo.getUserFieldValue(i)
+ n = self.dinfo.getUserFieldName(i)
+ v = self.dinfo.getUserFieldValue(i)
if caseDown(n) in self.userfields:
if v:
- metadata[caseDown(n)]=v
+ metadata[caseDown(n)] = v
return metadata
def _convert(self):
- '''convertion - only known mimetypes,store as a file with
- appropriate extension we append extension instead of
- replacing, because filename may have unusual extension or
- none at all'''
+ """
+ This method converts non-ODF format into an ODF
+ only known mimetypes,store as a file with
+ appropriate extension
+ we append extension instead of
+ replacing, because filename may have unusual extension or
+ none at all
+ """
self._checkMimeType(2)
- pm=mimemapper.convertable.get(self.metadata['MIMEType'],None)
+ pm = mimemapper.convertable.get(self.metadata['MIMEType'],None)
if pm is None:
if self.metadata['MIMEType'] in mimemapper.ooo_types: # we can convert OOo to itself
- ext=mimemapper.types_map[self.metadata['MIMEType']]['ext']
+ ext = mimemapper.types_map[self.metadata['MIMEType']]['ext']
else:
raise lib.IllegalMimeType(self.metadata['MIMEType'])
else:
- ext=pm['ext']
- self.destUrl=self.fileUrl+'.'+ext
- self.doc.storeAsURL(self.destUrl,())
+ ext = pm['ext']
+ self.destUrl = self.fileUrl + '.' + ext
+ self.doc.storeAsURL(self.destUrl, ())
# we change mimetype to OOo before sending metadata back
if pm is not None:
- self.metadata['MIMEType']=pm['mime_type']
+ self.metadata['MIMEType'] = pm['mime_type']
def _setMetadata(self,meta):
- '''set metadata given as a dictionary
- add new property if does not exist
- OOo uses titlecased property names
- '''
+ """
+ set metadata given as a dictionary
+ add new property if does not exist
+ OOo uses titlecased property names
+ """
self._checkMimeType(1)
- self.dinfo=self.doc.getDocumentInfo()
- userfieldcounter=0
- maxcount=self.dinfo.getUserFieldCount()
+ self.dinfo = self.doc.getDocumentInfo()
+ userfieldcounter = 0
+ maxcount = self.dinfo.getUserFieldCount()
for k,v in meta.items():
- K=caseUp(k)
+ K = caseUp(k)
if k in self.metafields:
try:
- self.dinfo.addProperty(K,0,v)
+ self.dinfo.addProperty(K, 0, v)
except PropertyExistException:
- self.dinfo.setPropertyValue(K,v)
+ self.dinfo.setPropertyValue(K, v)
elif k in self.userfields:
- if userfieldcounter<maxcount:
- self.dinfo.setUserFieldName(userfieldcounter,K)
- self.dinfo.setUserFieldValue(userfieldcounter,v)
- userfieldcounter+=1
+ if userfieldcounter < maxcount:
+ self.dinfo.setUserFieldName(userfieldcounter, K)
+ self.dinfo.setUserFieldValue(userfieldcounter, v)
+ userfieldcounter += 1
# we silently ignore if there is too many user fields
else:
raise lib.IllegalMetaAttribute(k)
- def getAllowedTargets(self,mimetype):
- '''list types which can be generated from given OOo type'''
+ def getAllowedTargetItemList(self,mimetype):
+ """
+ list types which can be generated from given OOo type
+ """
try:
return mimemapper.getAllowedTo(mimetype)
except KeyError:
return []
-if __name__=='__main__':
+if __name__ == '__main__':
print caseDown('Reference')
- #w=Worker(None)
+ #w = Worker(None)
#print w.__dict__
#w.run_convert('s')
#
More information about the Erp5-report
mailing list