[Erp5-report] r11524 - /erp5/trunk/utils/erp5mechanize/multiSmartRun.py
nobody at svn.erp5.org
nobody at svn.erp5.org
Thu Nov 30 10:29:10 CET 2006
Author: vincent
Date: Thu Nov 30 10:29:08 2006
New Revision: 11524
URL: http://svn.erp5.org?rev=11524&view=rev
Log:
Initial import of multiSmartRun.py, which is used to start multiple runBenchmarks in parallel to take advantage of SMP machines used to benchmark servers (python multithread does not).
Added:
erp5/trunk/utils/erp5mechanize/multiSmartRun.py (with props)
Added: erp5/trunk/utils/erp5mechanize/multiSmartRun.py
URL: http://svn.erp5.org/erp5/trunk/utils/erp5mechanize/multiSmartRun.py?rev=11524&view=auto
==============================================================================
--- erp5/trunk/utils/erp5mechanize/multiSmartRun.py (added)
+++ erp5/trunk/utils/erp5mechanize/multiSmartRun.py Thu Nov 30 10:29:08 2006
@@ -1,0 +1,234 @@
+#!/usr/bin/python
+##############################################################################
+#
+# Copyright (c) 2006 Nexedi SARL and Contributors. All Rights Reserved.
+# Vincent Pelletier <vincent at nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+import os
+import imp
+import sys
+import getopt
+import signal
+
+def log(level, message):
+ if verbose > level:
+ print message
+
+def execute(command):
+ log(2, command)
+ result = os.WEXITSTATUS(os.system(command))
+ if result == 127:
+ raise 'NotFound'
+ return result
+
+child_pid_dict = {}
+
+def background_execute(command):
+ pid = os.fork()
+ if pid != 0:
+ child_pid_dict[pid] = None
+ log(1, 'PID %s : %s' % (pid, command))
+ else:
+ execute(command)
+ sys.exit(0)
+
+def background_wait():
+ while len(child_pid_dict):
+ pid, status = os.wait()
+ del child_pid_dict[pid]
+ if status != 0:
+ log(1, 'PID %s ended with code %s.' % (pid, status))
+ kill_childs(0, None)
+
+def kill_childs(signum, frame):
+ for pid in child_pid_dict.iterkeys():
+ os.kill(pid)
+ sys.exit()
+
+signal.signal(signal.SIGTERM, kill_childs)
+
+def getUsercountList(instance_zopecount_list, usercount):
+ zopecount = sum(instance_zopecount_list)
+ result_list = []
+ base_value = usercount / zopecount
+ remain = usercount % zopecount
+ for pos in xrange(len(instance_zopecount_list)):
+ value = base_value * instance_zopecount_list[pos]
+ if remain > 0:
+ used = min(remain, instance_zopecount_list[pos])
+ value += used
+ remain -= used
+ result_list.append(value)
+ return result_list
+
+def getZopecountList(suitecount, zopecount):
+ result_list = [zopecount / suitecount] * suitecount
+ remain = zopecount % suitecount
+ if remain > 0:
+ for pos in xrange(remain):
+ result_list[pos] += 1
+ return [result for result in result_list if result > 0]
+
+min_zope = 1
+max_zope = None
+runBenchmarks = './runBenchmarks.py'
+userperzope = './userperzope.py'
+min_user = None
+max_user = None
+testsuite_list = []
+usercoef = None
+verbose = 0
+
+try:
+ opt_list, arg_list = getopt.getopt(sys.argv[1:], '', ['minzope=',
+ 'maxzope=', 'runBenchmarks=', 'userperzope=', 'minuser=',
+ 'maxuser=', 'testsuite=', 'usercoef='])
+except getopt.error, msg:
+ print 'Error: %s' % (msg, )
+ print \
+'''Synopsis:
+ %s --testsuite str [--testsuite str [...]] --usercoef int [--minzope int] [--maxzope int]
+ [--runBenchmarks str] [--userperzope str] [--minuser int] [--maxuser int]
+ [-- userperzope-arguments] [--verbose int]
+
+ testsuite si a mandatory argument giving the name of the suite test to run.
+ If there is more than one testsuite argument, each testsuite will be run in
+ parallel. The total number of zope and users will be balanced between each
+ testsuite.
+ usercoef is a mandatory argument giving the maximum number of users to
+ simulate per zope.
+ minzope is the number of zope to start with.
+ default: %s
+ maxzope is the number of zope to reach.
+ default: (same as minzope)
+ runBenchmarks is the name of the program to run a benchmark.
+ default: %s
+ userperzope is the name of the program to test if a result fit the
+ constraints.
+ default: %s
+ minuser is the number of users for the initial run.
+ default: (same as minzope)
+ maxuser is the absolute maximum number of users to test.
+ default: (no limit)
+ verbose:
+ Display more messages.
+ The higher the int, the more messages are displayed.
+
+ Optionnally at the end of the arugment list can be given a list of arguments
+ which will be directly passed to userperzope program.
+''' % (sys.argv[0], min_zope, runBenchmarks, userperzope)
+ sys.exit(2)
+
+for o, a in opt_list:
+ if o == '--minzope':
+ min_zope = int(a)
+ elif o == '--maxzope':
+ max_zope = int(a)
+ elif o == '--runBenchmarks':
+ runBenchmarks = a
+ elif o == '--userperzope':
+ userperzope = a
+ elif o == '--minuser':
+ min_user = int(a)
+ elif o == '--maxuser':
+ max_user = int(a)
+ elif o == '--testsuite':
+ testsuite_list.append(a)
+ elif o == '--usercoef':
+ usercoef = int(a)
+ elif o == '--verbose':
+ verbose = int(a)
+
+if max_zope is None:
+ max_zope = min_zope
+if min_user is None:
+ min_user = min_zope
+userperzope_args = ' '.join(arg_list)
+
+if len(testsuite_list) == 0:
+ print 'Mandatory testsuite agument not given.'
+ sys.exit(2)
+
+if usercoef is None:
+ print 'Mandatory usercoef agument not given.'
+ sys.exit(2)
+
+testsuite_resultpath_list = []
+
+sys.path.insert(0, os.environ['PWD'])
+for testsuite in testsuite_list:
+ file, path, description = imp.find_module(testsuite, sys.path)
+ module = imp.load_module(testsuite, file, path, description)
+ file.close()
+ if 'runBenchmarks_parameter_list' not in dir(module) or not module.runBenchmarks_parameter_list.has_key('results'):
+ raise KeyError, 'Test suite must contain its parameter set (runBenchmarks_parameter_list) to be automated and must define the value for key "results".'
+ testsuite_resultpath_list.append(module.runBenchmarks_parameter_list['results'])
+
+testsuite_count = len(testsuite_list)
+userperzope_arguments = ' '.join(arg_list)
+process_zopecount_list = []
+current_usercount = min_user
+for current_zopecount in xrange(min_zope, max_zope + 1):
+ if current_usercount < current_zopecount:
+ print 'FINAL LIMIT REACHED: Less successfull users than zope to test, continuing has no meaning.'
+ break
+ process_zopecount_list = getZopecountList(suitecount=testsuite_count, zopecount=current_zopecount)
+ process_count = len(process_zopecount_list)
+ try:
+ assert sum(process_zopecount_list) == current_zopecount
+ except AssertionError:
+ print '%s != %s' % (sum(process_zopecount_list), current_zopecount)
+ sys.exit(-1)
+ min_usercount = max(min_user, current_usercount)
+ max_usercount = current_zopecount * usercoef
+ if max_user is not None:
+ max_usercount = min(max_user, max_usercount)
+ for current_usercount in xrange(min_usercount, max_usercount + 1):
+ process_usercount_list = getUsercountList(instance_zopecount_list=process_zopecount_list, usercount=current_usercount)
+ try:
+ assert sum(process_usercount_list) == current_usercount
+ except AssertionError:
+ print '%s != %s' % (sum(process_usercount_list), current_usercount)
+ sys.exit(-1)
+ for process_number in xrange(process_count):
+ testsuite = testsuite_list[process_number]
+ background_execute('%s --config %s --usercount %s --zopecount %s' % \
+ (runBenchmarks, testsuite,
+ process_usercount_list[process_number],
+ process_zopecount_list[process_number]))
+ background_wait()
+ resultpath_list = []
+ for process_number in xrange(process_count):
+ resultpath = testsuite_resultpath_list[process_number]
+ if callable(resultpath):
+ resultpath = resultpath(kw={'usercount': process_usercount_list[process_number],
+ 'zopecount': process_zopecount_list[process_number]})
+ resultpath_list.append(resultpath)
+ if execute('%s %s %s' % (userperzope, userperzope_args, ' '.join(resultpath_list))) != 0:
+ print 'LIMIT REACHED: Test with %s users and %s zopes is out of limits.' % (current_usercount, current_zopecount)
+ break
+ else:
+ current_usercount += 1
+
Propchange: erp5/trunk/utils/erp5mechanize/multiSmartRun.py
------------------------------------------------------------------------------
svn:executable = *
More information about the Erp5-report
mailing list