[Erp5-report] r45977 guillaume.bottex - in /slapos/trunk/util: slapos.tool.nosqltester_mana...
nobody at svn.erp5.org
nobody at svn.erp5.org
Mon Jul 11 19:09:36 CEST 2011
Author: guillaume.bottex
Date: Mon Jul 11 19:09:36 2011
New Revision: 45977
URL: http://svn.erp5.org?rev=45977&view=rev
Log:
Changes for result publication in ERP5.
Modified:
slapos/trunk/util/slapos.tool.nosqltester_manager.kumo/src/slapos/tool/nosqltester_manager/kumo/__init__.py
slapos/trunk/util/slapos.tool.nosqltester_manager/setup.py
slapos/trunk/util/slapos.tool.nosqltester_manager/src/slapos/tool/nosqltester_manager/__init__.py
Modified: slapos/trunk/util/slapos.tool.nosqltester_manager.kumo/src/slapos/tool/nosqltester_manager/kumo/__init__.py
URL: http://svn.erp5.org/slapos/trunk/util/slapos.tool.nosqltester_manager.kumo/src/slapos/tool/nosqltester_manager/kumo/__init__.py?rev=45977&r1=45976&r2=45977&view=diff
==============================================================================
--- slapos/trunk/util/slapos.tool.nosqltester_manager.kumo/src/slapos/tool/nosqltester_manager/kumo/__init__.py [utf8] (original)
+++ slapos/trunk/util/slapos.tool.nosqltester_manager.kumo/src/slapos/tool/nosqltester_manager/kumo/__init__.py [utf8] Mon Jul 11 19:09:36 2011
@@ -38,9 +38,6 @@ class KumoTesterManager(NoSQLTesterManag
self.node_type = "server"
- self.nb_thread = self.argument_namespace.nb_thread
- self.nb_request = self.argument_namespace.nb_request
-
self.max_manager = 1
self.computer_guid_list = self.computer_partition.getConnectionParameter('computer_guid_list').split(':')
@@ -187,7 +184,8 @@ class KumoTesterManager(NoSQLTesterManag
return False
def process_report(self, inputnames):
- outputname = os.path.join(self.log_directory, 'report'+str(self.node_count)+'.csv')
+ filename = 'report'+str(self.node_count)+'.csv'
+ outputname = os.path.join(self.log_directory, filename)
output = csv.writer(open(outputname, 'wb'), delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL)
# Header
@@ -204,4 +202,6 @@ class KumoTesterManager(NoSQLTesterManag
l.append(re.findall(r'\d+\.\d+|\d+',lines[i])[0].replace('.', ','))
output.writerow(l)
+
+ return filename
Modified: slapos/trunk/util/slapos.tool.nosqltester_manager/setup.py
URL: http://svn.erp5.org/slapos/trunk/util/slapos.tool.nosqltester_manager/setup.py?rev=45977&r1=45976&r2=45977&view=diff
==============================================================================
--- slapos/trunk/util/slapos.tool.nosqltester_manager/setup.py [utf8] (original)
+++ slapos/trunk/util/slapos.tool.nosqltester_manager/setup.py [utf8] Mon Jul 11 19:09:36 2011
@@ -12,7 +12,7 @@ long_description=( read('README.txt')
read('CHANGES.txt')
)
-install_require_list = ['setuptools', 'Flask', 'slapos.core']
+install_require_list = ['setuptools', 'Flask', 'slapos.core', 'erp5.utils.benchmark']
import sys
python_major_version, python_minor_version = sys.version_info[:2]
Modified: slapos/trunk/util/slapos.tool.nosqltester_manager/src/slapos/tool/nosqltester_manager/__init__.py
URL: http://svn.erp5.org/slapos/trunk/util/slapos.tool.nosqltester_manager/src/slapos/tool/nosqltester_manager/__init__.py?rev=45977&r1=45976&r2=45977&view=diff
==============================================================================
--- slapos/trunk/util/slapos.tool.nosqltester_manager/src/slapos/tool/nosqltester_manager/__init__.py [utf8] (original)
+++ slapos/trunk/util/slapos.tool.nosqltester_manager/src/slapos/tool/nosqltester_manager/__init__.py [utf8] Mon Jul 11 19:09:36 2011
@@ -66,33 +66,22 @@ class NoSQLTesterManager(object):
"""
self.software_release_type = sys.argv.pop(1)
- self.argument_namespace = self._parse_arguments(argparse.ArgumentParser(
+ parameter_dict = vars(self._parse_arguments(argparse.ArgumentParser(
description='Manage scalability tester(s).',
# When adding arguments in the subclass having the same name, just
# override it
- conflict_handler='resolve'))
- parameter_dict = vars(self.argument_namespace)
+ conflict_handler='resolve')))
+
+ for attribut, value in parameter_dict.iteritems():
+ setattr(self, attribut, value)
- self.log_directory = parameter_dict['report_directory']
self.work_directory = os.path.split(os.path.split(self.log_directory)[0])[0]
self.tmp_directory = os.path.join(self.work_directory, 'tmp')
self.tester_urls = []
self.testers_status = []
self.todo = 0
self.reports = []
- self.address = parameter_dict['address']
- self.port = parameter_dict['port']
- self.nb_init_server = parameter_dict['nb_server_init']
- self.nb_init_tester = parameter_dict['nb_tester_init']
- self.max_server = parameter_dict['nb_server_max']
- self.max_tester = parameter_dict['nb_tester_max']
- self.node_increment = parameter_dict['node_increment']
- self.software_release_url = parameter_dict['software_release_url']
- self.server_url = parameter_dict['server_url']
- self.key_file = parameter_dict['key_file']
- self.cert_file = parameter_dict['cert_file']
- self.computer_id = parameter_dict['computer_id']
- self.computer_partition_id = parameter_dict['computer_partition_id']
+
self.erp5_publish_url = parameter_dict['erp5_publish_url']
self.erp5_publish_project = parameter_dict['erp5_publish_project']
@@ -105,9 +94,15 @@ class NoSQLTesterManager(object):
self.node_type = ""
+ # Needed for ERP5BenchmarkResult
+ self.repeat = 1
+ self.error_message_set = set()
+ self.user_tuple = ("user0")
+ self.benchmark_suite_name_list = []
+
# Logger initialization
self.logger = logging.getLogger("slap.tool.nosqltester_manager")
- if parameter_dict['debug']:
+ if self.enable_debug:
self.logger.setLevel(logging.DEBUG)
file_handler = logging.FileHandler(os.path.join(self.log_directory, "nosqltester_manager.log"))
file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
@@ -120,22 +115,22 @@ class NoSQLTesterManager(object):
parser.add_argument('--port', '-p', type=int, default=5000,
help='Manager port')
- parser.add_argument('--debug', '-d', type=bool, default=True,
+ parser.add_argument('--enable-debug', '-d', type=bool, default=True,
help='Enable debug messages')
- parser.add_argument('--report-directory', '-r', default='/var/log',
- help='Report directory')
+ parser.add_argument('--log-directory', '-r', default='/var/log',
+ help='Log/Report directory')
- parser.add_argument('--nb-server-max', '-s', type=int, default=3,
+ parser.add_argument('--max-server', '-s', type=int, default=3,
help='Maximum number of servers')
- parser.add_argument('--nb-tester-max', '-t', type=int, default=3,
+ parser.add_argument('--max-tester', '-t', type=int, default=3,
help='Maximum number of testers')
- parser.add_argument('--nb-server-init', '-n', type=int, default=1,
+ parser.add_argument('--nb-init-server', '-n', type=int, default=1,
help='Number of servers to start with')
- parser.add_argument('--nb-tester-init', '-m', type=int, default=3,
+ parser.add_argument('--nb-init-tester', '-m', type=int, default=3,
help='Number of testers to start with')
parser.add_argument('--node-increment', '-i', type=int, default=1,
@@ -227,27 +222,50 @@ class NoSQLTesterManager(object):
return True
def preRun(self):
- start_value_dict = {}
+ try:
+ start_value_dict = {}
- if self.erp5_publish_url and self.erp5_publish_project:
- if self.nb_init_tester != self.max_tester:
- nb_users = (self.nb_init_tester, self.max_tester)
- else:
- nb_users = self.max_tester
+ if self.erp5_publish_url and self.erp5_publish_project:
+ if self.nb_init_tester != self.max_tester:
+ nb_users = (self.nb_init_tester, self.max_tester)
+ else:
+ nb_users = self.max_tester
from erp5.utils.benchmark.benchmark import ERP5BenchmarkResult
new_document_id = ERP5BenchmarkResult.createResultDocument(
self.erp5_publish_url,
self.erp5_publish_project,
- getattr(self.argument_namespace, 'repeat', 1),
+ self.repeat,
nb_users)
- self.erp5_publish_url = '/'.join((self.erp5_publish_url, new_document_id))
- start_value_dict['erp5_publish_url'] = self.erp5_publish_url
- start_value_dict['erp5_publish_project'] = self.erp5_publish_project
+ self.erp5_publish_url = '/'.join((self.erp5_publish_url, new_document_id))
+ start_value_dict['erp5_publish_url'] = self.erp5_publish_url
+ start_value_dict['erp5_publish_project'] = self.erp5_publish_project
+ except:
+ self.logger.debug(Formatter().formatException(sys.exc_info()))
return start_value_dict
+ def push_to_erp5(self):
+ try:
+ if self.erp5_publish_url and self.erp5_publish_project:
+ from erp5.utils.benchmark.benchmark import ERP5BenchmarkResult
+ result_instance = ERP5BenchmarkResult(self, 1, 0)
+
+ with result_instance as result:
+ for filename in self.benchmark_suite_name_list:
+ current_suite = result.enterSuite(filename)
+ f = open(os.path.join(self.log_directory, filename))
+ i = 0
+ for line in f:
+ result(line.rstrip(), 0)
+ f.close()
+ result.exitSuite()
+
+ result_instance.iterationFinished()
+ except:
+ self.logger.debug(Formatter().formatException(sys.exc_info()))
+
def do_scheduling(self, action):
"""
Sends an action to all the testers know by the manager.
@@ -379,6 +397,9 @@ class NoSQLTesterManager(object):
os.write(self.write_pipe, "GO")
self.node_count += self.node_increment
else:
+ self.logger.debug("Pushing results to ERP5")
+ self.push_to_erp5()
+ self.benchmark_suite_name_list = []
self.logger.debug("Stopping")
os.write(self.write_pipe, "END")
os.close(self.write_pipe)
@@ -416,7 +437,7 @@ class NoSQLTesterManager(object):
self.todo -= 1
if self.todo == 0:
- self.process_report(self.reports)
+ self.benchmark_suite_name_list.append(self.process_report(self.reports))
self.reports = []
_allCurrentTesterExecuted()
@@ -462,10 +483,13 @@ class NoSQLTesterManager(object):
pass
def postRun(self):
- if self.erp5_publish_url and self.erp5_publish_project:
- from erp5.utils.benchmark.benchmark import ERP5BenchmarkResult
- ERP5BenchmarkResult.closeResultDocument(self.erp5_publish_url,
- tuple(self.error_message_set))
+ try:
+ if self.erp5_publish_url and self.erp5_publish_project:
+ from erp5.utils.benchmark.benchmark import ERP5BenchmarkResult
+ ERP5BenchmarkResult.closeResultDocument(self.erp5_publish_url,
+ tuple(self.error_message_set))
+ except:
+ self.logger.debug(Formatter().formatException(sys.exc_info()))
def scheduler_thread(scheduler, read):
s = ""
More information about the Erp5-report
mailing list