[Erp5-report] r9627 - /erp5/trunk/utils/erp5mechanize/userperzope.py

[nobody at svn.erp5.org]

Author: vincent
Date: Mon Sep  4 11:23:18 2006
New Revision: 9627

URL: http://svn.erp5.org?rev=9627&view=rev
Log:
Add maximum-failure parameter to specify the maximum total number of failure or the average number of failure that can be accepted in a valid test.
Sort results before displaying to make sure no result is missed.

Modified:
    erp5/trunk/utils/erp5mechanize/userperzope.py

Modified: erp5/trunk/utils/erp5mechanize/userperzope.py
URL: http://svn.erp5.org/erp5/trunk/utils/erp5mechanize/userperzope.py?rev=9627&r1=9626&r2=9627&view=diff
==============================================================================
--- erp5/trunk/utils/erp5mechanize/userperzope.py (original)
+++ erp5/trunk/utils/erp5mechanize/userperzope.py Mon Sep  4 11:23:18 2006
@@ -29,6 +29,21 @@
 import sys
 import getopt
 
+def compute_max_error_count(user_count, maximum_failure):
+  if maximum_failure is None:
+    return None
+  maximum_failure = int(maximum_failure)
+  if maximum_failure < 0:
+    return -maximum_failure / user_count
+  else:
+    return maximum_failure
+
+def result_cmp(a, b):
+  dif = cmp(a['user'], b['user'])
+  if dif != 0:
+    return dif
+  return cmp(a['duration'], b['duration'])
+
 def parse_csv_line(line, sep):
   line = line.replace('\n','')
   item_list = line.split(sep)
@@ -46,17 +61,23 @@
   sys.stdout.write('\n')
 
 try:
-  opt_list, arg_list = getopt.getopt(sys.argv[1:], '', ['maximum_duration=', 'csv_separator=', 'default_step_number=', 'ignore_first_run'])
+  opt_list, arg_list = getopt.getopt(sys.argv[1:], '', ['maximum_duration=', 'maximum_failure=', 'csv_separator=', 'default_step_number=', 'ignore_first_run'])
 except getopt.error, msg:
   print 'Error: %s' % (msg, )
   print \
 '''Synopsis:
-  %s [--maximum_duration=float] [--csv_separator=char] [--default_step_number=integer] [file [...]]
+  %s [--maximum_duration=float] [--maximum_failure=int] [--csv_separator=char] [--default_step_number=integer] [file [...]]
 
   Each given file must be a CSV file separated by separator (default = ,).
   First CSV column must be of the form user at host.
   If last collumn contains "FAILED" (without quotes) the following is ignored
     and the test immediatelly marked as failed.
+  If maximum_failure is given and positive, it will be the absolute maximum
+    of tolerated failures in a test, independently of user count.
+  If maximum_failure is given and negative, its absolute value will be the
+    maximum number of tolerated failures per user in average
+    (computed as -maximum_failure divided by the number of user for a given
+    test run).
   If maximum_duration is given, second and third columns must be floating
     point values (second is start time, third is stop time). The
     maximum_duration parameter must be a floating point value aswel. If the
@@ -75,11 +96,14 @@
 
 csv_separator = ','
 maximum_duration = None
+maximum_failure = None
 ignore_first_run = False
 default_step_number = None
 for o, a in opt_list:
   if o == '--maximum_duration':
     maximum_duration = float(a)
+  elif o == '--maximum_failure':
+    maximum_failure = a
   elif o == '--csv_separator':
     csv_separator = a
   elif o == '--default_step_number':
@@ -105,8 +129,8 @@
       if ignore_first_run: # This allows to avoid the impact of the first run for a given user (cache filling, ...)
         continue
     if field_list[-1] == 'FAILED':
-      failed = True
-      break
+      failed += 1
+      continue
     if len(field_list) < 4:
       if default_step_number is not None:
         duration_list.append(float(field_list[2])/default_step_number)
@@ -116,25 +140,36 @@
     else:
       duration_list.append(float(field_list[2])/int(field_list[3]))
   if len(line_list):
-    result = {'user': failed==False and len(known_user_list) or 0, 'duration': len(duration_list) and sum(duration_list)/len(duration_list) or 0, 'max_duration': len(duration_list) and max(duration_list) or 0, 'min_duration': len(duration_list) and min(duration_list) or 0}
+    result = {'user': len(known_user_list), 'duration': len(duration_list) and sum(duration_list)/len(duration_list) or 0, 'max_duration': len(duration_list) and max(duration_list) or 0, 'min_duration': len(duration_list) and min(duration_list) or 0, 'failures': failed, 'filename': filename}
     if not result_list.has_key(len(known_zope_list)):
       result_list[len(known_zope_list)] = [result]
     else:
       result_list[len(known_zope_list)].append(result)
   csv.close()
 
-output_csv(separator=csv_separator, fields=['Zope count', 'Total user count', 'Average test duration at maximum user count', 'Min', 'Max'])
+output_csv(separator=csv_separator, fields=['Zope count', 'Total user count', 'Average test duration at maximum user count', 'Min', 'Max', 'Limited by', 'File'])
 for k, v in result_list.items():
+  v.sort(cmp=result_cmp)
   maximum_user_number = 0
   avg_duration = 0
   min_duration = 0
   max_duration = 0
+  filename = ''
+  reason = '(no limit reached)'
   for result in v:
-    if result['user'] == 0 or result['user'] < maximum_user_number or maximum_duration is not None and result['duration'] > maximum_duration:
+    if result['user'] < maximum_user_number or (result['user'] == maximum_user_number and result['duration'] > avg_duration):
+      continue
+    max_failure_for_this_run = compute_max_error_count(result['user'], maximum_failure)
+    if max_failure_for_this_run is not None and result['failures'] > max_failure_for_this_run:
+      reason = 'Too many failures'
+      break
+    if maximum_duration is not None and result['duration'] > maximum_duration:
+      reason = 'Too slow'
       break
     maximum_user_number = result['user']
     avg_duration = result['duration']
     min_duration = result['min_duration']
     max_duration = result['max_duration']
-  output_csv(separator=csv_separator, fields=[k, maximum_user_number, avg_duration, min_duration, max_duration])
+    filename = result['filename']
+  output_csv(separator=csv_separator, fields=[k, maximum_user_number, avg_duration, min_duration, max_duration, reason, filename])