[Erp5-report] r17268 - in /erp5/trunk/products/CMFActivity/Activity: SQLDict.py SQLQueue.py

[nobody at svn.erp5.org]

Author: seb
Date: Tue Oct 30 00:37:21 2007
New Revision: 17268

URL: http://svn.erp5.org?rev=17268&view=rev
Log:
- do not stock new ids used by activities into zodb, this improve
  a lot performance under very high load
- add a round_robin_scheduling parameter for SQLQueue so that
  all messages on a same path will not be processed on the same
  node

Modified:
    erp5/trunk/products/CMFActivity/Activity/SQLDict.py
    erp5/trunk/products/CMFActivity/Activity/SQLQueue.py

Modified: erp5/trunk/products/CMFActivity/Activity/SQLDict.py
URL: http://svn.erp5.org/erp5/trunk/products/CMFActivity/Activity/SQLDict.py?rev=17268&r1=17267&r2=17268&view=diff
==============================================================================
--- erp5/trunk/products/CMFActivity/Activity/SQLDict.py (original)
+++ erp5/trunk/products/CMFActivity/Activity/SQLDict.py Tue Oct 30 00:37:21 2007
@@ -95,7 +95,8 @@
                               for message in registered_message_list]
       tag_list = [message.activity_kw.get('tag', '') for message in registered_message_list]
       order_validation_text_list = [self.getOrderValidationText(message) for message in registered_message_list]
-      uid_list = activity_tool.getPortalObject().portal_ids.generateNewLengthIdList(id_group='portal_activity', id_count=len(registered_message_list))
+      uid_list = activity_tool.getPortalObject().portal_ids.generateNewLengthIdList(id_group='portal_activity', 
+		           id_count=len(registered_message_list), store=0)
       activity_tool.SQLDict_writeMessageList( uid_list = uid_list,
                                               path_list = path_list,
                                               method_id_list = method_id_list,
@@ -484,12 +485,12 @@
                                    order_validation_text = line.order_validation_text)
         self.getExecutableMessageList(activity_tool, message, message_dict,
                                       validation_text_dict)
-
       # XXX probably this below can be optimized by assigning multiple messages at a time.
       path_dict = {}
       assignMessage = activity_tool.SQLDict_assignMessage
       processing_node = LAST_PROCESSING_NODE
       id_tool = activity_tool.getPortalObject().portal_ids
+  
       for message in message_dict.itervalues():
         path = '/'.join(message.object_path)
         broadcast = message.activity_kw.get('broadcast', 0)
@@ -499,7 +500,8 @@
           assignMessage(processing_node=1, uid=[uid])
           if node_count > 1:
             uid_list = id_tool.generateNewLengthIdList(id_group='portal_activity',
-                                                       id_count=node_count - 1)
+                                                       id_count=node_count - 1,
+						       store=0)
             path_list = [path] * (node_count - 1)
             method_id_list = [message.method_id] * (node_count - 1)
             priority_list = [message.activity_kw.get('priority', 1)] * (node_count - 1)

Modified: erp5/trunk/products/CMFActivity/Activity/SQLQueue.py
URL: http://svn.erp5.org/erp5/trunk/products/CMFActivity/Activity/SQLQueue.py?rev=17268&r1=17267&r2=17268&view=diff
==============================================================================
--- erp5/trunk/products/CMFActivity/Activity/SQLQueue.py (original)
+++ erp5/trunk/products/CMFActivity/Activity/SQLQueue.py Tue Oct 30 00:37:21 2007
@@ -64,7 +64,7 @@
   def prepareQueueMessage(self, activity_tool, m):
     if m.is_registered:
       id_tool = activity_tool.getPortalObject().portal_ids
-      uid = id_tool.generateNewLengthId(id_group='portal_activity_queue')
+      uid = id_tool.generateNewLengthId(id_group='portal_activity_queue', store=0)
       activity_tool.SQLQueue_writeMessage(uid = uid,
                                           path = '/'.join(m.object_path) ,
                                           method_id = m.method_id,
@@ -89,6 +89,10 @@
     next_processing_date = now_date + float(VALIDATION_ERROR_DELAY)/86400
     message_list = readMessage(processing_node=processing_node, to_date=now_date)
     for line in message_list:
+      # Do not process many messages if there are long
+      new_date = DateTime()
+      if ((new_date-now_date)*86400) > 10:
+        break
       path = line.path
       method_id = line.method_id
       # Make sure message can not be processed anylonger
@@ -331,7 +335,8 @@
           assignMessage(processing_node=1, uid=message.uid)
           if node_count > 1:
             uid_list = id_tool.generateNewLengthIdList(id_group='portal_activity_queue',
-                                                       id_count=node_count - 1)
+                                                       id_count=node_count - 1,
+						       store=0)
             priority = message.activity_kw.get('priority', 1)
             dumped_message = self.dumpMessage(message)
             date = message.activity_kw.get('at_date', now_date)
@@ -352,9 +357,11 @@
           # the same node, so that object caching is more efficient. Otherwise, apply a round
           # robin scheduling.
           node = path_dict.get(path)
+          round_robin_scheduling = message.activity_kw.get('round_robin_scheduling', 0)
           if node is None:
             node = processing_node
-            path_dict[path] = node
+	    if not round_robin_scheduling:
+              path_dict[path] = node
             processing_node += 1
             if processing_node > node_count:
               processing_node = 1