[Erp5-report] r16295 - in /erp5/trunk/products/ERP5Type: Core/ tests/
nobody at svn.erp5.org
nobody at svn.erp5.org
Wed Sep 12 14:04:54 CEST 2007
Author: aurel
Date: Wed Sep 12 14:04:54 2007
New Revision: 16295
URL: http://svn.erp5.org?rev=16295&view=rev
Log:
add integration of HBTreeFolder into ERP5 Folder class
add a new method to generate id per day
add unit test for migration from one btree to an hbtree
Added:
erp5/trunk/products/ERP5Type/tests/testFolderMigration.py (with props)
Modified:
erp5/trunk/products/ERP5Type/Core/Folder.py
Modified: erp5/trunk/products/ERP5Type/Core/Folder.py
URL: http://svn.erp5.org/erp5/trunk/products/ERP5Type/Core/Folder.py?rev=16295&r1=16294&r2=16295&view=diff
==============================================================================
--- erp5/trunk/products/ERP5Type/Core/Folder.py (original)
+++ erp5/trunk/products/ERP5Type/Core/Folder.py Wed Sep 12 14:04:54 2007
@@ -46,9 +46,16 @@
from Products.CMFCore.CMFBTreeFolder import CMFBTreeFolder
except ImportError:
from Products.BTreeFolder2.CMFBTreeFolder import CMFBTreeFolder
+
from Products.BTreeFolder2.BTreeFolder2 import BTreeFolder2Base
+
+from Products.HBTreeFolder2.CMFHBTreeFolder import CMFHBTreeFolder
+from Products.HBTreeFolder2.HBTreeFolder2 import HBTreeFolder2Base
+
from AccessControl import getSecurityManager
from Products.ERP5Type import Permissions
+from MethodObject import Method
+from DateTime import DateTime
from random import randint
import os
@@ -139,6 +146,22 @@
else:
raise TypeError, 'deleteContent only accepts string or list, '\
'not %s' % type(id)
+
+ def _generatePerDayId(self):
+ """
+ Generate id base on date, useful for HBTreeFolder
+ """
+ current_date = str(DateTime().Date()).replace("/", "")
+ try:
+ my_id = int(self.getLastId())
+ except TypeError:
+ my_id = 1
+ while self.hasContent("%s-%s" %(current_date, my_id)):
+ my_id = my_id + 1
+ my_id = str(my_id)
+ self._setLastId(my_id) # Make sure no reindexing happens
+ return "%s-%s" %(current_date, my_id)
+
def _generateRandomId(self):
"""
@@ -234,7 +257,7 @@
"Please use a domain dict instead.",
DeprecationWarning)
kw['selection_report'] = kw['selection_report'].asDomainDict()
- if kw['selection_report'].has_key('parent'):
+ if kw['selection_report'].has_key('parent'):
delete_parent_uid = 1
if delete_parent_uid:
del kw['parent_uid']
@@ -295,7 +318,23 @@
"""
return self.countFolder(**kw)[0][0]
-class Folder( CopyContainer, CMFBTreeFolder, Base, FolderMixIn, WebDAVFolder):
+class FolderMethodWrapper(Method):
+ """
+ This a wrapper between folder method and folder type method
+ """
+ def __init__(self, method_id):
+ self.method_id = method_id
+
+ def __call__(self, folder, *args, **kw):
+ try:
+ method = getattr(folder._plugin, self.method_id)
+ return method(folder, *args, **kw)
+ except AttributeError:
+ LOG("Folder call %s" %self.method_id, WARNING, "raise AttributeError on %s with plugin %s" %(folder, folder._plugin))
+ pass
+
+
+class Folder(CopyContainer, CMFBTreeFolder, CMFHBTreeFolder, Base, FolderMixIn, WebDAVFolder):
"""
A Folder is a subclass of Base but not of XMLObject.
Folders are not considered as documents and are therefore
@@ -358,13 +397,165 @@
# XXX Prevent inheritance from PortalFolderBase
description = None
-
+ # Per default we use BTree folder
+ _isHBTree = False
+ _isBTree = True
+ _plugin = None # plugin store the folder type class use by the folder
+
# Overload __init__ so that we do not take into account title
# This is required for test_23_titleIsNotDefinedByDefault
def __init__(self, id):
self.id = id
- BTreeFolder2Base.__init__(self, id)
-
+ # do not do any other initialisation here,
+ # wait for new content call to do init
+
+ method_id_list = ["initBTrees", "manage_fixCount", "manage_cleanup",
+ "getBatchObjectListing", "manage_object_workspace",
+ "tpValues", "objectCount", "has_key", "objectIds",
+ "objectItems", "objectMap", "objectIds_d", "objectMap_d",
+ "keys", "values", "items", "hasObject", "get", "generateId",
+ "__len__", "allowedContentTypes", "_delOb",
+ "_getOb", "_setObject", "_initBTrees", "_populateFromFolder",
+ "_fixCount", "_cleanup", "_setOb", "_checkId", "_delObject",]
+
+ def _generatePluginMethod(self):
+ """ Will generate alias to Tree method depending
+ on configuration """
+ for method_id in self.method_id_list:
+ method_wrapper = FolderMethodWrapper(method_id)
+ setattr(self, method_id, method_wrapper)
+
+ def newContent(self, *args, **kw):
+ """ Create a new content """
+ # Create data structure if none present
+ if self._plugin is None:
+ if self._isHBTree:
+ self._plugin = CMFHBTreeFolder
+ elif self._isBTree:
+ self._plugin = CMFBTreeFolder
+ else:
+ raise ValueError, "No plugin defined"
+ self._generatePluginMethod()
+ self._plugin.__init__(self, self.id)
+ return FolderMixIn.newContent(self, *args, **kw)
+
+ security.declareProtected(Permissions.View, 'isBTree')
+ def isBTree(self):
+ """ Return if folder is a BTree or not """
+ return self._isBTree
+
+ security.declareProtected(Permissions.View, 'isHBTree')
+ def isHBTree(self):
+ """ Return if folder is a HBTree or not """
+ return self._isHBTree
+
+ def _setHBTree(self,):
+ """ Define this folder use HBTree stucture """
+ self._isHBTree = True
+ self._isBTree = False
+
+ def _setBTree(self,):
+ """ Define this folder use BTree stucture """
+ self._isHBTree = False
+ self._isBTree = True
+
+ def hashId(self, id):
+ """ id hashing can be override with a script """
+ script = self._getTypeBasedMethod('hashId')
+ if script is not None:
+ return script(self, id)
+ return self._plugin.hashId(self, id)
+
+ security.declareProtected( Permissions.ManagePortal, 'migrateToHBTree' )
+ def migrateToHBTree(self, migration_generate_id_method=None, new_generate_id_method=None, REQUEST=None):
+ """
+ Function to migrate from a BTree folder to HBTree folder.
+ It will first call setId on all folder objects to have right id
+ to be used with an hbtreefolder.
+ Then it will migrate foder from btree to hbtree.
+ """
+ if self._plugin is None:
+ # make sure to have all method right defined
+ self._plugin = CMFBTreeFolder
+ self._generatePluginMethod()
+
+ BUNDLE_COUNT = 10
+ # we may want to change all objects ids before migrating to new folder type
+ if migration_generate_id_method not in (None, ''):
+ # set new id generator here so that object created while migration
+ # got a right id
+ self.setIdGenerator(new_generate_id_method)
+ tag = "%s/%s/migrate" %(self.getId(),migration_generate_id_method)
+ id_list = list(self.objectIds())
+ # set new id by bundle
+ for x in xrange(len(self) / BUNDLE_COUNT):
+ self.activate(activity="SQLQueue", tag=tag).ERP5Site_setNewIdPerBundle(
+ self.getPath(),
+ id_list[x*BUNDLE_COUNT:(x+1)*BUNDLE_COUNT],
+ migration_generate_id_method, tag)
+
+ remaining_id_count = len(self) % BUNDLE_COUNT
+ if remaining_id_count:
+ self.activate(activity="SQLQueue", tag=tag).ERP5Site_setNewIdPerBundle(
+ self.getPath(),
+ id_list[-remaining_id_count:],
+ migration_generate_id_method, tag)
+ else:
+ tag = ''
+ # copy from btree to hbtree
+ self.activate(activity="SQLQueue", after_tag=tag)._launchCopyObjectToHBTree(tag)
+
+ if REQUEST is not None:
+ psm = N_('Migration to HBTree is running.',)
+ ret_url = '%s/%s?portal_status_message=%s' % \
+ (self.absolute_url(),
+ REQUEST.get('form_id', 'view'), psm)
+ return REQUEST.RESPONSE.redirect( ret_url )
+
+ def _finishCopyObjectToHBTree(self):
+ """
+ Remove remaining attributes from previous btree
+ and migration
+ """
+ delattr(self, "_tree")
+ delattr(self, "_former_plugin")
+
+ def _launchCopyObjectToHBTree(self, tag):
+ """
+ Launch activity per bundle to move object
+ from a btree to an hbtree
+ """
+ # migrate folder from btree to hbtree
+ id_list = list(self.objectIds())
+ self._former_plugin = self._plugin
+ self._setHBTree()
+ self._plugin = CMFHBTreeFolder
+ self._generatePluginMethod()
+ self._plugin.__init__(self, self.id)
+ # launch activity per bundle to copy/paste to hbtree
+ BUNDLE_COUNT = 100
+ for x in xrange(len(id_list) / BUNDLE_COUNT):
+ self.activate(activity="SQLQueue", tag=tag)._copyObjectToHBTree(
+ id_list=id_list[x*BUNDLE_COUNT:(x+1)*BUNDLE_COUNT],)
+
+ remaining_id_count = len(id_list) % BUNDLE_COUNT
+ if remaining_id_count:
+ self.activate(activity="SQLQueue", tag=tag)._copyObjectToHBTree(
+ id_list=id_list[-remaining_id_count:],)
+ # remove uneeded attribute
+ self.activate(activity="SQLQueue", after_tag=tag)._finishCopyObjectToHBTree()
+
+ def _copyObjectToHBTree(self, id_list=None,):
+ """
+ Move object from a btree container to
+ a hbtree one
+ """
+ getOb = self._former_plugin._getOb
+ setOb = self._setOb
+ for id in id_list:
+ obj = getOb(self, id)
+ setOb(id, obj)
+
# Override Zope default by folder id generation
def _get_id(self, id):
if self._getOb(id, None) is None :
@@ -394,8 +585,11 @@
security.declareProtected(Permissions.View, 'hasContent')
def hasContent(self,id):
- return self.hasObject(id)
-
+ try:
+ return self._plugin.hasObject(self, id)
+ except AttributeError:
+ return False
+
security.declareProtected( Permissions.ModifyPortalContent, 'exportAll' )
def exportAll(self,dir=None):
"""
@@ -727,8 +921,12 @@
def _getVisibleAllowedContentTypeList():
hidden_type_list = portal.portal_types.getTypeInfo(self)\
.getHiddenContentTypeList()
- return [ ti.id for ti in CMFBTreeFolder.allowedContentTypes(self)
- if ti.id not in hidden_type_list ]
+ try:
+ return [ ti.id for ti in self._plugin.allowedContentTypes(self)
+ if ti.id not in hidden_type_list ]
+ except AttributeError:
+ return [ ti.id for ti in CMFBTreeFolder.allowedContentTypes(self)
+ if ti.id not in hidden_type_list ]
user = str(_getAuthenticatedUser(self))
portal_type = self.getPortalType()
@@ -772,7 +970,10 @@
# account i18n into consideration.
# XXX So sorting should be done in skins, after translation is performed.
def compareTypes(a, b): return cmp(a.title or a.id, b.title or b.id)
- type_list = CMFBTreeFolder.allowedContentTypes(self)
+ try:
+ type_list = self._plugin.allowedContentTypes(self)
+ except AttributeError:
+ type_list = CMFBTreeFolder.allowedContentTypes(self)
type_list.sort(compareTypes)
return ['/'.join(x.getPhysicalPath()) for x in type_list]
@@ -796,8 +997,15 @@
view = Base.view
# Aliases
- getObjectIds = CMFBTreeFolder.objectIds
-
+ security.declareProtected(Permissions.AccessContentsInformation,
+ 'getObjectIds')
+ def getObjectIds(self, *args, **kw):
+ try:
+ self._plugin.objectIds(args, kw)
+ except AttributeError:
+ CMFBTreeFolder.objectIds(args, kw)
+
+
# Overloading
security.declareProtected( Permissions.AccessContentsInformation,
'getParentSQLExpression' )
@@ -862,7 +1070,7 @@
security.declareProtected( Permissions.AccessContentsInformation,
'objectValues' )
def objectValues(self, spec=None, meta_type=None, portal_type=None,
- sort_on=None, sort_order=None, **kw):
+ sort_on=None, sort_order=None, base_id=None,**kw):
"""
Returns a list containing object contained in this folder.
"""
@@ -876,7 +1084,13 @@
except AttributeError:
from Products.BTreeFolder2.BTreeFolder2 import BTreeFolder2Base
BTreeFolder2Base.__init__(self, self.getId())
- object_list = CMFBTreeFolder.objectValues(self, spec=spec)
+ try:
+ if self.isBTree():
+ object_list = self._plugin.objectValues(self, spec=spec)
+ else:
+ object_list = self._plugin.objectValues(self, base_id=base_id)
+ except AttributeError:
+ object_list = CMFBTreeFolder.objectValues(self, spec=spec)
if portal_type is not None:
if type(portal_type) == type(''):
portal_type = (portal_type,)
@@ -899,7 +1113,10 @@
kw['portal_type'] = portal_type
filter = kw.pop('filter', {}) or {}
kw.update(filter)
- object_list = CMFBTreeFolder.contentValues(self, spec=spec, filter=kw)
+ try:
+ object_list = self._plugin.contentValues(self, spec=spec, filter=kw)
+ except AttributeError:
+ object_list = CMFBTreeFolder.contentValues(self, spec=spec, filter=kw)
object_list = sortValueList(object_list, sort_on, sort_order, **kw)
return object_list
Added: erp5/trunk/products/ERP5Type/tests/testFolderMigration.py
URL: http://svn.erp5.org/erp5/trunk/products/ERP5Type/tests/testFolderMigration.py?rev=16295&view=auto
==============================================================================
--- erp5/trunk/products/ERP5Type/tests/testFolderMigration.py (added)
+++ erp5/trunk/products/ERP5Type/tests/testFolderMigration.py Wed Sep 12 14:04:54 2007
@@ -1,0 +1,136 @@
+##############################################################################
+#
+# Copyright (c) 2007 Nexedi SARL and Contributors. All Rights Reserved.
+# Aurélien Calonne <aurel at nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+import os, sys
+if __name__ == '__main__':
+ execfile(os.path.join(sys.path[0], 'framework.py'))
+
+# Needed in order to have a log file inside the current folder
+os.environ['EVENT_LOG_FILE'] = os.path.join(os.getcwd(), 'zLOG.log')
+os.environ['EVENT_LOG_SEVERITY'] = '-300'
+
+from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
+from zLOG import LOG
+from Products.CMFCore.tests.base.testcase import LogInterceptor
+from Products.ERP5Type.tests.utils import createZODBPythonScript
+from Products.ERP5Type.ERP5Type import ERP5TypeInformation
+from Products.ERP5Type.Cache import clearCache
+
+class TestFolderMigration(ERP5TypeTestCase, LogInterceptor):
+
+ # Some helper methods
+
+ def getTitle(self):
+ return "Folder Migration"
+
+ def getBusinessTemplateList(self):
+ """
+ Return the list of business templates.
+ """
+ return tuple()
+
+ def afterSetUp(self):
+ """
+ Executed before each test_*.
+ """
+ self.login()
+ self.folder = self.getPortal().newContent(id='TestFolder',
+ portal_type='Folder')
+
+ def beforeTearDown(self):
+ """
+ Executed after each test_*.
+ """
+ self.getPortal().manage_delObjects(ids=[self.folder.getId(),])
+ clearCache()
+
+ def newContent(self):
+ """
+ Create an object in self.folder and return it.
+ """
+ return self.folder.newContent(portal_type='Folder')
+
+ def test_01_folderIsBtree(self, quiet=0, run=1):
+ """
+ Test the folder is a BTree
+ """
+ if not run : return
+ if not quiet:
+ message = 'Test folderIsBtree'
+ LOG('Testing... ', 0, message)
+ self.assertRaises(AttributeError, self.folder.getTreeIdList)
+ self.assertEqual(self.folder.isBTree(), True)
+ self.assertEqual(self.folder.isHBTree(), False)
+
+
+ def test_02_migrateFolder(self, quiet=0, run=1):
+ """
+ migrate folder from btree to hbtree
+ """
+ if not run : return
+ if not quiet:
+ message = 'Test migrateFolder'
+ LOG('Testing... ', 0, message)
+ # Create some objects
+ self.assertEquals(self.folder.getIdGenerator(), '')
+ self.assertEquals(len(self.folder), 0)
+ obj1 = self.newContent()
+ self.assertEquals(obj1.getId(), '1')
+ obj2 = self.newContent()
+ self.assertEquals(obj2.getId(), '2')
+ obj3 = self.newContent()
+ self.assertEquals(obj3.getId(), '3')
+ get_transaction().commit()
+ self.tic()
+ # call migration script
+ self.folder.migrateToHBTree(migration_generate_id_method="Base_generateIdFromStopDate",
+ new_generate_id_method="_generatePerDayId")
+ get_transaction().commit()
+ self.tic()
+ # check we now have a hbtree
+ self.assertEqual(self.folder.isBTree(), False)
+ self.assertEqual(self.folder.isHBTree(), True)
+ self.assertEqual(len(self.folder.getTreeIdList()), 1)
+ self.assertEqual(len(self.folder.objectIds()), 3)
+ # check object ids
+ from DateTime import DateTime
+ date = DateTime().Date()
+ date = date.replace("/", "")
+ self.assertEquals(obj1.getId(), '%s-1' %date)
+ self.assertEquals(obj2.getId(), '%s-2' %date)
+ self.assertEquals(obj3.getId(), '%s-3' %date)
+
+
+if __name__ == '__main__':
+ framework()
+else:
+ import unittest
+ def test_suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(TestFolderMigration))
+ return suite
Propchange: erp5/trunk/products/ERP5Type/tests/testFolderMigration.py
------------------------------------------------------------------------------
svn:executable = *
More information about the Erp5-report
mailing list