[Erp5-report] r11067 - in /erp5/trunk/products/ERP5Type: ./ Tool/ dtml/ tests/
nobody at svn.erp5.org
nobody at svn.erp5.org
Thu Nov 2 16:59:53 CET 2006
Author: ivan
Date: Thu Nov 2 16:59:45 2006
New Revision: 11067
URL: http://svn.erp5.org?rev=11067&view=rev
Log:
Moved CacheTool from ERP5 to ERP5Type
Added:
erp5/trunk/products/ERP5Type/Tool/CacheTool.py
erp5/trunk/products/ERP5Type/dtml/cache_tool_configure.dtml
erp5/trunk/products/ERP5Type/tests/testCache.py
Modified:
erp5/trunk/products/ERP5Type/__init__.py
erp5/trunk/products/ERP5Type/tests/testCacheTool.py
Added: erp5/trunk/products/ERP5Type/Tool/CacheTool.py
URL: http://svn.erp5.org/erp5/trunk/products/ERP5Type/Tool/CacheTool.py?rev=11067&view=auto
==============================================================================
--- erp5/trunk/products/ERP5Type/Tool/CacheTool.py (added)
+++ erp5/trunk/products/ERP5Type/Tool/CacheTool.py Thu Nov 2 16:59:45 2006
@@ -1,0 +1,202 @@
+##############################################################################
+#
+# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
+# Ivan Tyagov <ivan at nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+
+""" Cache Tool module for ERP5 """
+from AccessControl import ClassSecurityInfo
+from Products.ERP5Type.Tool.BaseTool import BaseTool
+from Products.ERP5Type import Permissions
+from Globals import InitializeClass, DTMLFile, PersistentMapping
+from Products.ERP5 import _dtmldir
+from Products.ERP5Type.Cache import CachingMethod, CacheFactory
+from Products.ERP5Type.CachePlugins.RamCache import RamCache
+from Products.ERP5Type.CachePlugins.DistributedRamCache import DistributedRamCache
+from Products.ERP5Type.CachePlugins.SQLCache import SQLCache
+
+## try to import needed modules for cache plugins
+try:
+ import memcache
+except ImportError:
+ pass
+
+class CacheTool(BaseTool):
+ """ Caches tool wrapper for ERP5 """
+
+ id = "portal_caches"
+ meta_type = "ERP5 Cache Tool"
+ portal_type = "Cache Tool"
+
+ security = ClassSecurityInfo()
+ manage_options = ({'label': 'Configure',
+ 'action': 'cache_tool_configure',
+ },) + BaseTool.manage_options
+
+ security.declareProtected( Permissions.ManagePortal, 'cache_tool_configure')
+ cache_tool_configure = DTMLFile('cache_tool_configure', _dtmldir)
+
+ def __init__(self):
+ BaseTool.__init__(self)
+
+ security.declareProtected(Permissions.AccessContentsInformation, 'getCacheFactoryList')
+ def getCacheFactoryList(self):
+ """ Return available cache factories """
+ rd ={}
+ for cf in self.objectValues('ERP5 Cache Factory'):
+ cache_scope = cf.getId()
+ rd[cache_scope] = {}
+ rd[cache_scope]['cache_plugins'] = []
+ rd[cache_scope]['cache_params'] = {}
+ for cp in cf.getCachePluginList():
+ cache_obj = None
+ cp_meta_type = cp.meta_type
+ if cp_meta_type == 'ERP5 Ram Cache Plugin':
+ cache_obj = RamCache()
+ elif cp_meta_type == 'ERP5 Distributed Ram Cache Plugin':
+ ## even thougn we have such plugin in ZODB that doens't mean
+ ## we have corresponding memcache module installed
+ if 'memcache' in globals().keys():
+ cache_obj = DistributedRamCache({'server':cp.getServer()})
+ else:
+ ## we don't have memcache python module installed
+ ## thus we can't use DistributedRamCache plugin
+ cache_obj = None
+ elif cp_meta_type == 'ERP5 SQL Cache Plugin':
+ ## use connection details from 'erp5_sql_transactionless_connection' ZMySLQDA object
+ connection_string = self.erp5_sql_transactionless_connection.connection_string
+ kw = self.parseDBConnectionString(connection_string)
+ kw['cache_table_name'] = cp.getCacheTableName()
+ cache_obj = SQLCache(kw)
+ if cache_obj:
+ ## set cache expire check interval
+ cache_obj.cache_expire_check_interval = cp.getCacheExpireCheckInterval()
+ rd[cache_scope]['cache_plugins'].append(cache_obj)
+ rd[cache_scope]['cache_params']['cache_duration'] = cf.getCacheDuration()
+ return rd
+
+ ##
+ ## DB structure
+ ##
+ security.declareProtected(Permissions.ModifyPortalContent, 'createDBCacheTable')
+ def createDBCacheTable(self, cache_table_name="cache", REQUEST=None):
+ """ create in MySQL DB cache table """
+ my_query = SQLCache.create_table_sql %cache_table_name
+ try:
+ self.erp5_sql_transactionless_connection.manage_test("DROP TABLE %s" %cache_table_name)
+ except:
+ pass
+ self.erp5_sql_transactionless_connection.manage_test(my_query)
+ if REQUEST:
+ self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache table successfully created.')
+
+ security.declareProtected(Permissions.AccessContentsInformation, 'parseDBConnectionString')
+ def parseDBConnectionString(self, connection_string):
+ """ Parse given connection string. Code "borrowed" from ZMySLQDA.db """
+ kwargs = {}
+ items = connection_string.split()
+ if not items:
+ return kwargs
+ lockreq, items = items[0], items[1:]
+ if lockreq[0] == "*":
+ db_host, items = items[0], items[1:]
+ else:
+ db_host = lockreq
+ if '@' in db_host:
+ db, host = split(db_host,'@',1)
+ kwargs['db'] = db
+ if ':' in host:
+ host, port = split(host,':',1)
+ kwargs['port'] = int(port)
+ kwargs['host'] = host
+ else:
+ kwargs['db'] = db_host
+ if kwargs['db'] and kwargs['db'][0] in ('+', '-'):
+ kwargs['db'] = kwargs['db'][1:]
+ if not kwargs['db']:
+ del kwargs['db']
+ if not items:
+ return kwargs
+ kwargs['user'], items = items[0], items[1:]
+ if not items:
+ return kwargs
+ kwargs['passwd'], items = items[0], items[1:]
+ if not items:
+ return kwargs
+ kwargs['unix_socket'], items = items[0], items[1:]
+ return kwargs
+
+ ##
+ ## RAM cache structure
+ ##
+ security.declareProtected(Permissions.AccessContentsInformation, 'getRamCacheRoot')
+ def getRamCacheRoot(self):
+ """ Return RAM based cache root """
+ return CachingMethod.factories
+
+ security.declareProtected(Permissions.ModifyPortalContent, 'updateCache')
+ def updateCache(self, REQUEST=None):
+ """ Clear and update cache structure """
+ #erp5_site_id = self.getPortalObject().getId()
+ for cf in CachingMethod.factories:
+ for cp in CachingMethod.factories[cf].getCachePluginList():
+ del cp
+ CachingMethod.factories = {}
+ ## read configuration from ZODB
+ for key,item in self.getCacheFactoryList().items():
+ if len(item['cache_plugins'])!=0:
+ CachingMethod.factories[key] = CacheFactory(item['cache_plugins'], item['cache_params'])
+ if REQUEST:
+ self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache updated.')
+
+ security.declareProtected(Permissions.ModifyPortalContent, 'clearCache')
+ def clearCache(self, REQUEST=None):
+ """ Clear whole cache structure """
+ ram_cache_root = self.getRamCacheRoot()
+ for cf in ram_cache_root:
+ for cp in ram_cache_root[cf].getCachePluginList():
+ cp.clearCache()
+ if REQUEST:
+ self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache cleared.')
+
+ security.declareProtected(Permissions.ModifyPortalContent, 'clearCacheFactory')
+ def clearCacheFactory(self, cache_factory_id, REQUEST=None):
+ """ Clear only cache factory. """
+ ram_cache_root = self.getRamCacheRoot()
+ if ram_cache_root.has_key(cache_factory_id):
+ ram_cache_root[cache_factory_id].clearCache()
+ if REQUEST:
+ self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache factory %s cleared.' %cache_factory_id)
+
+ security.declareProtected(Permissions.ModifyPortalContent, 'clearCacheFactoryScope')
+ def clearCacheFactoryScope(self, cache_factory_id, scope, REQUEST=None):
+ """ Clear only cache factory. """
+ ram_cache_root = self.getRamCacheRoot()
+ if ram_cache_root.has_key(cache_factory_id):
+ ram_cache_root[cache_factory_id].clearCacheForScope(scope)
+ if REQUEST:
+ self.REQUEST.RESPONSE.redirect('cache_tool_configure?portal_status_message=Cache factory scope %s cleared.' %cache_factory_id)
+
Modified: erp5/trunk/products/ERP5Type/__init__.py
URL: http://svn.erp5.org/erp5/trunk/products/ERP5Type/__init__.py?rev=11067&r1=11066&r2=11067&view=diff
==============================================================================
--- erp5/trunk/products/ERP5Type/__init__.py (original)
+++ erp5/trunk/products/ERP5Type/__init__.py Thu Nov 2 16:59:45 2006
@@ -53,7 +53,7 @@
def initialize( context ):
# Import Product Components
- from Tool import ClassTool
+ from Tool import ClassTool, CacheTool
import Document
import Base, XMLObject
from ERP5Type import ERP5TypeInformation
@@ -61,7 +61,8 @@
object_classes = ()
content_constructors = ()
content_classes = (Base.Base, XMLObject.XMLObject,)
- portal_tools = (ClassTool.ClassTool, )
+ portal_tools = (ClassTool.ClassTool,
+ CacheTool.CacheTool,)
# Do initialization step
initializeProduct(context, this_module, globals(),
document_module = Document,
Added: erp5/trunk/products/ERP5Type/dtml/cache_tool_configure.dtml
URL: http://svn.erp5.org/erp5/trunk/products/ERP5Type/dtml/cache_tool_configure.dtml?rev=11067&view=auto
==============================================================================
--- erp5/trunk/products/ERP5Type/dtml/cache_tool_configure.dtml (added)
+++ erp5/trunk/products/ERP5Type/dtml/cache_tool_configure.dtml Thu Nov 2 16:59:45 2006
@@ -1,0 +1,27 @@
+<dtml-var manage_page_header>
+<dtml-var manage_tabs>
+
+<b><br/>
+ <dtml-var expr="REQUEST.get('portal_status_message', '')">
+</b>
+
+<h3>Cache invalidation</h3>
+ <form action="clearCache" method="POST">
+ <input type="submit" value="Clear all cache factories"/>
+ </form>
+ <dtml-in expr="objectIds('ERP5 Cache Factory')">
+ <form action="clearCacheFactory" method="POST">
+ <input type="hidden" name="cache_factory_id" value="<dtml-var sequence-item>">
+ <input type="submit" value="Clear <dtml-var sequence-item>"/>
+ </form>
+ </dtml-in>
+
+<h3>SQLCache configuration</h3>
+<p>Create SQL cache table(Note: you need to adjust later each SQLCache plugin to use this cache table name manually. Generally it is a good idea to use default sql cache table name)</p>
+<form action="createDBCacheTable" method="POST">
+ <input name="cache_table_name" value="cache">
+ <br/>
+ <input type="submit" value="Create (Recreate) sql cache table"/>
+</form>
+
+<dtml-var manage_page_footer>
Added: erp5/trunk/products/ERP5Type/tests/testCache.py
URL: http://svn.erp5.org/erp5/trunk/products/ERP5Type/tests/testCache.py?rev=11067&view=auto
==============================================================================
--- erp5/trunk/products/ERP5Type/tests/testCache.py (added)
+++ erp5/trunk/products/ERP5Type/tests/testCache.py Thu Nov 2 16:59:45 2006
@@ -1,0 +1,192 @@
+##############################################################################
+#
+# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
+# Ivan Tyagov <ivan at nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+import random
+import unittest
+import time
+import base64, md5
+from ERP5Cache.CachePlugins.RamCache import RamCache
+from ERP5Cache.CachePlugins.DistributedRamCache import DistributedRamCache
+from ERP5Cache.CachePlugins.SQLCache import SQLCache
+from ERP5Cache.CachePlugins.BaseCache import CacheEntry
+
+
+class Foo:
+ my_field = (1,2,3,4,5)
+
+class TestRamCache(unittest.TestCase):
+
+ def setUp(self):
+ self.cache_plugins = (RamCache(),
+ DistributedRamCache({'servers': '127.0.0.1:11211',
+ 'debugLevel': 7,}),
+ SQLCache( {'server': '',
+ 'user': '',
+ 'passwd': '',
+ 'db': 'test',
+ 'cache_table_name': 'cache',
+ }),
+ )
+
+ def testScope(self):
+ """ test scope functions """
+ ## create some sample scopes
+ iterations = 10
+ test_scopes = []
+ for i in range(0, iterations):
+ test_scopes.append("my_scope_%s" %i)
+ test_scopes.sort()
+
+ ## remove DistributedRamCache since it's a flat storage
+ filtered_cache_plugins = filter(lambda x: not isinstance(x, DistributedRamCache), self.cache_plugins)
+
+ for cache_plugin in filtered_cache_plugins:
+ print "TESTING (scope): ", cache_plugin
+
+ ## clear cache for this plugin
+ cache_plugin.clearCache()
+
+ ## should exists no scopes in cache
+ self.assertEqual([], cache_plugin.getScopeList())
+
+ ## set some sample values
+ for scope in test_scopes:
+ cache_id = '%s_cache_id' %scope
+ cache_plugin.set(cache_id, scope, scope*10)
+
+ ## we set ONLY one value per scope -> check if we get the same cache_id
+ self.assertEqual([cache_id], cache_plugin.getScopeKeyList(scope))
+ print "\t", cache_id, scope, "\t\tOK"
+
+ ## get list of scopes which must be the same as test_scopes since we clear cache initially
+ scopes_from_cache = cache_plugin.getScopeList()
+ scopes_from_cache.sort()
+ self.assertEqual(test_scopes, scopes_from_cache)
+
+ ## remove scope one by one
+ count = 1
+ for scope in test_scopes:
+ cache_plugin.clearCacheForScope(scope)
+ ## .. and check that we should have 1 less cache scope
+ scopes_from_cache = cache_plugin.getScopeList()
+ self.assertEqual(iterations - count, len(scopes_from_cache))
+ count = count + 1
+
+ ## .. we shouldn't have any cache scopes
+ scopes_from_cache = cache_plugin.getScopeList()
+ self.assertEqual([], scopes_from_cache)
+
+
+ def testSetGet(self):
+ """ set value to cache and then get it back """
+ for cache_plugin in self.cache_plugins:
+ self.generaltestSetGet(cache_plugin, 100)
+
+ def testExpire(self):
+ """ Check expired by setting a key, wit for its timeout and check if in cache"""
+ for cache_plugin in self.cache_plugins:
+ self.generalExpire(cache_plugin, 2)
+
+
+ def generalExpire(self, cache_plugin, iterations):
+ print "TESTING (expire): ", cache_plugin
+ base_timeout = 1
+ values = self.prepareValues(iterations)
+ scope = "peter"
+ count = 0
+ for value in values:
+ count = count +1
+ cache_timeout = base_timeout + random.random()*2
+ cache_id = "mycache_id_to_expire_%s" %(count)
+ print "\t", cache_id, " ==> timeout (s) = ", cache_timeout,
+
+ ## set to cache
+ cache_plugin.set(cache_id, scope, value, cache_timeout)
+
+ ## sleep for timeout +1
+ time.sleep(cache_timeout + 1)
+
+ ## should remove from cache expired cache entries
+ cache_plugin.expireOldCacheEntries(forceCheck=True)
+
+ ## check it, we MUST NOT have this key any more in cache
+ self.assertEqual(False, cache_plugin.has_key(cache_id, scope))
+ print "\t\tOK"
+
+ def generaltestSetGet(self, cache_plugin, iterations):
+ print "TESTING (set/get/has/del): ", cache_plugin
+ values = self.prepareValues(iterations)
+ cache_duration = 30
+ scope = "peter"
+ count = 0
+ for value in values:
+ count = count +1
+ cache_id = "mycache_id_to_set_get_has_del_%s" %(count)
+
+ ## set to cache
+ cache_plugin.set(cache_id, scope, value, cache_duration)
+ print "\t", cache_id,
+
+ ## check has_key()
+ self.assertEqual(True, cache_plugin.has_key(cache_id, scope))
+
+ ## check get()
+ cache_entry = cache_plugin.get(cache_id, scope)
+ if isinstance(value, Foo):
+ ## when memcached or sql cached we have a new object created for user
+ ## just compare one field from it
+ self.assertEqual(value.my_field, cache_entry.getValue().my_field)
+ else:
+ ## primitive types, direct comparision
+ self.assertEqual(value, cache_entry.getValue())
+
+ ## is returned result proper cache entry?
+ self.assertEqual(True, isinstance(cache_entry, CacheEntry))
+
+ ## is returned result proper type?
+ self.assertEqual(type(value), type(cache_entry.getValue()))
+
+ ## check delete(), key should be removed from there
+ cache_plugin.delete(cache_id, scope)
+ self.assertEqual(False, cache_plugin.has_key(cache_id, scope))
+
+ print "\t\tOK"
+
+ def prepareValues(self, iterations):
+ """ generate a big list of values """
+ values = []
+ my_text = "".join(map(chr, range(50,200))) * 10 ## long string (150*x)
+ for i in range(0, iterations):
+ values.append(random.random()*i)
+ values.append(random.random()*i/1000)
+ values.append(my_text)
+ values.append(Foo())
+ return values
+
+if __name__ == '__main__':
+ unittest.main()
Modified: erp5/trunk/products/ERP5Type/tests/testCacheTool.py
URL: http://svn.erp5.org/erp5/trunk/products/ERP5Type/tests/testCacheTool.py?rev=11067&r1=11066&r2=11067&view=diff
==============================================================================
--- erp5/trunk/products/ERP5Type/tests/testCacheTool.py (original)
+++ erp5/trunk/products/ERP5Type/tests/testCacheTool.py Thu Nov 2 16:59:45 2006
@@ -70,37 +70,39 @@
user = uf.getUserById('seb').__of__(uf)
newSecurityManager(None, user)
- def test_01_CheckCacheTool(self, quiet=0, run=run_all_test):
- if not run:
- return
- if not quiet:
- message = '\nCheck CacheTool '
- ZopeTestCase._print(message)
- LOG('Testing... ',0,message)
-
- portal = self.getPortal()
- self.assertNotEqual(None,getattr(portal,'portal_caches',None))
- get_transaction().commit()
-
-
- def test_02_CheckPortalTypes(self, quiet=0, run=run_all_test):
- if not run:
- return
- if not quiet:
- message = '\nCheck Portal Types'
+ def test_01_CreateCacheTool(self, quiet=0, run=run_all_test):
+ if not run:
+ return
+ if not quiet:
+ message = '\nCreate CacheTool '
+ ZopeTestCase._print(message)
+ LOG('Testing... ',0,message)
+
+ portal = self.getPortal()
+ addTool = portal.manage_addProduct['ERP5'].manage_addTool
+ addTool("ERP5 Cache Tool", None)
+ get_transaction().commit()
+
+
+ def test_02_CreatePortalTypes(self, quiet=0, run=run_all_test):
+ if not run:
+ return
+ if not quiet:
+ message = '\nCreate Portal Types'
ZopeTestCase._print(message)
LOG('Testing... ',0,message)
portal = self.getPortal()
portal_types = portal.portal_types
- typeinfo_names = ("Cache Factory",
- "Ram Cache Plugin",
- "Distributed Ram Cache Plugin",
- "SQL Cache Plugin",
+ typeinfo_names = ("ERP5Type: Cache Factory (ERP5 Cache Factory)",
+ "ERP5Type: Ram Cache Plugin (ERP5 Ram Cache Plugin)",
+ "ERP5Type: Distributed Ram Cache Plugin (ERP5 Distributed Ram Cache Plugin)",
+ "ERP5Type: SQL Cache Plugin (ERP5 SQL Cache Plugin)",
)
for typeinfo_name in typeinfo_names:
- portal_type = getattr(portal_types,typeinfo_name,None)
- self.assertNotEqual(None,portal_type)
+ portal_types.manage_addTypeInformation(add_meta_type = "ERP5 Type Information",
+ id = "",
+ typeinfo_name = typeinfo_name)
get_transaction().commit()
def test_03_CreateCacheFactories(self, quiet=0, run=run_all_test):
@@ -156,11 +158,15 @@
portal_caches.updateCache()
from Products.ERP5Type.Cache import CachingMethod
+ ## do we have cache enabled for this site?
+ erp5_site_id = portal.getId()
+ self.assert_(CachingMethod.factories.has_key(erp5_site_id))
+
## do we have the same structure we created above?
- self.assert_('ram_cache_factory' in CachingMethod.factories)
- self.assert_('distributed_ram_cache_factory' in CachingMethod.factories)
- self.assert_('sql_cache_factory' in CachingMethod.factories)
- self.assert_('erp5_user_factory' in CachingMethod.factories)
+ self.assert_('ram_cache_factory' in CachingMethod.factories[erp5_site_id])
+ self.assert_('distributed_ram_cache_factory' in CachingMethod.factories[erp5_site_id])
+ self.assert_('sql_cache_factory' in CachingMethod.factories[erp5_site_id])
+ self.assert_('erp5_user_factory' in CachingMethod.factories[erp5_site_id])
def test_04_CreateCachedMethod(self, quiet=0, run=run_all_test):
if not run:
More information about the Erp5-report
mailing list