[Erp5-report] r25635 - /erp5/trunk/utils/zodb-cutter/
nobody at svn.erp5.org
nobody at svn.erp5.org
Fri Feb 20 11:53:50 CET 2009
Author: luke
Date: Fri Feb 20 11:53:49 2009
New Revision: 25635
URL: http://svn.erp5.org?rev=25635&view=rev
Log:
- utility to cut ZODB database into pieces
Added:
erp5/trunk/utils/zodb-cutter/
erp5/trunk/utils/zodb-cutter/README.txt (with props)
erp5/trunk/utils/zodb-cutter/cutter.py (with props)
erp5/trunk/utils/zodb-cutter/cutter_cluster_config_generator.py (with props)
erp5/trunk/utils/zodb-cutter/cutter_config.py
Added: erp5/trunk/utils/zodb-cutter/README.txt
URL: http://svn.erp5.org/erp5/trunk/utils/zodb-cutter/README.txt?rev=25635&view=auto
==============================================================================
--- erp5/trunk/utils/zodb-cutter/README.txt (added)
+++ erp5/trunk/utils/zodb-cutter/README.txt [utf8] Fri Feb 20 11:53:49 2009
@@ -1,0 +1,10 @@
+ZODB cutter
+===========
+
+Small utility to cut Data.fs in smaller parts and generate configuration for
+ZEO Clients and Servers cluster.
+
+Configure with cutter_config.py
+
+Use cutter.py to cut ZODB into pieces.
+Use cutter_cluster_config_generator.py to generate configurations.
Propchange: erp5/trunk/utils/zodb-cutter/README.txt
------------------------------------------------------------------------------
svn:eol-style = native
Added: erp5/trunk/utils/zodb-cutter/cutter.py
URL: http://svn.erp5.org/erp5/trunk/utils/zodb-cutter/cutter.py?rev=25635&view=auto
==============================================================================
--- erp5/trunk/utils/zodb-cutter/cutter.py (added)
+++ erp5/trunk/utils/zodb-cutter/cutter.py [utf8] Fri Feb 20 11:53:49 2009
@@ -1,0 +1,166 @@
+#!/usr/bin/python2.4
+# -*- coding: utf8 -*-
+##############################################################################
+#
+# Copyright (c) 2009 Nexedi SA and Contributors. All Rights Reserved.
+# Łukasz Nowak <luke at nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+# Script to generate many ZODB files from one
+# Excepts configuration in same directory as cutter_config.py file
+
+from time import time
+import os.path, sys, shutil
+if sys.version_info[:2] != (2,4):
+ raise AttributeError("Use only with python 2.4")
+
+from stat import S_ISDIR, ST_MODE
+
+sys.path.insert(0, '.')
+import cutter_config as config
+
+sys.path.insert(0, config.erp5_software_home)
+from ZODB import FileStorage, DB
+from Products.ERP5Type.CopySupport import CopyContainer
+from ZODB.broken import PersistentBroken
+import transaction
+
+# patch manage_beforeDelete to be able to use _delObject
+def manage_beforeDelete(self, item, container):
+ """Nothing to do before deletion"""
+ pass
+
+def getApp(config, root):
+ """Get application to operate on"""
+ app = root
+ for rc in config.application:
+ app = app[rc]
+ return app
+
+def checkCutList(config):
+ """Fetch list with ids to cut, check nothing repeated"""
+ cut_set = set()
+ for module_name in config.cut_list:
+ if module_name in cut_set:
+ raise AttributeError('%s repeated in definition' % module_name)
+ cut_set.add(module_name)
+
+def checkConfig(config):
+ """Check if configuration is ok"""
+ mode = os.stat(config.data_directory)[ST_MODE]
+ if not S_ISDIR(mode):
+ raise AttributeError('%s directory do not exists'
+ % config.data_directory)
+
+def checkSourceFile(config, cut_list):
+ """Check if source file contain everything for cut"""
+ print '%s checking that all off %s is available...' % (config.source_file,
+ ', '.join(cut_list)),
+ b = time()
+ source_storage = FileStorage.FileStorage(config.source_file)
+ db = DB(source_storage)
+ conn = db.open()
+
+ dbroot = conn.root()
+
+ app = getApp(config, dbroot)
+
+ # check original has everything
+ for c in cut_list:
+ app[c]
+
+ db.close()
+ print 'done (%.3fs).' % (time() - b)
+
+def cutAll(config, filename, l, remove_l):
+ """Cut ZODB. If remove_l than remove list l, else remove eveyrthing else"""
+ destination_file = os.path.join(config.data_directory, filename)
+ b = time()
+ print '%s creating...' % destination_file,
+ for suffix in '', '.index', '.tmp':
+ shutil.copy2(config.source_file + suffix,
+ destination_file + suffix)
+ storage = FileStorage.FileStorage(destination_file)
+ print 'done (%.3fs).' % (time() - b)
+
+ print '%s %s...' % (destination_file, ', '.join(l)),
+ b = time()
+ db = DB(storage)
+ conn = db.open()
+ dbroot = conn.root()
+ app = getApp(config, dbroot)
+ for object_id in app.objectIds():
+ if (object_id in l) == bool(remove_l):
+ print object_id,
+ app._delObject(object_id)
+
+ print 'removed...',
+ transaction.commit()
+
+ print 'done (%.3fs).' % (time() - b)
+
+ print '%s packing...' % destination_file,
+ b = time()
+ db.pack()
+ print 'done (%.3fs).' % (time() - b)
+ db.close()
+ print '%s removing old...' % destination_file,
+ b = time()
+ os.remove('%s.old' % destination_file)
+ print 'done (%.3fs).' % (time() - b)
+
+
+def main():
+ # Work begin
+ global_time = time()
+
+ # reopen stdout file descriptor with write mode
+ # and 0 as the buffer size (unbuffered)
+ sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+
+ # patch manage_beforeDelete to be able to use _delObject
+ CopyContainer.manage_beforeDelete = manage_beforeDelete
+ PersistentBroken.manage_beforeDelete = manage_beforeDelete
+
+ # Initalise variables, check configuration
+ checkCutList(config)
+ checkConfig(config)
+ cut_list = sorted(config.cut_list)
+ checkSourceFile(config, cut_list)
+
+ # Remove everything except wanted module
+ for module_name in cut_list:
+ cutAll(config, module_name + '.fs', [module_name], False)
+ # create root-mount file
+ cutAll(config, config.destination_file, cut_list, True)
+
+ # Works ends
+ print "Global execution time is %.3fs" % (time() - global_time)
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
Propchange: erp5/trunk/utils/zodb-cutter/cutter.py
------------------------------------------------------------------------------
svn:executable = *
Added: erp5/trunk/utils/zodb-cutter/cutter_cluster_config_generator.py
URL: http://svn.erp5.org/erp5/trunk/utils/zodb-cutter/cutter_cluster_config_generator.py?rev=25635&view=auto
==============================================================================
--- erp5/trunk/utils/zodb-cutter/cutter_cluster_config_generator.py (added)
+++ erp5/trunk/utils/zodb-cutter/cutter_cluster_config_generator.py [utf8] Fri Feb 20 11:53:49 2009
@@ -1,0 +1,111 @@
+#!/usr/bin/python
+# -*- coding: utf8 -*-
+##############################################################################
+#
+# Copyright (c) 2009 Nexedi SA and Contributors. All Rights Reserved.
+# Łukasz Nowak <luke at nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+# Script to generate ZEO servers and clients configuration
+# Excepts configuration in same directory as cutter_config.py file
+
+import sys, os
+from stat import S_ISDIR, ST_MODE
+
+sys.path.insert(0, '.')
+import cutter_config as config
+
+def checkConfig(config):
+ """Check if configuration is ok"""
+ mode = os.stat(config.config_directory)[ST_MODE]
+ if not S_ISDIR(mode):
+ raise AttributeError('%s directory do not exists' % config.config_directory)
+
+def main():
+ # do checks for consistency
+ checkConfig(config)
+
+ if len(config.cluster_mount_configuration) != len(config.cut_list) + 1:
+ raise ValueError('Mount configuration is not covering every cut case')
+ if sorted(config.zeo_server_list) != sorted(config.zeo_server_config.keys()):
+ raise ValueError('Zeo server list is not corresponding its configuration')
+ for zeo_server in config.zeo_server_list:
+ zeo_found = False
+ for v in config.cluster_mount_configuration.values():
+ if v['zeo'] == zeo_server:
+ zeo_found = True
+ break
+ if v['zeo'] not in config.zeo_server_list:
+ raise ValueError('Zeo server %s not in server list' % v['zeo'])
+ if not zeo_found:
+ raise ValueError('Zeo server %s not configured' % zeo_server)
+
+ # create zeo server and client configurations
+ zope_config_file = file(config.config_directory+'/zope.conf', 'w')
+ for zeo_server in config.zeo_server_list:
+ zeo_config_file = file(config.config_directory+'/zeo-%s.conf' % zeo_server,
+ 'w')
+ zeo_config_file.write("""<zeo>
+ address %(address)s
+ read-only %(read-only)s
+ invalidation-queue-size %(invalidation-queue-size)s
+</zeo>
+
+""" % (config.zeo_server_config[zeo_server]))
+ for module_name, cluster_config in config.cluster_mount_configuration.iteritems():
+ if cluster_config['zeo'] == zeo_server:
+ if module_name == '/': # special case
+ mount_point = module_name
+ module_name = config.destination_file.split('.fs')[0]
+ else:
+ mount_point = config.root_mount + module_name
+ zeo_config_file.write("""<filestorage %s>
+ path $INSTANCE/var/%s.fs
+</filestorage>
+
+""" % (module_name,module_name))
+ zope_config_file.write("""<zodb_db %(module-name)s>
+ mount-point %(mount-point)s
+ # ZODB cache, in number of objects
+ cache-size 5000
+ <zeoclient>
+ server %(address)s
+ storage %(module-name)s
+ name zeostorage
+ var $INSTANCE/var
+ cache-size %(cache-size)s
+ </zeoclient>
+</zodb_db>
+
+""" % ({
+ 'module-name': module_name,
+ 'mount-point': mount_point,
+ 'address': config.zeo_server_config[zeo_server]['address'],
+ 'cache-size': cluster_config['cache']}))
+ print 'Written %s' % config.config_directory + '/zeo-%s.conf' % zeo_server
+ print 'Written %s' % config.config_directory + '/zope.conf'
+
+if __name__ == '__main__':
+ sys.exit(main())
Propchange: erp5/trunk/utils/zodb-cutter/cutter_cluster_config_generator.py
------------------------------------------------------------------------------
svn:executable = *
Added: erp5/trunk/utils/zodb-cutter/cutter_config.py
URL: http://svn.erp5.org/erp5/trunk/utils/zodb-cutter/cutter_config.py?rev=25635&view=auto
==============================================================================
--- erp5/trunk/utils/zodb-cutter/cutter_config.py (added)
+++ erp5/trunk/utils/zodb-cutter/cutter_config.py [utf8] Fri Feb 20 11:53:49 2009
@@ -1,0 +1,103 @@
+#!/usr/bin/python
+# -*- coding: utf8 -*-
+##############################################################################
+#
+# Copyright (c) 2009 Nexedi SA and Contributors. All Rights Reserved.
+# Łukasz Nowak <luke at nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+
+
+# configuration file for cutter
+
+# where is ERP5 installed
+erp5_software_home = '/usr/lib/erp5/lib/python'
+
+# path to file on wich cut will be based
+source_file = 'Data.fs'
+
+# directory for Data.fs files - needs much of space
+data_directory = 'data'
+
+# directory for configuration
+config_directory = 'config'
+
+# file name which will be generated to mount as root, it will be put
+# in destination_directory
+destination_file = 'root.fs'
+
+# handle to application
+application = ['Application', 'erp5']
+root_mount = '/' + '/'.join(application[1:]) + '/'
+
+# what to cut, file name will be module_name.fs
+
+cut_list = [
+ 'accounting_module',
+ 'organisation_module',
+ 'person_module',
+]
+
+# list of nicknames of used zeo servers
+zeo_server_list = ['zeo1', 'zeo2']
+
+# configuration of each zeo server, <zeo></zeo>
+zeo_server_config = {
+ 'zeo1': {
+ 'address': '127.0.0.1:8101',
+ 'read-only': 'false',
+ 'invalidation-queue-size': '100',
+ },
+ 'zeo2': {
+ 'address': '127.0.0.1:8102',
+ 'read-only': 'false',
+ 'invalidation-queue-size': '100',
+ },
+}
+
+# where is what, how much cache
+small_cache = '10MB'
+default_cache = '20MB'
+big_cache = '40MB'
+huge_cache = '80MB'
+
+# key is a mount - only / will be untouched,rest will be prepended with root_mount
+cluster_mount_configuration = {
+ '/': {
+ 'cache': default_cache,
+ 'zeo': 'zeo1',
+ },
+ 'person_module': {
+ 'cache': huge_cache,
+ 'zeo': 'zeo1',
+ },
+ 'organisation_module': {
+ 'cache': huge_cache,
+ 'zeo': 'zeo1',
+ },
+ 'accounting_module': {
+ 'cache': default_cache,
+ 'zeo': 'zeo2',
+ },
+}
More information about the Erp5-report
mailing list