[Erp5-report] r37871 kazuhiko - in /erp5/trunk/utils/erp5.recipe.sphinxserver: ./ src/ src/...
nobody at svn.erp5.org
nobody at svn.erp5.org
Wed Aug 18 10:21:50 CEST 2010
Author: kazuhiko
Date: Wed Aug 18 10:21:48 2010
New Revision: 37871
URL: http://svn.erp5.org?rev=37871&view=rev
Log:
1.0 (2010-08-18)
------------------
- Initial version based on erp5.recipe.mysqlserver-1.1.7.
[Kazuhiko Shiozaki]
Added:
erp5/trunk/utils/erp5.recipe.sphinxserver/
erp5/trunk/utils/erp5.recipe.sphinxserver/CHANGES.txt
erp5/trunk/utils/erp5.recipe.sphinxserver/README.txt
erp5/trunk/utils/erp5.recipe.sphinxserver/setup.py
erp5/trunk/utils/erp5.recipe.sphinxserver/src/
erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/
erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/__init__.py
erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/
erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/__init__.py
erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/
erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/__init__.py
erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/ctl.py
erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/sphinx.conf.in
Added: erp5/trunk/utils/erp5.recipe.sphinxserver/CHANGES.txt
URL: http://svn.erp5.org/erp5/trunk/utils/erp5.recipe.sphinxserver/CHANGES.txt?rev=37871&view=auto
==============================================================================
--- erp5/trunk/utils/erp5.recipe.sphinxserver/CHANGES.txt (added)
+++ erp5/trunk/utils/erp5.recipe.sphinxserver/CHANGES.txt [utf8] Wed Aug 18 10:21:48 2010
@@ -0,0 +1,8 @@
+Changelog
+=========
+
+1.0 (2010-08-18)
+------------------
+
+- Initial version based on erp5.recipe.mysqlserver-1.1.7.
+ [Kazuhiko Shiozaki]
Added: erp5/trunk/utils/erp5.recipe.sphinxserver/README.txt
URL: http://svn.erp5.org/erp5/trunk/utils/erp5.recipe.sphinxserver/README.txt?rev=37871&view=auto
==============================================================================
--- erp5/trunk/utils/erp5.recipe.sphinxserver/README.txt (added)
+++ erp5/trunk/utils/erp5.recipe.sphinxserver/README.txt [utf8] Wed Aug 18 10:21:48 2010
@@ -0,0 +1,42 @@
+Introduction
+============
+
+This recipe generates a new server setup for Sphinx, and allow to have multiple
+Sphinx servers running in the same machine. This does not compile new Sphinx
+Software.
+
+Example
+=======
+
+You can use it with a part like this::
+
+ [default-server]
+ recipe = erp5.recipe.sphinxserver
+ sphinx_software_bin = /usr/bin
+ sphinx_bin_folder = ${buildout:bin-directory}
+ sphinx_data_dir = ${buildout:directory}/var/sphinx
+ sphinx_conf_file = ${buildout:etc-directory}/sphinx.cnf
+
+Options
+=======
+
+sphinx_software_bin
+
+ Where the Orginal Software Binaries are installed, like /usr/bin when they
+ are provided by Linux Packages.
+
+sphinx_bin_folder
+
+ Where the Executable Wrapper scripts like searched will be created.
+
+sphinx_data_dir
+
+ Where the Sphinx data will be kept.
+
+sphinx_conf_file
+
+ Define the configuration file path generated and used by the server.
+
+sphinx_conf_template (optional)
+
+ Optional own template which will be used to generate sphinx.cnf file.
Added: erp5/trunk/utils/erp5.recipe.sphinxserver/setup.py
URL: http://svn.erp5.org/erp5/trunk/utils/erp5.recipe.sphinxserver/setup.py?rev=37871&view=auto
==============================================================================
--- erp5/trunk/utils/erp5.recipe.sphinxserver/setup.py (added)
+++ erp5/trunk/utils/erp5.recipe.sphinxserver/setup.py [utf8] Wed Aug 18 10:21:48 2010
@@ -0,0 +1,33 @@
+from setuptools import setup, find_packages
+
+name = 'erp5.recipe.sphinxserver'
+version = '1.0'
+
+def read(name):
+ return open(name).read()
+
+long_description=( read('README.txt')
+ + '\n' +
+ read('CHANGES.txt')
+ )
+
+setup(
+ name = name,
+ version = version,
+ author = 'Kazuhiko Shiozaki',
+ author_email = 'kazuhiko at nexedi.com',
+ description = 'ZC Buildout recipe for create a sphinx database',
+ long_description=long_description,
+ license = 'ZPL 2.1',
+ keywords = 'sphinx server buildout',
+ classifiers=[
+ 'License :: OSI Approved :: Zope Public License',
+ 'Framework :: Buildout',
+ ],
+ packages = find_packages('src'),
+ package_dir = {'': 'src'},
+ include_package_data = True,
+ install_requires = ['zc.recipe.egg', 'setuptools'],
+ namespace_packages = ['erp5', 'erp5.recipe'],
+ entry_points = {'zc.buildout': ['default = %s:Recipe' % name]},
+ )
Added: erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/__init__.py
URL: http://svn.erp5.org/erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/__init__.py?rev=37871&view=auto
==============================================================================
--- erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/__init__.py (added)
+++ erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/__init__.py [utf8] Wed Aug 18 10:21:48 2010
@@ -0,0 +1,6 @@
+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+ __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+ from pkgutil import extend_path
+ __path__ = extend_path(__path__, __name__)
Added: erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/__init__.py
URL: http://svn.erp5.org/erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/__init__.py?rev=37871&view=auto
==============================================================================
--- erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/__init__.py (added)
+++ erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/__init__.py [utf8] Wed Aug 18 10:21:48 2010
@@ -0,0 +1,6 @@
+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+ __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+ from pkgutil import extend_path
+ __path__ = extend_path(__path__, __name__)
Added: erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/__init__.py
URL: http://svn.erp5.org/erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/__init__.py?rev=37871&view=auto
==============================================================================
--- erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/__init__.py (added)
+++ erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/__init__.py [utf8] Wed Aug 18 10:21:48 2010
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2010 Nexedi SA and Contributors. All Rights Reserved.
+# Copyright (c) 2006-2008 Zope Corporation and Contributors.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import os
+from string import Template
+import logging
+from time import sleep
+import subprocess
+import zc.buildout
+import zc.buildout.easy_install
+import zc.recipe.egg
+import pkg_resources
+
+class WithMinusTemplate(Template):
+ idpattern = '[_a-z][-_a-z0-9]*'
+
+class Recipe(object):
+ def __init__(self, buildout, name, options):
+ self.buildout, self.options, self.name = buildout, options, name
+ self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options)
+ self.logger=logging.getLogger(self.name)
+
+ options['location'] = os.path.join(
+ buildout['buildout']['parts-directory'],
+ self.name)
+
+ options.setdefault('sphinx_bin_folder',
+ buildout['buildout']['bin-directory'])
+
+
+ def install_script(self, script_name):
+ """ Create default scripts
+ """
+ options = self.options
+ sphinx_software_bin = options.get("sphinx_software_bin")
+ sphinx_conf_file = options.get("sphinx_conf_file")
+ requirements, ws = self.egg.working_set(['erp5.recipe.sphinxserver'])
+ scripts = zc.buildout.easy_install.scripts(
+ [(script_name,'erp5.recipe.sphinxserver.ctl', 'main')],
+ ws, options['executable'], options.get("sphinx_bin_folder"),
+ arguments = ("\n ['%s/%s' ,"
+ "\n '-c', '%s'] "
+ "\n + sys.argv[1:]" % (sphinx_software_bin,
+ script_name,
+ sphinx_conf_file)))
+
+ def build_configuration_file(self):
+ template_file = self.options.get('sphinx_conf_template', '').strip()
+ if template_file:
+ template_input_data = file(template_file).read()
+ else:
+ template_input_data = pkg_resources.resource_stream(__name__,
+ 'sphinx.conf.in').read()
+
+ file(self.options['sphinx_conf_file'], 'w').write(
+ WithMinusTemplate(template_input_data).substitute(
+ self.options.copy()))
+
+ def install(self):
+ options = self.options
+ self.build_configuration_file()
+ data_dir = options.get('sphinx_data_dir')
+ if not os.path.exists(data_dir):
+ os.mkdir(data_dir)
+ log_dir = options.get('sphinx_log_dir')
+ if not os.path.exists(log_dir):
+ os.mkdir(log_dir)
+ self.install_script('searchd')
+ return []
+
+ update = install
Added: erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/ctl.py
URL: http://svn.erp5.org/erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/ctl.py?rev=37871&view=auto
==============================================================================
--- erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/ctl.py (added)
+++ erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/ctl.py [utf8] Wed Aug 18 10:21:48 2010
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2010 Nexedi SA and Contributors. All Rights Reserved.
+# Copyright (c) 2006-2008 Zope Corporation and Contributors.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+from os import execl
+import sys
+
+def main(args):
+ sys.stdout.flush()
+ execl(args[0], args[0], *args[1:])
+
+if __name__ == "__main__":
+ main(*sys.argv[1:])
Added: erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/sphinx.conf.in
URL: http://svn.erp5.org/erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/sphinx.conf.in?rev=37871&view=auto
==============================================================================
--- erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/sphinx.conf.in (added)
+++ erp5/trunk/utils/erp5.recipe.sphinxserver/src/erp5/recipe/sphinxserver/sphinx.conf.in [utf8] Wed Aug 18 10:21:48 2010
@@ -0,0 +1,225 @@
+#############################################################################
+## index definition
+#############################################################################
+
+# realtime index
+#
+# you can run INSERT, REPLACE, and DELETE on this index on the fly
+# using MySQL protocol (see 'listen' directive below)
+index erp5
+{
+ # 'rt' index type must be specified to use RT index
+ type = rt
+
+ # index files path and file name, without extension
+ # mandatory, path must be writable, extensions will be auto-appended
+ path = ${sphinx_data_dir}/erp5
+
+ # RAM chunk size limit
+ # RT index will keep at most this much data in RAM, then flush to disk
+ # optional, default is 32M
+ #
+ # rt_mem_limit = 512M
+
+ # full-text field declaration
+ # multi-value, mandatory
+ rt_field = SearchableText
+
+ # unsigned integer attribute declaration
+ # multi-value (an arbitrary number of attributes is allowed), optional
+ # declares an unsigned 32-bit attribute
+ rt_attr_uint = security_uid
+
+ # RT indexes currently support the following attribute types:
+ # uint, bigint, float, timestamp, string
+ #
+ # rt_attr_bigint = guid
+ # rt_attr_float = gpa
+ # rt_attr_timestamp = ts_added
+ # rt_attr_string = author
+
+ # charset encoding type
+ # optional, default is 'sbcs'
+ # known types are 'sbcs' (Single Byte CharSet) and 'utf-8'
+ charset_type = utf-8
+
+ # charset definition and case folding rules "table"
+ # optional, default value depends on charset_type
+ #
+ # defaults are configured to include English and Russian characters only
+ # you need to change the table to include additional ones
+ # this behavior MAY change in future versions
+ #
+ # 'sbcs' default value is
+ # charset_table = 0..9, A..Z->a..z, _, a..z, U+A8->U+B8, U+B8, U+C0..U+DF->U+E0..U+FF, U+E0..U+FF
+ #
+ # 'utf-8' default value is
+ # charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F
+
+ # n-gram length to index, for CJK indexing
+ # only supports 0 and 1 for now, other lengths to be implemented
+ # optional, default is 0 (disable n-grams)
+ #
+ ngram_len = 1
+
+ # n-gram characters list, for CJK indexing
+ # optional, default is empty
+ #
+ ngram_chars = U+3000..U+2FA1F
+}
+
+#############################################################################
+## searchd settings
+#############################################################################
+
+searchd
+{
+ # [hostname:]port[:protocol], or /unix/socket/path to listen on
+ # known protocols are 'sphinx' (SphinxAPI) and 'mysql41' (SphinxQL)
+ #
+ # multi-value, multiple listen points are allowed
+ # optional, defaults are 9312:sphinx and 9306:mysql41, as below
+ #
+ # listen = 127.0.0.1
+ # listen = 192.168.0.1:9312
+ # listen = 9312
+ # listen = /var/run/searchd.sock
+ listen = ${sphinx_address}:${sphinx_port}:sphinx
+ listen = ${sphinx_address}:${sphinx_sql_port}:mysql41
+
+ # log file, searchd run info is logged here
+ # optional, default is 'searchd.log'
+ log = ${sphinx_log_dir}/sphinx-searchd.log
+
+ # query log file, all search queries are logged here
+ # optional, default is empty (do not log queries)
+ query_log = ${sphinx_log_dir}/sphinx-query.log
+
+ # client read timeout, seconds
+ # optional, default is 5
+ read_timeout = 5
+
+ # request timeout, seconds
+ # optional, default is 5 minutes
+ client_timeout = 300
+
+ # maximum amount of children to fork (concurrent searches to run)
+ # optional, default is 0 (unlimited)
+ max_children = 30
+
+ # PID file, searchd process ID file name
+ # mandatory
+ pid_file = ${sphinx_data_dir}/sphinx-searchd.pid
+
+ # max amount of matches the daemon ever keeps in RAM, per-index
+ # WARNING, THERE'S ALSO PER-QUERY LIMIT, SEE SetLimits() API CALL
+ # default is 1000 (just like Google)
+ max_matches = 1000
+
+ # seamless rotate, prevents rotate stalls if precaching huge datasets
+ # optional, default is 1
+ seamless_rotate = 1
+
+ # whether to forcibly preopen all indexes on startup
+ # optional, default is 0 (do not preopen)
+ preopen_indexes = 0
+
+ # whether to unlink .old index copies on succesful rotation.
+ # optional, default is 1 (do unlink)
+ unlink_old = 1
+
+ # attribute updates periodic flush timeout, seconds
+ # updates will be automatically dumped to disk this frequently
+ # optional, default is 0 (disable periodic flush)
+ #
+ # attr_flush_period = 900
+
+ # instance-wide ondisk_dict defaults (per-index value take precedence)
+ # optional, default is 0 (precache all dictionaries in RAM)
+ #
+ # ondisk_dict_default = 1
+
+ # MVA updates pool size
+ # shared between all instances of searchd, disables attr flushes!
+ # optional, default size is 1M
+ mva_updates_pool = 1M
+
+ # max allowed network packet size
+ # limits both query packets from clients, and responses from agents
+ # optional, default size is 8M
+ max_packet_size = 8M
+
+ # crash log path
+ # searchd will (try to) log crashed query to 'crash_log_path.PID' file
+ # optional, default is empty (do not create crash logs)
+ #
+ # crash_log_path = ${sphinx_log_dir}
+
+ # max allowed per-query filter count
+ # optional, default is 256
+ max_filters = 256
+
+ # max allowed per-filter values count
+ # optional, default is 4096
+ max_filter_values = 4096
+
+ # socket listen queue length
+ # optional, default is 5
+ #
+ # listen_backlog = 5
+
+ # per-keyword read buffer size
+ # optional, default is 256K
+ #
+ # read_buffer = 256K
+
+ # unhinted read size (currently used when reading hits)
+ # optional, default is 32K
+ #
+ # read_unhinted = 32K
+
+ # max allowed per-batch query count (aka multi-query count)
+ # optional, default is 32
+ max_batch_queries = 32
+
+ # max common subtree document cache size, per-query
+ # optional, default is 0 (disable subtree optimization)
+ #
+ # subtree_docs_cache = 4M
+
+ # max common subtree hit cache size, per-query
+ # optional, default is 0 (disable subtree optimization)
+ #
+ # subtree_hits_cache = 8M
+
+ # multi-processing mode (MPM)
+ # known values are none, fork, prefork, and threads
+ # optional, default is fork
+ #
+ workers = threads # for RT to work
+
+ # max threads to create for searching local parts of a distributed index
+ # optional, default is 0, which means disable multi-threaded searching
+ # should work with all MPMs (ie. does NOT require workers=threads)
+ #
+ # dist_threads = 4
+
+ # binlog files path; use empty string to disable binlog
+ # optional, default is build-time configured data directory
+ #
+ # binlog_path = # disable logging
+ # binlog_path = ${sphinx_data_dir} # binlog.001 etc will be created there
+
+ # binlog flush/sync mode
+ # 0 means flush and sync every second
+ # 1 means flush and sync every transaction
+ # 2 means flush every transaction, sync every second
+ # optional, default is 2
+ #
+ # binlog_flush = 2
+
+ # binlog per-file size limit
+ # optional, default is 128M, 0 means no limit
+ #
+ # binlog_max_log_size = 256M
+}
More information about the Erp5-report
mailing list