[Erp5-report] r45657 luke - /erp5/trunk/utils/z2loganalyser/analyzeZ2log.py
nobody at svn.erp5.org
nobody at svn.erp5.org
Fri Apr 22 16:35:42 CEST 2011
Author: luke
Date: Fri Apr 22 16:35:42 2011
New Revision: 45657
URL: http://svn.erp5.org?rev=45657&view=rev
Log:
- original version of analytic tool of Zope access log files created by Romain COURTEAUD
Added:
erp5/trunk/utils/z2loganalyser/analyzeZ2log.py
Added: erp5/trunk/utils/z2loganalyser/analyzeZ2log.py
URL: http://svn.erp5.org/erp5/trunk/utils/z2loganalyser/analyzeZ2log.py?rev=45657&view=auto
==============================================================================
--- erp5/trunk/utils/z2loganalyser/analyzeZ2log.py (added)
+++ erp5/trunk/utils/z2loganalyser/analyzeZ2log.py [utf8] Fri Apr 22 16:35:42 2011
@@ -0,0 +1,202 @@
+##############################################################################
+#
+# Copyright (c) 2008-2010 Nexedi SA and Contributors. All Rights Reserved.
+# Romain COURTEAUD <romain at nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+##############################################################################
+__prog__ = "analyzeZ2log.py"
+__version__ = "0.1"
+
+from optparse import Option, OptionParser
+import sys
+import os
+import re
+import datetime
+import gzip
+
+class Parser(OptionParser):
+ """
+ Parse all arguments.
+ """
+ def __init__(self, usage=None, version=None):
+ """
+ Initialize all options possibles.
+ """
+ OptionParser.__init__(self, usage=usage, version=version,
+ option_list=[
+ Option("-v", "--verbose",
+ help="be verbose",
+ action="store_true",
+ default=0,
+ dest="verbose"),
+ Option("-n", "--dry-run",
+ help="simulate",
+ dest="dry_run",
+ action="store_true"),
+ Option("-f", "--force",
+ help="write in output file even if it exists",
+ action="store_true",
+ default=0,
+ dest="force"),
+ ])
+
+ def check_args(self):
+ """
+ Check arguments
+ """
+ (options, args) = self.parse_args()
+ if len(args) != 2:
+ self.error("Incorrect number of arguments")
+ elif not os.path.isdir(args[0]):
+ # Check that directory exists
+ self.error("Not existing directory '%s'" % args[0])
+ elif os.path.exists(args[1]):
+ # Check that output files do not exists
+ if not options.force:
+ self.error("Existing output file '%s'" % args[1])
+ return options, args[0], args[1]
+
+class Config:
+ """
+ Hold all config parameters
+ """
+ def __init__(self, directory_path, output_path):
+ """
+ Initialize all config options.
+ """
+ self.directory_path = directory_path
+ self.output_path = output_path
+
+ def setConfig(self, option_dict):
+ """
+ Set options given by parameters.
+ """
+ # Set options parameters
+ for option, value in option_dict.__dict__.items():
+ setattr(self, option, value)
+
+class Executor:
+ """
+ Program executor.
+ """
+ def __init__(self, config):
+ """
+ Store config file.
+ """
+ self.config = config
+
+ def printMessage(self, message, error=0):
+ """
+ Print message to STDOUT if verbose is requested.
+ """
+ if self.config.verbose or error:
+ print message
+
+ def run(self):
+ """
+ Exec.
+ """
+# 127.0.0.1 - zope [23/May/2008:12:40:04 +0200] "GET /erp5/erp5.css HTTP/1.1" 200 17249 "http://localhost:9180/erp5/login_form" "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.11) Gecko/20070914 Mandriva/2.0.0.11-1.1mdv2007.1 (2007.1) Firefox/2.0.0.11"
+
+ rg = "^(?P<ip>([0-9]+\.){3}[0-9]+) - (?P<user>[A-Za-z0-9\.@]*) " \
+ "\[(?P<date>[0-9]+/[a-zA-Z]+/[0-9]+(:[0-9]{2}){3}) .*\]"
+# "\[(?P<day>[0-9]+)/(?P<month>[a-zA-Z]+)/(?P<year>[0-9]+):.*\]"
+ regexp = re.compile(rg)
+ regexp_match = regexp.match
+ result_dict = {}
+
+ top_directory_path = self.config.directory_path
+ for root, dirs, files in os.walk(top_directory_path):
+ # Z2.log-20090430.gz
+ for file in files:
+ if file.startswith('Z2.log') and file.endswith('.gz'):
+ log_path = '%s/%s' % (root, file)
+ file = gzip.open(log_path, 'rb')
+ # file = open(log_path, 'rb')
+
+ for line in file:
+ m = regexp_match(line)
+ try:
+ result = m.groupdict()
+ except AttributeError:
+ continue
+ else:
+ # date = datetime.date(int(result['year']), 12, int(result['day']))
+ format = '%d/%b/%Y:%H:%M:%S'
+ date = datetime.datetime.strptime(result['date'], format)
+ user = result['user']
+ if user in ('', 'Anonymous', None):
+ continue
+
+ # ordinal = date.date().toordinal()
+ ordinal = date.date().isoformat()
+ time = date.time().isoformat()
+ try:
+ user_dict = result_dict[ordinal]
+ except KeyError:
+ result_dict[ordinal] = {}
+ user_dict = result_dict[ordinal]
+
+ if user in user_dict:
+ time_frame = user_dict[user]
+ user_dict[user] = [min(time, time_frame[0]),
+ max(time, time_frame[1]),
+ time_frame[2] + 1]
+ else:
+ user_dict[user] = [time, time, 1]
+ file.close()
+
+ if not self.config.dry_run:
+ logfile = open(self.config.output_path, 'w')
+ logfile.write('"date";"count";"user";"connection";"deconnection";"get"\n')
+
+ date_list = result_dict.keys()
+ date_list.sort()
+ for date in date_list:
+ if not self.config.dry_run:
+ logfile.write('"%s";%i;;;;\n' % (date, len(result_dict[date])))
+ user_list = result_dict[date].keys()
+ user_list.sort()
+ for user in user_list:
+ logfile.write('"%s";;"%s";"%s";"%s";"%s"\n' % (date, user,
+ result_dict[date][user][0], result_dict[date][user][1],
+ result_dict[date][user][2]))
+ if not self.config.dry_run:
+ logfile.close()
+
+def main(argv=None):
+ usage = "usage: %s [options] directory_to_parse output_file" % __prog__
+ try:
+ parser = Parser(usage=usage, version="%s %s" % (__prog__, __version__))
+ options, directory_path, file_path = parser.check_args()
+ config = Config(directory_path, file_path)
+ config.setConfig(options)
+ exe = Executor(config)
+ exe.run()
+ except SystemExit, err:
+ # Catch exception raise by optparse
+ return err
+
+if __name__ == "__main__":
+ sys.exit(main())
More information about the Erp5-report
mailing list