[Erp5-report] r37471 nicolas - in /erp5/trunk/utils/erp5.utils.web_checker: ./ src/erp5/uti...
nobody at svn.erp5.org
nobody at svn.erp5.org
Wed Aug 4 14:01:48 CEST 2010
Author: nicolas
Date: Wed Aug 4 14:01:45 2010
New Revision: 37471
URL: http://svn.erp5.org?rev=37471&view=rev
Log:
realease 0.0.2beta
- Bug fix in status_search_in_wget_regex
- Enhance output report (fetch all errors per url)
- Add 2 options to check Urls:
prohibited_file_name_list
prohibited_folder_name_list
Modified:
erp5/trunk/utils/erp5.utils.web_checker/CHANGES.txt
erp5/trunk/utils/erp5.utils.web_checker/README.txt
erp5/trunk/utils/erp5.utils.web_checker/setup.py
erp5/trunk/utils/erp5.utils.web_checker/src/erp5/utils/web_checker/__init__.py
Modified: erp5/trunk/utils/erp5.utils.web_checker/CHANGES.txt
URL: http://svn.erp5.org/erp5/trunk/utils/erp5.utils.web_checker/CHANGES.txt?rev=37471&r1=37470&r2=37471&view=diff
==============================================================================
--- erp5/trunk/utils/erp5.utils.web_checker/CHANGES.txt [utf8] (original)
+++ erp5/trunk/utils/erp5.utils.web_checker/CHANGES.txt [utf8] Wed Aug 4 14:01:45 2010
@@ -1,3 +1,11 @@
+2010-08-04 Nicolas Delaby
+-------------------------
+Bug fix in status_search_in_wget_regex
+Enhance output report (fetch all errors per url)
+Add 2 options to check Urls
+ prohibited_file_name_list
+ prohibited_folder_name_list
+
2010-08-03 Nicolas Delaby
-------------------------
-First release
\ No newline at end of file
+First release
Modified: erp5/trunk/utils/erp5.utils.web_checker/README.txt
URL: http://svn.erp5.org/erp5/trunk/utils/erp5.utils.web_checker/README.txt?rev=37471&r1=37470&r2=37471&view=diff
==============================================================================
--- erp5/trunk/utils/erp5.utils.web_checker/README.txt [utf8] (original)
+++ erp5/trunk/utils/erp5.utils.web_checker/README.txt [utf8] Wed Aug 4 14:01:45 2010
@@ -21,20 +21,36 @@ Vary = Accept-Language, Cookie, Accept-E
Accept-Language,Cookie
Expires = True
+[erp5_extension_list]
+prohibited_file_name_list = WebSection_viewAsWeb
+ Base_viewHistory
+ list
+prohibited_folder_name_list = web_page_module
+ document_module
+
+
with
url : website to check
working_directory : fetched data will be downloaded
varnishlog_binary_path : path to varnishlog
- header_list : Key == Header id.
- value: if equals to True, it means that header needs to be present in RESPONSE
- if it is a tuple, the Header value must sastify at least one of the proposed values
email_address : email address to send result
smtp_host : smtp host to use
debug_level : log level of this utility (debug =>very verbose,
info=>normal,
warning=>nothing)
+ header_list : Key == Header id.
+ value: if equals to True, it means that header needs to be present in RESPONSE
+ if it is a tuple, the Header value must sastify at least one of the proposed values
+
+ erp5_extension_list: Optional section.
+ prohibited_file_name_list: which check that any links redirect to prohibited forms
+ like WebSection_viewAsWeb, Base_viewHistory, list, ...
+ prohibited_folder_name_list: usefull to detect that links does not redirect to
+ specified modules like, web_page_module, document_module, ...
+
+
This utility requires wget => 1.12
And a callable varnishlog.
The utility must be run on same server where varnish is running.
Modified: erp5/trunk/utils/erp5.utils.web_checker/setup.py
URL: http://svn.erp5.org/erp5/trunk/utils/erp5.utils.web_checker/setup.py?rev=37471&r1=37470&r2=37471&view=diff
==============================================================================
--- erp5/trunk/utils/erp5.utils.web_checker/setup.py [utf8] (original)
+++ erp5/trunk/utils/erp5.utils.web_checker/setup.py [utf8] Wed Aug 4 14:01:45 2010
@@ -2,7 +2,7 @@
from setuptools import setup, find_packages
import sys, os
-version = '0.0.1b'
+version = '0.0.2b'
name = 'erp5.utils.web_checker'
def read(name):
Modified: erp5/trunk/utils/erp5.utils.web_checker/src/erp5/utils/web_checker/__init__.py
URL: http://svn.erp5.org/erp5/trunk/utils/erp5.utils.web_checker/src/erp5/utils/web_checker/__init__.py?rev=37471&r1=37470&r2=37471&view=diff
==============================================================================
--- erp5/trunk/utils/erp5.utils.web_checker/src/erp5/utils/web_checker/__init__.py [utf8] (original)
+++ erp5/trunk/utils/erp5.utils.web_checker/src/erp5/utils/web_checker/__init__.py [utf8] Wed Aug 4 14:01:45 2010
@@ -62,7 +62,7 @@ class HTTPCacheCheckerTestSuite(object):
url_search_in_wget_regex = re.compile('^--\d{4}.*--\s+(?P<%s>.*)$' %\
URL_CODE)
- status_search_in_wget_regex = re.compile('^HTTP request sent, awaiting'\
+ status_search_in_wget_regex = re.compile('^HTTP request sent, awaiting '\
'response\.\.\. (?P<%s>\d+).+$' % STATUS_CODE)
file_save_search_regex = re.compile("^Saving to: `(?P<%s>.*)'" %\
FILE_PATH_CODE)
@@ -71,7 +71,7 @@ class HTTPCacheCheckerTestSuite(object):
x_varnish_header_search_regex = re.compile('X-Varnish:\s(\d+)', re.MULTILINE)
generic_header_search_regex = '%s:\s(.*)\s$'
- ACCEPTABLE_STATUS_LIST = ('200', '304',)
+ ACCEPTABLE_STATUS_LIST = ('200', '304', '302',)
def __init__(self, root_url, working_directory, varnishlog_binary_path,
header_list, email_address, smtp_host, debug_level):
@@ -98,7 +98,7 @@ class HTTPCacheCheckerTestSuite(object):
self.smtp_host = smtp_host
level = self.LOG_LEVEL_DICT.get(debug_level, logging.INFO)
logging.basicConfig(filename='erp5_web_checker.log', level=level)
- self.report_list = []
+ self.report_dict = {}
self._timeout = 30
def _initFolder(self):
@@ -237,7 +237,9 @@ class HTTPCacheCheckerTestSuite(object):
stdout, stderr = wget_process.communicate()
return stdout
- def _parseWgetLogs(self, wget_log_file, discarded_url_list=_MARKER):
+ def _parseWgetLogs(self, wget_log_file, discarded_url_list=_MARKER,
+ prohibited_file_name_list=None,
+ prohibited_folder_name_list=None):
"""read wget logs and test Caching configuration
"""
if discarded_url_list is _MARKER:
@@ -263,12 +265,25 @@ class HTTPCacheCheckerTestSuite(object):
continue
if code == self.STATUS_CODE:
if value not in self.ACCEPTABLE_STATUS_LIST:
- message = 'Page in error:%r status:%r' % (url, value)
- if message not in self.report_list:
- self.report_list.append(message)
+ message = 'Page in error status:%r' % (value)
+ self.report_dict.setdefault(url, []).append(message)
if code == self.FILE_PATH_CODE:
# Here we check if Response was cached
file_path = os.path.sep.join((self.working_directory, value))
+ folder_path , filename = os.path.split(file_path)
+ if prohibited_file_name_list:
+ if '?' in filename:
+ filename = filename.rpartition('?')[0]
+ if filename in prohibited_file_name_list:
+ # Error
+ message = '%r is prohibited as filename' % filename
+ self.report_dict.setdefault(url, []).append(message)
+ if prohibited_folder_name_list:
+ for prohibited_folder_name in prohibited_folder_name_list:
+ if prohibited_folder_name in folder_path.split(os.path.sep):
+ message = '%r is prohibited as folder name' % prohibited_folder_name
+ self.report_dict.setdefault(url, []).append(message)
+ break
file_object = None
try:
file_object = open(file_path, 'r')
@@ -292,8 +307,8 @@ class HTTPCacheCheckerTestSuite(object):
self.x_cache_header_search_regex.search(fetched_data)
if x_cache_header_match_object is None:
# This RESPONSE is not cached by Varnish
- message = 'X-Cache header not found for %r' % url
- self.report_list.append(message)
+ message = 'X-Cache header not found'
+ self.report_dict.setdefault(url, []).append(message)
else:
# means X-Cache header is present in reponse
# Read the X-Varnish header to know if backend has been touched
@@ -317,22 +332,23 @@ class HTTPCacheCheckerTestSuite(object):
re.MULTILINE | re.IGNORECASE)
match_object = re_compiled.search(fetched_data)
if match_object is None:
- message = 'header:%r not found for %r' % (header, url)
- self.report_list.append(message)
+ message = 'header:%r not found' % (header)
+ self.report_dict.setdefault(url, []).append(message)
else:
read_value = match_object.group(1)
if reference_value is True and not read_value:
- message = 'value of header:%r not found for %r' % (header, url)
- self.report_list.append(message)
+ message = 'value of header:%r not found' % (header)
+ self.report_dict.setdefault(url, []).append(message)
elif isinstance(reference_value, (tuple,list)):
if read_value not in reference_value:
message = 'value of header:%r does not match'\
- ' for %r (%r not in %r)' %\
- (header, url, read_value, reference_value)
- self.report_list.append(message)
+ ' (%r not in %r)' %\
+ (header, read_value, reference_value)
+ self.report_dict.setdefault(url, []).append(message)
return x_varnish_reference_list, discarded_url_list[:]
- def start(self):
+ def start(self, prohibited_file_name_list=None,
+ prohibited_folder_name_list=None):
"""Run test suite
return errors
"""
@@ -356,13 +372,25 @@ class HTTPCacheCheckerTestSuite(object):
wget_log_file = self._runSpider()
varnishlog_data = self._readVarnishLog(varnishlog_reading_process)
x_varnish_reference_list, discarded_url_list =\
- self._parseWgetLogs(wget_log_file,
- discarded_url_list=discarded_url_list)
+ self._parseWgetLogs(wget_log_file,
+ discarded_url_list=discarded_url_list,
+ prohibited_file_name_list=prohibited_file_name_list,
+ prohibited_folder_name_list=prohibited_folder_name_list)
self._readVarnishLogAndGetIsBackendTouched(varnishlog_data,
x_varnish_reference_list)
logging.info('End of second pass\n')
- if self.report_list:
- report_message = 'Errors:\n\t%s' % '\n\t'.join(self.report_list)
+ if self.report_dict:
+ report_message_list = ['*Errors*:']
+ for url, message_list in self.report_dict.iteritems():
+ unique_message_list = []
+ for message in message_list:
+ if message not in unique_message_list:
+ unique_message_list.append(message)
+ report_message_list.append('\n')
+ report_message_list.append(url)
+ report_message_list.extend(['\t%s' % message for message\
+ in unique_message_list])
+ report_message = '\n'.join(report_message_list)
signal = 'PROBLEM'
else:
report_message = 'No errors'
@@ -376,7 +404,7 @@ class HTTPCacheCheckerTestSuite(object):
server = smtplib.SMTP(self.smtp_host)
server.sendmail(self.email_address, self.email_address, message)
server.quit()
- return 'Email sends to %s' % self.email_address
+ return 'Email sended to %s' % self.email_address
else:
return report_message
@@ -400,6 +428,9 @@ def web_checker_utility():
working_directory = config.get('web_checker', 'working_directory')
url = config.get('web_checker', 'url')
varnishlog_binary_path = config.get('web_checker' , 'varnishlog_binary_path')
+ email_address = config.get('web_checker' , 'email_address')
+ smtp_host = config.get('web_checker' , 'smtp_host')
+ debug_level = config.get('web_checker' , 'debug_level')
header_list = {}
for header, configuration in config.items('header_list'):
if configuration in ('True', 'true', 'yes'):
@@ -407,17 +438,22 @@ def web_checker_utility():
else:
value = configuration.splitlines()
header_list[header] = value
- email_address = config.get('web_checker' , 'email_address')
- smtp_host = config.get('web_checker' , 'smtp_host')
- debug_level = config.get('web_checker' , 'debug_level')
+ if config.has_section('erp5_extension_list'):
+ prohibited_file_name_list = config.get('erp5_extension_list',
+ 'prohibited_file_name_list').splitlines()
+ prohibited_folder_name_list = config.get('erp5_extension_list',
+ 'prohibited_folder_name_list').splitlines()
+ else:
+ prohibited_file_name_list = prohibited_folder_name_list = []
instance = HTTPCacheCheckerTestSuite(url,
- working_directory,
- varnishlog_binary_path,
- header_list,
- email_address,
- smtp_host,
- debug_level)
+ working_directory,
+ varnishlog_binary_path,
+ header_list,
+ email_address,
+ smtp_host,
+ debug_level)
- print instance.start()
+ print instance.start(prohibited_file_name_list=prohibited_file_name_list,
+ prohibited_folder_name_list=prohibited_folder_name_list)
More information about the Erp5-report
mailing list