Author: ctheune
Date: Fri Aug  1 19:22:00 2008
New Revision: 6377

Log:
Create area for httop - a top for HTTP.



Added:
   httop/

Author: ctheune
Date: Fri Aug  1 19:22:27 2008
New Revision: 6378

Log:
import code



Added:
   httop/trunk/
   httop/trunk/httop.py   (contents, props changed)

Added: httop/trunk/httop.py
==============================================================================
--- (empty file)
+++ httop/trunk/httop.py	Fri Aug  1 19:22:27 2008
(at)(at) -0,0 +1,110 (at)(at)
+# vim:fileencoding=utf-8
+# Copyright (c) 2008 gocept gmbh & co. kg
+# See also LICENSE.txt
+# $Id$
+""" XXX """
+
+URLS = ['http://www.assembly.org/summer08/',
+        'http://alive.assembly.org',
+        'http://www.assemblytv.net']
+
+urlwidth = 35 # min(60, max(filter(len, URLS)))
+
+import threading
+import urllib2
+import time
+import curses
+
+
+def float_format(f):
+    if f is None:
+        return 'n/a '
+    return '%2.2f' % f
+
+
+class URLChecker(threading.Thread):
+
+    min = None
+    max = None
+    avg = None
+    avg50 = None
+    avg500 = None
+    code = ''
+
+    def __init__(self, url):
+        super(URLChecker, self).__init__()
+        self.url = url
+        self.history = []
+
+    def run(self):
+        while True:
+            self.update()
+            time.sleep(3)
+
+    def update(self):
+        start = time.time()
+        response = urllib2.urlopen(self.url)
+        end = time.time()
+        duration = end-start
+
+        # Update statistics
+        self.history.insert(0, duration)
+        if self.min is None or duration < self.min:
+            self.min = duration
+        if self.max is None or duration > self.max:
+            self.max = duration
+        self.avg = sum(self.history) / len(self.history)
+        if len(self.history) >= 50:
+            self.avg50 = sum(self.history[:50]) / 50
+        if len(self.history) >= 500:
+            self.avg500 = sum(self.history[:500]) / 500
+        self.code = response.code
+
+
+def mainloop():
+    checkers = [URLChecker(url) for url in URLS]
+    for checker in checkers:
+        checker.setDaemon(True)
+        checker.start()
+
+    while True:
+        mainwin.clear()
+        url_head = '%s %s %s %s %s %s %s %s\n' % (
+            'URL'.ljust(urlwidth),
+            'Last', 'Min ', 'Max ', 'Avg ', 'Avg50', 'Avg500', 'Code')
+        mainwin.addstr(url_head)
+        for checker in checkers:
+            url_col = checker.url.ljust(urlwidth)[:urlwidth]
+            if not checker.history:
+                info = '%s n/a\n' % url_col
+            else:
+                last_col = '%2.2f' % checker.history[0]
+                min_col = float_format(checker.min)
+                max_col = float_format(checker.max)
+                avg_col = float_format(checker.avg)
+                avg50_col = float_format(checker.avg50)
+                avg500_col = float_format(checker.avg500)
+
+                info = '%s %s %s %s %s %s  %s   %s\n' % (url_col, last_col,
min_col,
+                                                max_col, avg_col, avg50_col,
+                                                      avg500_col,
checker.code)
+            mainwin.addstr(info)
+        mainwin.refresh()
+        time.sleep(0.1)
+
+screen = curses.initscr()
+curses.start_color()
+curses.curs_set(0)
+curses.noecho()
+curses.cbreak()
+
+mainwin =  curses.newwin(0,0)
+
+try:
+    mainloop()
+except:
+    curses.nocbreak()
+    curses.curs_set(1)
+    curses.echo()
+    curses.endwin()
+    raise

Author: ctheune
Date: Fri Aug  1 19:50:08 2008
New Revision: 6379

Log:
more data, flexible loading of urls from a file



Modified:
   httop/trunk/httop.py

Modified: httop/trunk/httop.py
==============================================================================
--- httop/trunk/httop.py	(original)
+++ httop/trunk/httop.py	Fri Aug  1 19:50:08 2008
(at)(at) -4,10 +4,6 (at)(at)
 # $Id$
 """ XXX """
 
-URLS = ['http://www.assembly.org/summer08/',
-        'http://alive.assembly.org',
-        'http://www.assemblytv.net']
-
 urlwidth = 35 # min(60, max(filter(len, URLS)))
 
 import threading
(at)(at) -19,7 +15,7 (at)(at)
 def float_format(f):
     if f is None:
         return 'n/a '
-    return '%2.2f' % f
+    return '%1.2f' % f
 
 
 class URLChecker(threading.Thread):
(at)(at) -27,23 +23,29 (at)(at)
     min = None
     max = None
     avg = None
-    avg50 = None
-    avg500 = None
+    errors = 0
+    avg5 = None
+    avg100 = None
     code = ''
 
     def __init__(self, url):
         super(URLChecker, self).__init__()
         self.url = url
         self.history = []
+        self.running = True
 
     def run(self):
-        while True:
+        while self.running:
             self.update()
             time.sleep(3)
 
     def update(self):
         start = time.time()
-        response = urllib2.urlopen(self.url)
+        response = None
+        try:
+            response = urllib2.urlopen(self.url)
+        except:
+            self.errors += 1
         end = time.time()
         duration = end-start
 
(at)(at) -54,40 +56,65 (at)(at)
         if self.max is None or duration > self.max:
             self.max = duration
         self.avg = sum(self.history) / len(self.history)
-        if len(self.history) >= 50:
-            self.avg50 = sum(self.history[:50]) / 50
-        if len(self.history) >= 500:
-            self.avg500 = sum(self.history[:500]) / 500
-        self.code = response.code
-
+        if len(self.history) >= 5:
+            self.avg5 = sum(self.history[:5]) / 5
+        if len(self.history) >= 100:
+            self.avg100 = sum(self.history[:100]) / 100
+        if response is None:
+            self.code = 'ERR'
+        else:
+            self.code = response.code
+
+def update_config(checkers):
+    try:
+        urls = open('/home/ctheune/.httop')
+    except OSError:
+        return
+
+    urls = urls.readlines()
+    urls = [url[:-1] for url in urls]
+
+    # Add new checkers
+    for url in urls:
+        if url not in [x.url for x in checkers]:
+            checker = URLChecker(url)
+            checker.setDaemon(True)
+            checker.start()
+            checkers.append(checker)
+        checkers.sort(key=lambda x:x.url)
 
-def mainloop():
-    checkers = [URLChecker(url) for url in URLS]
+    # Remove old checkers
     for checker in checkers:
-        checker.setDaemon(True)
-        checker.start()
+        if checker.url not in urls:
+            checker.run = False
+            checkers.remove(checker)
 
+def mainloop():
+    checkers = []
     while True:
+        update_config(checkers)
         mainwin.clear()
-        url_head = '%s %s %s %s %s %s %s %s\n' % (
+        url_head = '%s %s %s %s %s %s %s %s %s\n' % (
             'URL'.ljust(urlwidth),
-            'Last', 'Min ', 'Max ', 'Avg ', 'Avg50', 'Avg500', 'Code')
+            'Last', 'Min ', 'Max ', 'Avg ', 'Avg5', 'Avg100', 'Code',
+            'Errors')
         mainwin.addstr(url_head)
         for checker in checkers:
             url_col = checker.url.ljust(urlwidth)[:urlwidth]
             if not checker.history:
                 info = '%s n/a\n' % url_col
             else:
-                last_col = '%2.2f' % checker.history[0]
+                last_col = float_format(checker.history[0])
                 min_col = float_format(checker.min)
                 max_col = float_format(checker.max)
                 avg_col = float_format(checker.avg)
-                avg50_col = float_format(checker.avg50)
-                avg500_col = float_format(checker.avg500)
-
-                info = '%s %s %s %s %s %s  %s   %s\n' % (url_col, last_col,
min_col,
-                                                max_col, avg_col, avg50_col,
-                                                      avg500_col,
checker.code)
+                avg5_col = float_format(checker.avg5)
+                avg100_col = float_format(checker.avg100)
+                code_col = str(checker.code).ljust(4)
+                info = '%s %s %s %s %s %s  %s   %s %s\n' % (
+                    url_col, last_col, min_col, max_col,
+                    avg_col, avg5_col, avg100_col, code_col,
+                    checker.errors)
             mainwin.addstr(info)
         mainwin.refresh()
         time.sleep(0.1)

Author: zagy
Date: Fri Aug  1 21:06:57 2008
New Revision: 6380

Log:
ope.traversing = 3.5.0a4




Modified:
   zopeversions/zope-stable.cfg

Modified: zopeversions/zope-stable.cfg
==============================================================================
--- zopeversions/zope-stable.cfg	(original)
+++ zopeversions/zope-stable.cfg	Fri Aug  1 21:06:57 2008
(at)(at) -130,6 +130,6 (at)(at)
 zope.testbrowser = 3.4.0
 zope.testing = 3.6.0
 zope.thread = 3.4
-zope.traversing = 3.5.0a3
+zope.traversing = 3.5.0a4
 zope.viewlet = 3.4.2
 zope.xmlpicle = 3.4.0

Author: ctheune
Date: Fri Aug  1 21:47:40 2008
New Revision: 6381

Log:
update the display more often

highlight a url whether it is currently being checked




Modified:
   httop/trunk/httop.py

Modified: httop/trunk/httop.py
==============================================================================
--- httop/trunk/httop.py	(original)
+++ httop/trunk/httop.py	Fri Aug  1 21:47:40 2008
(at)(at) -27,6 +27,7 (at)(at)
     avg5 = None
     avg100 = None
     code = ''
+    checking = False
 
     def __init__(self, url):
         super(URLChecker, self).__init__()
(at)(at) -42,10 +43,12 (at)(at)
     def update(self):
         start = time.time()
         response = None
+        self.checking = True
         try:
             response = urllib2.urlopen(self.url)
         except:
             self.errors += 1
+        self.checking = False
         end = time.time()
         duration = end-start
 
(at)(at) -68,11 +71,12 (at)(at)
 def update_config(checkers):
     try:
         urls = open('/home/ctheune/.httop')
-    except OSError:
+    except Exception:
         return
 
     urls = urls.readlines()
     urls = [url[:-1] for url in urls]
+    urls = [url for url in urls if not url.startswith('#')]
 
     # Add new checkers
     for url in urls:
(at)(at) -100,7 +104,8 (at)(at)
             'Errors')
         mainwin.addstr(url_head)
         for checker in checkers:
-            url_col = checker.url.ljust(urlwidth)[:urlwidth]
+            url = '%s%s' % ((checker.checking and '*' or ' '), checker.url)
+            url_col = url.ljust(urlwidth)[:urlwidth]
             if not checker.history:
                 info = '%s n/a\n' % url_col
             else:
(at)(at) -117,7 +122,7 (at)(at)
                     checker.errors)
             mainwin.addstr(info)
         mainwin.refresh()
-        time.sleep(0.1)
+        time.sleep(0.05)
 
 screen = curses.initscr()
 curses.start_color()

Author: ctheune
Date: Fri Aug  1 22:30:19 2008
New Revision: 6382

Log:
better justification



Modified:
   httop/trunk/httop.py

Modified: httop/trunk/httop.py
==============================================================================
--- httop/trunk/httop.py	(original)
+++ httop/trunk/httop.py	Fri Aug  1 22:30:19 2008
(at)(at) -14,8 +14,8 (at)(at)
 
 def float_format(f):
     if f is None:
-        return 'n/a '
-    return '%1.2f' % f
+        return 'n/a  '
+    return ('%.2f' % f).rjust(5, '0')
 
 
 class URLChecker(threading.Thread):
(at)(at) -98,7 +98,7 (at)(at)
     while True:
         update_config(checkers)
         mainwin.clear()
-        url_head = '%s %s %s %s %s %s %s %s %s\n' % (
+        url_head = '%s  %s   %s   %s   %s   %s   %s   %s   %s\n' % (
             'URL'.ljust(urlwidth),
             'Last', 'Min ', 'Max ', 'Avg ', 'Avg5', 'Avg100', 'Code',
             'Errors')
(at)(at) -107,7 +107,7 (at)(at)
             url = '%s%s' % ((checker.checking and '*' or ' '), checker.url)
             url_col = url.ljust(urlwidth)[:urlwidth]
             if not checker.history:
-                info = '%s n/a\n' % url_col
+                info = '%s  n/a\n' % url_col
             else:
                 last_col = float_format(checker.history[0])
                 min_col = float_format(checker.min)
(at)(at) -116,7 +116,7 (at)(at)
                 avg5_col = float_format(checker.avg5)
                 avg100_col = float_format(checker.avg100)
                 code_col = str(checker.code).ljust(4)
-                info = '%s %s %s %s %s %s  %s   %s %s\n' % (
+                info = '%s  %s  %s  %s  %s  %s  %s   %s %s\n' % (
                     url_col, last_col, min_col, max_col,
                     avg_col, avg5_col, avg100_col, code_col,
                     checker.errors)

Author: ctheune
Date: Sat Aug  2 00:03:04 2008
New Revision: 6383

Log:
adjustments and highlighting



Modified:
   httop/trunk/httop.py

Modified: httop/trunk/httop.py
==============================================================================
--- httop/trunk/httop.py	(original)
+++ httop/trunk/httop.py	Sat Aug  2 00:03:04 2008
(at)(at) -53,6 +53,11 (at)(at)
         duration = end-start
 
         # Update statistics
+        if response is None:
+            self.code = 'ERR'
+            return
+
+        self.code = response.code
         self.history.insert(0, duration)
         if self.min is None or duration < self.min:
             self.min = duration
(at)(at) -63,10 +68,6 (at)(at)
             self.avg5 = sum(self.history[:5]) / 5
         if len(self.history) >= 100:
             self.avg100 = sum(self.history[:100]) / 100
-        if response is None:
-            self.code = 'ERR'
-        else:
-            self.code = response.code
 
 def update_config(checkers):
     try:
(at)(at) -98,14 +99,15 (at)(at)
     while True:
         update_config(checkers)
         mainwin.clear()
-        url_head = '%s  %s   %s   %s   %s   %s   %s   %s   %s\n' % (
-            'URL'.ljust(urlwidth),
+        url_head = '%s  %s   %s   %s   %s   %s   %s  %s %s\n' % (
+            ' URL'.ljust(urlwidth),
             'Last', 'Min ', 'Max ', 'Avg ', 'Avg5', 'Avg100', 'Code',
             'Errors')
         mainwin.addstr(url_head)
         for checker in checkers:
             url = '%s%s' % ((checker.checking and '*' or ' '), checker.url)
             url_col = url.ljust(urlwidth)[:urlwidth]
+            attributes = curses.A_NORMAL
             if not checker.history:
                 info = '%s  n/a\n' % url_col
             else:
(at)(at) -120,7 +122,9 (at)(at)
                     url_col, last_col, min_col, max_col,
                     avg_col, avg5_col, avg100_col, code_col,
                     checker.errors)
-            mainwin.addstr(info)
+                if checker.history[0] > 2:
+                    attributes = curses.A_BOLD
+            mainwin.addstr(info, attributes)
         mainwin.refresh()
         time.sleep(0.05)

Author: ctheune
Date: Sat Aug  2 00:20:05 2008
New Revision: 6384

Log:
remove handling of redirects to allow better analysis


Modified:
   httop/trunk/httop.py

Modified: httop/trunk/httop.py
==============================================================================
--- httop/trunk/httop.py	(original)
+++ httop/trunk/httop.py	Sat Aug  2 00:20:05 2008
(at)(at) -17,6 +17,14 (at)(at)
         return 'n/a  '
     return ('%.2f' % f).rjust(5, '0')
 
+http_opener = urllib2.OpenerDirector()
+for handler in [urllib2.ProxyHandler,
+                urllib2.HTTPHandler,
+                urllib2.HTTPSHandler,
+                urllib2.HTTPDefaultErrorHandler,
+                urllib2.HTTPErrorProcessor]:
+    http_opener.add_handler(handler())
+
 
 class URLChecker(threading.Thread):
 
(at)(at) -40,25 +48,37 (at)(at)
             self.update()
             time.sleep(3)
 
+    (at)property
+    def last(self):
+        if self.history:
+            return self.history[0]
+        return None
+
     def update(self):
         start = time.time()
         response = None
+        request = urllib2.Request(self.url)
+        request.add_header('user-agent', 'Bot/httop')
         self.checking = True
         try:
-            response = urllib2.urlopen(self.url)
+            response = http_opener.open(request)
+            code = response.code
+        except urllib2.HTTPError, e:
+            code = e.code
+            response = 'failed' # Satisfy the not None condition further down
         except:
+            code = None
             self.errors += 1
         self.checking = False
         end = time.time()
         duration = end-start
 
+        self.code = code
+
         # Update statistics
-        if response is None:
-            self.code = 'ERR'
-            return
 
-        self.code = response.code
-        self.history.insert(0, duration)
+        if response is not None:
+            self.history.insert(0, duration)
         if self.min is None or duration < self.min:
             self.min = duration
         if self.max is None or duration > self.max:
(at)(at) -108,22 +128,19 (at)(at)
             url = '%s%s' % ((checker.checking and '*' or ' '), checker.url)
             url_col = url.ljust(urlwidth)[:urlwidth]
             attributes = curses.A_NORMAL
-            if not checker.history:
-                info = '%s  n/a\n' % url_col
-            else:
-                last_col = float_format(checker.history[0])
-                min_col = float_format(checker.min)
-                max_col = float_format(checker.max)
-                avg_col = float_format(checker.avg)
-                avg5_col = float_format(checker.avg5)
-                avg100_col = float_format(checker.avg100)
-                code_col = str(checker.code).ljust(4)
-                info = '%s  %s  %s  %s  %s  %s  %s   %s %s\n' % (
-                    url_col, last_col, min_col, max_col,
-                    avg_col, avg5_col, avg100_col, code_col,
-                    checker.errors)
-                if checker.history[0] > 2:
-                    attributes = curses.A_BOLD
+            last_col = float_format(checker.last)
+            min_col = float_format(checker.min)
+            max_col = float_format(checker.max)
+            avg_col = float_format(checker.avg)
+            avg5_col = float_format(checker.avg5)
+            avg100_col = float_format(checker.avg100)
+            code_col = str(checker.code).ljust(4)
+            info = '%s  %s  %s  %s  %s  %s  %s   %s %s\n' % (
+                url_col, last_col, min_col, max_col,
+                avg_col, avg5_col, avg100_col, code_col,
+                checker.errors)
+            if checker.last > 2:
+                attributes = curses.A_BOLD
             mainwin.addstr(info, attributes)
         mainwin.refresh()
         time.sleep(0.05)