Stowage/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-01 06:01:29 +00:00
Code Issues Projects Releases Packages Wiki Activity

Give in to tabnanny

Browse source
This commit is contained in:
Guido van Rossum 1998-04-06 14:29:28 +00:00
parent 36dfbcf323
commit 986abac1ba
7 changed files with 907 additions and 1098 deletions
Download patch file Download diff file Expand all files Collapse all files

120
Lib/robotparser.py
View file

@ -9,79 +9,79 @@
class RobotFileParser: class RobotFileParser:
def __init__(self): def __init__(self):
self.rules = {} self.rules = {}
self.debug = 0 self.debug = 0
self.url = '' self.url = ''
self.last_checked = 0 self.last_checked = 0
def mtime(self): def mtime(self):
return self.last_checked return self.last_checked
def modified(self): def modified(self):
import time import time
self.last_checked = time.time() self.last_checked = time.time()
def set_url(self, url): def set_url(self, url):
self.url = url self.url = url
## import urlmisc ## import urlmisc
## self.url = urlmisc.canonical_url(url) ## self.url = urlmisc.canonical_url(url)
def read(self): def read(self):
import urllib import urllib
self.parse(urllib.urlopen(self.url).readlines()) self.parse(urllib.urlopen(self.url).readlines())
def parse(self, lines): def parse(self, lines):
import regsub, string, regex import regsub, string, regex
active = [] active = []
for line in lines: for line in lines:
if self.debug: print '>', line, if self.debug: print '>', line,
# blank line terminates current record # blank line terminates current record
if not line[:-1]: if not line[:-1]:
active = [] active = []
continue continue
# remove optional comment and strip line # remove optional comment and strip line
line = string.strip(line[:string.find(line, '#')]) line = string.strip(line[:string.find(line, '#')])
if not line: if not line:
continue continue
line = regsub.split(line, ' *: *') line = regsub.split(line, ' *: *')
if len(line) == 2: if len(line) == 2:
line[0] = string.lower(line[0]) line[0] = string.lower(line[0])
if line[0] == 'user-agent': if line[0] == 'user-agent':
# this record applies to this user agent # this record applies to this user agent
if self.debug: print '>> user-agent:', line[1] if self.debug: print '>> user-agent:', line[1]
active.append(line[1]) active.append(line[1])
if not self.rules.has_key(line[1]): if not self.rules.has_key(line[1]):
self.rules[line[1]] = [] self.rules[line[1]] = []
elif line[0] == 'disallow': elif line[0] == 'disallow':
if line[1]: if line[1]:
if self.debug: print '>> disallow:', line[1] if self.debug: print '>> disallow:', line[1]
for agent in active: for agent in active:
self.rules[agent].append(regex.compile(line[1])) self.rules[agent].append(regex.compile(line[1]))
else: else:
pass pass
for agent in active: for agent in active:
if self.debug: print '>> allow', agent if self.debug: print '>> allow', agent
self.rules[agent] = [] self.rules[agent] = []
else: else:
if self.debug: print '>> unknown:', line if self.debug: print '>> unknown:', line
self.modified() self.modified()
# returns true if agent is allowed to fetch url # returns true if agent is allowed to fetch url
def can_fetch(self, agent, url): def can_fetch(self, agent, url):
import urlparse import urlparse
ag = agent ag = agent
if not self.rules.has_key(ag): ag = '*' if not self.rules.has_key(ag): ag = '*'
if not self.rules.has_key(ag): if not self.rules.has_key(ag):
if self.debug: print '>> allowing', url, 'fetch by', agent if self.debug: print '>> allowing', url, 'fetch by', agent
return 1 return 1
path = urlparse.urlparse(url)[2] path = urlparse.urlparse(url)[2]
for rule in self.rules[ag]: for rule in self.rules[ag]:
if rule.match(path) != -1: if rule.match(path) != -1:
if self.debug: print '>> disallowing', url, 'fetch by', agent if self.debug: print '>> disallowing', url, 'fetch by', agent
return 0 return 0
if self.debug: print '>> allowing', url, 'fetch by', agent if self.debug: print '>> allowing', url, 'fetch by', agent
return 1 return 1
def test(): def test():
rp = RobotFileParser() rp = RobotFileParser()
@ -91,7 +91,7 @@ def test():
print rp.rules print rp.rules
print rp.can_fetch('*', 'http://www.calendar.com/concerts/') print rp.can_fetch('*', 'http://www.calendar.com/concerts/')
print rp.can_fetch('Musi-Cal-Robot', print rp.can_fetch('Musi-Cal-Robot',
'http://dolphin:80/cgi-bin/music-search?performer=Rolling+Stones') 'http://dolphin:80/cgi-bin/music-search?performer=Rolling+Stones')
print rp.can_fetch('Lycos', 'http://www/~skip/volkswagen/') print rp.can_fetch('Lycos', 'http://www/~skip/volkswagen/')
print rp.can_fetch('Lycos', 'http://www/~skip/volkswagen/vanagon-list-001') print rp.can_fetch('Lycos', 'http://www/~skip/volkswagen/vanagon-list-001')

191
Tools/webchecker/mimetypes.py
View file

@ -1,191 +0,0 @@
"""Guess the MIME type of a file.
This module defines one useful function:
guess_type(url) -- guess the MIME type and encoding of a URL.
It also contains the following, for tuning the behavior:
Data:
knownfiles -- list of files to parse
inited -- flag set when init() has been called
suffixes_map -- dictionary mapping suffixes to suffixes
encodings_map -- dictionary mapping suffixes to encodings
types_map -- dictionary mapping suffixes to types
Functions:
init([files]) -- parse a list of files, default knownfiles
read_mime_types(file) -- parse one file, return a dictionary or None
"""
import string
import posixpath
knownfiles = [
"/usr/local/etc/httpd/conf/mime.types",
"/usr/local/lib/netscape/mime.types",
]
inited = 0
def guess_type(url):
"""Guess the type of a file based on its URL.
Return value is a tuple (type, encoding) where type is None if the
type can't be guessed (no or unknown suffix) or a string of the
form type/subtype, usable for a MIME Content-type header; and
encoding is None for no encoding or the name of the program used
to encode (e.g. compress or gzip). The mappings are table
driven. Encoding suffixes are case sensitive; type suffixes are
first tried case sensitive, then case insensitive.
The suffixes .tgz, .taz and .tz (case sensitive!) are all mapped
to ".tar.gz". (This is table-driven too, using the dictionary
suffixes_map).
"""
if not inited:
init()
base, ext = posixpath.splitext(url)
while suffix_map.has_key(ext):
base, ext = posixpath.splitext(base + suffix_map[ext])
if encodings_map.has_key(ext):
encoding = encodings_map[ext]
base, ext = posixpath.splitext(base)
else:
encoding = None
if types_map.has_key(ext):
return types_map[ext], encoding
elif types_map.has_key(string.lower(ext)):
return types_map[string.lower(ext)], encoding
else:
return None, encoding
def init(files=None):
global inited
for file in files or knownfiles:
s = read_mime_types(file)
if s:
for key, value in s.items():
types_map[key] = value
inited = 1
def read_mime_types(file):
try:
f = open(file)
except IOError:
return None
map = {}
while 1:
line = f.readline()
if not line: break
words = string.split(line)
for i in range(len(words)):
if words[i][0] == '#':
del words[i:]
break
if not words: continue
type, suffixes = words[0], words[1:]
for suff in suffixes:
map['.'+suff] = type
f.close()
return map
suffix_map = {
'.tgz': '.tar.gz',
'.taz': '.tar.gz',
'.tz': '.tar.gz',
}
encodings_map = {
'.gz': 'gzip',
'.Z': 'compress',
}
types_map = {
'.a': 'application/octet-stream',
'.ai': 'application/postscript',
'.aif': 'audio/x-aiff',
'.aifc': 'audio/x-aiff',
'.aiff': 'audio/x-aiff',
'.au': 'audio/basic',
'.avi': 'video/x-msvideo',
'.bcpio': 'application/x-bcpio',
'.bin': 'application/octet-stream',
'.cdf': 'application/x-netcdf',
'.cpio': 'application/x-cpio',
'.csh': 'application/x-csh',
'.dll': 'application/octet-stream',
'.dvi': 'application/x-dvi',
'.exe': 'application/octet-stream',
'.eps': 'application/postscript',
'.etx': 'text/x-setext',
'.gif': 'image/gif',
'.gtar': 'application/x-gtar',
'.hdf': 'application/x-hdf',
'.htm': 'text/html',
'.html': 'text/html',
'.shtml': 'text/html',
'.ief': 'image/ief',
'.jpe': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.jpg': 'image/jpeg',
'.latex': 'application/x-latex',
'.man': 'application/x-troff-man',
'.me': 'application/x-troff-me',
'.mif': 'application/x-mif',
'.mov': 'video/quicktime',
'.movie': 'video/x-sgi-movie',
'.mpe': 'video/mpeg',
'.mpeg': 'video/mpeg',
'.mpg': 'video/mpeg',
'.ms': 'application/x-troff-ms',
'.nc': 'application/x-netcdf',
'.o': 'application/octet-stream',
'.obj': 'application/octet-stream',
'.oda': 'application/oda',
'.pbm': 'image/x-portable-bitmap',
'.pdf': 'application/pdf',
'.pgm': 'image/x-portable-graymap',
'.pnm': 'image/x-portable-anymap',
'.png': 'image/png',
'.ppm': 'image/x-portable-pixmap',
'.py': 'text/x-python',
'.pyc': 'application/x-python-code',
'.ps': 'application/postscript',
'.qt': 'video/quicktime',
'.ras': 'image/x-cmu-raster',
'.rgb': 'image/x-rgb',
'.roff': 'application/x-troff',
'.rtf': 'application/rtf',
'.rtx': 'text/richtext',
'.sgm': 'text/x-sgml',
'.sgml': 'text/x-sgml',
'.sh': 'application/x-sh',
'.shar': 'application/x-shar',
'.snd': 'audio/basic',
'.so': 'application/octet-stream',
'.src': 'application/x-wais-source',
'.sv4cpio': 'application/x-sv4cpio',
'.sv4crc': 'application/x-sv4crc',
'.t': 'application/x-troff',
'.tar': 'application/x-tar',
'.tcl': 'application/x-tcl',
'.tex': 'application/x-tex',
'.texi': 'application/x-texinfo',
'.texinfo': 'application/x-texinfo',
'.tif': 'image/tiff',
'.tiff': 'image/tiff',
'.tr': 'application/x-troff',
'.tsv': 'text/tab-separated-values',
'.txt': 'text/plain',
'.ustar': 'application/x-ustar',
'.wav': 'audio/x-wav',
'.xbm': 'image/x-xbitmap',
'.xpm': 'image/x-xpixmap',
'.xwd': 'image/x-xwindowdump',
'.zip': 'application/zip',
}

120
Tools/webchecker/robotparser.py
View file

@ -9,79 +9,79 @@
class RobotFileParser: class RobotFileParser:
def __init__(self): def __init__(self):
self.rules = {} self.rules = {}
self.debug = 0 self.debug = 0
self.url = '' self.url = ''
self.last_checked = 0 self.last_checked = 0
def mtime(self): def mtime(self):
return self.last_checked return self.last_checked
def modified(self): def modified(self):
import time import time
self.last_checked = time.time() self.last_checked = time.time()
def set_url(self, url): def set_url(self, url):
self.url = url self.url = url
## import urlmisc ## import urlmisc
## self.url = urlmisc.canonical_url(url) ## self.url = urlmisc.canonical_url(url)
def read(self): def read(self):
import urllib import urllib
self.parse(urllib.urlopen(self.url).readlines()) self.parse(urllib.urlopen(self.url).readlines())
def parse(self, lines): def parse(self, lines):
import regsub, string, regex import regsub, string, regex
active = [] active = []
for line in lines: for line in lines:
if self.debug: print '>', line, if self.debug: print '>', line,
# blank line terminates current record # blank line terminates current record
if not line[:-1]: if not line[:-1]:
active = [] active = []
continue continue
# remove optional comment and strip line # remove optional comment and strip line
line = string.strip(line[:string.find(line, '#')]) line = string.strip(line[:string.find(line, '#')])
if not line: if not line:
continue continue
line = regsub.split(line, ' *: *') line = regsub.split(line, ' *: *')
if len(line) == 2: if len(line) == 2:
line[0] = string.lower(line[0]) line[0] = string.lower(line[0])
if line[0] == 'user-agent': if line[0] == 'user-agent':
# this record applies to this user agent # this record applies to this user agent
if self.debug: print '>> user-agent:', line[1] if self.debug: print '>> user-agent:', line[1]
active.append(line[1]) active.append(line[1])
if not self.rules.has_key(line[1]): if not self.rules.has_key(line[1]):
self.rules[line[1]] = [] self.rules[line[1]] = []
elif line[0] == 'disallow': elif line[0] == 'disallow':
if line[1]: if line[1]:
if self.debug: print '>> disallow:', line[1] if self.debug: print '>> disallow:', line[1]
for agent in active: for agent in active:
self.rules[agent].append(regex.compile(line[1])) self.rules[agent].append(regex.compile(line[1]))
else: else:
pass pass
for agent in active: for agent in active:
if self.debug: print '>> allow', agent if self.debug: print '>> allow', agent
self.rules[agent] = [] self.rules[agent] = []
else: else:
if self.debug: print '>> unknown:', line if self.debug: print '>> unknown:', line
self.modified() self.modified()
# returns true if agent is allowed to fetch url # returns true if agent is allowed to fetch url
def can_fetch(self, agent, url): def can_fetch(self, agent, url):
import urlparse import urlparse
ag = agent ag = agent
if not self.rules.has_key(ag): ag = '*' if not self.rules.has_key(ag): ag = '*'
if not self.rules.has_key(ag): if not self.rules.has_key(ag):
if self.debug: print '>> allowing', url, 'fetch by', agent if self.debug: print '>> allowing', url, 'fetch by', agent
return 1 return 1
path = urlparse.urlparse(url)[2] path = urlparse.urlparse(url)[2]
for rule in self.rules[ag]: for rule in self.rules[ag]:
if rule.match(path) != -1: if rule.match(path) != -1:
if self.debug: print '>> disallowing', url, 'fetch by', agent if self.debug: print '>> disallowing', url, 'fetch by', agent
return 0 return 0
if self.debug: print '>> allowing', url, 'fetch by', agent if self.debug: print '>> allowing', url, 'fetch by', agent
return 1 return 1
def test(): def test():
rp = RobotFileParser() rp = RobotFileParser()
@ -91,7 +91,7 @@ def test():
print rp.rules print rp.rules
print rp.can_fetch('*', 'http://www.calendar.com/concerts/') print rp.can_fetch('*', 'http://www.calendar.com/concerts/')
print rp.can_fetch('Musi-Cal-Robot', print rp.can_fetch('Musi-Cal-Robot',
'http://dolphin:80/cgi-bin/music-search?performer=Rolling+Stones') 'http://dolphin:80/cgi-bin/music-search?performer=Rolling+Stones')
print rp.can_fetch('Lycos', 'http://www/~skip/volkswagen/') print rp.can_fetch('Lycos', 'http://www/~skip/volkswagen/')
print rp.can_fetch('Lycos', 'http://www/~skip/volkswagen/vanagon-list-001') print rp.can_fetch('Lycos', 'http://www/~skip/volkswagen/vanagon-list-001')

164
Tools/webchecker/tktools.py
View file

@ -7,8 +7,8 @@
def _clear_entry_widget(event): def _clear_entry_widget(event):
try: try:
widget = event.widget widget = event.widget
widget.delete(0, INSERT) widget.delete(0, INSERT)
except: pass except: pass
def install_keybindings(root): def install_keybindings(root):
root.bind_class('Entry', '<Control-u>', _clear_entry_widget) root.bind_class('Entry', '<Control-u>', _clear_entry_widget)
@ -23,12 +23,12 @@ def make_toplevel(master, title=None, class_=None):
""" """
if class_: if class_:
widget = Toplevel(master, class_=class_) widget = Toplevel(master, class_=class_)
else: else:
widget = Toplevel(master) widget = Toplevel(master)
if title: if title:
widget.title(title) widget.title(title)
widget.iconname(title) widget.iconname(title)
return widget return widget
def set_transient(widget, master, relx=0.5, rely=0.3, expose=1): def set_transient(widget, master, relx=0.5, rely=0.3, expose=1):
@ -43,26 +43,26 @@ def set_transient(widget, master, relx=0.5, rely=0.3, expose=1):
widget.transient(master) widget.transient(master)
widget.update_idletasks() # Actualize geometry information widget.update_idletasks() # Actualize geometry information
if master.winfo_ismapped(): if master.winfo_ismapped():
m_width = master.winfo_width() m_width = master.winfo_width()
m_height = master.winfo_height() m_height = master.winfo_height()
m_x = master.winfo_rootx() m_x = master.winfo_rootx()
m_y = master.winfo_rooty() m_y = master.winfo_rooty()
else: else:
m_width = master.winfo_screenwidth() m_width = master.winfo_screenwidth()
m_height = master.winfo_screenheight() m_height = master.winfo_screenheight()
m_x = m_y = 0 m_x = m_y = 0
w_width = widget.winfo_reqwidth() w_width = widget.winfo_reqwidth()
w_height = widget.winfo_reqheight() w_height = widget.winfo_reqheight()
x = m_x + (m_width - w_width) * relx x = m_x + (m_width - w_width) * relx
y = m_y + (m_height - w_height) * rely y = m_y + (m_height - w_height) * rely
widget.geometry("+%d+%d" % (x, y)) widget.geometry("+%d+%d" % (x, y))
if expose: if expose:
widget.deiconify() # Become visible at the desired location widget.deiconify() # Become visible at the desired location
return widget return widget
def make_scrollbars(parent, hbar, vbar, pack=1, class_=None, name=None, def make_scrollbars(parent, hbar, vbar, pack=1, class_=None, name=None,
takefocus=0): takefocus=0):
"""Subroutine to create a frame with scrollbars. """Subroutine to create a frame with scrollbars.
@ -76,38 +76,38 @@ def make_scrollbars(parent, hbar, vbar, pack=1, class_=None, name=None,
""" """
if class_: if class_:
if name: frame = Frame(parent, class_=class_, name=name) if name: frame = Frame(parent, class_=class_, name=name)
else: frame = Frame(parent, class_=class_) else: frame = Frame(parent, class_=class_)
else: else:
if name: frame = Frame(parent, name=name) if name: frame = Frame(parent, name=name)
else: frame = Frame(parent) else: frame = Frame(parent)
if pack: if pack:
frame.pack(fill=BOTH, expand=1) frame.pack(fill=BOTH, expand=1)
corner = None corner = None
if vbar: if vbar:
if not hbar: if not hbar:
vbar = Scrollbar(frame, takefocus=takefocus) vbar = Scrollbar(frame, takefocus=takefocus)
vbar.pack(fill=Y, side=RIGHT) vbar.pack(fill=Y, side=RIGHT)
else: else:
vbarframe = Frame(frame, borderwidth=0) vbarframe = Frame(frame, borderwidth=0)
vbarframe.pack(fill=Y, side=RIGHT) vbarframe.pack(fill=Y, side=RIGHT)
vbar = Scrollbar(frame, name="vbar", takefocus=takefocus) vbar = Scrollbar(frame, name="vbar", takefocus=takefocus)
vbar.pack(in_=vbarframe, expand=1, fill=Y, side=TOP) vbar.pack(in_=vbarframe, expand=1, fill=Y, side=TOP)
sbwidth = vbar.winfo_reqwidth() sbwidth = vbar.winfo_reqwidth()
corner = Frame(vbarframe, width=sbwidth, height=sbwidth) corner = Frame(vbarframe, width=sbwidth, height=sbwidth)
corner.propagate(0) corner.propagate(0)
corner.pack(side=BOTTOM) corner.pack(side=BOTTOM)
else: else:
vbar = None vbar = None
if hbar: if hbar:
hbar = Scrollbar(frame, orient=HORIZONTAL, name="hbar", hbar = Scrollbar(frame, orient=HORIZONTAL, name="hbar",
takefocus=takefocus) takefocus=takefocus)
hbar.pack(fill=X, side=BOTTOM) hbar.pack(fill=X, side=BOTTOM)
else: else:
hbar = None hbar = None
return hbar, vbar, frame return hbar, vbar, frame
@ -121,20 +121,20 @@ def set_scroll_commands(widget, hbar, vbar):
""" """
if vbar: if vbar:
widget['yscrollcommand'] = (vbar, 'set') widget['yscrollcommand'] = (vbar, 'set')
vbar['command'] = (widget, 'yview') vbar['command'] = (widget, 'yview')
if hbar: if hbar:
widget['xscrollcommand'] = (hbar, 'set') widget['xscrollcommand'] = (hbar, 'set')
hbar['command'] = (widget, 'xview') hbar['command'] = (widget, 'xview')
widget.vbar = vbar widget.vbar = vbar
widget.hbar = hbar widget.hbar = hbar
def make_text_box(parent, width=0, height=0, hbar=0, vbar=1, def make_text_box(parent, width=0, height=0, hbar=0, vbar=1,
fill=BOTH, expand=1, wrap=WORD, pack=1, fill=BOTH, expand=1, wrap=WORD, pack=1,
class_=None, name=None, takefocus=None): class_=None, name=None, takefocus=None):
"""Subroutine to create a text box. """Subroutine to create a text box.
@ -148,8 +148,8 @@ def make_text_box(parent, width=0, height=0, hbar=0, vbar=1,
""" """
hbar, vbar, frame = make_scrollbars(parent, hbar, vbar, pack, hbar, vbar, frame = make_scrollbars(parent, hbar, vbar, pack,
class_=class_, name=name, class_=class_, name=name,
takefocus=takefocus) takefocus=takefocus)
widget = Text(frame, wrap=wrap, name="text") widget = Text(frame, wrap=wrap, name="text")
if width: widget.config(width=width) if width: widget.config(width=width)
@ -162,16 +162,16 @@ def make_text_box(parent, width=0, height=0, hbar=0, vbar=1,
def make_list_box(parent, width=0, height=0, hbar=0, vbar=1, def make_list_box(parent, width=0, height=0, hbar=0, vbar=1,
fill=BOTH, expand=1, pack=1, class_=None, name=None, fill=BOTH, expand=1, pack=1, class_=None, name=None,
takefocus=None): takefocus=None):
"""Subroutine to create a list box. """Subroutine to create a list box.
Like make_text_box(). Like make_text_box().
""" """
hbar, vbar, frame = make_scrollbars(parent, hbar, vbar, pack, hbar, vbar, frame = make_scrollbars(parent, hbar, vbar, pack,
class_=class_, name=name, class_=class_, name=name,
takefocus=takefocus) takefocus=takefocus)
widget = Listbox(frame, name="listbox") widget = Listbox(frame, name="listbox")
if width: widget.config(width=width) if width: widget.config(width=width)
@ -184,8 +184,8 @@ def make_list_box(parent, width=0, height=0, hbar=0, vbar=1,
def make_canvas(parent, width=0, height=0, hbar=1, vbar=1, def make_canvas(parent, width=0, height=0, hbar=1, vbar=1,
fill=BOTH, expand=1, pack=1, class_=None, name=None, fill=BOTH, expand=1, pack=1, class_=None, name=None,
takefocus=None): takefocus=None):
"""Subroutine to create a canvas. """Subroutine to create a canvas.
@ -194,8 +194,8 @@ def make_canvas(parent, width=0, height=0, hbar=1, vbar=1,
""" """
hbar, vbar, frame = make_scrollbars(parent, hbar, vbar, pack, hbar, vbar, frame = make_scrollbars(parent, hbar, vbar, pack,
class_=class_, name=name, class_=class_, name=name,
takefocus=takefocus) takefocus=takefocus)
widget = Canvas(frame, scrollregion=(0, 0, width, height), name="canvas") widget = Canvas(frame, scrollregion=(0, 0, width, height), name="canvas")
if width: widget.config(width=width) if width: widget.config(width=width)
@ -228,9 +228,9 @@ def make_form_entry(parent, label, borderwidth=None):
label.pack(side=LEFT) label.pack(side=LEFT)
if borderwidth is None: if borderwidth is None:
entry = Entry(frame, relief=SUNKEN) entry = Entry(frame, relief=SUNKEN)
else: else:
entry = Entry(frame, relief=SUNKEN, borderwidth=borderwidth) entry = Entry(frame, relief=SUNKEN, borderwidth=borderwidth)
entry.pack(side=LEFT, fill=X, expand=1) entry.pack(side=LEFT, fill=X, expand=1)
return entry, frame return entry, frame
@ -243,8 +243,8 @@ def make_form_entry(parent, label, borderwidth=None):
# expandable while still aligning the colons. This doesn't work yet. # expandable while still aligning the colons. This doesn't work yet.
# #
def make_labeled_form_entry(parent, label, entrywidth=20, entryheight=1, def make_labeled_form_entry(parent, label, entrywidth=20, entryheight=1,
labelwidth=0, borderwidth=None, labelwidth=0, borderwidth=None,
takefocus=None): takefocus=None):
"""Subroutine to create a form entry. """Subroutine to create a form entry.
Create: Create:
@ -261,32 +261,32 @@ def make_labeled_form_entry(parent, label, entrywidth=20, entryheight=1,
label = Label(frame, text=label, width=labelwidth, anchor=E) label = Label(frame, text=label, width=labelwidth, anchor=E)
label.pack(side=LEFT) label.pack(side=LEFT)
if entryheight == 1: if entryheight == 1:
if borderwidth is None: if borderwidth is None:
entry = Entry(frame, relief=SUNKEN, width=entrywidth) entry = Entry(frame, relief=SUNKEN, width=entrywidth)
else: else:
entry = Entry(frame, relief=SUNKEN, width=entrywidth, entry = Entry(frame, relief=SUNKEN, width=entrywidth,
borderwidth=borderwidth) borderwidth=borderwidth)
entry.pack(side=RIGHT, expand=1, fill=X) entry.pack(side=RIGHT, expand=1, fill=X)
frame.pack(fill=X) frame.pack(fill=X)
else: else:
entry = make_text_box(frame, entrywidth, entryheight, 1, 1, entry = make_text_box(frame, entrywidth, entryheight, 1, 1,
takefocus=takefocus) takefocus=takefocus)
frame.pack(fill=BOTH, expand=1) frame.pack(fill=BOTH, expand=1)
return entry, frame, label return entry, frame, label
def make_double_frame(master=None, class_=None, name=None, relief=RAISED, def make_double_frame(master=None, class_=None, name=None, relief=RAISED,
borderwidth=1): borderwidth=1):
"""Create a pair of frames suitable for 'hosting' a dialog.""" """Create a pair of frames suitable for 'hosting' a dialog."""
if name: if name:
if class_: frame = Frame(master, class_=class_, name=name) if class_: frame = Frame(master, class_=class_, name=name)
else: frame = Frame(master, name=name) else: frame = Frame(master, name=name)
else: else:
if class_: frame = Frame(master, class_=class_) if class_: frame = Frame(master, class_=class_)
else: frame = Frame(master) else: frame = Frame(master)
top = Frame(frame, name="topframe", relief=relief, top = Frame(frame, name="topframe", relief=relief,
borderwidth=borderwidth) borderwidth=borderwidth)
bottom = Frame(frame, name="bottomframe") bottom = Frame(frame, name="bottomframe")
bottom.pack(fill=X, padx='1m', pady='1m', side=BOTTOM) bottom.pack(fill=X, padx='1m', pady='1m', side=BOTTOM)
top.pack(expand=1, fill=BOTH, padx='1m', pady='1m') top.pack(expand=1, fill=BOTH, padx='1m', pady='1m')
@ -298,7 +298,7 @@ def make_double_frame(master=None, class_=None, name=None, relief=RAISED,
def make_group_frame(master, name=None, label=None, fill=Y, def make_group_frame(master, name=None, label=None, fill=Y,
side=None, expand=None, font=None): side=None, expand=None, font=None):
"""Create nested frames with a border and optional label. """Create nested frames with a border and optional label.
The outer frame is only used to provide the decorative border, to The outer frame is only used to provide the decorative border, to
@ -311,7 +311,7 @@ def make_group_frame(master, name=None, label=None, fill=Y,
outer = Frame(master, borderwidth=2, relief=GROOVE) outer = Frame(master, borderwidth=2, relief=GROOVE)
outer.pack(expand=expand, fill=fill, side=side) outer.pack(expand=expand, fill=fill, side=side)
if label: if label:
Label(outer, text=label, font=font, anchor=W).pack(fill=X) Label(outer, text=label, font=font, anchor=W).pack(fill=X)
inner = Frame(master, borderwidth='1m', name=name) inner = Frame(master, borderwidth='1m', name=name)
inner.pack(expand=1, fill=BOTH, in_=outer) inner.pack(expand=1, fill=BOTH, in_=outer)
inner.forget = outer.forget inner.forget = outer.forget
@ -326,20 +326,20 @@ def unify_button_widths(*buttons):
""" """
wid = 0 wid = 0
for btn in buttons: for btn in buttons:
wid = max(wid, len(btn["text"])) wid = max(wid, len(btn["text"]))
for btn in buttons: for btn in buttons:
btn["width"] = wid btn["width"] = wid
def flatten(msg): def flatten(msg):
"""Turn a list or tuple into a single string -- recursively.""" """Turn a list or tuple into a single string -- recursively."""
t = type(msg) t = type(msg)
if t in (ListType, TupleType): if t in (ListType, TupleType):
msg = string.join(map(flatten, msg)) msg = string.join(map(flatten, msg))
elif t is ClassType: elif t is ClassType:
msg = msg.__name__ msg = msg.__name__
else: else:
msg = str(msg) msg = str(msg)
return msg return msg
@ -356,8 +356,8 @@ def test():
entry, eframe = make_form_entry(root, 'Boolean:') entry, eframe = make_form_entry(root, 'Boolean:')
text, tframe = make_text_box(root) text, tframe = make_text_box(root)
def enter(event, entry=entry, text=text): def enter(event, entry=entry, text=text):
s = boolean(entry.get()) and '\nyes' or '\nno' s = boolean(entry.get()) and '\nyes' or '\nno'
text.insert('end', s) text.insert('end', s)
entry.bind('<Return>', enter) entry.bind('<Return>', enter)
entry.insert(END, flatten(sys.argv)) entry.insert(END, flatten(sys.argv))
root.mainloop() root.mainloop()

542
Tools/webchecker/wcgui.py
View file

@ -72,365 +72,365 @@
def main(): def main():
try: try:
opts, args = getopt.getopt(sys.argv[1:], 'm:qv') opts, args = getopt.getopt(sys.argv[1:], 'm:qv')
except getopt.error, msg: except getopt.error, msg:
sys.stdout = sys.stderr sys.stdout = sys.stderr
print msg print msg
print __doc__%vars(webchecker) print __doc__%vars(webchecker)
sys.exit(2) sys.exit(2)
for o, a in opts: for o, a in opts:
if o == '-m': if o == '-m':
webchecker.maxpage = string.atoi(a) webchecker.maxpage = string.atoi(a)
if o == '-q': if o == '-q':
webchecker.verbose = 0 webchecker.verbose = 0
if o == '-v': if o == '-v':
webchecker.verbose = webchecker.verbose + 1 webchecker.verbose = webchecker.verbose + 1
root = Tk(className='Webchecker') root = Tk(className='Webchecker')
root.protocol("WM_DELETE_WINDOW", root.quit) root.protocol("WM_DELETE_WINDOW", root.quit)
c = CheckerWindow(root) c = CheckerWindow(root)
if args: if args:
for arg in args[:-1]: for arg in args[:-1]:
c.addroot(arg) c.addroot(arg)
c.suggestroot(args[-1]) c.suggestroot(args[-1])
root.mainloop() root.mainloop()
class CheckerWindow(webchecker.Checker): class CheckerWindow(webchecker.Checker):
def __init__(self, parent, root=webchecker.DEFROOT): def __init__(self, parent, root=webchecker.DEFROOT):
self.__parent = parent self.__parent = parent
self.__topcontrols = Frame(parent) self.__topcontrols = Frame(parent)
self.__topcontrols.pack(side=TOP, fill=X) self.__topcontrols.pack(side=TOP, fill=X)
self.__label = Label(self.__topcontrols, text="Root URL:") self.__label = Label(self.__topcontrols, text="Root URL:")
self.__label.pack(side=LEFT) self.__label.pack(side=LEFT)
self.__rootentry = Entry(self.__topcontrols, width=60) self.__rootentry = Entry(self.__topcontrols, width=60)
self.__rootentry.pack(side=LEFT) self.__rootentry.pack(side=LEFT)
self.__rootentry.bind('<Return>', self.enterroot) self.__rootentry.bind('<Return>', self.enterroot)
self.__rootentry.focus_set() self.__rootentry.focus_set()
self.__controls = Frame(parent) self.__controls = Frame(parent)
self.__controls.pack(side=TOP, fill=X) self.__controls.pack(side=TOP, fill=X)
self.__running = 0 self.__running = 0
self.__start = Button(self.__controls, text="Run", command=self.start) self.__start = Button(self.__controls, text="Run", command=self.start)
self.__start.pack(side=LEFT) self.__start.pack(side=LEFT)
self.__stop = Button(self.__controls, text="Stop", command=self.stop, self.__stop = Button(self.__controls, text="Stop", command=self.stop,
state=DISABLED) state=DISABLED)
self.__stop.pack(side=LEFT) self.__stop.pack(side=LEFT)
self.__step = Button(self.__controls, text="Check one", self.__step = Button(self.__controls, text="Check one",
command=self.step) command=self.step)
self.__step.pack(side=LEFT) self.__step.pack(side=LEFT)
self.__cv = BooleanVar(parent) self.__cv = BooleanVar(parent)
self.__cv.set(self.checkext) self.__cv.set(self.checkext)
self.__checkext = Checkbutton(self.__controls, variable=self.__cv, self.__checkext = Checkbutton(self.__controls, variable=self.__cv,
command=self.update_checkext, command=self.update_checkext,
text="Check nonlocal links",) text="Check nonlocal links",)
self.__checkext.pack(side=LEFT) self.__checkext.pack(side=LEFT)
self.__reset = Button(self.__controls, text="Start over", command=self.reset) self.__reset = Button(self.__controls, text="Start over", command=self.reset)
self.__reset.pack(side=LEFT) self.__reset.pack(side=LEFT)
if __name__ == '__main__': # No Quit button under Grail! if __name__ == '__main__': # No Quit button under Grail!
self.__quit = Button(self.__controls, text="Quit", self.__quit = Button(self.__controls, text="Quit",
command=self.__parent.quit) command=self.__parent.quit)
self.__quit.pack(side=RIGHT) self.__quit.pack(side=RIGHT)
self.__status = Label(parent, text="Status: initial", anchor=W) self.__status = Label(parent, text="Status: initial", anchor=W)
self.__status.pack(side=TOP, fill=X) self.__status.pack(side=TOP, fill=X)
self.__checking = Label(parent, text="Idle", anchor=W) self.__checking = Label(parent, text="Idle", anchor=W)
self.__checking.pack(side=TOP, fill=X) self.__checking.pack(side=TOP, fill=X)
self.__mp = mp = MultiPanel(parent) self.__mp = mp = MultiPanel(parent)
sys.stdout = self.__log = LogPanel(mp, "Log") sys.stdout = self.__log = LogPanel(mp, "Log")
self.__todo = ListPanel(mp, "To check", self.showinfo) self.__todo = ListPanel(mp, "To check", self.showinfo)
self.__done = ListPanel(mp, "Checked", self.showinfo) self.__done = ListPanel(mp, "Checked", self.showinfo)
self.__bad = ListPanel(mp, "Bad links", self.showinfo) self.__bad = ListPanel(mp, "Bad links", self.showinfo)
self.__errors = ListPanel(mp, "Pages w/ bad links", self.showinfo) self.__errors = ListPanel(mp, "Pages w/ bad links", self.showinfo)
self.__details = LogPanel(mp, "Details") self.__details = LogPanel(mp, "Details")
webchecker.Checker.__init__(self) webchecker.Checker.__init__(self)
if root: if root:
root = string.strip(str(root)) root = string.strip(str(root))
if root: if root:
self.suggestroot(root) self.suggestroot(root)
self.newstatus() self.newstatus()
def reset(self): def reset(self):
webchecker.Checker.reset(self) webchecker.Checker.reset(self)
for p in self.__todo, self.__done, self.__bad, self.__errors: for p in self.__todo, self.__done, self.__bad, self.__errors:
p.clear() p.clear()
def suggestroot(self, root): def suggestroot(self, root):
self.__rootentry.delete(0, END) self.__rootentry.delete(0, END)
self.__rootentry.insert(END, root) self.__rootentry.insert(END, root)
self.__rootentry.select_range(0, END) self.__rootentry.select_range(0, END)
def enterroot(self, event=None): def enterroot(self, event=None):
root = self.__rootentry.get() root = self.__rootentry.get()
root = string.strip(root) root = string.strip(root)
if root: if root:
self.__checking.config(text="Adding root "+root) self.__checking.config(text="Adding root "+root)
self.__checking.update_idletasks() self.__checking.update_idletasks()
self.addroot(root) self.addroot(root)
self.__checking.config(text="Idle") self.__checking.config(text="Idle")
try: try:
i = self.__todo.items.index(root) i = self.__todo.items.index(root)
except (ValueError, IndexError): except (ValueError, IndexError):
pass pass
else: else:
self.__todo.list.select_clear(0, END) self.__todo.list.select_clear(0, END)
self.__todo.list.select_set(i) self.__todo.list.select_set(i)
self.__todo.list.yview(i) self.__todo.list.yview(i)
self.__rootentry.delete(0, END) self.__rootentry.delete(0, END)
def start(self): def start(self):
self.__start.config(state=DISABLED, relief=SUNKEN) self.__start.config(state=DISABLED, relief=SUNKEN)
self.__stop.config(state=NORMAL) self.__stop.config(state=NORMAL)
self.__step.config(state=DISABLED) self.__step.config(state=DISABLED)
self.enterroot() self.enterroot()
self.__running = 1 self.__running = 1
self.go() self.go()
def stop(self): def stop(self):
self.__stop.config(state=DISABLED, relief=SUNKEN) self.__stop.config(state=DISABLED, relief=SUNKEN)
self.__running = 0 self.__running = 0
def step(self): def step(self):
self.__start.config(state=DISABLED) self.__start.config(state=DISABLED)
self.__step.config(state=DISABLED, relief=SUNKEN) self.__step.config(state=DISABLED, relief=SUNKEN)
self.enterroot() self.enterroot()
self.__running = 0 self.__running = 0
self.dosomething() self.dosomething()
def go(self): def go(self):
if self.__running: if self.__running:
self.__parent.after_idle(self.dosomething) self.__parent.after_idle(self.dosomething)
else: else:
self.__checking.config(text="Idle") self.__checking.config(text="Idle")
self.__start.config(state=NORMAL, relief=RAISED) self.__start.config(state=NORMAL, relief=RAISED)
self.__stop.config(state=DISABLED, relief=RAISED) self.__stop.config(state=DISABLED, relief=RAISED)
self.__step.config(state=NORMAL, relief=RAISED) self.__step.config(state=NORMAL, relief=RAISED)
__busy = 0 __busy = 0
def dosomething(self): def dosomething(self):
if self.__busy: return if self.__busy: return
self.__busy = 1 self.__busy = 1
if self.todo: if self.todo:
l = self.__todo.selectedindices() l = self.__todo.selectedindices()
if l: if l:
i = l[0] i = l[0]
else: else:
i = 0 i = 0
self.__todo.list.select_set(i) self.__todo.list.select_set(i)
self.__todo.list.yview(i) self.__todo.list.yview(i)
url = self.__todo.items[i] url = self.__todo.items[i]
self.__checking.config(text="Checking "+url) self.__checking.config(text="Checking "+url)
self.__parent.update() self.__parent.update()
self.dopage(url) self.dopage(url)
else: else:
self.stop() self.stop()
self.__busy = 0 self.__busy = 0
self.go() self.go()
def showinfo(self, url): def showinfo(self, url):
d = self.__details d = self.__details
d.clear() d.clear()
d.put("URL: %s\n" % url) d.put("URL: %s\n" % url)
if self.bad.has_key(url): if self.bad.has_key(url):
d.put("Error: %s\n" % str(self.bad[url])) d.put("Error: %s\n" % str(self.bad[url]))
if url in self.roots: if url in self.roots:
d.put("Note: This is a root URL\n") d.put("Note: This is a root URL\n")
if self.done.has_key(url): if self.done.has_key(url):
d.put("Status: checked\n") d.put("Status: checked\n")
o = self.done[url] o = self.done[url]
elif self.todo.has_key(url): elif self.todo.has_key(url):
d.put("Status: to check\n") d.put("Status: to check\n")
o = self.todo[url] o = self.todo[url]
else: else:
d.put("Status: unknown (!)\n") d.put("Status: unknown (!)\n")
o = [] o = []
if self.errors.has_key(url): if self.errors.has_key(url):
d.put("Bad links from this page:\n") d.put("Bad links from this page:\n")
for triple in self.errors[url]: for triple in self.errors[url]:
link, rawlink, msg = triple link, rawlink, msg = triple
d.put(" HREF %s" % link) d.put(" HREF %s" % link)
if link != rawlink: d.put(" (%s)" %rawlink) if link != rawlink: d.put(" (%s)" %rawlink)
d.put("\n") d.put("\n")
d.put(" error %s\n" % str(msg)) d.put(" error %s\n" % str(msg))
self.__mp.showpanel("Details") self.__mp.showpanel("Details")
for source, rawlink in o: for source, rawlink in o:
d.put("Origin: %s" % source) d.put("Origin: %s" % source)
if rawlink != url: if rawlink != url:
d.put(" (%s)" % rawlink) d.put(" (%s)" % rawlink)
d.put("\n") d.put("\n")
d.text.yview("1.0") d.text.yview("1.0")
def setbad(self, url, msg): def setbad(self, url, msg):
webchecker.Checker.setbad(self, url, msg) webchecker.Checker.setbad(self, url, msg)
self.__bad.insert(url) self.__bad.insert(url)
self.newstatus() self.newstatus()
def setgood(self, url): def setgood(self, url):
webchecker.Checker.setgood(self, url) webchecker.Checker.setgood(self, url)
self.__bad.remove(url) self.__bad.remove(url)
self.newstatus() self.newstatus()
def newlink(self, url, origin): def newlink(self, url, origin):
webchecker.Checker.newlink(self, url, origin) webchecker.Checker.newlink(self, url, origin)
if self.done.has_key(url): if self.done.has_key(url):
self.__done.insert(url) self.__done.insert(url)
elif self.todo.has_key(url): elif self.todo.has_key(url):
self.__todo.insert(url) self.__todo.insert(url)
self.newstatus() self.newstatus()
def markdone(self, url): def markdone(self, url):
webchecker.Checker.markdone(self, url) webchecker.Checker.markdone(self, url)
self.__done.insert(url) self.__done.insert(url)
self.__todo.remove(url) self.__todo.remove(url)
self.newstatus() self.newstatus()
def seterror(self, url, triple): def seterror(self, url, triple):
webchecker.Checker.seterror(self, url, triple) webchecker.Checker.seterror(self, url, triple)
self.__errors.insert(url) self.__errors.insert(url)
self.newstatus() self.newstatus()
def newstatus(self): def newstatus(self):
self.__status.config(text="Status: "+self.status()) self.__status.config(text="Status: "+self.status())
self.__parent.update() self.__parent.update()
def update_checkext(self): def update_checkext(self):
self.checkext = self.__cv.get() self.checkext = self.__cv.get()
class ListPanel: class ListPanel:
def __init__(self, mp, name, showinfo=None): def __init__(self, mp, name, showinfo=None):
self.mp = mp self.mp = mp
self.name = name self.name = name
self.showinfo = showinfo self.showinfo = showinfo
self.panel = mp.addpanel(name) self.panel = mp.addpanel(name)
self.list, self.frame = tktools.make_list_box( self.list, self.frame = tktools.make_list_box(
self.panel, width=60, height=5) self.panel, width=60, height=5)
self.list.config(exportselection=0) self.list.config(exportselection=0)
if showinfo: if showinfo:
self.list.bind('<Double-Button-1>', self.doubleclick) self.list.bind('<Double-Button-1>', self.doubleclick)
self.items = [] self.items = []
def clear(self): def clear(self):
self.items = [] self.items = []
self.list.delete(0, END) self.list.delete(0, END)
self.mp.hidepanel(self.name) self.mp.hidepanel(self.name)
def doubleclick(self, event): def doubleclick(self, event):
l = self.selectedindices() l = self.selectedindices()
if l: if l:
self.showinfo(self.list.get(l[0])) self.showinfo(self.list.get(l[0]))
def selectedindices(self): def selectedindices(self):
l = self.list.curselection() l = self.list.curselection()
if not l: return [] if not l: return []
return map(string.atoi, l) return map(string.atoi, l)
def insert(self, url): def insert(self, url):
if url not in self.items: if url not in self.items:
if not self.items: if not self.items:
self.mp.showpanel(self.name) self.mp.showpanel(self.name)
# (I tried sorting alphabetically, but the display is too jumpy) # (I tried sorting alphabetically, but the display is too jumpy)
i = len(self.items) i = len(self.items)
self.list.insert(i, url) self.list.insert(i, url)
self.list.yview(i) self.list.yview(i)
self.items.insert(i, url) self.items.insert(i, url)
def remove(self, url): def remove(self, url):
try: try:
i = self.items.index(url) i = self.items.index(url)
except (ValueError, IndexError): except (ValueError, IndexError):
pass pass
else: else:
was_selected = i in self.selectedindices() was_selected = i in self.selectedindices()
self.list.delete(i) self.list.delete(i)
del self.items[i] del self.items[i]
if not self.items: if not self.items:
self.mp.hidepanel(self.name) self.mp.hidepanel(self.name)
elif was_selected: elif was_selected:
if i >= len(self.items): if i >= len(self.items):
i = len(self.items) - 1 i = len(self.items) - 1
self.list.select_set(i) self.list.select_set(i)
class LogPanel: class LogPanel:
def __init__(self, mp, name): def __init__(self, mp, name):
self.mp = mp self.mp = mp
self.name = name self.name = name
self.panel = mp.addpanel(name) self.panel = mp.addpanel(name)
self.text, self.frame = tktools.make_text_box(self.panel, height=10) self.text, self.frame = tktools.make_text_box(self.panel, height=10)
self.text.config(wrap=NONE) self.text.config(wrap=NONE)
def clear(self): def clear(self):
self.text.delete("1.0", END) self.text.delete("1.0", END)
self.text.yview("1.0") self.text.yview("1.0")
def put(self, s): def put(self, s):
self.text.insert(END, s) self.text.insert(END, s)
if '\n' in s: if '\n' in s:
self.text.yview(END) self.text.yview(END)
def write(self, s): def write(self, s):
self.text.insert(END, s) self.text.insert(END, s)
if '\n' in s: if '\n' in s:
self.text.yview(END) self.text.yview(END)
self.panel.update() self.panel.update()
class MultiPanel: class MultiPanel:
def __init__(self, parent): def __init__(self, parent):
self.parent = parent self.parent = parent
self.frame = Frame(self.parent) self.frame = Frame(self.parent)
self.frame.pack(expand=1, fill=BOTH) self.frame.pack(expand=1, fill=BOTH)
self.topframe = Frame(self.frame, borderwidth=2, relief=RAISED) self.topframe = Frame(self.frame, borderwidth=2, relief=RAISED)
self.topframe.pack(fill=X) self.topframe.pack(fill=X)
self.botframe = Frame(self.frame) self.botframe = Frame(self.frame)
self.botframe.pack(expand=1, fill=BOTH) self.botframe.pack(expand=1, fill=BOTH)
self.panelnames = [] self.panelnames = []
self.panels = {} self.panels = {}
def addpanel(self, name, on=0): def addpanel(self, name, on=0):
v = StringVar(self.parent) v = StringVar(self.parent)
if on: if on:
v.set(name) v.set(name)
else: else:
v.set("") v.set("")
check = Checkbutton(self.topframe, text=name, check = Checkbutton(self.topframe, text=name,
offvalue="", onvalue=name, variable=v, offvalue="", onvalue=name, variable=v,
command=self.checkpanel) command=self.checkpanel)
check.pack(side=LEFT) check.pack(side=LEFT)
panel = Frame(self.botframe) panel = Frame(self.botframe)
label = Label(panel, text=name, borderwidth=2, relief=RAISED, anchor=W) label = Label(panel, text=name, borderwidth=2, relief=RAISED, anchor=W)
label.pack(side=TOP, fill=X) label.pack(side=TOP, fill=X)
t = v, check, panel t = v, check, panel
self.panelnames.append(name) self.panelnames.append(name)
self.panels[name] = t self.panels[name] = t
if on: if on:
panel.pack(expand=1, fill=BOTH) panel.pack(expand=1, fill=BOTH)
return panel return panel
def showpanel(self, name): def showpanel(self, name):
v, check, panel = self.panels[name] v, check, panel = self.panels[name]
v.set(name) v.set(name)
panel.pack(expand=1, fill=BOTH) panel.pack(expand=1, fill=BOTH)
def hidepanel(self, name): def hidepanel(self, name):
v, check, panel = self.panels[name] v, check, panel = self.panels[name]
v.set("") v.set("")
panel.pack_forget() panel.pack_forget()
def checkpanel(self): def checkpanel(self):
for name in self.panelnames: for name in self.panelnames:
v, check, panel = self.panels[name] v, check, panel = self.panels[name]
panel.pack_forget() panel.pack_forget()
for name in self.panelnames: for name in self.panelnames:
v, check, panel = self.panels[name] v, check, panel = self.panels[name]
if v.get(): if v.get():
panel.pack(expand=1, fill=BOTH) panel.pack(expand=1, fill=BOTH)
if __name__ == '__main__': if __name__ == '__main__':

758
Tools/webchecker/webchecker.py
View file

@ -116,17 +116,17 @@
if __version__[0] == '$': if __version__[0] == '$':
_v = string.split(__version__) _v = string.split(__version__)
if len(_v) == 3: if len(_v) == 3:
__version__ = _v[1] __version__ = _v[1]
# Tunable parameters # Tunable parameters
DEFROOT = "file:/usr/local/etc/httpd/htdocs/" # Default root URL DEFROOT = "file:/usr/local/etc/httpd/htdocs/" # Default root URL
CHECKEXT = 1 # Check external references (1 deep) CHECKEXT = 1 # Check external references (1 deep)
VERBOSE = 1 # Verbosity level (0-3) VERBOSE = 1 # Verbosity level (0-3)
MAXPAGE = 150000 # Ignore files bigger than this MAXPAGE = 150000 # Ignore files bigger than this
ROUNDSIZE = 50 # Number of links processed per round ROUNDSIZE = 50 # Number of links processed per round
DUMPFILE = "@webchecker.pickle" # Pickled checkpoint DUMPFILE = "@webchecker.pickle" # Pickled checkpoint
AGENTNAME = "webchecker" # Agent name for robots.txt parser AGENTNAME = "webchecker" # Agent name for robots.txt parser
# Global variables # Global variables
@ -142,76 +142,76 @@ def main():
norun = 0 norun = 0
try: try:
opts, args = getopt.getopt(sys.argv[1:], 'Rd:m:nqr:vx') opts, args = getopt.getopt(sys.argv[1:], 'Rd:m:nqr:vx')
except getopt.error, msg: except getopt.error, msg:
sys.stdout = sys.stderr sys.stdout = sys.stderr
print msg print msg
print __doc__%globals() print __doc__%globals()
sys.exit(2) sys.exit(2)
for o, a in opts: for o, a in opts:
if o == '-R': if o == '-R':
restart = 1 restart = 1
if o == '-d': if o == '-d':
dumpfile = a dumpfile = a
if o == '-m': if o == '-m':
maxpage = string.atoi(a) maxpage = string.atoi(a)
if o == '-n': if o == '-n':
norun = 1 norun = 1
if o == '-q': if o == '-q':
verbose = 0 verbose = 0
if o == '-r': if o == '-r':
roundsize = string.atoi(a) roundsize = string.atoi(a)
if o == '-v': if o == '-v':
verbose = verbose + 1 verbose = verbose + 1
if o == '-x': if o == '-x':
checkext = not checkext checkext = not checkext
if verbose > 0: if verbose > 0:
print AGENTNAME, "version", __version__ print AGENTNAME, "version", __version__
if restart: if restart:
c = load_pickle(dumpfile=dumpfile, verbose=verbose) c = load_pickle(dumpfile=dumpfile, verbose=verbose)
else: else:
c = Checker() c = Checker()
c.setflags(checkext=checkext, verbose=verbose, c.setflags(checkext=checkext, verbose=verbose,
maxpage=maxpage, roundsize=roundsize) maxpage=maxpage, roundsize=roundsize)
if not restart and not args: if not restart and not args:
args.append(DEFROOT) args.append(DEFROOT)
for arg in args: for arg in args:
c.addroot(arg) c.addroot(arg)
if not norun: if not norun:
try: try:
c.run() c.run()
except KeyboardInterrupt: except KeyboardInterrupt:
if verbose > 0: if verbose > 0:
print "[run interrupted]" print "[run interrupted]"
try: try:
c.report() c.report()
except KeyboardInterrupt: except KeyboardInterrupt:
if verbose > 0: if verbose > 0:
print "[report interrupted]" print "[report interrupted]"
if c.save_pickle(dumpfile): if c.save_pickle(dumpfile):
if dumpfile == DUMPFILE: if dumpfile == DUMPFILE:
print "Use ``%s -R'' to restart." % sys.argv[0] print "Use ``%s -R'' to restart." % sys.argv[0]
else: else:
print "Use ``%s -R -d %s'' to restart." % (sys.argv[0], dumpfile) print "Use ``%s -R -d %s'' to restart." % (sys.argv[0], dumpfile)
def load_pickle(dumpfile=DUMPFILE, verbose=VERBOSE): def load_pickle(dumpfile=DUMPFILE, verbose=VERBOSE):
if verbose > 0: if verbose > 0:
print "Loading checkpoint from %s ..." % dumpfile print "Loading checkpoint from %s ..." % dumpfile
f = open(dumpfile, "rb") f = open(dumpfile, "rb")
c = pickle.load(f) c = pickle.load(f)
f.close() f.close()
if verbose > 0: if verbose > 0:
print "Done." print "Done."
print "Root:", string.join(c.roots, "\n ") print "Root:", string.join(c.roots, "\n ")
return c return c
@ -225,364 +225,364 @@ class Checker:
validflags = tuple(dir()) validflags = tuple(dir())
def __init__(self): def __init__(self):
self.reset() self.reset()
def setflags(self, **kw): def setflags(self, **kw):
for key in kw.keys(): for key in kw.keys():
if key not in self.validflags: if key not in self.validflags:
raise NameError, "invalid keyword argument: %s" % str(key) raise NameError, "invalid keyword argument: %s" % str(key)
for key, value in kw.items(): for key, value in kw.items():
setattr(self, key, value) setattr(self, key, value)
def reset(self): def reset(self):
self.roots = [] self.roots = []
self.todo = {} self.todo = {}
self.done = {} self.done = {}
self.bad = {} self.bad = {}
self.round = 0 self.round = 0
# The following are not pickled: # The following are not pickled:
self.robots = {} self.robots = {}
self.errors = {} self.errors = {}
self.urlopener = MyURLopener() self.urlopener = MyURLopener()
self.changed = 0 self.changed = 0
def __getstate__(self): def __getstate__(self):
return (self.roots, self.todo, self.done, self.bad, self.round) return (self.roots, self.todo, self.done, self.bad, self.round)
def __setstate__(self, state): def __setstate__(self, state):
self.reset() self.reset()
(self.roots, self.todo, self.done, self.bad, self.round) = state (self.roots, self.todo, self.done, self.bad, self.round) = state
for root in self.roots: for root in self.roots:
self.addrobot(root) self.addrobot(root)
for url in self.bad.keys(): for url in self.bad.keys():
self.markerror(url) self.markerror(url)
def addroot(self, root): def addroot(self, root):
if root not in self.roots: if root not in self.roots:
troot = root troot = root
scheme, netloc, path, params, query, fragment = \ scheme, netloc, path, params, query, fragment = \
urlparse.urlparse(root) urlparse.urlparse(root)
i = string.rfind(path, "/") + 1 i = string.rfind(path, "/") + 1
if 0 < i < len(path): if 0 < i < len(path):
path = path[:i] path = path[:i]
troot = urlparse.urlunparse((scheme, netloc, path, troot = urlparse.urlunparse((scheme, netloc, path,
params, query, fragment)) params, query, fragment))
self.roots.append(troot) self.roots.append(troot)
self.addrobot(root) self.addrobot(root)
self.newlink(root, ("<root>", root)) self.newlink(root, ("<root>", root))
def addrobot(self, root): def addrobot(self, root):
root = urlparse.urljoin(root, "/") root = urlparse.urljoin(root, "/")
if self.robots.has_key(root): return if self.robots.has_key(root): return
url = urlparse.urljoin(root, "/robots.txt") url = urlparse.urljoin(root, "/robots.txt")
self.robots[root] = rp = robotparser.RobotFileParser() self.robots[root] = rp = robotparser.RobotFileParser()
if self.verbose > 2: if self.verbose > 2:
print "Parsing", url print "Parsing", url
rp.debug = self.verbose > 3 rp.debug = self.verbose > 3
rp.set_url(url) rp.set_url(url)
try: try:
rp.read() rp.read()
except IOError, msg: except IOError, msg:
if self.verbose > 1: if self.verbose > 1:
print "I/O error parsing", url, ":", msg print "I/O error parsing", url, ":", msg
def run(self): def run(self):
while self.todo: while self.todo:
self.round = self.round + 1 self.round = self.round + 1
if self.verbose > 0: if self.verbose > 0:
print print
print "Round %d (%s)" % (self.round, self.status()) print "Round %d (%s)" % (self.round, self.status())
print print
urls = self.todo.keys()[:self.roundsize] urls = self.todo.keys()[:self.roundsize]
for url in urls: for url in urls:
self.dopage(url) self.dopage(url)
def status(self): def status(self):
return "%d total, %d to do, %d done, %d bad" % ( return "%d total, %d to do, %d done, %d bad" % (
len(self.todo)+len(self.done), len(self.todo)+len(self.done),
len(self.todo), len(self.done), len(self.todo), len(self.done),
len(self.bad)) len(self.bad))
def report(self): def report(self):
print print
if not self.todo: print "Final", if not self.todo: print "Final",
else: print "Interim", else: print "Interim",
print "Report (%s)" % self.status() print "Report (%s)" % self.status()
self.report_errors() self.report_errors()
def report_errors(self): def report_errors(self):
if not self.bad: if not self.bad:
print print
print "No errors" print "No errors"
return return
print print
print "Error Report:" print "Error Report:"
sources = self.errors.keys() sources = self.errors.keys()
sources.sort() sources.sort()
for source in sources: for source in sources:
triples = self.errors[source] triples = self.errors[source]
print print
if len(triples) > 1: if len(triples) > 1:
print len(triples), "Errors in", source print len(triples), "Errors in", source
else: else:
print "Error in", source print "Error in", source
for url, rawlink, msg in triples: for url, rawlink, msg in triples:
print " HREF", url, print " HREF", url,
if rawlink != url: print "(%s)" % rawlink, if rawlink != url: print "(%s)" % rawlink,
print print
print " msg", msg print " msg", msg
def dopage(self, url): def dopage(self, url):
if self.verbose > 1: if self.verbose > 1:
if self.verbose > 2: if self.verbose > 2:
self.show("Check ", url, " from", self.todo[url]) self.show("Check ", url, " from", self.todo[url])
else: else:
print "Check ", url print "Check ", url
page = self.getpage(url) page = self.getpage(url)
if page: if page:
for info in page.getlinkinfos(): for info in page.getlinkinfos():
link, rawlink = info link, rawlink = info
origin = url, rawlink origin = url, rawlink
self.newlink(link, origin) self.newlink(link, origin)
self.markdone(url) self.markdone(url)
def newlink(self, url, origin): def newlink(self, url, origin):
if self.done.has_key(url): if self.done.has_key(url):
self.newdonelink(url, origin) self.newdonelink(url, origin)
else: else:
self.newtodolink(url, origin) self.newtodolink(url, origin)
def newdonelink(self, url, origin): def newdonelink(self, url, origin):
self.done[url].append(origin) self.done[url].append(origin)
if self.verbose > 3: if self.verbose > 3:
print " Done link", url print " Done link", url
def newtodolink(self, url, origin): def newtodolink(self, url, origin):
if self.todo.has_key(url): if self.todo.has_key(url):
self.todo[url].append(origin) self.todo[url].append(origin)
if self.verbose > 3: if self.verbose > 3:
print " Seen todo link", url print " Seen todo link", url
else: else:
self.todo[url] = [origin] self.todo[url] = [origin]
if self.verbose > 3: if self.verbose > 3:
print " New todo link", url print " New todo link", url
def markdone(self, url): def markdone(self, url):
self.done[url] = self.todo[url] self.done[url] = self.todo[url]
del self.todo[url] del self.todo[url]
self.changed = 1 self.changed = 1
def inroots(self, url): def inroots(self, url):
for root in self.roots: for root in self.roots:
if url[:len(root)] == root: if url[:len(root)] == root:
root = urlparse.urljoin(root, "/") root = urlparse.urljoin(root, "/")
return self.robots[root].can_fetch(AGENTNAME, url) return self.robots[root].can_fetch(AGENTNAME, url)
return 0 return 0
def getpage(self, url): def getpage(self, url):
if url[:7] == 'mailto:' or url[:5] == 'news:': if url[:7] == 'mailto:' or url[:5] == 'news:':
if self.verbose > 1: print " Not checking mailto/news URL" if self.verbose > 1: print " Not checking mailto/news URL"
return None return None
isint = self.inroots(url) isint = self.inroots(url)
if not isint: if not isint:
if not self.checkext: if not self.checkext:
if self.verbose > 1: print " Not checking ext link" if self.verbose > 1: print " Not checking ext link"
return None return None
f = self.openpage(url) f = self.openpage(url)
if f: if f:
self.safeclose(f) self.safeclose(f)
return None return None
text, nurl = self.readhtml(url) text, nurl = self.readhtml(url)
if nurl != url: if nurl != url:
if self.verbose > 1: if self.verbose > 1:
print " Redirected to", nurl print " Redirected to", nurl
url = nurl url = nurl
if text: if text:
return Page(text, url, verbose=self.verbose, maxpage=self.maxpage) return Page(text, url, verbose=self.verbose, maxpage=self.maxpage)
def readhtml(self, url): def readhtml(self, url):
text = None text = None
f, url = self.openhtml(url) f, url = self.openhtml(url)
if f: if f:
text = f.read() text = f.read()
f.close() f.close()
return text, url return text, url
def openhtml(self, url): def openhtml(self, url):
f = self.openpage(url) f = self.openpage(url)
if f: if f:
url = f.geturl() url = f.geturl()
info = f.info() info = f.info()
if not self.checkforhtml(info, url): if not self.checkforhtml(info, url):
self.safeclose(f) self.safeclose(f)
f = None f = None
return f, url return f, url
def openpage(self, url): def openpage(self, url):
try: try:
return self.urlopener.open(url) return self.urlopener.open(url)
except IOError, msg: except IOError, msg:
msg = self.sanitize(msg) msg = self.sanitize(msg)
if self.verbose > 0: if self.verbose > 0:
print "Error ", msg print "Error ", msg
if self.verbose > 0: if self.verbose > 0:
self.show(" HREF ", url, " from", self.todo[url]) self.show(" HREF ", url, " from", self.todo[url])
self.setbad(url, msg) self.setbad(url, msg)
return None return None
def checkforhtml(self, info, url): def checkforhtml(self, info, url):
if info.has_key('content-type'): if info.has_key('content-type'):
ctype = string.lower(info['content-type']) ctype = string.lower(info['content-type'])
else: else:
if url[-1:] == "/": if url[-1:] == "/":
return 1 return 1
ctype, encoding = mimetypes.guess_type(url) ctype, encoding = mimetypes.guess_type(url)
if ctype == 'text/html': if ctype == 'text/html':
return 1 return 1
else: else:
if self.verbose > 1: if self.verbose > 1:
print " Not HTML, mime type", ctype print " Not HTML, mime type", ctype
return 0 return 0
def setgood(self, url): def setgood(self, url):
if self.bad.has_key(url): if self.bad.has_key(url):
del self.bad[url] del self.bad[url]
self.changed = 1 self.changed = 1
if self.verbose > 0: if self.verbose > 0:
print "(Clear previously seen error)" print "(Clear previously seen error)"
def setbad(self, url, msg): def setbad(self, url, msg):
if self.bad.has_key(url) and self.bad[url] == msg: if self.bad.has_key(url) and self.bad[url] == msg:
if self.verbose > 0: if self.verbose > 0:
print "(Seen this error before)" print "(Seen this error before)"
return return
self.bad[url] = msg self.bad[url] = msg
self.changed = 1 self.changed = 1
self.markerror(url) self.markerror(url)
def markerror(self, url): def markerror(self, url):
try: try:
origins = self.todo[url] origins = self.todo[url]
except KeyError: except KeyError:
origins = self.done[url] origins = self.done[url]
for source, rawlink in origins: for source, rawlink in origins:
triple = url, rawlink, self.bad[url] triple = url, rawlink, self.bad[url]
self.seterror(source, triple) self.seterror(source, triple)
def seterror(self, url, triple): def seterror(self, url, triple):
try: try:
self.errors[url].append(triple) self.errors[url].append(triple)
except KeyError: except KeyError:
self.errors[url] = [triple] self.errors[url] = [triple]
# The following used to be toplevel functions; they have been # The following used to be toplevel functions; they have been
# changed into methods so they can be overridden in subclasses. # changed into methods so they can be overridden in subclasses.
def show(self, p1, link, p2, origins): def show(self, p1, link, p2, origins):
print p1, link print p1, link
i = 0 i = 0
for source, rawlink in origins: for source, rawlink in origins:
i = i+1 i = i+1
if i == 2: if i == 2:
p2 = ' '*len(p2) p2 = ' '*len(p2)
print p2, source, print p2, source,
if rawlink != link: print "(%s)" % rawlink, if rawlink != link: print "(%s)" % rawlink,
print print
def sanitize(self, msg): def sanitize(self, msg):
if isinstance(IOError, ClassType) and isinstance(msg, IOError): if isinstance(IOError, ClassType) and isinstance(msg, IOError):
# Do the other branch recursively # Do the other branch recursively
msg.args = self.sanitize(msg.args) msg.args = self.sanitize(msg.args)
elif isinstance(msg, TupleType): elif isinstance(msg, TupleType):
if len(msg) >= 4 and msg[0] == 'http error' and \ if len(msg) >= 4 and msg[0] == 'http error' and \
isinstance(msg[3], InstanceType): isinstance(msg[3], InstanceType):
# Remove the Message instance -- it may contain # Remove the Message instance -- it may contain
# a file object which prevents pickling. # a file object which prevents pickling.
msg = msg[:3] + msg[4:] msg = msg[:3] + msg[4:]
return msg return msg
def safeclose(self, f): def safeclose(self, f):
try: try:
url = f.geturl() url = f.geturl()
except AttributeError: except AttributeError:
pass pass
else: else:
if url[:4] == 'ftp:' or url[:7] == 'file://': if url[:4] == 'ftp:' or url[:7] == 'file://':
# Apparently ftp connections don't like to be closed # Apparently ftp connections don't like to be closed
# prematurely... # prematurely...
text = f.read() text = f.read()
f.close() f.close()
def save_pickle(self, dumpfile=DUMPFILE): def save_pickle(self, dumpfile=DUMPFILE):
if not self.changed: if not self.changed:
if self.verbose > 0: if self.verbose > 0:
print print
print "No need to save checkpoint" print "No need to save checkpoint"
elif not dumpfile: elif not dumpfile:
if self.verbose > 0: if self.verbose > 0:
print "No dumpfile, won't save checkpoint" print "No dumpfile, won't save checkpoint"
else: else:
if self.verbose > 0: if self.verbose > 0:
print print
print "Saving checkpoint to %s ..." % dumpfile print "Saving checkpoint to %s ..." % dumpfile
newfile = dumpfile + ".new" newfile = dumpfile + ".new"
f = open(newfile, "wb") f = open(newfile, "wb")
pickle.dump(self, f) pickle.dump(self, f)
f.close() f.close()
try: try:
os.unlink(dumpfile) os.unlink(dumpfile)
except os.error: except os.error:
pass pass
os.rename(newfile, dumpfile) os.rename(newfile, dumpfile)
if self.verbose > 0: if self.verbose > 0:
print "Done." print "Done."
return 1 return 1
class Page: class Page:
def __init__(self, text, url, verbose=VERBOSE, maxpage=MAXPAGE): def __init__(self, text, url, verbose=VERBOSE, maxpage=MAXPAGE):
self.text = text self.text = text
self.url = url self.url = url
self.verbose = verbose self.verbose = verbose
self.maxpage = maxpage self.maxpage = maxpage
def getlinkinfos(self): def getlinkinfos(self):
size = len(self.text) size = len(self.text)
if size > self.maxpage: if size > self.maxpage:
if self.verbose > 0: if self.verbose > 0:
print "Skip huge file", self.url print "Skip huge file", self.url
print " (%.0f Kbytes)" % (size*0.001) print " (%.0f Kbytes)" % (size*0.001)
return [] return []
if self.verbose > 2: if self.verbose > 2:
print " Parsing", self.url, "(%d bytes)" % size print " Parsing", self.url, "(%d bytes)" % size
parser = MyHTMLParser(verbose=self.verbose) parser = MyHTMLParser(verbose=self.verbose)
parser.feed(self.text) parser.feed(self.text)
parser.close() parser.close()
rawlinks = parser.getlinks() rawlinks = parser.getlinks()
base = urlparse.urljoin(self.url, parser.getbase() or "") base = urlparse.urljoin(self.url, parser.getbase() or "")
infos = [] infos = []
for rawlink in rawlinks: for rawlink in rawlinks:
t = urlparse.urlparse(rawlink) t = urlparse.urlparse(rawlink)
t = t[:-1] + ('',) t = t[:-1] + ('',)
rawlink = urlparse.urlunparse(t) rawlink = urlparse.urlunparse(t)
link = urlparse.urljoin(base, rawlink) link = urlparse.urljoin(base, rawlink)
infos.append((link, rawlink)) infos.append((link, rawlink))
return infos return infos
class MyStringIO(StringIO.StringIO): class MyStringIO(StringIO.StringIO):
def __init__(self, url, info): def __init__(self, url, info):
self.__url = url self.__url = url
self.__info = info self.__info = info
StringIO.StringIO.__init__(self) StringIO.StringIO.__init__(self)
def info(self): def info(self):
return self.__info return self.__info
def geturl(self): def geturl(self):
return self.__url return self.__url
class MyURLopener(urllib.FancyURLopener): class MyURLopener(urllib.FancyURLopener):
@ -590,81 +590,81 @@ class MyURLopener(urllib.FancyURLopener):
http_error_default = urllib.URLopener.http_error_default http_error_default = urllib.URLopener.http_error_default
def __init__(*args): def __init__(*args):
self = args[0] self = args[0]
apply(urllib.FancyURLopener.__init__, args) apply(urllib.FancyURLopener.__init__, args)
self.addheaders = [ self.addheaders = [
('User-agent', 'Python-webchecker/%s' % __version__), ('User-agent', 'Python-webchecker/%s' % __version__),
] ]
def http_error_401(self, url, fp, errcode, errmsg, headers): def http_error_401(self, url, fp, errcode, errmsg, headers):
return None return None
def open_file(self, url): def open_file(self, url):
path = urllib.url2pathname(urllib.unquote(url)) path = urllib.url2pathname(urllib.unquote(url))
if path[-1] != os.sep: if path[-1] != os.sep:
url = url + '/' url = url + '/'
if os.path.isdir(path): if os.path.isdir(path):
indexpath = os.path.join(path, "index.html") indexpath = os.path.join(path, "index.html")
if os.path.exists(indexpath): if os.path.exists(indexpath):
return self.open_file(url + "index.html") return self.open_file(url + "index.html")
try: try:
names = os.listdir(path) names = os.listdir(path)
except os.error, msg: except os.error, msg:
raise IOError, msg, sys.exc_traceback raise IOError, msg, sys.exc_traceback
names.sort() names.sort()
s = MyStringIO("file:"+url, {'content-type': 'text/html'}) s = MyStringIO("file:"+url, {'content-type': 'text/html'})
s.write('<BASE HREF="file:%s">\n' % s.write('<BASE HREF="file:%s">\n' %
urllib.quote(os.path.join(path, ""))) urllib.quote(os.path.join(path, "")))
for name in names: for name in names:
q = urllib.quote(name) q = urllib.quote(name)
s.write('<A HREF="%s">%s</A>\n' % (q, q)) s.write('<A HREF="%s">%s</A>\n' % (q, q))
s.seek(0) s.seek(0)
return s return s
return urllib.FancyURLopener.open_file(self, path) return urllib.FancyURLopener.open_file(self, path)
class MyHTMLParser(sgmllib.SGMLParser): class MyHTMLParser(sgmllib.SGMLParser):
def __init__(self, verbose=VERBOSE): def __init__(self, verbose=VERBOSE):
self.base = None self.base = None
self.links = {} self.links = {}
self.myverbose = verbose self.myverbose = verbose
sgmllib.SGMLParser.__init__(self) sgmllib.SGMLParser.__init__(self)
def start_a(self, attributes): def start_a(self, attributes):
self.link_attr(attributes, 'href') self.link_attr(attributes, 'href')
def end_a(self): pass def end_a(self): pass
def do_area(self, attributes): def do_area(self, attributes):
self.link_attr(attributes, 'href') self.link_attr(attributes, 'href')
def do_img(self, attributes): def do_img(self, attributes):
self.link_attr(attributes, 'src', 'lowsrc') self.link_attr(attributes, 'src', 'lowsrc')
def do_frame(self, attributes): def do_frame(self, attributes):
self.link_attr(attributes, 'src') self.link_attr(attributes, 'src')
def link_attr(self, attributes, *args): def link_attr(self, attributes, *args):
for name, value in attributes: for name, value in attributes:
if name in args: if name in args:
if value: value = string.strip(value) if value: value = string.strip(value)
if value: self.links[value] = None if value: self.links[value] = None
def do_base(self, attributes): def do_base(self, attributes):
for name, value in attributes: for name, value in attributes:
if name == 'href': if name == 'href':
if value: value = string.strip(value) if value: value = string.strip(value)
if value: if value:
if self.myverbose > 1: if self.myverbose > 1:
print " Base", value print " Base", value
self.base = value self.base = value
def getlinks(self): def getlinks(self):
return self.links.keys() return self.links.keys()
def getbase(self): def getbase(self):
return self.base return self.base
if __name__ == '__main__': if __name__ == '__main__':

110
Tools/webchecker/websucker.py
View file

@ -16,29 +16,29 @@
if __version__[0] == '$': if __version__[0] == '$':
_v = string.split(__version__) _v = string.split(__version__)
if len(_v) == 3: if len(_v) == 3:
__version__ = _v[1] __version__ = _v[1]
def main(): def main():
verbose = webchecker.VERBOSE verbose = webchecker.VERBOSE
try: try:
opts, args = getopt.getopt(sys.argv[1:], "qv") opts, args = getopt.getopt(sys.argv[1:], "qv")
except getopt.error, msg: except getopt.error, msg:
print msg print msg
print "usage:", sys.argv[0], "[-qv] ... [rooturl] ..." print "usage:", sys.argv[0], "[-qv] ... [rooturl] ..."
return 2 return 2
for o, a in opts: for o, a in opts:
if o == "-q": if o == "-q":
verbose = 0 verbose = 0
if o == "-v": if o == "-v":
verbose = verbose + 1 verbose = verbose + 1
c = Sucker() c = Sucker()
c.setflags(verbose=verbose) c.setflags(verbose=verbose)
c.urlopener.addheaders = [ c.urlopener.addheaders = [
('User-agent', 'websucker/%s' % __version__), ('User-agent', 'websucker/%s' % __version__),
] ]
for arg in args: for arg in args:
print "Adding root", arg print "Adding root", arg
c.addroot(arg) c.addroot(arg)
print "Run..." print "Run..."
c.run() c.run()
@ -47,57 +47,57 @@ class Sucker(webchecker.Checker):
checkext = 0 checkext = 0
def readhtml(self, url): def readhtml(self, url):
text = None text = None
path = self.savefilename(url) path = self.savefilename(url)
try: try:
f = open(path, "rb") f = open(path, "rb")
except IOError: except IOError:
f = self.openpage(url) f = self.openpage(url)
if f: if f:
info = f.info() info = f.info()
nurl = f.geturl() nurl = f.geturl()
if nurl != url: if nurl != url:
url = nurl url = nurl
path = self.savefilename(url) path = self.savefilename(url)
text = f.read() text = f.read()
f.close() f.close()
self.savefile(text, path) self.savefile(text, path)
if not self.checkforhtml(info, url): if not self.checkforhtml(info, url):
text = None text = None
else: else:
if self.checkforhtml({}, url): if self.checkforhtml({}, url):
text = f.read() text = f.read()
f.close() f.close()
return text, url return text, url
def savefile(self, text, path): def savefile(self, text, path):
dir, base = os.path.split(path) dir, base = os.path.split(path)
makedirs(dir) makedirs(dir)
f = open(path, "wb") f = open(path, "wb")
f.write(text) f.write(text)
f.close() f.close()
print "saved", path print "saved", path
def savefilename(self, url): def savefilename(self, url):
type, rest = urllib.splittype(url) type, rest = urllib.splittype(url)
host, path = urllib.splithost(rest) host, path = urllib.splithost(rest)
while path[:1] == "/": path = path[1:] while path[:1] == "/": path = path[1:]
user, host = urllib.splituser(host) user, host = urllib.splituser(host)
host, port = urllib.splitnport(host) host, port = urllib.splitnport(host)
host = string.lower(host) host = string.lower(host)
path = os.path.join(host, path) path = os.path.join(host, path)
if path[-1] == "/": path = path + "index.html" if path[-1] == "/": path = path + "index.html"
if os.sep != "/": if os.sep != "/":
path = string.join(string.split(path, "/"), os.sep) path = string.join(string.split(path, "/"), os.sep)
return path return path
def makedirs(dir): def makedirs(dir):
if not dir or os.path.exists(dir): if not dir or os.path.exists(dir):
return return
head, tail = os.path.split(dir) head, tail = os.path.split(dir)
if not tail: if not tail:
print "Huh? Don't know how to make dir", dir print "Huh? Don't know how to make dir", dir
return return
makedirs(head) makedirs(head)
os.mkdir(dir, 0777) os.mkdir(dir, 0777)