deadlink-crawler

commit 72a3b889125ea2ee59c4220c72f82c3b554e7b5f
parent 4c7a665a5cc210d0daa0c0ddff0a8182939b4f91
Author: Stefan <stefan@eliteinformatiker.de>
Date:   Wed, 23 Jan 2013 13:50:38 +0100

added comment for future ideas

Diffstat:
M
crawler.py
 | 
2
++

1 file changed, 2 insertions(+), 0 deletions(-)
diff --git a/crawler.py b/crawler.py
@@ -3,6 +3,8 @@ import urllib2, urlparse
 import re
 import time
 
+# TODO: Do not apply wait time to external links
+
 class Crawler(object):
 	def __init__(self, init_url):
 		# A list of to be crawled urls, where the second element