Onionbot is updated.

juhanurmi · juhanurmi · commit 3a0643e822f5 · 2014-07-30T09:48:11.000+03:00
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@
 *.bak
 venv/
 ahmia_db
+onionbot/*.json
diff --git a/onionbot/README.rst b/onionbot/README.rst
@@ -48,14 +48,9 @@ The items scraped by this project are websites, and the item is defined in the
 
     dirbot.items.CrawledWebsiteItem
 
-Which is:
-
-    domain = Field()
-    url = Field()
-    tor2web_url = Field()
-    title = Field()
-    text = Field()
-    date_inserted = Field()
+and the data is
+
+    domain, url, tor2web_url, title, text, date_inserted
 
 Spiders
 =======
diff --git a/onionbot/dirbot/settings.py b/onionbot/dirbot/settings.py
@@ -6,7 +6,7 @@
 
 DOWNLOAD_TIMEOUT = 60 # 1 min
 DOWNLOAD_DELAY = 2 # 2 sec
-DEPTH_LIMIT = 10
+DEPTH_LIMIT = 1
 DEPTH_STATS = True
 DNSCACHE_ENABLED = True
 ROBOTSTXT_OBEY = True
