Refactor max idle time (#220)

* Add exception handle for refactor max_idle_time

* Add unittest for refactor max_idle_time

* Add default MAX_IDLE_TIME value

Author: LuckyPigeon

src/scrapy_redis/defaults.py

@@ -26,3 +26,4 @@
 START_URLS_KEY = '%(name)s:start_urls'
 START_URLS_AS_SET = False
 START_URLS_AS_ZSET = False
+MAX_IDLE_TIME = 0

src/scrapy_redis/spiders.py

@@ -86,7 +86,16 @@ def setup_redis(self, crawler=None):
             self.fetch_data = self.pop_list_queue
             self.count_size = self.server.llen
 
-        self.max_idle_time = settings.getint("MAX_IDLE_TIME_BEFORE_CLOSE")
+        if self.max_idle_time is None:
+            self.max_idle_time = settings.getint(
+                "MAX_IDLE_TIME_BEFORE_CLOSE",
+                defaults.MAX_IDLE_TIME
+            )
+
+        try:
+            self.max_idle_time = int(self.max_idle_time)
+        except (TypeError, ValueError):
+            raise ValueError("max_idle_time must be an integer")
 
         # The idle signal is called when the spider has no requests left,
         # that's when we will schedule new requests from redis queue

tests/test_spiders.py

@@ -56,6 +56,13 @@ def test_invalid_batch_size(self):
             self.myspider.setup_redis()
         assert "redis_batch_size" in str(excinfo.value)
 
+    def test_invalid_idle_time(self):
+        self.myspider.max_idle_time = 'x'
+        self.myspider.crawler = get_crawler()
+        with pytest.raises(ValueError) as excinfo:
+            self.myspider.setup_redis()
+        assert "max_idle_time" in str(excinfo.value)
+
     @mock.patch('scrapy_redis.spiders.connection')
     def test_via_from_crawler(self, connection):
         server = connection.from_settings.return_value = mock.Mock()
@@ -82,12 +89,13 @@ def test_from_crawler_with_spider_arguments(spider_cls):
         crawler, 'foo',
         redis_key='key:%(name)s',
         redis_batch_size='2000',
+        max_idle_time='100',
     )
     assert spider.name == 'foo'
     assert spider.redis_key == 'key:foo'
     assert spider.redis_batch_size == 2000
-
-
+    assert spider.max_idle_time == 100
+    
 class MockRequest(mock.Mock):
     def __init__(self, url, **kwargs):
         super(MockRequest, self).__init__()