[dev] Default value for json support data (#240)

* give default value to variable for push json

* add make_request_from_data comment

* update json supported data guide

* update json supported data guide

Co-authored-by: LuckyPigeon <lucky90322@gmail.com>

Author: songhao8080

src/scrapy_redis/spiders.py

@@ -139,55 +139,56 @@ def next_requests(self):
             self.logger.debug("Read %s requests from '%s'", found, self.redis_key)
 
     def make_request_from_data(self, data):
-        """Returns a Request instance from data coming from Redis.
+        """
+        Returns a `Request` instance for data coming from Redis.
 
-        Overriding this function to support the 'json' requested ``data`` that contains
+        Overriding this function to support the `json` requested `data` that contains
         `url` ,`meta` and other optional parameters. `meta` is a nested json which contains sub-data.
 
         Along with:
-        After accessing the data, sending the FormRequest with `url`, `meta` and addition `formdata`
-
+        After accessing the data, sending the FormRequest with `url`, `meta` and addition `formdata`, `method`
         For example:
         {
             "url": "https://exaple.com",
             "meta": {
                 'job-id':'123xsd',
                 'start-date':'dd/mm/yy'
             },
-            "url_cookie_key":"fertxsas"
+            "url_cookie_key":"fertxsas",
+            "method":"POST"
         }
 
-        this data can be accessed from 'scrapy.spider' through response.
-        'request.url', 'request.meta', 'request.cookies'
+        If `url` is empty, return []. So you should verify the `url` in the data.
+        If `method` is empty, the request object will set method to 'GET', optional.
+        If `meta` is empty, the request object will set `meta` to {}, optional.
+
+        This json supported data can be accessed from 'scrapy.spider' through response.
+        'request.url', 'request.meta', 'request.cookies', 'request.method'
 
         Parameters
         ----------
         data : bytes
             Message from redis.
 
         """
-        # url = bytes_to_str(data, self.redis_encoding)
         formatted_data = bytes_to_str(data, self.redis_encoding)
 
-        # change to json array
-        parameter = {}
         if is_dict(formatted_data):
             parameter = json.loads(formatted_data)
         else:
-            print(TextColor.WARNING + "WARNING: String request is deprecated, please use JSON data format. \
+            self.logger.warning(TextColor.WARNING + "WARNING: String request is deprecated, please use JSON data format. \
                 Detail information, please check https://github.com/rmax/scrapy-redis#features" + TextColor.ENDC)
             return FormRequest(formatted_data, dont_filter=True)
 
-        url = parameter['url']
-        del parameter['url']
-        metadata = {}
-        try:
-            metadata = parameter['meta']
-            del parameter['meta']
-        except KeyError as e:
-            print('Failed to delete metadata: ', e)
+        if parameter.get('url', None) is None:
+            self.logger.warning(TextColor.WARNING + "The data from Redis has no url key in push data" + TextColor.ENDC)
+            return []
+
+        url = parameter.pop("url")
+        method = parameter.pop("method").upper() if "method" in parameter else "GET"
+        metadata = parameter.pop("meta") if "meta" in parameter else {}
 
-        return FormRequest(url, dont_filter=True, formdata=parameter, meta=metadata)
+        return FormRequest(url, dont_filter=True, method=method, formdata=parameter, meta=metadata)
 
     def schedule_next_requests(self):
         """Schedules a request if available"""