File tree Expand file tree Collapse file tree 1 file changed +13
-0
lines changed
tests/unit/request_loaders Expand file tree Collapse file tree 1 file changed +13
-0
lines changed Original file line number Diff line number Diff line change @@ -188,9 +188,22 @@ def transform_request(request_options: RequestOptions) -> RequestOptions | Reque
188188 transform_request_function = transform_request ,
189189 )
190190
191+ extracted_urls = set ()
192+
191193 while not await sitemap_loader .is_finished ():
192194 request = await sitemap_loader .fetch_next_request ()
193195 assert request is not None
194196 assert request .user_data .get ('transformed' ) is True
195197
198+ extracted_urls .add (request .url )
199+
196200 await sitemap_loader .mark_request_as_handled (request )
201+
202+ assert len (extracted_urls ) == 5
203+ assert extracted_urls == {
204+ 'http://not-exists.com/' ,
205+ 'http://not-exists.com/catalog?item=12&desc=vacation_hawaii' ,
206+ 'http://not-exists.com/catalog?item=73&desc=vacation_new_zealand' ,
207+ 'http://not-exists.com/catalog?item=74&desc=vacation_newfoundland' ,
208+ 'http://not-exists.com/catalog?item=83&desc=vacation_usa' ,
209+ }
You can’t perform that action at this time.
0 commit comments