python - Scrapy not working as expected -
this scrapy spider. i'm trying scrape data web. don't know how force scrapy follow links recursively. mistake? import re scrapy.selector import htmlxpathselector scrapy.contrib.spiders import crawlspider, rule scrapy.contrib.linkextractors.sgml import sgmllinkextractor scrapy.selector import selector yellowpagesofmoldova.items import yellowpagesofmoldovaitem scrapy.item import item class yellowspider(crawlspider): name = 'yellow' allowed_domains = ['yellowpages.md'] start_urls = [ 'http://www.yellowpages.md/eng/companies/info/8939-arc-publishing-house'] rules = ( rule(sgmllinkextractor(allow=('eng.+')), follow=true), ) def parse(self, response): sel = selector(response) = yellowpagesofmoldovaitem() i['url'] = response.url i['locality'] = sel.xpath("//tr[3]/td/p[1]/span[1]/text()").extract() i['title'] ...