How can I send scrapy result to django views so that frontend can get scrapy result by axios?

I am planning making backend using django + scrapy. my goal is this.

1. frontend(react) send 'get' methods by axios to django views endpoint.

2. this activate scrapy to start crawling (spiders)

3. send scraping result to django views.

4. frontend get json result (scraped result, not jobid or log file)

from twisted.internet import reactor
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapyApp.items import ScrapyappItem 
from scrapy.utils.project import get_project_settings




class MySpider(scrapy.Spider):
    name = "quotes"

    def start_requests(self):
        urls = [
            'https://www.google.com',
           
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        
        item = ScrapyappItem()
        item['title'] = response.css('title::text').get()
            
        yield item 
        
   



def show1(request):
    # configure_logging({'LOG_FORMAT': '%(levelname)s: %(message)s'})
    configure_logging({'LOG_FORMAT': '%(levelname)s: %(message)s'})
    runner = CrawlerRunner()

    d = runner.crawl(MySpider)
    d.addBoth(lambda _: reactor.stop())
    reactor.run() # the script will block here until the crawling is finished

    return HttpResponse({"result":d})

    

Back to Top