Skip to content

web_scraper

__all__ = ['BaseWebScraperDriver'] module-attribute

BaseWebScraperDriver

Bases: ABC

Source code in griptape/drivers/web_scraper/base_web_scraper_driver.py 
 6
 7
 8
 9
10
11
12
13
14
15
16
class BaseWebScraperDriver(ABC):
    def scrape_url(self, url: str) -> TextArtifact:
        source = self.fetch_url(url)

        return self.extract_page(source)

    @abstractmethod
    def fetch_url(self, url: str) -> str: ...

    @abstractmethod
    def extract_page(self, page: str) -> TextArtifact: ...

extract_page(page) abstractmethod

Source code in griptape/drivers/web_scraper/base_web_scraper_driver.py 
15
16
@abstractmethod
def extract_page(self, page: str) -> TextArtifact: ...

fetch_url(url) abstractmethod

Source code in griptape/drivers/web_scraper/base_web_scraper_driver.py 
12
13
@abstractmethod
def fetch_url(self, url: str) -> str: ...

scrape_url(url)

Source code in griptape/drivers/web_scraper/base_web_scraper_driver.py 
 7
 8
 9
10
def scrape_url(self, url: str) -> TextArtifact:
    source = self.fetch_url(url)

    return self.extract_page(source)