INNER CODE UNIT · Python

CrawlerRequest

SPThole/CoexistAI · app.py:422

class CrawlerRequest(BaseModel):
    url_or_urls: Union[str, List[str]]  # Single URL to crawl or list of URLs to scrape
    keywords: Optional[List[str]] = [""]  # Optional keywords to filter content
    depth: Optional[int] = None  # Crawl depth for crawling (None for full website crawl)
    crawl: bool = True  # Whether to crawl (True) or process URLs directly (False)
    min_delay: float = 1.0  # Minimum delay between requests in seconds
    max_delay: float = 2.0  # Maximum delay between requests in seconds
    max_pages: int = 10000  # Maximum number of pages to collect during crawling
    url_keyword: Optional[str] = ""  # Optional keyword to filter URLs by presence in the URL string

@app.post('/clickable-elements', operation_id="get_website_structure")
async def get_website_structure(request: ClickableElementRequest):
    """
    Retrieves the top-k clickable elements from a given URL based on a query.
    This will help you to find out if there are any clickable elements on the page that match the query.
    You can use this to find deeper links since connected pieces of information are often linked together.
    RECOMMENDATION: Be specific with the query to get the most relevant clickable elements.
    Args:

View source record →