INNER CODE UNIT · TypeScript

crawl

philschmid/clipper.js · src/crawler.ts:6

export async function crawl(url: string, output: any, additionalGlobalUrls: string[] = [], headless: boolean = true,) {
  // PlaywrightCrawler crawls the web using a headless
  // browser controlled by the Playwright library.
  const crawler = new PlaywrightCrawler({
    // maxRequestsPerCrawl: 2,
    launchContext: {
      userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36'
    },
    sessionPoolOptions: {
      blockedStatusCodes: [401, 403, 429],
    },
    // Use the requestHandler to process each of the crawled pages.
    async requestHandler({ request, page, response, enqueueLinks, log }) {
      if (!await response?.ok()) {
        log.error(`Got ${response?.status()} for ${request.loadedUrl}`)
        return;
      }
      const title = await page.title()

View source record →