github.com/MontFerret/ferret@v0.18.0/examples/crawler.fql (about)

     1  LET doc = DOCUMENT('https://www.theverge.com/tech', {
     2      driver: "cdp",
     3      ignore: {
     4          resources: [
     5              {
     6                  url: "*",
     7                  type: "image"
     8              }
     9          ]
    10      }
    11  })
    12  
    13  WAIT_ELEMENT(doc, '.c-compact-river__entry', 5000)
    14  LET articles = ELEMENTS(doc, '.c-entry-box--compact__image-wrapper')
    15  LET links = (
    16      FOR article IN articles
    17          FILTER article.attributes?.href LIKE 'https://www.theverge.com/*'
    18          RETURN article.attributes.href
    19  )
    20  
    21  FOR link IN links
    22      // The Verge has pretty heavy pages, so let's increase the navigation wait time
    23      NAVIGATE(doc, link, 20000)
    24      WAIT_ELEMENT(doc, '.c-entry-content', 15000)
    25      LET texter = ELEMENT(doc, '.c-entry-content')
    26      RETURN texter.innerText