Add a shouldStopCallback hook for graceful external stops

README.md

@@ -52,6 +52,23 @@ Crawler::create('https://example.com')
     ->foundUrls();
 ```
 
+If you need to stop a crawl based on external state, you can register a callback that receives the current crawler instance and is checked before scheduling each next request:
+
+```php
+use Spatie\Crawler\Crawler;
+
+$shouldStop = false;
+
+Crawler::create('https://example.com')
+    ->shouldStopCallback(function (Crawler $crawler) use (&$shouldStop) {
+        return $shouldStop;
+    })
+    ->onCrawled(function (string $url) use (&$shouldStop) {
+        $shouldStop = true;
+    })
+    ->start();
+```
+
 ## Support us
 
 [<img src="https://github-ads.s3.eu-central-1.amazonaws.com/crawler.jpg?t=1" width="419px" />](https://spatie.be/github-ad-click/crawler)

src/Crawler.php

@@ -75,6 +75,8 @@ class Crawler
 
     protected bool $shouldStop = false;
 
+    protected ?\Closure $shouldStopCallback = null;
+
     protected int $crawledUrlCount = 0;
 
     protected int $failedUrlCount = 0;
@@ -249,6 +251,21 @@ public function fake(array $fakes): self
         return $this;
     }
 
+    /**
+     * Register a callback that is evaluated before scheduling each next request.
+     * Returning true interrupts the crawl and makes start() return FinishReason::Interrupted.
+     *
+     * @param  callable(self):bool  $shouldStopCallback
+     */
+    public function shouldStopCallback(callable $shouldStopCallback): self
+    {
+        $this->shouldStopCallback = $shouldStopCallback instanceof \Closure
+            ? $shouldStopCallback
+            : \Closure::fromCallable($shouldStopCallback);
+
+        return $this;
+    }
+
     public function start(): FinishReason
     {
         $this->shouldStop = false;
@@ -513,6 +530,12 @@ protected function getCrawlRequests(): Generator
             $this->reachedTimeLimits() === false &&
             $crawlUrl = $this->crawlQueue->getPendingUrl()
         ) {
+            if ($this->shouldStopCallback !== null && ($this->shouldStopCallback)($this)) {
+                $this->shouldStop = true;
+
+                break;
+            }
+
             if ($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)) {
                 $this->crawlQueue->markAsProcessed($crawlUrl);
 

tests/Crawler/GracefulShutdownTest.php

@@ -32,6 +32,47 @@
     expect($reason)->toBe(FinishReason::Interrupted);
 });
 
+it('stops crawling when shouldStopCallback returns true immediately', function () {
+    $crawled = [];
+
+    $reason = Crawler::create('https://example.com')
+        ->fake(fullSiteFakes())
+        ->depth(3)
+        ->concurrency(1)
+        ->shouldStopCallback(fn (Crawler $crawler) => true)
+        ->onCrawled(function (string $url) use (&$crawled) {
+            $crawled[] = $url;
+        })
+        ->start();
+
+    expect($crawled)->toBeEmpty();
+    expect($reason)->toBe(FinishReason::Interrupted);
+});
+
+it('stops crawling when shouldStopCallback becomes true between requests', function () {
+    $crawled = [];
+    $shouldStop = false;
+
+    $reason = Crawler::create('https://example.com')
+        ->fake(fullSiteFakes())
+        ->depth(3)
+        ->concurrency(1)
+        ->shouldStopCallback(function (Crawler $crawler) use (&$shouldStop) {
+            return $shouldStop;
+        })
+        ->onCrawled(function (string $url) use (&$crawled, &$shouldStop) {
+            $crawled[] = $url;
+
+            if (count($crawled) === 1) {
+                $shouldStop = true;
+            }
+        })
+        ->start();
+
+    expect(count($crawled))->toBe(1);
+    expect($reason)->toBe(FinishReason::Interrupted);
+});
+
 it('calls finishedCrawling after graceful shutdown', function () {
     $finishedCalled = false;