URL check tool is time limited

+ unchecked or too old external links cause page rendering + url checker use a time limit : 3s for page/read, 8s for page/render and 12s for multi-render if checkbox is ticked + Serviceurlchecker is instanciated once for multi render + url cache expire in 90 days + unchecked URLs are listed in home view
vincent-peugnet · Nov 25, 2024 · d6b8fff · d6b8fff
1 parent d3c5cf2
commit d6b8fff
Show file tree

Hide file tree

Showing 12 changed files with 126 additions and 46 deletions.
diff --git a/app/class/Controllerhome.php b/app/class/Controllerhome.php
@@ -327,13 +327,14 @@ public function multiedit()
     public function multirender(): void
     {
         $pagelist = $_POST['pagesid'] ?? [];
-        $checkurl = $_POST['checkurl'] ?? false;
+        $checkurl = boolval($_POST['checkurl']);
         $total = count($pagelist);
         $pagelist = $this->pagemanager->pagelistbyid($pagelist);
         $count = 0;
+        $urlchecker = $checkurl ? new Serviceurlchecker(12) : null; // time to check URLs is limited to 12s
         foreach ($pagelist as $page) {
             try {
-                $page = $this->pagemanager->renderpage($page, $this->router, $checkurl);
+                $page = $this->pagemanager->renderpage($page, $this->router, $urlchecker);
                 if ($this->pagemanager->update($page)) {
                     $count++;
                 }

diff --git a/app/class/Controllerpage.php b/app/class/Controllerpage.php
@@ -75,7 +75,7 @@ public function render(string $page): void
 
         if ($this->importpage() && $this->user->iseditor()) {
             try {
-                $this->page = $this->pagemanager->renderpage($this->page, $this->router, true);
+                $this->page = $this->pagemanager->renderpage($this->page, $this->router, new Serviceurlchecker(8));
             } catch (RuntimeException $e) {
                 Logger::errorex($e);
             }
@@ -116,7 +116,7 @@ private function templaterender(Page $page): void
             $templates = $this->pagemanager->getpagecsstemplates($page);
             foreach ($templates as $page) {
                 if ($this->pagemanager->needtoberendered($page)) {
-                    $page = $this->pagemanager->renderpage($page, $this->router, false);
+                    $page = $this->pagemanager->renderpage($page, $this->router, null);
                     $this->pagemanager->update($page);
                 }
             }
@@ -127,7 +127,7 @@ private function templaterender(Page $page): void
             try {
                 $templatejs = $this->pagemanager->get($page->templatejavascript());
                 if ($this->pagemanager->needtoberendered($templatejs)) {
-                    $templatejs = $this->pagemanager->renderpage($templatejs, $this->router, false);
+                    $templatejs = $this->pagemanager->renderpage($templatejs, $this->router, null);
                     $this->pagemanager->update($templatejs);
                 }
             } catch (RuntimeException $e) {
@@ -186,7 +186,7 @@ public function read(string $page): void
                 $oldlinkto = $this->page->linkto();
             }
             try {
-                $this->page = $this->pagemanager->renderpage($this->page, $this->router, true);
+                $this->page = $this->pagemanager->renderpage($this->page, $this->router, new Serviceurlchecker(3));
             } catch (RuntimeException $e) {
                 Logger::errorex($e);
             }

diff --git a/app/class/Modelpage.php b/app/class/Modelpage.php
@@ -403,22 +403,38 @@ protected function reset(Page $page, array $reset): Page
      *
      * 1. This will compare edit and render dates
      * 2. then if render file exists
-     * 3. then if the templatebody is set and has been updated
+     * 3. then if page have external links and
+     *     - if some haven't been checked yet
+     *     - or if it's been a long time
+     *     - or if url cache is deleted
+     * 4. then if the templatebody is set and has been updated
      *
      * @param Page $page                    Page to be checked
      *
      * @return bool                         true if the page need to be rendered otherwise false
      */
     public function needtoberendered(Page $page): bool
     {
+        if ($page->daterender() <= $page->datemodif()) {
+            return true;
+        }
         if (
-            $page->daterender() <= $page->datemodif() ||
-            !file_exists(self::HTML_RENDER_DIR . $page->id() . '.html') ||
-            !file_exists(self::ASSETS_RENDER_DIR . $page->id() . '.css') ||
-            !file_exists(self::ASSETS_RENDER_DIR . $page->id() . '.js')
+            !file_exists(self::HTML_RENDER_DIR . $page->id() . '.html')
+            || !file_exists(self::ASSETS_RENDER_DIR . $page->id() . '.css')
+            || !file_exists(self::ASSETS_RENDER_DIR . $page->id() . '.js')
         ) {
             return true;
-        } elseif (!empty($page->templatebody())) {
+        }
+        if (count($page->externallinks()) > 0) {
+            $now = new DateTimeImmutable("now", timezone_open("Europe/Paris"));
+            if (
+                $page->daterender()->diff($now)->days > Serviceurlchecker::CACHE_EXPIRE_TIME
+                || $page->uncheckedlinkcount() > 0
+            ) {
+                return true;
+            }
+        }
+        if (!empty($page->templatebody())) {
             try {
                 $bodytemplate = $this->get($page->templatebody());
                 return $page->daterender() <= $bodytemplate->datemodif();
@@ -437,17 +453,17 @@ public function needtoberendered(Page $page): bool
      *
      * @param Page $page
      *
-     * @param bool $checkurl                If true, URLs of rendered page will be checked
+     * @param ?Serviceurlchecker $urlchecker
      *
      * @return Page rendered $page
      *
      * @throws Runtimeexception if writing files to filesystem failed
      */
-    public function renderpage(Page $page, AltoRouter $router, bool $checkurl = false): Page
+    public function renderpage(Page $page, AltoRouter $router, ?Serviceurlchecker $urlchecker = null): Page
     {
         $now = new DateTimeImmutable("now", timezone_open("Europe/Paris"));
 
-        $params = [$router, $this, Config::externallinkblank(), Config::internallinkblank()];
+        $params = [$router, $this, Config::externallinkblank(), Config::internallinkblank(), $urlchecker];
 
         switch ($page->version()) {
             case Page::V1:
@@ -460,7 +476,7 @@ public function renderpage(Page $page, AltoRouter $router, bool $checkurl = fals
                 throw new DomainException('Page version is out of range');
         }
 
-        $html = $renderengine->render($page, $checkurl);
+        $html = $renderengine->render($page);
 
         Fs::dircheck(Model::ASSETS_RENDER_DIR, true, 0775);
         Fs::dircheck(Model::HTML_RENDER_DIR, true, 0775);

diff --git a/app/class/Page.php b/app/class/Page.php
@@ -806,8 +806,8 @@ public function addtag($tag)
 
     public function deadlinkcount(): int
     {
-        $deadurls = array_filter($this->externallinks, function ($ok): bool {
-            return !$ok;
+        $deadurls = array_filter($this->externallinks, function ($status): bool {
+            return $status === false;
         });
         return count($deadurls);
     }
@@ -821,12 +821,24 @@ public function externallinkstitle(): string
     {
         $links = $this->externallinks;
         array_walk($links, function (&$value, string $key) {
-            $symbol = $value ? '✅' : '💀';
+            if (is_null($value)) {
+                $symbol = '🔍️';
+            } else {
+                $symbol = $value ? '✅' : '💀';
+            }
             $value = $key . ' ' . $symbol;
         });
         return implode("\n", $links);
     }
 
+    public function uncheckedlinkcount(): int
+    {
+        $uncheckedurls = array_filter($this->externallinks, function ($status): bool {
+            return is_null($status);
+        });
+        return count($uncheckedurls);
+    }
+
 
     // _________________________________ T O O L S ______________________________________
 

diff --git a/app/class/Servicerender.php b/app/class/Servicerender.php
@@ -60,12 +60,14 @@ public function __construct(
         AltoRouter $router,
         Modelpage $pagemanager,
         bool $externallinkblank = false,
-        bool $internallinkblank = false
+        bool $internallinkblank = false,
+        ?Serviceurlchecker $urlchecker = null
     ) {
         $this->router = $router;
         $this->pagemanager = $pagemanager;
         $this->externallinkblank = $externallinkblank;
         $this->internallinkblank = $internallinkblank;
+        $this->urlchecker = $urlchecker;
     }
 
 
@@ -76,15 +78,13 @@ public function __construct(
      *
      * @return string                       HTML render of the page
      */
-    public function render(Page $page, bool $checkurl): string
+    public function render(Page $page): string
     {
         $this->page = $page;
 
-        $this->urlchecker = $checkurl ? new Serviceurlchecker() : null;
-
         $html = $this->gethmtl();
 
-        if ($checkurl) {
+        if (!is_null($this->urlchecker)) {
             try {
                 $this->urlchecker->savecache();
             } catch (RuntimeException $e) {
@@ -395,9 +395,13 @@ protected function htmlparser(string $html): string
                 $url = filter_var($href, FILTER_SANITIZE_URL);
                 $this->urls[$url] = null;
                 if ($this->urlchecker !== null) {
-                    $dead = $this->urlchecker->isdead($url);
-                    $classes[] = $dead ? 'dead' : 'ok';
-                    $this->urls[$url] = !$dead;
+                    try {
+                        $dead = $this->urlchecker->isdead($url);
+                        $classes[] = $dead ? 'dead' : 'ok';
+                        $this->urls[$url] = !$dead;
+                    } catch (RuntimeException $e) {
+                        // Web search limit reached
+                    }
                 }
             } elseif (preg_match('~^([a-z0-9-_]+)((\/?#[a-z0-9-_]+)|(\/([\w\-\%\[\]\=\?\&]*)))?$~', $href, $out)) {
                 $classes[] = 'internal';

diff --git a/app/class/Servicerenderv1.php b/app/class/Servicerenderv1.php
@@ -18,12 +18,12 @@ class Servicerenderv1 extends Servicerender
      *
      * @return string                       HTML render of the page
      */
-    public function render(Page $page, bool $checkurl): string
+    public function render(Page $page): string
     {
         if (!$page instanceof Pagev1) {
             throw new DomainException('Page should be only Pagev1');
         }
-        return parent::render($page, $checkurl);
+        return parent::render($page);
     }
 
     public function renderprimary(Page $page): string

diff --git a/app/class/Servicerenderv2.php b/app/class/Servicerenderv2.php
@@ -17,12 +17,12 @@ class Servicerenderv2 extends Servicerender
      *
      * @return string                       HTML render of the page
      */
-    public function render(Page $page, bool $checkurl): string
+    public function render(Page $page): string
     {
         if (!$page instanceof Pagev2) {
             throw new DomainException('Page should be only Pagev2');
         }
-        return parent::render($page, $checkurl);
+        return parent::render($page);
     }
 
     public function renderprimary(Page $page): string

diff --git a/app/class/Serviceurlchecker.php b/app/class/Serviceurlchecker.php
@@ -2,6 +2,7 @@
 
 namespace Wcms;
 
+use RuntimeException;
 use Wcms\Exception\Filesystemexception;
 
 /**
@@ -12,21 +13,43 @@ class Serviceurlchecker
     /** @var array[] $urls */
     protected array $urls = [];
 
+    /** @var int $starttimestamp timestamp lauched when object is build (in seconds) */
+    protected int $starttimestamp;
+
+    /** @var int $webchecktime time before stopping Web check (in seconds) */
+    protected int $webchecktime;
+
+    /** @var bool $cacheonly Limit URL checking to cache */
+    protected bool $cacheonly = false;
+
     /** @var int MAX_BOUNCE limit of redirections to follow */
     public const MAX_BOUNCE = 8;
 
     /** @var int CACHE_EXPIRE_TIME in days */
-    public const CACHE_EXPIRE_TIME = 30;
+    public const CACHE_EXPIRE_TIME = 90;
 
     /** @var null[] URL response code considered as not dead */
-    public const ACCEPTED_CODES = [
+    public const ACCEPTED_RESPONSE_CODES = [
         200 => null,
         401 => null,
         403 => null,
     ];
 
-    public function __construct()
+    /**
+     * Tool that check for urls status, first in the cache, then on the Web
+     * The cache expires according to CACHE_EXPIRE_TIME constant
+     * A time limite have to be set to limit Web checking time
+     *
+     * @param int $webchecktime             allocated time for looking URL on the Web (in seconds)
+     *                                      if set to `0`, Check on the Web is disabled: only the cache is used
+     */
+    public function __construct(int $webchecktime)
     {
+        $this->webchecktime = $webchecktime;
+        if ($webchecktime === 0) {
+            $this->cacheonly = true;
+        }
+        $this->starttimestamp = time();
         try {
             $urlfile = Fs::readfile(Model::URLS_FILE);
             $this->urls = json_decode($urlfile, true);
@@ -36,15 +59,21 @@ public function __construct()
     }
 
     /**
-     * Check if URL is dead according to ACCEPTED CODES
+     * Check if URL is dead according to ACCEPTED_RESPONSE_CODES
+     *
+     * @throws RuntimeException             If time limit is reached and URL status is expired or not stored in cache
      */
     public function isdead(string $url): bool
     {
-        if (!$this->iscached($url)) {
+        if ($this->iscachedandvalid($url)) {
+            return !key_exists($this->urls[$url]['response'], self::ACCEPTED_RESPONSE_CODES);
+        }
+        if (!$this->cacheonly && time() < ($this->starttimestamp + $this->webchecktime)) {
             $this->urls[$url]['response'] = $this->getresponse($url);
             $this->urls[$url]['timestamp'] = time();
+            return !key_exists($this->urls[$url]['response'], self::ACCEPTED_RESPONSE_CODES);
         }
-        return !key_exists($this->urls[$url]['response'], self::ACCEPTED_CODES);
+        throw new RuntimeException('Impossible to give a status about this URL');
     }
 
     /**
@@ -70,12 +99,20 @@ protected function getresponse(string $url): int
         return 0;
     }
 
-    protected function iscached(string $url): bool
+    /**
+     * Check if the status of URL is cached and has not expired
+     * If cache is expired, the entry is deleted
+     */
+    protected function iscachedandvalid(string $url): bool
     {
         if (!key_exists($url, $this->urls)) {
             return false;
         }
-        return !($this->urls[$url]['timestamp'] < (time() - self::CACHE_EXPIRE_TIME * 3600 * 24));
+        if (($this->urls[$url]['timestamp'] + self::CACHE_EXPIRE_TIME * 3600 * 24) < time()) {
+            unset($this->urls[$url]);
+            return false;
+        }
+        return true;
     }
 
     /**

diff --git a/app/view/templates/home.php b/app/view/templates/home.php
@@ -325,10 +325,13 @@ class="redirection"
                                     <td class="linkto"><?= $opt->linktolink($item->linkto('array')) ?></td>
                                 <?php endif ?>
                                 <?php if ($columns['externallinks']) : ?>
-                                    <td class="linkto" title="<?= $item->externallinkstitle() ?>">
+                                    <td class="externallinks" title="<?= $item->externallinkstitle() ?>">
                                         <?= $item->externallinks('sort') ?>
                                         <?php if (!empty($deadlinks = $item->deadlinkcount())) : ?>
-                                            <span class="deadlinks"><?= $deadlinks ?></span>
+                                            <span class="deadlinkcount"><?= $deadlinks ?></span>
+                                        <?php endif ?>
+                                        <?php if (!empty($uncheckedlinkcount = $item->uncheckedlinkcount())) : ?>
+                                            <span class="uncheckedlinkcount"><?= $uncheckedlinkcount ?></span>
                                         <?php endif ?>
                                     </td>
                                 <?php endif ?>

diff --git a/assets/css/home.css b/assets/css/home.css
@@ -129,14 +129,21 @@ table .favicon img {
     max-width: 32px;
 }
 
-table .deadlinks {
-    background-color: red;
-    color: white;
+table .deadlinkcount, table .uncheckedlinkcount {
     border-radius: 15px;
     display: inline-block;
     height: 17px;
     width: 17px;
     text-align: center;
+    color: white;
+}
+
+table .deadlinkcount {
+    background-color: red;
+}
+
+table .uncheckedlinkcount {
+    background-color: rgb(65, 65, 65);
 }
 
 td.title {

diff --git a/tests/Servicerenderv1Test.php b/tests/Servicerenderv1Test.php
@@ -45,7 +45,7 @@ public function renderTest(string $name): void
     {
         $pagedata = json_decode(file_get_contents(__DIR__ . "/data/Servicerenderv1Test/$name.json"), true);
         $page = new Pagev1($pagedata);
-        $html = $this->renderengine->render($page, false);
+        $html = $this->renderengine->render($page);
 
         $expected = __DIR__ . "/data/Servicerenderv1Test/$name.html";
         $actual = self::$tmpdir . "/$name.html";