From 5487f5ca403bc54c44af2d4f4586f919f002e691 Mon Sep 17 00:00:00 2001 From: Tom Dickman Date: Fri, 24 Jan 2020 10:51:29 +1100 Subject: [PATCH] tests: Add unit tests for issue #108 --- classes/robot/crawler.php | 7 +-- tests/phpunit/robot_crawler_test.php | 66 +++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/classes/robot/crawler.php b/classes/robot/crawler.php index 497519b2..a7b329d5 100644 --- a/classes/robot/crawler.php +++ b/classes/robot/crawler.php @@ -916,16 +916,17 @@ private function link_from_node_to_url($from, $url, $text, $idattr) { global $DB; // Ascertain the correct node level based on parent node level. - if ($from->level == TOOL_CRAWLER_NODE_LEVEL_PARENT) { + if (!empty($from->level) && $from->level == TOOL_CRAWLER_NODE_LEVEL_PARENT) { $level = TOOL_CRAWLER_NODE_LEVEL_DIRECT_CHILD; } else { $level = TOOL_CRAWLER_NODE_LEVEL_INDIRECT_CHILD; } - $priority = $from->priority ? $from->priority : TOOL_CRAWLER_PRIORITY_DEFAULT; + $priority = isset($from->priority) ? $from->priority : TOOL_CRAWLER_PRIORITY_DEFAULT; + $courseid = isset($from->courseid) ? $from->courseid : null; // Add the node URL to the queue. - $to = $this->mark_for_crawl($from->url, $url, $from->courseid, $priority, $level); + $to = $this->mark_for_crawl($from->url, $url, $courseid, $priority, $level); if ($to === false) { return false; } diff --git a/tests/phpunit/robot_crawler_test.php b/tests/phpunit/robot_crawler_test.php index 84a2dc46..08c80d28 100644 --- a/tests/phpunit/robot_crawler_test.php +++ b/tests/phpunit/robot_crawler_test.php @@ -295,6 +295,7 @@ public function test_should_be_excluded() { $node->contents = $page . $linktoexclude; $node->url = $url; $node->id = $insertid; + $node->level = TOOL_CRAWLER_NODE_LEVEL_PARENT; $this->resetAfterTest(true); @@ -310,6 +311,67 @@ public function test_should_be_excluded() { self::assertFalse($found); } + /** + * Test for issue #108 - passing node crawl priority to child nodes when parsing html. + */ + public function test_parse_html_priority_inheritance() { + global $CFG, $DB; + + $parentlocalurl = 'course/view.php?id=1§ion=2'; + $directchildlocalurl = 'mod/book/view.php?id=7'; + $indirectchildexternalurl = 'http://someexternalsite.net.au'; + $nodes = []; + + // Internal parent node. + $node = $this->robot->mark_for_crawl($CFG->wwwroot, $parentlocalurl, 1, TOOL_CRAWLER_PRIORITY_HIGH); + $node->httpcode = 200; + $node->mimetype = 'text/html'; + $node->external = 0; + $node->contents = << + + + + Test title + + + Direct child node + + +HTML; + // Parse the parent node, to add create the direct child node. + $parentnode = $this->robot->parse_html($node, $node->external); + + // Internal node direct child. + $url = new moodle_url('/' . $directchildlocalurl); + $node = $DB->get_record('tool_crawler_url', array('url' => $url->raw_out()) ); + $node->url = $CFG->wwwroot.'/'.$directchildlocalurl; + $node->httpcode = 200; + $node->mimetype = 'text/html'; + $node->external = 0; + $node->contents = << + + + + Test title + + + Indirect child node + + +HTML; + // Parse the direct child, to create the indirect child node. + $directchildnode = $this->robot->parse_html($node, $node->external); + $indirectchildnode = $DB->get_record('tool_crawler_url', ['url' => $indirectchildexternalurl]); + + // Direct child nodes should inherit priority from parent node (super node). + $this->assertEquals($parentnode->priority, $directchildnode->priority); + // Indirect child nodes should not inherit a high priority from parent node (super node). + $this->assertGreaterThanOrEqual($indirectchildnode->priority, $parentnode->priority); + // Indirect child nodes should not inherit a high priority from parent node (super node). + $this->assertGreaterThanOrEqual($indirectchildnode->priority, $directchildnode->priority); + // Indirect child nodes should not be able to have a high priority. + $this->assertLessThan(TOOL_CRAWLER_PRIORITY_HIGH, $indirectchildnode->priority); + } } - -