beenull/orcinus-search
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Prevent endless loop

Browse source 
If an orphan URL is blocked by a user rule, then remove it from the 'sp_exist' list so it doesn't keep coming back again and again. This only happens the next crawl after the user adds new rules.
Other misc edits.
This commit is contained in:
Brian Huisman 2023-06-01 12:20:09 -04:00
parent 3f9d713633
commit 56c84a89cb
3 changed files with 9 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
orcinus/admin.php
View file

@ -3099,15 +3099,16 @@ document.write(mustache.render(
<tr class="text-nowrap user-select-none"> <tr class="text-nowrap user-select-none">
<th class="fs-5" scope="col"> <th class="fs-5" scope="col">
<span role="button" id="os_queries_query">Query</span> <span role="button" id="os_queries_query">Query</span>
<img src="img/arrow-down.svg" alt="Sort" class="align-middle svg-icon-sm mb-1"> <img src="img/arrow-down.svg" alt="Sort" title="Sort order" class="align-middle svg-icon-sm mb-1">
</th> </th>
<th class="fs-5 text-center os_sorting os_desc" scope="col"> <th class="fs-5 text-center os_sorting os_desc" scope="col">
<span role="button" id="os_queries_hits">Hits</span> <span data-bs-toggle="tooltip" data-bs-placement="top" title="The number of times this query has been searched for with (unique users / IP addresses) in brackets"
<img src="img/arrow-down.svg" alt="Sort" class="align-middle svg-icon-sm mb-1"> role="button" id="os_queries_hits">Hits</span>
<img src="img/arrow-down.svg" alt="Sort" title="Sort order" class="align-middle svg-icon-sm mb-1">
</th> </th>
<th class="fs-5 text-center d-none d-sm-table-cell" scope="col"> <th class="fs-5 text-center d-none d-sm-table-cell" scope="col">
<span role="button" id="os_queries_results">Results</span> <span role="button" id="os_queries_results" title="Search results returned">Results</span>
<img src="img/arrow-down.svg" alt="Sort" class="align-middle svg-icon-sm mb-1"> <img src="img/arrow-down.svg" alt="Sort" title="Sort order" class="align-middle svg-icon-sm mb-1">
</th> </th>
<th class="fs-5 text-center" colspan="2" scope="col"> <th class="fs-5 text-center" colspan="2" scope="col">
<span role="button" id="os_queries_stamp"> <span role="button" id="os_queries_stamp">
@ -3115,7 +3116,7 @@ document.write(mustache.render(
<img src="img/clock.svg" alt="" class="d-md-none align-middle svg-icon mb-1" <img src="img/clock.svg" alt="" class="d-md-none align-middle svg-icon mb-1"
data-bs-toggle="tooltip" data-bs-placement="left" title="Time since this query was last requested"> data-bs-toggle="tooltip" data-bs-placement="left" title="Time since this query was last requested">
</span> </span>
<img src="img/arrow-down.svg" alt="Sort" class="align-middle svg-icon-sm mb-1"> <img src="img/arrow-down.svg" alt="Sort" title="Sort order" class="align-middle svg-icon-sm mb-1">
</th> </th>
</tr> </tr>
</thead> </thead>

2
orcinus/config.php
View file

@ -2,6 +2,7 @@
$_DDATA = array(); $_DDATA = array();
$_ODATA = array();
$_RDATA = array(); $_RDATA = array();
require __DIR__.'/config.ini.php'; require __DIR__.'/config.ini.php';
@ -365,7 +366,6 @@ $err = $odata->errorInfo();
if ($err[0] == '00000') { if ($err[0] == '00000') {
$odata = $odata->fetchAll(); $odata = $odata->fetchAll();
if (count($odata)) { if (count($odata)) {
$_ODATA = array();
foreach ($odata[0] as $key => $value) { foreach ($odata[0] as $key => $value) {
$json = json_decode($value, true); $json = json_decode($value, true);
$_ODATA[$key] = (!is_null($json)) ? $json : $value; $_ODATA[$key] = (!is_null($json)) ? $json : $value;

1
orcinus/crawler.php
View file

@ -1687,6 +1687,7 @@ while ($_cURL && count($_RDATA['sp_queue'])) {
if ($nx = OS_filterURL($link, $data['base'])) { if ($nx = OS_filterURL($link, $data['base'])) {
OS_crawlLog('Orphan URL ignored due to noindex rule \''.$nx.'\': '.$link, 0); OS_crawlLog('Orphan URL ignored due to noindex rule \''.$nx.'\': '.$link, 0);
$_RDATA['sp_status']['Blocked']++; $_RDATA['sp_status']['Blocked']++;
unset($_RDATA['sp_exist'][$key]);
continue; continue;
} }