beenull/orcinus-search
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Javascript crawl support enhancement

Browse source 
Don't require reloading the page after a crawl has completed.
Javascript will dynamically update the Crawler Information values if we are on the Crawler Management page.
This commit is contained in:
Brian Huisman 2023-04-28 13:55:26 -04:00
parent ddc601697c
commit 83f8fc9ed2
4 changed files with 123 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
orcinus/admin.php
View file

@ -12,7 +12,7 @@ require __DIR__.'/config.php';
* Display a 'time since' HTML/Javascript counter
*
*/
function OS_countUp($time) {
function OS_countUp($time, $id = '') {
$since = time() - $time;
$periods = array(
array('d', 'day', 'days'),
@ -24,7 +24,8 @@ function OS_countUp($time) {
$hours = floor($since / 3600); $since %= 3600;
$minutes = floor($since / 60);
$seconds = $since % 60; ?>
<span class="countup_timer" data-start="<?php echo $time; ?>" title="<?php echo date('r', $time); ?>">
<span class="countup_timer" data-start="<?php echo $time; ?>" title="<?php echo date('r', $time); ?>"<?php
if (!empty($id)) echo ' id="'.htmlspecialchars($id).'"'; ?>>
<span data-period="days"<?php
if (!$days) echo ' class="d-none"'; ?>>
<var><?php echo $days; ?></var>
@ -1798,7 +1799,7 @@ document.write(mustache.render(
</span>
</label>
<div><?php
OS_countUp(($_ODATA['sp_time_end']) ? $_ODATA['sp_time_end'] : time());
OS_countUp(($_ODATA['sp_time_end']) ? $_ODATA['sp_time_end'] : time(), 'os_countup_time_end');
?></div><?php
if ($_ODATA['sp_time_end'] != $_ODATA['sp_time_end_success']) { ?>
<p class="data-text text-danger">
@ -1810,7 +1811,7 @@ document.write(mustache.render(
<li class="list-group-item">
<label class="d-flex w-100">
<strong class="pe-2">Crawl Time</strong>
<var class="flex-grow-1 text-end"><?php
<var class="flex-grow-1 text-end" id="os_crawl_time_last"><?php
echo $_ODATA['sp_time_last'];
?> <abbr title="seconds">s</abbr></var>
</label>
@ -1818,7 +1819,7 @@ document.write(mustache.render(
<li class="list-group-item">
<label class="d-flex w-100">
<strong class="pe-2">Data Transferred</strong>
<var class="flex-grow-1 text-end"><?php
<var class="flex-grow-1 text-end" id="os_crawl_data_transferred"><?php
echo OS_readSize($_ODATA['sp_data_transferred'], true);
?></var>
</label>
@ -1826,7 +1827,7 @@ document.write(mustache.render(
<li class="list-group-item">
<label class="d-flex w-100">
<strong class="pe-2">Data Stored</strong>
<var class="flex-grow-1 text-end"><?php
<var class="flex-grow-1 text-end" id="os_crawl_data_stored"><?php
if ($_ODATA['sp_data_transferred']) { ?>
<small data-bs-toggle="tooltip" data-bs-placement="bottom" title="Efficiency percentage of data stored vs. data downloaded"><?php
echo '('.round(($_ODATA['sp_data_stored'] / $_ODATA['sp_data_transferred']) * 100, 1).'%)';
@ -1839,7 +1840,7 @@ document.write(mustache.render(
<li class="list-group-item">
<label class="d-flex w-100">
<strong class="pe-2">Links Crawled</strong>
<var class="flex-grow-1 text-end"><?php
<var class="flex-grow-1 text-end" id="os_crawl_links_crawled"><?php
echo $_ODATA['sp_links_crawled'];
?></var>
</label>
@ -1847,7 +1848,7 @@ document.write(mustache.render(
<li class="list-group-item">
<label class="d-flex w-100">
<strong class="pe-2">Pages Stored</strong>
<var class="flex-grow-1 text-end"><?php
<var class="flex-grow-1 text-end" id="os_crawl_pages_stored"><?php
if ($_ODATA['sp_links_crawled']) { ?>
<small data-bs-toggle="tooltip" data-bs-placement="bottom" title="Efficiency percentage of pages stored vs. links crawled"><?php
echo '('.round(($_ODATA['sp_pages_stored'] / $_ODATA['sp_links_crawled']) * 100, 1).'%)';

8
orcinus/config.php
View file

@ -243,10 +243,10 @@ if (!in_array($_DDATA['tbprefix'].'query', $_DDATA['tables'])) {
*/
function OS_readSize($bytes, $abbr = false) {
$bytes = (int)$bytes;
if ($bytes >= 1020054733) return round(($bytes / 1073741824), 1).' '.(($abbr) ? '<abbr title="gibibytes">GiB</abbr>' : 'GiB');
if ($bytes >= 996148) return round(($bytes / 1048576), 1).' '.(($abbr) ? '<abbr title="mebibytes">MiB</abbr>' : 'MiB');
if ($bytes >= 973) return round(($bytes / 1024), 1).' '.(($abbr) ? '<abbr title="kibibytes">kiB</abbr>' : 'kiB');
if ($bytes >= 0) return $bytes.' '.(($abbr) ? '<abbr title="bytes">B</abbr>' : 'B');
if ($bytes >= 1020054733) return round(($bytes / 1073741824), 1).(($abbr) ? ' <abbr title="gibibytes">GiB</abbr>' : ' GiB');
if ($bytes >= 996148) return round(($bytes / 1048576), 1).(($abbr) ? ' <abbr title="mebibytes">MiB</abbr>' : ' MiB');
if ($bytes >= 973) return round(($bytes / 1024), 1).(($abbr) ? ' <abbr title="kibibytes">kiB</abbr>' : ' kiB');
if ($bytes >= 0) return $bytes.(($abbr) ? ' <abbr title="bytes">B</abbr>' : ' B');
return '';
}

7
orcinus/crawler.php
View file

@ -506,7 +506,14 @@ switch ($_SERVER['REQUEST_METHOD']) {
'status' => ($_ODATA['sp_crawling']) ? 'Crawling' : 'Complete',
'progress' => $_ODATA['sp_progress'],
'time_crawl' => time() - $_ODATA['sp_time_start'],
'time_end' => $_ODATA['sp_time_end'],
'time_end_success' => $_ODATA['sp_time_end_success'],
'time_last' => $_ODATA['sp_time_last'],
'timeout_crawl' => $_ODATA['sp_timeout_crawl'],
'data_transferred' => $_ODATA['sp_data_transferred'],
'data_stored' => $_ODATA['sp_data_stored'],
'links_crawled' => $_ODATA['sp_links_crawled'],
'pages_stored' => $_ODATA['sp_pages_stored'],
'tail' => trim(implode("\n", $lines))
);
break;

130
orcinus/js/admin.js
View file

@ -40,6 +40,20 @@ let os_download = function(defaultFilename, postValues) {
}
/**
* Generates a readable filesize string from an integer byte-count
* abbr => Optional <abbr> tag with title attribute added
*/
let os_readSize = function(bytes, abbr) {
bytes = parseInt(bytes);
if (bytes >= 1020054733) return (Math.round((bytes * 10 / 1073741824)) / 10) + ((abbr) ? ' <abbr title="gibibytes">GiB</abbr>' : ' GiB');
if (bytes >= 996148) return (Math.round((bytes * 10 / 1048576)) / 10) + ((abbr) ? ' <abbr title="mebibytes">MiB</abbr>' : ' MiB');
if (bytes >= 973) return (Math.round((bytes * 10 / 1024)) / 10) + ((abbr) ? ' <abbr title="kibibytes">kiB</abbr>' : ' kiB');
if (bytes >= 0) return bytes + ((abbr) ? ' <abbr title="bytes">B</abbr>' : ' B');
return '';
}
// Enable Popper.js tooltips
let toolTipElems = document.querySelectorAll('[data-bs-toggle="tooltip"]');
let toolTipList = [...toolTipElems].map(elem => new bootstrap.Tooltip(elem));
@ -67,6 +81,10 @@ for (let x = 0; x < countUpTimers.length; x++) {
if (timeTracker + 5000 < timeNow || this.originalStart != dataStart) {
this.originalStart = dataStart;
// Update the title attribute
let startDate = new Date(dataStart * 1000);
this.title = startDate.toString();
let since = Math.round((new Date()).getTime() / 1000) - dataStart;
let periods = [];
periods[0] = Math.floor(since / 86400); since %= 86400;
@ -372,27 +390,92 @@ let os_get_crawl_progress = function(getLog) {
return;
}
// If crawl is complete, we are checking to see if a new one has
// started; if one has, trigger progress checks every second
if (os_crawl_start.complete && data.status == 'Crawling') {
os_crawl_start.disabled = 'disabled';
// If crawl is complete...
if (os_crawl_start.complete) {
os_crawl_progress.value = 0
os_crawl_progress.max = 1;
os_crawl_progress.setAttribute('data-progress', '');
os_crawl_progress.innerHTML = '0%';
// Check if a new crawl has started and is ongoing
if (data.status == 'Crawling') {
os_crawl_start.disabled = 'disabled';
os_crawl_start.allow_grep = false;
os_crawl_start.complete = false;
os_crawl_progress.value = 0
os_crawl_progress.max = 1;
os_crawl_progress.setAttribute('data-progress', '');
os_crawl_progress.innerHTML = '0%';
os_crawl_cancel.disabled = '';
os_crawl_log_download.disabled = 'disabled';
os_crawl_start.innerHTML = 'Crawling...';
os_crawl_navbar.innerHTML = 'Crawling...';
os_crawl_start.allow_grep = false;
os_crawl_start.complete = false;
// Start an interval progress check
clearInterval(os_crawl_interval);
os_crawl_interval = setInterval(os_get_crawl_progress, 1000);
os_crawl_cancel.disabled = '';
os_crawl_log_download.disabled = 'disabled';
os_crawl_start.innerHTML = 'Crawling...';
os_crawl_navbar.innerHTML = 'Crawling...';
// Start an interval progress check
clearInterval(os_crawl_interval);
os_crawl_interval = setInterval(os_get_crawl_progress, 1000);
// Else check if the given time_end is later than the time this
// page was loaded; if so, a crawl has finished after this page
// was loaded; if we are on the Crawler Management page, update
// all the info there
} else if (os_crawl_loaded < data.time_end) {
os_crawl_loaded = parseInt((new Date()).getTime() / 1000);
os_crawl_start.disabled = '';
os_crawl_start.innerHTML = 'Start Crawl';
let os_countup_time_end = document.getElementById('os_countup_time_end');
if (os_countup_time_end) {
os_countup_time_end.setAttribute('data-start', data.time_end);
// Try to locate the warning <p> element
let pDanger = os_countup_time_end.parentNode.parentNode.querySelector('p.data-text.text-danger');
// If the time_end does not match the time_end_success, then
// the last crawl did not succeed; show the error message
if (data.time_end != data.time_end_success) {
if (!pDanger) {
let pDanger = document.createElement('p');
pDanger.classList.add('data-text', 'text-danger');
let strong = document.createElement('strong');
strong.appendChild(document.createTextNode('Warning:'));
pDanger.appendChild(strong);
pDanger.appendChild(document.createTextNode(' The previous crawl did not complete successfully. Please check the crawl log for more details.'));
os_countup_time_end.parentNode.parentNode.appendChild(pDanger);
}
// Else if it matches, it was successful, remove any warning
} else if (pDanger) pDanger.parentNode.removeChild(pDanger);
// Update the Crawl information items
let os_crawl_time_last = document.getElementById('os_crawl_time_last');
os_crawl_time_last.innerHTML = data.time_last + ' <abbr title="seconds">s</abbr>';
let os_crawl_data_transferred = document.getElementById('os_crawl_data_transferred');
os_crawl_data_transferred.innerHTML = os_readSize(data.data_transferred, true);
let os_crawl_data_stored = document.getElementById('os_crawl_data_stored');
let text = '';
if (data.data_transferred) {
text += '<small data-bs-toggle="tooltip" data-bs-placement="bottom" title="Efficiency percentage of data stored vs. data downloaded">';
text += '(' + (Math.round(data.data_stored * 1000 / data.data_transferred) / 10) + '%)';
text += '</small> ';
}
os_crawl_data_stored.innerHTML = text + os_readSize(data.data_stored, true);
let os_crawl_links_crawled = document.getElementById('os_crawl_links_crawled');
os_crawl_links_crawled.innerHTML = data.links_crawled;
let os_crawl_pages_stored = document.getElementById('os_crawl_pages_stored');
text = '';
if (data.links_crawled) {
text += '<small data-bs-toggle="tooltip" data-bs-placement="bottom" title="Efficiency percentage of pages stored vs. links crawled">';
text += '(' + (Math.round(data.pages_stored * 1000 / data.links_crawled) / 10) + '%)';
text += '</small> ';
}
os_crawl_pages_stored.innerHTML = text + data.pages_stored;
}
}
}
if (!data.tail) return;
@ -420,8 +503,6 @@ let os_get_crawl_progress = function(getLog) {
}
if (!os_crawl_start.complete && data.status == 'Complete') {
clearInterval(os_crawl_interval);
os_crawl_cancel.disabled = 'disabled';
os_crawl_log_download.disabled = '';
os_crawl_start.complete = true;
@ -430,19 +511,14 @@ let os_get_crawl_progress = function(getLog) {
os_crawl_navbar.innerHTML = 'Crawler';
os_crawl_interval = false;
// Check if the crawler modal window is open
if (crawlerModal && crawlerModal.classList.contains('show')) {
// Don't refresh the page until the user closes the modal
crawlerModal.addEventListener('hide.bs.modal', function() {
window.location.reload();
}, false);
} else window.location.reload();
clearInterval(os_crawl_interval);
os_crawl_interval = setInterval(os_get_crawl_progress, 5000);
}
});
};
let os_crawl_interval;
let os_crawl_loaded = parseInt((new Date()).getTime() / 1000);
let os_crawl_start = document.getElementById('os_crawl_start');
let os_crawl_navbar = document.getElementById('os_crawl_navbar');
let os_crawl_cancel = document.getElementById('os_crawl_cancel');