Javascript crawl support enhancement
Don't require reloading the page after a crawl has completed. Javascript will dynamically update the Crawler Information values if we are on the Crawler Management page.
This commit is contained in:
parent
ddc601697c
commit
83f8fc9ed2
|
@ -12,7 +12,7 @@ require __DIR__.'/config.php';
|
|||
* Display a 'time since' HTML/Javascript counter
|
||||
*
|
||||
*/
|
||||
function OS_countUp($time) {
|
||||
function OS_countUp($time, $id = '') {
|
||||
$since = time() - $time;
|
||||
$periods = array(
|
||||
array('d', 'day', 'days'),
|
||||
|
@ -24,7 +24,8 @@ function OS_countUp($time) {
|
|||
$hours = floor($since / 3600); $since %= 3600;
|
||||
$minutes = floor($since / 60);
|
||||
$seconds = $since % 60; ?>
|
||||
<span class="countup_timer" data-start="<?php echo $time; ?>" title="<?php echo date('r', $time); ?>">
|
||||
<span class="countup_timer" data-start="<?php echo $time; ?>" title="<?php echo date('r', $time); ?>"<?php
|
||||
if (!empty($id)) echo ' id="'.htmlspecialchars($id).'"'; ?>>
|
||||
<span data-period="days"<?php
|
||||
if (!$days) echo ' class="d-none"'; ?>>
|
||||
<var><?php echo $days; ?></var>
|
||||
|
@ -1798,7 +1799,7 @@ document.write(mustache.render(
|
|||
</span>
|
||||
</label>
|
||||
<div><?php
|
||||
OS_countUp(($_ODATA['sp_time_end']) ? $_ODATA['sp_time_end'] : time());
|
||||
OS_countUp(($_ODATA['sp_time_end']) ? $_ODATA['sp_time_end'] : time(), 'os_countup_time_end');
|
||||
?></div><?php
|
||||
if ($_ODATA['sp_time_end'] != $_ODATA['sp_time_end_success']) { ?>
|
||||
<p class="data-text text-danger">
|
||||
|
@ -1810,7 +1811,7 @@ document.write(mustache.render(
|
|||
<li class="list-group-item">
|
||||
<label class="d-flex w-100">
|
||||
<strong class="pe-2">Crawl Time</strong>
|
||||
<var class="flex-grow-1 text-end"><?php
|
||||
<var class="flex-grow-1 text-end" id="os_crawl_time_last"><?php
|
||||
echo $_ODATA['sp_time_last'];
|
||||
?> <abbr title="seconds">s</abbr></var>
|
||||
</label>
|
||||
|
@ -1818,7 +1819,7 @@ document.write(mustache.render(
|
|||
<li class="list-group-item">
|
||||
<label class="d-flex w-100">
|
||||
<strong class="pe-2">Data Transferred</strong>
|
||||
<var class="flex-grow-1 text-end"><?php
|
||||
<var class="flex-grow-1 text-end" id="os_crawl_data_transferred"><?php
|
||||
echo OS_readSize($_ODATA['sp_data_transferred'], true);
|
||||
?></var>
|
||||
</label>
|
||||
|
@ -1826,7 +1827,7 @@ document.write(mustache.render(
|
|||
<li class="list-group-item">
|
||||
<label class="d-flex w-100">
|
||||
<strong class="pe-2">Data Stored</strong>
|
||||
<var class="flex-grow-1 text-end"><?php
|
||||
<var class="flex-grow-1 text-end" id="os_crawl_data_stored"><?php
|
||||
if ($_ODATA['sp_data_transferred']) { ?>
|
||||
<small data-bs-toggle="tooltip" data-bs-placement="bottom" title="Efficiency percentage of data stored vs. data downloaded"><?php
|
||||
echo '('.round(($_ODATA['sp_data_stored'] / $_ODATA['sp_data_transferred']) * 100, 1).'%)';
|
||||
|
@ -1839,7 +1840,7 @@ document.write(mustache.render(
|
|||
<li class="list-group-item">
|
||||
<label class="d-flex w-100">
|
||||
<strong class="pe-2">Links Crawled</strong>
|
||||
<var class="flex-grow-1 text-end"><?php
|
||||
<var class="flex-grow-1 text-end" id="os_crawl_links_crawled"><?php
|
||||
echo $_ODATA['sp_links_crawled'];
|
||||
?></var>
|
||||
</label>
|
||||
|
@ -1847,7 +1848,7 @@ document.write(mustache.render(
|
|||
<li class="list-group-item">
|
||||
<label class="d-flex w-100">
|
||||
<strong class="pe-2">Pages Stored</strong>
|
||||
<var class="flex-grow-1 text-end"><?php
|
||||
<var class="flex-grow-1 text-end" id="os_crawl_pages_stored"><?php
|
||||
if ($_ODATA['sp_links_crawled']) { ?>
|
||||
<small data-bs-toggle="tooltip" data-bs-placement="bottom" title="Efficiency percentage of pages stored vs. links crawled"><?php
|
||||
echo '('.round(($_ODATA['sp_pages_stored'] / $_ODATA['sp_links_crawled']) * 100, 1).'%)';
|
||||
|
|
|
@ -243,10 +243,10 @@ if (!in_array($_DDATA['tbprefix'].'query', $_DDATA['tables'])) {
|
|||
*/
|
||||
function OS_readSize($bytes, $abbr = false) {
|
||||
$bytes = (int)$bytes;
|
||||
if ($bytes >= 1020054733) return round(($bytes / 1073741824), 1).' '.(($abbr) ? '<abbr title="gibibytes">GiB</abbr>' : 'GiB');
|
||||
if ($bytes >= 996148) return round(($bytes / 1048576), 1).' '.(($abbr) ? '<abbr title="mebibytes">MiB</abbr>' : 'MiB');
|
||||
if ($bytes >= 973) return round(($bytes / 1024), 1).' '.(($abbr) ? '<abbr title="kibibytes">kiB</abbr>' : 'kiB');
|
||||
if ($bytes >= 0) return $bytes.' '.(($abbr) ? '<abbr title="bytes">B</abbr>' : 'B');
|
||||
if ($bytes >= 1020054733) return round(($bytes / 1073741824), 1).(($abbr) ? ' <abbr title="gibibytes">GiB</abbr>' : ' GiB');
|
||||
if ($bytes >= 996148) return round(($bytes / 1048576), 1).(($abbr) ? ' <abbr title="mebibytes">MiB</abbr>' : ' MiB');
|
||||
if ($bytes >= 973) return round(($bytes / 1024), 1).(($abbr) ? ' <abbr title="kibibytes">kiB</abbr>' : ' kiB');
|
||||
if ($bytes >= 0) return $bytes.(($abbr) ? ' <abbr title="bytes">B</abbr>' : ' B');
|
||||
return '';
|
||||
}
|
||||
|
||||
|
|
|
@ -506,7 +506,14 @@ switch ($_SERVER['REQUEST_METHOD']) {
|
|||
'status' => ($_ODATA['sp_crawling']) ? 'Crawling' : 'Complete',
|
||||
'progress' => $_ODATA['sp_progress'],
|
||||
'time_crawl' => time() - $_ODATA['sp_time_start'],
|
||||
'time_end' => $_ODATA['sp_time_end'],
|
||||
'time_end_success' => $_ODATA['sp_time_end_success'],
|
||||
'time_last' => $_ODATA['sp_time_last'],
|
||||
'timeout_crawl' => $_ODATA['sp_timeout_crawl'],
|
||||
'data_transferred' => $_ODATA['sp_data_transferred'],
|
||||
'data_stored' => $_ODATA['sp_data_stored'],
|
||||
'links_crawled' => $_ODATA['sp_links_crawled'],
|
||||
'pages_stored' => $_ODATA['sp_pages_stored'],
|
||||
'tail' => trim(implode("\n", $lines))
|
||||
);
|
||||
break;
|
||||
|
|
|
@ -40,6 +40,20 @@ let os_download = function(defaultFilename, postValues) {
|
|||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generates a readable filesize string from an integer byte-count
|
||||
* abbr => Optional <abbr> tag with title attribute added
|
||||
*/
|
||||
let os_readSize = function(bytes, abbr) {
|
||||
bytes = parseInt(bytes);
|
||||
if (bytes >= 1020054733) return (Math.round((bytes * 10 / 1073741824)) / 10) + ((abbr) ? ' <abbr title="gibibytes">GiB</abbr>' : ' GiB');
|
||||
if (bytes >= 996148) return (Math.round((bytes * 10 / 1048576)) / 10) + ((abbr) ? ' <abbr title="mebibytes">MiB</abbr>' : ' MiB');
|
||||
if (bytes >= 973) return (Math.round((bytes * 10 / 1024)) / 10) + ((abbr) ? ' <abbr title="kibibytes">kiB</abbr>' : ' kiB');
|
||||
if (bytes >= 0) return bytes + ((abbr) ? ' <abbr title="bytes">B</abbr>' : ' B');
|
||||
return '';
|
||||
}
|
||||
|
||||
|
||||
// Enable Popper.js tooltips
|
||||
let toolTipElems = document.querySelectorAll('[data-bs-toggle="tooltip"]');
|
||||
let toolTipList = [...toolTipElems].map(elem => new bootstrap.Tooltip(elem));
|
||||
|
@ -67,6 +81,10 @@ for (let x = 0; x < countUpTimers.length; x++) {
|
|||
if (timeTracker + 5000 < timeNow || this.originalStart != dataStart) {
|
||||
this.originalStart = dataStart;
|
||||
|
||||
// Update the title attribute
|
||||
let startDate = new Date(dataStart * 1000);
|
||||
this.title = startDate.toString();
|
||||
|
||||
let since = Math.round((new Date()).getTime() / 1000) - dataStart;
|
||||
let periods = [];
|
||||
periods[0] = Math.floor(since / 86400); since %= 86400;
|
||||
|
@ -372,27 +390,92 @@ let os_get_crawl_progress = function(getLog) {
|
|||
return;
|
||||
}
|
||||
|
||||
// If crawl is complete, we are checking to see if a new one has
|
||||
// started; if one has, trigger progress checks every second
|
||||
if (os_crawl_start.complete && data.status == 'Crawling') {
|
||||
os_crawl_start.disabled = 'disabled';
|
||||
// If crawl is complete...
|
||||
if (os_crawl_start.complete) {
|
||||
|
||||
os_crawl_progress.value = 0
|
||||
os_crawl_progress.max = 1;
|
||||
os_crawl_progress.setAttribute('data-progress', '');
|
||||
os_crawl_progress.innerHTML = '0%';
|
||||
// Check if a new crawl has started and is ongoing
|
||||
if (data.status == 'Crawling') {
|
||||
os_crawl_start.disabled = 'disabled';
|
||||
|
||||
os_crawl_start.allow_grep = false;
|
||||
os_crawl_start.complete = false;
|
||||
os_crawl_progress.value = 0
|
||||
os_crawl_progress.max = 1;
|
||||
os_crawl_progress.setAttribute('data-progress', '');
|
||||
os_crawl_progress.innerHTML = '0%';
|
||||
|
||||
os_crawl_cancel.disabled = '';
|
||||
os_crawl_log_download.disabled = 'disabled';
|
||||
os_crawl_start.innerHTML = 'Crawling...';
|
||||
os_crawl_navbar.innerHTML = 'Crawling...';
|
||||
os_crawl_start.allow_grep = false;
|
||||
os_crawl_start.complete = false;
|
||||
|
||||
// Start an interval progress check
|
||||
clearInterval(os_crawl_interval);
|
||||
os_crawl_interval = setInterval(os_get_crawl_progress, 1000);
|
||||
os_crawl_cancel.disabled = '';
|
||||
os_crawl_log_download.disabled = 'disabled';
|
||||
os_crawl_start.innerHTML = 'Crawling...';
|
||||
os_crawl_navbar.innerHTML = 'Crawling...';
|
||||
|
||||
// Start an interval progress check
|
||||
clearInterval(os_crawl_interval);
|
||||
os_crawl_interval = setInterval(os_get_crawl_progress, 1000);
|
||||
|
||||
// Else check if the given time_end is later than the time this
|
||||
// page was loaded; if so, a crawl has finished after this page
|
||||
// was loaded; if we are on the Crawler Management page, update
|
||||
// all the info there
|
||||
} else if (os_crawl_loaded < data.time_end) {
|
||||
os_crawl_loaded = parseInt((new Date()).getTime() / 1000);
|
||||
|
||||
os_crawl_start.disabled = '';
|
||||
os_crawl_start.innerHTML = 'Start Crawl';
|
||||
|
||||
let os_countup_time_end = document.getElementById('os_countup_time_end');
|
||||
if (os_countup_time_end) {
|
||||
os_countup_time_end.setAttribute('data-start', data.time_end);
|
||||
|
||||
// Try to locate the warning <p> element
|
||||
let pDanger = os_countup_time_end.parentNode.parentNode.querySelector('p.data-text.text-danger');
|
||||
|
||||
// If the time_end does not match the time_end_success, then
|
||||
// the last crawl did not succeed; show the error message
|
||||
if (data.time_end != data.time_end_success) {
|
||||
if (!pDanger) {
|
||||
let pDanger = document.createElement('p');
|
||||
pDanger.classList.add('data-text', 'text-danger');
|
||||
let strong = document.createElement('strong');
|
||||
strong.appendChild(document.createTextNode('Warning:'));
|
||||
pDanger.appendChild(strong);
|
||||
pDanger.appendChild(document.createTextNode(' The previous crawl did not complete successfully. Please check the crawl log for more details.'));
|
||||
os_countup_time_end.parentNode.parentNode.appendChild(pDanger);
|
||||
}
|
||||
|
||||
// Else if it matches, it was successful, remove any warning
|
||||
} else if (pDanger) pDanger.parentNode.removeChild(pDanger);
|
||||
|
||||
// Update the Crawl information items
|
||||
let os_crawl_time_last = document.getElementById('os_crawl_time_last');
|
||||
os_crawl_time_last.innerHTML = data.time_last + ' <abbr title="seconds">s</abbr>';
|
||||
|
||||
let os_crawl_data_transferred = document.getElementById('os_crawl_data_transferred');
|
||||
os_crawl_data_transferred.innerHTML = os_readSize(data.data_transferred, true);
|
||||
|
||||
let os_crawl_data_stored = document.getElementById('os_crawl_data_stored');
|
||||
let text = '';
|
||||
if (data.data_transferred) {
|
||||
text += '<small data-bs-toggle="tooltip" data-bs-placement="bottom" title="Efficiency percentage of data stored vs. data downloaded">';
|
||||
text += '(' + (Math.round(data.data_stored * 1000 / data.data_transferred) / 10) + '%)';
|
||||
text += '</small> ';
|
||||
}
|
||||
os_crawl_data_stored.innerHTML = text + os_readSize(data.data_stored, true);
|
||||
|
||||
let os_crawl_links_crawled = document.getElementById('os_crawl_links_crawled');
|
||||
os_crawl_links_crawled.innerHTML = data.links_crawled;
|
||||
|
||||
let os_crawl_pages_stored = document.getElementById('os_crawl_pages_stored');
|
||||
text = '';
|
||||
if (data.links_crawled) {
|
||||
text += '<small data-bs-toggle="tooltip" data-bs-placement="bottom" title="Efficiency percentage of pages stored vs. links crawled">';
|
||||
text += '(' + (Math.round(data.pages_stored * 1000 / data.links_crawled) / 10) + '%)';
|
||||
text += '</small> ';
|
||||
}
|
||||
os_crawl_pages_stored.innerHTML = text + data.pages_stored;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!data.tail) return;
|
||||
|
@ -420,8 +503,6 @@ let os_get_crawl_progress = function(getLog) {
|
|||
}
|
||||
|
||||
if (!os_crawl_start.complete && data.status == 'Complete') {
|
||||
clearInterval(os_crawl_interval);
|
||||
|
||||
os_crawl_cancel.disabled = 'disabled';
|
||||
os_crawl_log_download.disabled = '';
|
||||
os_crawl_start.complete = true;
|
||||
|
@ -430,19 +511,14 @@ let os_get_crawl_progress = function(getLog) {
|
|||
os_crawl_navbar.innerHTML = 'Crawler';
|
||||
os_crawl_interval = false;
|
||||
|
||||
// Check if the crawler modal window is open
|
||||
if (crawlerModal && crawlerModal.classList.contains('show')) {
|
||||
|
||||
// Don't refresh the page until the user closes the modal
|
||||
crawlerModal.addEventListener('hide.bs.modal', function() {
|
||||
window.location.reload();
|
||||
}, false);
|
||||
} else window.location.reload();
|
||||
clearInterval(os_crawl_interval);
|
||||
os_crawl_interval = setInterval(os_get_crawl_progress, 5000);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
let os_crawl_interval;
|
||||
let os_crawl_loaded = parseInt((new Date()).getTime() / 1000);
|
||||
let os_crawl_start = document.getElementById('os_crawl_start');
|
||||
let os_crawl_navbar = document.getElementById('os_crawl_navbar');
|
||||
let os_crawl_cancel = document.getElementById('os_crawl_cancel');
|
||||
|
|
Loading…
Reference in a new issue