zoraxy/src/web/components/stats.html
Toby Chui 2574d0504e Updates v2.6.2
+ Added advance stats operation tab
+ Added statistic reset #13
+ Added statistic export to csv and json (please use json)
+ Make subdomain clickable (not vdir) #12
+ Added TCP Proxy
+ Updates SMTP setup UI to make it more straight forward to setup
2023-06-04 23:59:56 +08:00

805 lines
29 KiB
HTML

<script src="./script/useragent.js"></script>
<link href="./script/datepicker/dp-light.css" rel="stylesheet">
<script defer src="./script/datepicker/datepicker.js"></script>
<div class="standardContainer">
<div class="ui basic segment">
<h2>Statistical Analysis</h2>
<p>Statistic of your server in every aspects</p>
<div style="margin-bottom: 0.4em;">
<div class="ui small input">
<input type="text" id="statsRangeStart" placeholder="From date">
</div>
<span style="padding-left: 0.6em; padding-right: 0.6em;"> <i class="ui right arrow icon"></i> </span>
<div class="ui small input">
<input type="text" id="statsRangeEnd" placeholder="End date">
</div>
</div>
<button onclick="handleLoadStatisticButtonPress();" class="ui basic button"><i class="blue search icon"></i> Load</button>
<button onclick="clearStatisticDateRange();" class="ui basic button"><i class="eraser icon"></i> Clear Search</button>
<br>
<small>Leave end range as empty for showing starting day only statistic</small>
</div>
<div class="ui divider"></div>
<div id="statisticRenderNotEnoughData" class="ui segment" style="padding: 2em;" align="center">
<h4 class="ui icon header">
<i class="medium icons" style="color: #dbdbdb !important;">
<i class="chart pie icon"></i>
<i class="small corner remove icon" style="
font-size: 1.4em;
margin-right: -0.2em;
margin-bottom: -0em;
"></i>
</i>
<div class="content" style="margin-top: 1em; color: #7c7c7c !important;">
No Data
<div class="sub header" style="color: #adadad !important;">The selected period contains too little or no request data collected. <br>
Please select a larger range or make sure there are enough traffic routing through this site.</div>
</div>
</h4>
</div>
<div id="statisticRenderElement" class="ui basic segment">
<!-- View Counts Statistics -->
<div class="ui three small statistics">
<div class="statistic">
<div class="value totalViewCount">
</div>
<div class="label">
Total Requests
</div>
</div>
<div class="statistic">
<div class="value">
<i class="ui green check circle icon"></i> <span class="totalSuccCount"></span>
</div>
<div class="label">
Success Requests
</div>
</div>
<div class="statistic">
<div class="value">
<i class="ui red times circle icon"></i> <span class="totalErrorCount"></span>
</div>
<div class="label">
Error Requests
</div>
</div>
</div>
<!-- Forward Type Data -->
<h3>Forward Traffic Types</h3>
<p>Traffic forwarding type classified by their protocols and proxy rules.</p>
<table class="ui celled unstackable table">
<thead>
<tr><th>Forward Type</th>
<th>Counts</th>
<th>Percentage</th>
</tr></thead>
<tbody class="forwardTypeCounts">
</tbody>
</table>
<!-- Client Geolocation Analysis-->
<h3>Visitors Countries</h3>
<p>Distributions of visitors by country code. Access origin are estimated using open source GeoIP database and might not be accurate.</p>
<div style="min-height: 400px;">
<canvas id="stats_visitors"></canvas>
</div>
<div class="ui divider"></div>
<!-- Client IP Analysis -->
<div class="ui stackable grid">
<div class="eight wide column">
<h3>Requests IP Version</h3>
<p>The version of Internet Protocol that the client is using. If the request client is pass through a DNS proxy like CloudFlare,
the CloudFlare proxy server address will be filtered and only the first value in the RemoteAddress field will be analysised.</p>
<div>
<canvas id="stats_ipver"></canvas>
</div>
</div>
<div class="eight wide column">
<h3>Request Origins</h3>
<p>Top 25 request origin sorted by request count</p>
<div style="height: 500px; overflow-y: auto;">
<table class="ui unstackable striped celled table">
<thead>
<tr>
<th class="no-sort">Request Origin</th>
<th class="no-sort">No of Requests</th>
</tr></thead>
<tbody id="stats_requestCountlist">
</tbody>
</table>
</div>
</div>
</div>
<div class="ui divider"></div>
<!-- Client Device Analysis -->
<div class="ui stackable grid">
<div class="eight wide column">
<h3>Client Devices</h3>
<p>Device type analysis by its request interactions.The number of iteration count does not means the number unique device, as no cookie is used to track the devices identify.</p>
<div>
<canvas id="stats_device"></canvas>
</div>
</div>
<div class="eight wide column">
<h3>Client Browsers</h3>
<p>The browsers where your client is using to visit your site</p>
<div>
<canvas id="stats_browsers"></canvas>
</div>
</div>
</div>
<div class="ui stackable grid">
<div class="eight wide column">
<h3>Client OS</h3>
<p>The OS where your client is using. Estimated using the UserAgent header sent by the client browser while requesting a resources from one of your host.</p>
<div>
<canvas id="stats_OS"></canvas>
</div>
</div>
<div class="eight wide column">
<h3>OS Versions</h3>
<p>The OS versions most commonly used by your site's visitors.</p>
<div style="height: 500px; overflow-y: auto;">
<table class="ui unstackable striped celled table">
<thead>
<tr>
<th>OS Version</th>
<th>Request Counts</th>
<th>Percentage</th>
</tr></thead>
<tbody id="stats_OSVersionList">
</tbody>
</table>
</div>
</div>
</div>
<div class="ui divider"></div>
<div class="ui stackable grid">
<div class="eight wide column">
<h3>Request File Types</h3>
<p>The file types being served by this proxy</p>
<div>
<canvas id="stats_filetype"></canvas>
</div>
</div>
<div class="eight wide column">
<h3>Referring Sites</h3>
<p>The Top 100 sources of traffic according to referer header</p>
<div>
<div style="height: 500px; overflow-y: auto;">
<table class="ui unstackable striped celled table">
<thead>
<tr>
<th class="no-sort">Referer</th>
<th class="no-sort">Requests</th>
</tr></thead>
<tbody id="stats_RefererTable">
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class="ui divider"></div>
<div class="ui basic segment" id="trendGraphs">
<h3>Visitor Trend Analysis</h3>
<p>Request trends in the selected time range</p>
<div>
<canvas id="requestTrends"></canvas>
</div>
</div>
<button onclick="showSideWrapper('snippet/advanceStatsOprs.html?t=' + Date.now() + '#' + encodeURIComponent(JSON.stringify(getStatisticDateRange())));" class="ui basic right floated black button"><i class="external square alternate icon"></i> Advance Operations</button>
</div>
<!-- <button class="ui icon right floated basic button" onclick="initStatisticSummery();"><i class="green refresh icon"></i> Refresh</button> -->
<br><br>
</div>
<script>
var statisticCharts = [];
//Start day must be earlier than end date
function initStatisticSummery(startdate="", enddate=""){
if (startdate == "" && enddate == "" ){
let todaykey = getTodayStatisticKey();
loadStatisticByDate(todaykey);
}else if ((startdate != "" && enddate == "") || startdate == enddate){
//Load the target date
let targetDate = startdate.trim();
loadStatisticByDate(targetDate);
}else{
//Two dates are given and they are not identical
loadStatisticByRange(startdate, enddate);
//console.log(startdate, enddate);
}
}
function loadStatisticByDate(dateid){
$.getJSON("/api/analytic/load?id=" + dateid, function(data){
if (data.error != undefined){
msgbox(data.error, false, 6000);
return;
}
//Destroy all the previous charts
statisticCharts.forEach(function(thisChart){
thisChart.destroy();
})
if (data.TotalRequest == 0){
//No data to analysis
$("#statisticRenderElement").hide()
$("#statisticRenderNotEnoughData").show();
return;
}else{
$("#statisticRenderElement").show();
$("#statisticRenderNotEnoughData").hide();
}
//Render the text values
$("#statisticRenderElement").find(".totalViewCount").text(abbreviateNumber(data.TotalRequest));
$("#statisticRenderElement").find(".totalSuccCount").text(abbreviateNumber(data.ValidRequest));
$("#statisticRenderElement").find(".totalErrorCount").text(abbreviateNumber(data.ErrorRequest));
//Render forward type data
renderForwardTypeCounts(data.ForwardTypes);
//Render visitor data
renderVisitorChart(data.RequestOrigin);
//Render IP versions
renderIPVersionChart(data.RequestClientIp);
//Render user agent analysis
renderUserAgentCharts(data.UserAgent);
//Render file type by analysising request URL paths
renderFileTypeGraph(data.RequestURL);
//Render Referer header
renderRefererTable(data.Referer);
//Hide the trend graphs
$("#trendGraphs").hide();
});
}
initStatisticSummery();
$("#statsRangeStart").val(getTodayStatisticKey().split("_").join("-"));
function loadStatisticByRange(startdate, endDate){
$.getJSON("/api/analytic/loadRange?start=" + startdate + "&end=" + endDate, function(data){
//console.log(data);
//Destroy all the previous charts
statisticCharts.forEach(function(thisChart){
thisChart.destroy();
})
if (data.Summary.TotalRequest == 0){
//No data to analysis
$("#statisticRenderElement").hide()
$("#statisticRenderNotEnoughData").show();
return;
}else{
$("#statisticRenderElement").show();
$("#statisticRenderNotEnoughData").hide();
}
//Render the text values
$("#statisticRenderElement").find(".totalViewCount").text(abbreviateNumber(data.Summary.TotalRequest));
$("#statisticRenderElement").find(".totalSuccCount").text(abbreviateNumber(data.Summary.ValidRequest));
$("#statisticRenderElement").find(".totalErrorCount").text(abbreviateNumber(data.Summary.ErrorRequest));
//Render forward type data
renderForwardTypeCounts(data.Summary.ForwardTypes);
//Render visitor data
renderVisitorChart(data.Summary.RequestOrigin);
//Render IP versions
renderIPVersionChart(data.Summary.RequestClientIp);
//Render user agent analysis
renderUserAgentCharts(data.Summary.UserAgent);
//Render file type by analysising request URL paths
renderFileTypeGraph(data.Summary.RequestURL);
//Render Referer header
renderRefererTable(data.Summary.Referer);
//Render the trend graph
$("#trendGraphs").show();
renderTrendGraph(data.Records);
});
}
picker.attach({ target: document.getElementById("statsRangeStart") });
picker.attach({ target: document.getElementById("statsRangeEnd") });
function renderForwardTypeCounts(forwardTypes){
let tablBody = $("#statisticRenderElement").find(".forwardTypeCounts");
tablBody.empty();
let totalForwardCounts = 0;
for (let [key, value] of Object.entries(forwardTypes)) {
totalForwardCounts += value;
}
for (let [key, value] of Object.entries(forwardTypes)) {
tablBody.append(`<tr>
<td>${key}</td>
<td>${abbreviateNumber(value)} (${value})</td>
<td>${((value/totalForwardCounts)*100).toFixed(3)}%</td>
</tr>
`);
}
}
function getTodayStatisticKey(){
var today = new Date();
var year = today.getFullYear();
var month = String(today.getMonth() + 1).padStart(2, '0');
var day = String(today.getDate()).padStart(2, '0');
var formattedDate = year + '_' + month + '_' + day;
return formattedDate
}
function handleLoadStatisticButtonPress(){
var sd = $("#statsRangeStart").val();
var ed = $("#statsRangeEnd").val();
//Swap them if sd is later than ed
if (sd != "" && ed != "" && sd > ed) {
ed = [sd, sd = ed][0];
$("#statsRangeStart").val(sd);
$("#statsRangeEnd").val(ed);
}
initStatisticSummery(sd, ed);
}
function getStatisticDateRange(){
var sd = $("#statsRangeStart").val();
var ed = $("#statsRangeEnd").val();
if (ed == ""){
ed = sd;
}
if (sd == "" && ed == ""){
var sk = getTodayStatisticKey();
sd = sk;
ed = sk;
}
//Swap them if sd is later than ed
if (sd != "" && ed != "" && sd > ed) {
ed = [sd, sd = ed][0];
}
return [sd, ed];
}
function clearStatisticDateRange(){
$("#statsRangeStart").val("");
$("#statsRangeEnd").val("");
}
function renderRefererTable(refererList){
const sortedEntries = Object.entries(refererList).sort(([, valueA], [, valueB]) => valueB - valueA);
$("#stats_RefererTable").html("");
let endStop = 100;
if (sortedEntries.length < 100){
endStop = sortedEntries.length;
}
for (var i = 0; i < endStop; i++) {
let referer = (decodeURIComponent(sortedEntries[i][0])).replace(/(<([^>]+)>)/ig,"");
if (sortedEntries[i][0] == ""){
//Root
referer = `<span style="color: #b5b5b5;">(<i class="eye slash outline icon"></i> Unknown or Hidden)</span>`;
}
$("#stats_RefererTable").append(`<tr>
<td>${referer}</td>
<td>${sortedEntries[i][1]}</td>
</tr>`);
}
}
function renderFileTypeGraph(requestURLs){
//Create the device chart
let fileExtensions = {};
for (const [url, count] of Object.entries(requestURLs)) {
let filename = url.split("/").pop();
let ext = "";
if (filename == ""){
//Loading from a folder
ext = "Folder path"
}else{
if (filename.includes(".")){
ext = filename.split(".").pop();
}else{
ext = "API call"
}
}
if (fileExtensions[ext] != undefined){
fileExtensions[ext] = fileExtensions[ext] + count;
}else{
//First time this ext show up
fileExtensions[ext] = count;
}
}
//Convert the key-value pairs to array for graph render
let fileTypes = [];
let fileCounts = [];
let colors = [];
for (const [ftype, count] of Object.entries(fileExtensions)) {
fileTypes.push(ftype);
fileCounts.push(count);
colors.push(generateColorFromHash(ftype));
}
let filetypeChart = new Chart(document.getElementById("stats_filetype"), {
type: 'pie',
data: {
labels: fileTypes,
datasets: [{
data: fileCounts,
backgroundColor: colors,
hoverBackgroundColor: colors,
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
}
});
statisticCharts.push(filetypeChart);
}
function renderTrendGraph(dailySummary){
// Get the canvas element
const canvas = document.getElementById('requestTrends');
//Generate the X axis labels
let datesLabel = [];
let succData = [];
let errorData = [];
let totalData = [];
for (var i = 0; i < dailySummary.length; i++){
let thisDayData = dailySummary[i];
datesLabel.push("Day " + i);
succData.push(thisDayData.ValidRequest);
errorData.push(thisDayData.ErrorRequest);
totalData.push(thisDayData.TotalRequest);
}
// Create the chart
let TrendChart = new Chart(canvas, {
type: 'line',
data: {
labels: datesLabel,
datasets: [
{
label: 'All Requests',
data: totalData,
borderColor: '#7d99f7',
backgroundColor: 'rgba(125, 153, 247, 0.4)',
fill: false
},
{
label: 'Success Requests',
data: succData,
borderColor: '#6dad7c',
backgroundColor: "rgba(109, 173, 124, 0.4)",
fill: true
},
{
label: 'Error Requests',
data: errorData,
borderColor: '#de7373',
backgroundColor: "rgba(222, 115, 115, 0.4)",
fill: true
},
]
},
options: {
responsive: true,
maintainAspectRatio: true,
title: {
display: true,
//text: 'Line Chart Example'
},
scales: {
x: {
display: true,
title: {
display: false,
text: 'Time'
}
},
y: {
display: true,
title: {
display: true,
text: 'Request Counts'
}
}
}
}
});
statisticCharts.push(TrendChart);
}
function renderUserAgentCharts(userAgentsEntries){
let userAgents = Object.keys(userAgentsEntries);
let requestCounts = Object.values(userAgentsEntries);
let mobileUser = 0;
let desktopUser = 0;
let osTypes = {};
let osVersion = {};
let browserTypes = {};
let totalRequestCounts = 0;
requestCounts.forEach(function(rc){
totalRequestCounts += rc;
})
//Building a statistic summary
userAgents.forEach(function(thisUA){
var uaInfo = parseUserAgent(thisUA);
if (uaInfo.isMobile){
mobileUser+=userAgentsEntries[thisUA];
}else{
desktopUser+=userAgentsEntries[thisUA];
}
let currentNo = osTypes[uaInfo.os];
let osVersionKey = uaInfo.os + " " + uaInfo.version.split("_").join(".");
if (currentNo == undefined){
osTypes[uaInfo.os] = userAgentsEntries[thisUA];
}else{
osTypes[uaInfo.os] = currentNo + userAgentsEntries[thisUA];
}
let p = osVersion[osVersionKey];
if (p == undefined){
osVersion[osVersionKey] = userAgentsEntries[thisUA];
}else{
osVersion[osVersionKey] = p + userAgentsEntries[thisUA];
}
let browserTypeKey = uaInfo.browser;
if (browserTypeKey.indexOf("//") >= 0){
//This is a uncatergorize browser, mostly a bot
browserTypeKey = "Bots";
}else if (browserTypeKey == ""){
//No information
browserTypeKey = "Unknown";
}
let b = browserTypes[browserTypeKey];
if (b == undefined){
browserTypes[browserTypeKey] = userAgentsEntries[thisUA];
}else{
browserTypes[browserTypeKey] = b + userAgentsEntries[thisUA];
}
});
//Create the device chart
let deviceTypeChart = new Chart(document.getElementById("stats_device"), {
type: 'pie',
data: {
labels: ['Desktop', 'Mobile'],
datasets: [{
data: [desktopUser, mobileUser],
backgroundColor: ['#e56b5e', '#6eb9c1'],
hoverBackgroundColor: ['#e56b5e', '#6eb9c1']
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
}
});
statisticCharts.push(deviceTypeChart);
//Create the OS chart
let OSnames = [];
let OSCounts = [];
let OSColors = [];
for (const [key, value] of Object.entries(osTypes)) {
OSnames.push(key);
OSCounts.push(value);
OSColors.push(getOSColorCode(key));
}
let osTypeChart = new Chart(document.getElementById("stats_OS"), {
type: 'pie',
data: {
labels: OSnames,
datasets: [{
data: OSCounts,
backgroundColor: OSColors,
hoverBackgroundColor: OSColors
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
}
});
statisticCharts.push(osTypeChart);
//Populate the OS version table
let sortedOSVersion = Object.entries(osVersion).sort((a, b) => b[1] - a[1]);
$("#stats_OSVersionList").html("");
// Loop through the sorted data and populate the table
for (let i = 0; i < sortedOSVersion.length; i++) {
let osVersion = sortedOSVersion[i][0];
let requestcount = abbreviateNumber(sortedOSVersion[i][1]);
let percentage = (sortedOSVersion[i][1] / totalRequestCounts * 100).toFixed(3);
// Create a new row in the table
let row = $("<tr>");
// Add the OS version and percentage as columns in the row
$("<td>").text(osVersion).appendTo(row);
$("<td>").text(requestcount).appendTo(row);
$("<td>").text(percentage + "%").appendTo(row);
// Add the row to the table body
$("#stats_OSVersionList").append(row);
}
//Create the browser charts
let browserNames = [];
let broserCounts = [];
let browserColors = [];
let sortedBrowserTypes = Object.entries(browserTypes).sort((a, b) => b[1] - a[1]);
console.log(sortedBrowserTypes);
sortedBrowserTypes.forEach(function(entry){
browserNames.push(entry[0]);
broserCounts.push(entry[1]);
browserColors.push(generateColorFromHash(entry[0]));
});
let browserTypeChart = new Chart(document.getElementById("stats_browsers"), {
type: 'pie',
data: {
labels: browserNames,
datasets: [{
data: broserCounts,
backgroundColor: browserColors,
hoverBackgroundColor: browserColors
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
}
});
statisticCharts.push(browserTypeChart);
}
//Generate the IPversion pie chart
function renderIPVersionChart(RequestClientIp){
let ipv4Count = Object.keys(RequestClientIp).filter(ip => ip.includes('.')).length;
let ipv6Count = Object.keys(RequestClientIp).filter(ip => ip.includes(':')).length;
let totalCount = ipv4Count + ipv6Count;
let ipv4Percent = ((ipv4Count / totalCount) * 100).toFixed(2);
let ipv6Percent = ((ipv6Count / totalCount) * 100).toFixed(2);
// Create the chart data object
let chartData = {
labels: ['IPv4', 'IPv6'],
datasets: [{
data: [ipv4Percent, ipv6Percent],
backgroundColor: ['#9295f0', '#bff092'],
hoverBackgroundColor: ['#9295f0', '#bff092']
}]
};
// Create the chart options object
let ipvChartOption = {
responsive: true,
maintainAspectRatio: false,
};
// Create the pie chart
let ipTypeChart = new Chart(document.getElementById("stats_ipver"), {
type: 'pie',
data: chartData,
options: ipvChartOption
});
statisticCharts.push(ipTypeChart);
//Populate the request count table
let requestCounts = Object.entries(RequestClientIp);
// Sort the array by the value (count)
requestCounts.sort((a, b) => b[1] - a[1]);
// Select the table body and empty it
let tableBody = $('#stats_requestCountlist');
tableBody.empty();
// Loop through the sorted array and add the top 25 requested IPs to the table
for (let i = 0; i < 25 && i < requestCounts.length; i++) {
let [ip, count] = requestCounts[i];
let row = $('<tr>').appendTo(tableBody);
$('<td style="word-break: break-all;">').text(ip).appendTo(row);
$('<td>').text(count).appendTo(row);
}
}
//Generate a fixed color code from string hash
function generateColorFromHash(stringToHash){
let hash = 0;
for (let i = 0; i < stringToHash.length; i++) {
hash = stringToHash.charCodeAt(i) + ((hash << 5) - hash);
}
const hue = hash % 300;
const saturation = Math.floor(Math.random() * 20) + 70;
const lightness = Math.floor(Math.random() * 20) + 60;
return `hsl(${hue}, ${saturation}%, ${lightness}%)`;
}
//Generate the visitor country pie chart
function renderVisitorChart(visitorData){
// Extract the labels and data from the visitor data object
let labels = [];
let data = Object.values(visitorData);
Object.keys(visitorData).forEach(function(cc){
if (cc == ""){
labels.push("Local / Unknown")
}else{
labels.push(`${getCountryName(cc)} [${cc.toUpperCase()}]` );
}
});
// Define the colors to be used in the pie chart
let colors = [];
labels.forEach(function(cc){
colors.push(generateColorFromHash(cc));
});
// Create the chart data object
let CCchartData = {
labels: labels,
datasets: [{
data: data,
backgroundColor: colors
}]
};
// Create the chart options object
let CCchartOptions = {
responsive: true,
maintainAspectRatio: false,
};
// Create the pie chart
const visitorsChart = new Chart(document.getElementById("stats_visitors"), {
type: 'pie',
data: CCchartData,
options: CCchartOptions
});
statisticCharts.push(visitorsChart);
}
</script>