marcel/zoraxy
1
0
Fork 0
mirror of https://github.com/tobychui/zoraxy.git synced 2025-06-21 06:53:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fixed #201 and #608

Browse Source 
- Added domain / host name specific statistics
- Added upstream forward request count
This commit is contained in:
Toby Chui 2025-04-03 13:35:34 +08:00
parent 05f1743ecd
commit ac91a3fef1
7 changed files with 198 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/def.go
View File

@ -45,7 +45,7 @@ const (
/* Build Constants */
SYSTEM_NAME = "Zoraxy"
SYSTEM_VERSION = "3.2.1"
DEVELOPMENT_BUILD = false /* Development: Set to false to use embedded web fs */
DEVELOPMENT_BUILD = true /* Development: Set to false to use embedded web fs */
/* System Constants */
TMP_FOLDER = "./tmp"

13
src/mod/dynamicproxy/dpcore/dpcore.go
View File

@ -105,7 +105,6 @@ func NewDynamicProxyCore(target *url.URL, prepender string, dpcOptions *DpcoreOp
thisTransporter := http.DefaultTransport
//Hack the default transporter to handle more connections
optimalConcurrentConnection := 256
if dpcOptions.MaxConcurrentConnection > 0 {
optimalConcurrentConnection = dpcOptions.MaxConcurrentConnection
@ -137,18 +136,6 @@ func NewDynamicProxyCore(target *url.URL, prepender string, dpcOptions *DpcoreOp
}
}
func singleJoiningSlash(a, b string) string {
aslash := strings.HasSuffix(a, "/")
bslash := strings.HasPrefix(b, "/")
switch {
case aslash && bslash:
return a + b[1:]
case !aslash && !bslash:
return a + "/" + b
}
return a + b
}
func joinURLPath(a, b *url.URL) (path, rawpath string) {
apath, bpath := a.EscapedPath(), b.EscapedPath()
aslash, bslash := strings.HasSuffix(apath, "/"), strings.HasPrefix(bpath, "/")

1
src/mod/dynamicproxy/proxyRequestHandler.go
View File

@ -320,6 +320,7 @@ func (router *Router) logRequest(r *http.Request, succ bool, statusCode int, for
UserAgent: r.UserAgent(),
RequestURL: r.Host + r.RequestURI,
Target: originalHostname,
Upstream: upstreamHostname,
}
router.Option.StatisticCollector.RecordRequest(requestInfo)
}()

10
src/mod/statistic/analytic/utils.go
View File

@ -36,6 +36,8 @@ func mergeDailySummaryExports(exports []*statistic.DailySummaryExport) *statisti
Referer: make(map[string]int),
UserAgent: make(map[string]int),
RequestURL: make(map[string]int),
Downstreams: make(map[string]int),
Upstreams: make(map[string]int),
}
for _, export := range exports {
@ -66,6 +68,14 @@ func mergeDailySummaryExports(exports []*statistic.DailySummaryExport) *statisti
for key, value := range export.RequestURL {
mergedExport.RequestURL[key] += value
}
for key, value := range export.Downstreams {
mergedExport.Downstreams[key] += value
}
for key, value := range export.Upstreams {
mergedExport.Upstreams[key] += value
}
}
return mergedExport

53
src/mod/statistic/statistic.go
View File

@ -24,12 +24,14 @@ type DailySummary struct {
ErrorRequest int64 //Invalid request of the day, including error or not found
ValidRequest int64 //Valid request of the day
//Type counters
ForwardTypes *sync.Map //Map that hold the forward types
RequestOrigin *sync.Map //Map that hold [country ISO code]: visitor counter
RequestClientIp *sync.Map //Map that hold all unique request IPs
Referer *sync.Map //Map that store where the user was refered from
UserAgent *sync.Map //Map that store the useragent of the request
RequestURL *sync.Map //Request URL of the request object
ForwardTypes *sync.Map //Map that hold the forward types
RequestOrigin *sync.Map //Map that hold [country ISO code]: visitor counter
RequestClientIp *sync.Map //Map that hold all unique request IPs
Referer *sync.Map //Map that store where the user was refered from
UserAgent *sync.Map //Map that store the useragent of the request
RequestURL *sync.Map //Request URL of the request object
DownstreamHostnames *sync.Map //Request count of downstream hostname
UpstreamHostnames *sync.Map //Forwarded request count of upstream hostname
}
type RequestInfo struct {
@ -42,6 +44,7 @@ type RequestInfo struct {
UserAgent string //UserAgent of the downstream request
RequestURL string //Request URL
Target string //Target domain or hostname
Upstream string ////Upstream domain or hostname, if the request is forwarded to upstream
}
type CollectorOption struct {
@ -233,6 +236,24 @@ func (c *Collector) RecordRequest(ri RequestInfo) {
} else {
c.DailySummary.RequestURL.Store(ri.RequestURL, ru.(int)+1)
}
//Record the downstream hostname
//This is the hostname that the user visited, not the target domain
ds, ok := c.DailySummary.DownstreamHostnames.Load(ri.Target)
if !ok {
c.DailySummary.DownstreamHostnames.Store(ri.Target, 1)
} else {
c.DailySummary.DownstreamHostnames.Store(ri.Target, ds.(int)+1)
}
//Record the upstream hostname
//This is the selected load balancer upstream hostname or ip
us, ok := c.DailySummary.UpstreamHostnames.Load(ri.Upstream)
if !ok {
c.DailySummary.UpstreamHostnames.Store(ri.Upstream, 1)
} else {
c.DailySummary.UpstreamHostnames.Store(ri.Upstream, us.(int)+1)
}
}()
//ADD MORE HERE IF NEEDED
@ -271,15 +292,17 @@ func (c *Collector) ScheduleResetRealtimeStats() chan bool {
func NewDailySummary() *DailySummary {
return &DailySummary{
TotalRequest: 0,
ErrorRequest: 0,
ValidRequest: 0,
ForwardTypes: &sync.Map{},
RequestOrigin: &sync.Map{},
RequestClientIp: &sync.Map{},
Referer: &sync.Map{},
UserAgent: &sync.Map{},
RequestURL: &sync.Map{},
TotalRequest: 0,
ErrorRequest: 0,
ValidRequest: 0,
ForwardTypes: &sync.Map{},
RequestOrigin: &sync.Map{},
RequestClientIp: &sync.Map{},
Referer: &sync.Map{},
UserAgent: &sync.Map{},
RequestURL: &sync.Map{},
DownstreamHostnames: &sync.Map{},
UpstreamHostnames: &sync.Map{},
}
}

113
src/mod/statistic/structconv.go
View File

@ -13,6 +13,21 @@ type DailySummaryExport struct {
Referer map[string]int
UserAgent map[string]int
RequestURL map[string]int
Downstreams map[string]int
Upstreams map[string]int
}
func SyncMapToMapStringInt(syncMap *sync.Map) map[string]int {
result := make(map[string]int)
syncMap.Range(func(key, value interface{}) bool {
strKey, okKey := key.(string)
intValue, okValue := value.(int)
if okKey && okValue {
result[strKey] = intValue
}
return true
})
return result
}
func DailySummaryToExport(summary DailySummary) DailySummaryExport {
@ -26,77 +41,53 @@ func DailySummaryToExport(summary DailySummary) DailySummaryExport {
Referer: make(map[string]int),
UserAgent: make(map[string]int),
RequestURL: make(map[string]int),
Downstreams: make(map[string]int),
Upstreams: make(map[string]int),
}
summary.ForwardTypes.Range(func(key, value interface{}) bool {
export.ForwardTypes[key.(string)] = value.(int)
return true
})
summary.RequestOrigin.Range(func(key, value interface{}) bool {
export.RequestOrigin[key.(string)] = value.(int)
return true
})
summary.RequestClientIp.Range(func(key, value interface{}) bool {
export.RequestClientIp[key.(string)] = value.(int)
return true
})
summary.Referer.Range(func(key, value interface{}) bool {
export.Referer[key.(string)] = value.(int)
return true
})
summary.UserAgent.Range(func(key, value interface{}) bool {
export.UserAgent[key.(string)] = value.(int)
return true
})
summary.RequestURL.Range(func(key, value interface{}) bool {
export.RequestURL[key.(string)] = value.(int)
return true
})
export.ForwardTypes = SyncMapToMapStringInt(summary.ForwardTypes)
export.RequestOrigin = SyncMapToMapStringInt(summary.RequestOrigin)
export.RequestClientIp = SyncMapToMapStringInt(summary.RequestClientIp)
export.Referer = SyncMapToMapStringInt(summary.Referer)
export.UserAgent = SyncMapToMapStringInt(summary.UserAgent)
export.RequestURL = SyncMapToMapStringInt(summary.RequestURL)
export.Downstreams = SyncMapToMapStringInt(summary.DownstreamHostnames)
export.Upstreams = SyncMapToMapStringInt(summary.UpstreamHostnames)
return export
}
func MapStringIntToSyncMap(m map[string]int) *sync.Map {
syncMap := &sync.Map{}
for k, v := range m {
syncMap.Store(k, v)
}
return syncMap
}
func DailySummaryExportToSummary(export DailySummaryExport) DailySummary {
summary := DailySummary{
TotalRequest: export.TotalRequest,
ErrorRequest: export.ErrorRequest,
ValidRequest: export.ValidRequest,
ForwardTypes: &sync.Map{},
RequestOrigin: &sync.Map{},
RequestClientIp: &sync.Map{},
Referer: &sync.Map{},
UserAgent: &sync.Map{},
RequestURL: &sync.Map{},
TotalRequest: export.TotalRequest,
ErrorRequest: export.ErrorRequest,
ValidRequest: export.ValidRequest,
ForwardTypes: &sync.Map{},
RequestOrigin: &sync.Map{},
RequestClientIp: &sync.Map{},
Referer: &sync.Map{},
UserAgent: &sync.Map{},
RequestURL: &sync.Map{},
DownstreamHostnames: &sync.Map{},
UpstreamHostnames: &sync.Map{},
}
for k, v := range export.ForwardTypes {
summary.ForwardTypes.Store(k, v)
}
for k, v := range export.RequestOrigin {
summary.RequestOrigin.Store(k, v)
}
for k, v := range export.RequestClientIp {
summary.RequestClientIp.Store(k, v)
}
for k, v := range export.Referer {
summary.Referer.Store(k, v)
}
for k, v := range export.UserAgent {
summary.UserAgent.Store(k, v)
}
for k, v := range export.RequestURL {
summary.RequestURL.Store(k, v)
}
summary.ForwardTypes = MapStringIntToSyncMap(export.ForwardTypes)
summary.RequestOrigin = MapStringIntToSyncMap(export.RequestOrigin)
summary.RequestClientIp = MapStringIntToSyncMap(export.RequestClientIp)
summary.Referer = MapStringIntToSyncMap(export.Referer)
summary.UserAgent = MapStringIntToSyncMap(export.UserAgent)
summary.RequestURL = MapStringIntToSyncMap(export.RequestURL)
summary.DownstreamHostnames = MapStringIntToSyncMap(export.Downstreams)
summary.UpstreamHostnames = MapStringIntToSyncMap(export.Upstreams)
return summary
}

95
src/web/components/stats.html
View File

@ -186,6 +186,45 @@
</div>
</div>
<div class="ui divider"></div>
<div class="ui stackable grid">
<div class="eight wide column">
<h3>Requested Hostnames</h3>
<p>Most requested hostnames from downstream</p>
<div>
<div style="height: 500px; overflow-y: auto;">
<table class="ui unstackable striped celled table">
<thead>
<tr>
<th class="no-sort">Hostname</th>
<th class="no-sort">Requests</th>
</tr></thead>
<tbody id="stats_downstreamTable">
</tbody>
</table>
</div>
</div>
</div>
<div class="eight wide column">
<h3>Forwarded Upstreams</h3>
<p>The Top 100 upstreams where the requests are forwarded to</p>
<div>
<div style="height: 500px; overflow-y: auto;">
<table class="ui unstackable striped celled table">
<thead>
<tr>
<th class="no-sort">Upstream Endpoint</th>
<th class="no-sort">Requests</th>
</tr></thead>
<tbody id="stats_upstreamTable">
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class="ui divider"></div>
<div class="ui basic segment" id="trendGraphs">
<h3>Visitor Trend Analysis</h3>
<p>Request trends in the selected time range</p>
@ -263,6 +302,22 @@
//Render Referer header
renderRefererTable(data.Referer);
if (data.Downstreams == null){
//No downstream data to show
$("#stats_downstreamTable").html("<tr><td colspan='2'>No data</td></tr>");
}else{
//Render the downstream table
renderDownstreamTable(data.Downstreams);
}
if (data.Upstreams == null){
//No upstream data to show
$("#stats_upstreamTable").html("<tr><td colspan='2'>No data</td></tr>");
}else{
//Render the upstream table
renderUpstreamTable(data.Upstreams);
}
//Hide the trend graphs
$("#trendGraphs").hide();
});
@ -410,6 +465,46 @@
}
}
function renderDownstreamTable(downstreamList){
const sortedEntries = Object.entries(downstreamList).sort(([, valueA], [, valueB]) => valueB - valueA);
$("#stats_downstreamTable").html("");
let endStop = 100;
if (sortedEntries.length < 100){
endStop = sortedEntries.length;
}
for (var i = 0; i < endStop; i++) {
let referer = (decodeURIComponent(sortedEntries[i][0])).replace(/(<([^>]+)>)/ig,"");
if (sortedEntries[i][0] == ""){
//Root
referer = `<span style="color: #b5b5b5;">(<i class="eye slash outline icon"></i> Unknown or Hidden)</span>`;
}
$("#stats_downstreamTable").append(`<tr>
<td>${referer}</td>
<td>${sortedEntries[i][1]}</td>
</tr>`);
}
}
function renderUpstreamTable(upstreamList){
const sortedEntries = Object.entries(upstreamList).sort(([, valueA], [, valueB]) => valueB - valueA);
$("#stats_upstreamTable").html("");
let endStop = 100;
if (sortedEntries.length < 100){
endStop = sortedEntries.length;
}
for (var i = 0; i < endStop; i++) {
let referer = (decodeURIComponent(sortedEntries[i][0])).replace(/(<([^>]+)>)/ig,"");
if (sortedEntries[i][0] == ""){
//Root
referer = `<span style="color: #b5b5b5;">(<i class="eye slash outline icon"></i> Unknown or Hidden)</span>`;
}
$("#stats_upstreamTable").append(`<tr>
<td>${referer}</td>
<td>${sortedEntries[i][1]}</td>
</tr>`);
}
}
function renderFileTypeGraph(requestURLs){
//Create the device chart
let fileExtensions = {};