Fixed #201 and #608

- Added domain / host name specific statistics
- Added upstream forward request count

src/def.go

@@ -45,7 +45,7 @@ const (
 	/* Build Constants */
 	SYSTEM_NAME       = "Zoraxy"
 	SYSTEM_VERSION    = "3.2.1"
-	DEVELOPMENT_BUILD = false /* Development: Set to false to use embedded web fs */
+	DEVELOPMENT_BUILD = true /* Development: Set to false to use embedded web fs */
 
 	/* System Constants */
 	TMP_FOLDER                   = "./tmp"

src/mod/dynamicproxy/dpcore/dpcore.go

@@ -105,7 +105,6 @@ func NewDynamicProxyCore(target *url.URL, prepender string, dpcOptions *DpcoreOp
 	thisTransporter := http.DefaultTransport
 
 	//Hack the default transporter to handle more connections
-
 	optimalConcurrentConnection := 256
 	if dpcOptions.MaxConcurrentConnection > 0 {
 		optimalConcurrentConnection = dpcOptions.MaxConcurrentConnection
@@ -137,18 +136,6 @@ func NewDynamicProxyCore(target *url.URL, prepender string, dpcOptions *DpcoreOp
 	}
 }
 
-func singleJoiningSlash(a, b string) string {
-	aslash := strings.HasSuffix(a, "/")
-	bslash := strings.HasPrefix(b, "/")
-	switch {
-	case aslash && bslash:
-		return a + b[1:]
-	case !aslash && !bslash:
-		return a + "/" + b
-	}
-	return a + b
-}
-
 func joinURLPath(a, b *url.URL) (path, rawpath string) {
 	apath, bpath := a.EscapedPath(), b.EscapedPath()
 	aslash, bslash := strings.HasSuffix(apath, "/"), strings.HasPrefix(bpath, "/")

src/mod/dynamicproxy/proxyRequestHandler.go

@@ -320,6 +320,7 @@ func (router *Router) logRequest(r *http.Request, succ bool, statusCode int, for
 				UserAgent:                     r.UserAgent(),
 				RequestURL:                    r.Host + r.RequestURI,
 				Target:                        originalHostname,
+				Upstream:                      upstreamHostname,
 			}
 			router.Option.StatisticCollector.RecordRequest(requestInfo)
 		}()

src/mod/statistic/analytic/utils.go

@@ -36,6 +36,8 @@ func mergeDailySummaryExports(exports []*statistic.DailySummaryExport) *statisti
 		Referer:         make(map[string]int),
 		UserAgent:       make(map[string]int),
 		RequestURL:      make(map[string]int),
+		Downstreams:     make(map[string]int),
+		Upstreams:       make(map[string]int),
 	}
 
 	for _, export := range exports {
@@ -66,6 +68,14 @@ func mergeDailySummaryExports(exports []*statistic.DailySummaryExport) *statisti
 		for key, value := range export.RequestURL {
 			mergedExport.RequestURL[key] += value
 		}
+
+		for key, value := range export.Downstreams {
+			mergedExport.Downstreams[key] += value
+		}
+
+		for key, value := range export.Upstreams {
+			mergedExport.Upstreams[key] += value
+		}
 	}
 
 	return mergedExport

src/mod/statistic/statistic.go

@@ -24,12 +24,14 @@ type DailySummary struct {
 	ErrorRequest int64 //Invalid request of the day, including error or not found
 	ValidRequest int64 //Valid request of the day
 	//Type counters
-	ForwardTypes    *sync.Map //Map that hold the forward types
+	ForwardTypes        *sync.Map //Map that hold the forward types
-	RequestOrigin   *sync.Map //Map that hold [country ISO code]: visitor counter
+	RequestOrigin       *sync.Map //Map that hold [country ISO code]: visitor counter
-	RequestClientIp *sync.Map //Map that hold all unique request IPs
+	RequestClientIp     *sync.Map //Map that hold all unique request IPs
-	Referer         *sync.Map //Map that store where the user was refered from
+	Referer             *sync.Map //Map that store where the user was refered from
-	UserAgent       *sync.Map //Map that store the useragent of the request
+	UserAgent           *sync.Map //Map that store the useragent of the request
-	RequestURL      *sync.Map //Request URL of the request object
+	RequestURL          *sync.Map //Request URL of the request object
+	DownstreamHostnames *sync.Map //Request count of downstream hostname
+	UpstreamHostnames   *sync.Map //Forwarded request count of upstream hostname
 }
 
 type RequestInfo struct {
@@ -42,6 +44,7 @@ type RequestInfo struct {
 	UserAgent                     string //UserAgent of the downstream request
 	RequestURL                    string //Request URL
 	Target                        string //Target domain or hostname
+	Upstream                      string ////Upstream domain or hostname, if the request is forwarded to upstream
 }
 
 type CollectorOption struct {
@@ -233,6 +236,24 @@ func (c *Collector) RecordRequest(ri RequestInfo) {
 		} else {
 			c.DailySummary.RequestURL.Store(ri.RequestURL, ru.(int)+1)
 		}
+
+		//Record the downstream hostname
+		//This is the hostname that the user visited, not the target domain
+		ds, ok := c.DailySummary.DownstreamHostnames.Load(ri.Target)
+		if !ok {
+			c.DailySummary.DownstreamHostnames.Store(ri.Target, 1)
+		} else {
+			c.DailySummary.DownstreamHostnames.Store(ri.Target, ds.(int)+1)
+		}
+
+		//Record the upstream hostname
+		//This is the selected load balancer upstream hostname or ip
+		us, ok := c.DailySummary.UpstreamHostnames.Load(ri.Upstream)
+		if !ok {
+			c.DailySummary.UpstreamHostnames.Store(ri.Upstream, 1)
+		} else {
+			c.DailySummary.UpstreamHostnames.Store(ri.Upstream, us.(int)+1)
+		}
 	}()
 
 	//ADD MORE HERE IF NEEDED
@@ -271,15 +292,17 @@ func (c *Collector) ScheduleResetRealtimeStats() chan bool {
 
 func NewDailySummary() *DailySummary {
 	return &DailySummary{
-		TotalRequest:    0,
+		TotalRequest:        0,
-		ErrorRequest:    0,
+		ErrorRequest:        0,
-		ValidRequest:    0,
+		ValidRequest:        0,
-		ForwardTypes:    &sync.Map{},
+		ForwardTypes:        &sync.Map{},
-		RequestOrigin:   &sync.Map{},
+		RequestOrigin:       &sync.Map{},
-		RequestClientIp: &sync.Map{},
+		RequestClientIp:     &sync.Map{},
-		Referer:         &sync.Map{},
+		Referer:             &sync.Map{},
-		UserAgent:       &sync.Map{},
+		UserAgent:           &sync.Map{},
-		RequestURL:      &sync.Map{},
+		RequestURL:          &sync.Map{},
+		DownstreamHostnames: &sync.Map{},
+		UpstreamHostnames:   &sync.Map{},
 	}
 }
 

src/mod/statistic/structconv.go

@@ -13,6 +13,21 @@ type DailySummaryExport struct {
 	Referer         map[string]int
 	UserAgent       map[string]int
 	RequestURL      map[string]int
+	Downstreams     map[string]int
+	Upstreams       map[string]int
+}
+
+func SyncMapToMapStringInt(syncMap *sync.Map) map[string]int {
+	result := make(map[string]int)
+	syncMap.Range(func(key, value interface{}) bool {
+		strKey, okKey := key.(string)
+		intValue, okValue := value.(int)
+		if okKey && okValue {
+			result[strKey] = intValue
+		}
+		return true
+	})
+	return result
 }
 
 func DailySummaryToExport(summary DailySummary) DailySummaryExport {
@@ -26,77 +41,53 @@ func DailySummaryToExport(summary DailySummary) DailySummaryExport {
 		Referer:         make(map[string]int),
 		UserAgent:       make(map[string]int),
 		RequestURL:      make(map[string]int),
+		Downstreams:     make(map[string]int),
+		Upstreams:       make(map[string]int),
 	}
 
-	summary.ForwardTypes.Range(func(key, value interface{}) bool {
+	export.ForwardTypes = SyncMapToMapStringInt(summary.ForwardTypes)
-		export.ForwardTypes[key.(string)] = value.(int)
+	export.RequestOrigin = SyncMapToMapStringInt(summary.RequestOrigin)
-		return true
+	export.RequestClientIp = SyncMapToMapStringInt(summary.RequestClientIp)
-	})
+	export.Referer = SyncMapToMapStringInt(summary.Referer)
-
+	export.UserAgent = SyncMapToMapStringInt(summary.UserAgent)
-	summary.RequestOrigin.Range(func(key, value interface{}) bool {
+	export.RequestURL = SyncMapToMapStringInt(summary.RequestURL)
-		export.RequestOrigin[key.(string)] = value.(int)
+	export.Downstreams = SyncMapToMapStringInt(summary.DownstreamHostnames)
-		return true
+	export.Upstreams = SyncMapToMapStringInt(summary.UpstreamHostnames)
-	})
-
-	summary.RequestClientIp.Range(func(key, value interface{}) bool {
-		export.RequestClientIp[key.(string)] = value.(int)
-		return true
-	})
-
-	summary.Referer.Range(func(key, value interface{}) bool {
-		export.Referer[key.(string)] = value.(int)
-		return true
-	})
-
-	summary.UserAgent.Range(func(key, value interface{}) bool {
-		export.UserAgent[key.(string)] = value.(int)
-		return true
-	})
-
-	summary.RequestURL.Range(func(key, value interface{}) bool {
-		export.RequestURL[key.(string)] = value.(int)
-		return true
-	})
 
 	return export
 }
 
+func MapStringIntToSyncMap(m map[string]int) *sync.Map {
+	syncMap := &sync.Map{}
+	for k, v := range m {
+		syncMap.Store(k, v)
+	}
+	return syncMap
+}
+
 func DailySummaryExportToSummary(export DailySummaryExport) DailySummary {
 	summary := DailySummary{
-		TotalRequest:    export.TotalRequest,
+		TotalRequest:        export.TotalRequest,
-		ErrorRequest:    export.ErrorRequest,
+		ErrorRequest:        export.ErrorRequest,
-		ValidRequest:    export.ValidRequest,
+		ValidRequest:        export.ValidRequest,
-		ForwardTypes:    &sync.Map{},
+		ForwardTypes:        &sync.Map{},
-		RequestOrigin:   &sync.Map{},
+		RequestOrigin:       &sync.Map{},
-		RequestClientIp: &sync.Map{},
+		RequestClientIp:     &sync.Map{},
-		Referer:         &sync.Map{},
+		Referer:             &sync.Map{},
-		UserAgent:       &sync.Map{},
+		UserAgent:           &sync.Map{},
-		RequestURL:      &sync.Map{},
+		RequestURL:          &sync.Map{},
+		DownstreamHostnames: &sync.Map{},
+		UpstreamHostnames:   &sync.Map{},
 	}
 
-	for k, v := range export.ForwardTypes {
+	summary.ForwardTypes = MapStringIntToSyncMap(export.ForwardTypes)
-		summary.ForwardTypes.Store(k, v)
+	summary.RequestOrigin = MapStringIntToSyncMap(export.RequestOrigin)
-	}
+	summary.RequestClientIp = MapStringIntToSyncMap(export.RequestClientIp)
-
+	summary.Referer = MapStringIntToSyncMap(export.Referer)
-	for k, v := range export.RequestOrigin {
+	summary.UserAgent = MapStringIntToSyncMap(export.UserAgent)
-		summary.RequestOrigin.Store(k, v)
+	summary.RequestURL = MapStringIntToSyncMap(export.RequestURL)
-	}
+	summary.DownstreamHostnames = MapStringIntToSyncMap(export.Downstreams)
-
+	summary.UpstreamHostnames = MapStringIntToSyncMap(export.Upstreams)
-	for k, v := range export.RequestClientIp {
-		summary.RequestClientIp.Store(k, v)
-	}
-
-	for k, v := range export.Referer {
-		summary.Referer.Store(k, v)
-	}
-
-	for k, v := range export.UserAgent {
-		summary.UserAgent.Store(k, v)
-	}
-
-	for k, v := range export.RequestURL {
-		summary.RequestURL.Store(k, v)
-	}
 
 	return summary
 }

src/web/components/stats.html

@@ -184,7 +184,46 @@
                     </div>
                 </div>
             </div>
-        </div>  
+        </div> 
+        <div class="ui divider"></div>
+        <div class="ui stackable grid">
+            <div class="eight wide column">
+                <h3>Requested Hostnames</h3>
+                <p>Most requested hostnames from downstream</p>
+                <div>
+                    <div style="height: 500px; overflow-y: auto;">
+                        <table class="ui unstackable striped celled table">
+                            <thead>
+                              <tr>
+                                <th class="no-sort">Hostname</th>
+                                <th class="no-sort">Requests</th>
+                            </tr></thead>
+                            <tbody id="stats_downstreamTable">
+                             
+                            </tbody>
+                        </table>
+                    </div>
+                </div>
+            </div>
+            <div class="eight wide column">
+                <h3>Forwarded Upstreams</h3>
+                <p>The Top 100 upstreams where the requests are forwarded to</p>
+                <div>
+                    <div style="height: 500px; overflow-y: auto;">
+                        <table class="ui unstackable striped celled table">
+                            <thead>
+                              <tr>
+                                <th class="no-sort">Upstream Endpoint</th>
+                                <th class="no-sort">Requests</th>
+                            </tr></thead>
+                            <tbody id="stats_upstreamTable">
+                             
+                            </tbody>
+                        </table>
+                    </div>
+                </div>
+            </div>
+        </div> 
         <div class="ui divider"></div>
         <div class="ui basic segment" id="trendGraphs">
             <h3>Visitor Trend Analysis</h3>
@@ -263,6 +302,22 @@
             //Render Referer header
             renderRefererTable(data.Referer);
 
+            if (data.Downstreams == null){
+                //No downstream data to show
+                $("#stats_downstreamTable").html("<tr><td colspan='2'>No data</td></tr>");
+            }else{
+                //Render the downstream table
+                renderDownstreamTable(data.Downstreams);
+            }
+
+            if (data.Upstreams == null){
+                //No upstream data to show
+                $("#stats_upstreamTable").html("<tr><td colspan='2'>No data</td></tr>");
+            }else{
+                //Render the upstream table
+                renderUpstreamTable(data.Upstreams);
+            }
+
             //Hide the trend graphs
             $("#trendGraphs").hide();
         });
@@ -410,6 +465,46 @@
         }
    }
 
+   function renderDownstreamTable(downstreamList){
+        const sortedEntries = Object.entries(downstreamList).sort(([, valueA], [, valueB]) => valueB - valueA);
+        $("#stats_downstreamTable").html("");
+        let endStop = 100;
+        if (sortedEntries.length < 100){
+            endStop = sortedEntries.length;
+        }
+        for (var i = 0; i < endStop; i++) {
+            let referer = (decodeURIComponent(sortedEntries[i][0])).replace(/(<([^>]+)>)/ig,"");
+            if (sortedEntries[i][0] == ""){
+                //Root
+                referer = `<span style="color: #b5b5b5;">(<i class="eye slash outline icon"></i> Unknown or Hidden)</span>`;
+            }
+            $("#stats_downstreamTable").append(`<tr>
+                    <td>${referer}</td>
+                    <td>${sortedEntries[i][1]}</td>
+                </tr>`);
+        }
+   }
+
+    function renderUpstreamTable(upstreamList){
+          const sortedEntries = Object.entries(upstreamList).sort(([, valueA], [, valueB]) => valueB - valueA);
+          $("#stats_upstreamTable").html("");
+          let endStop = 100;
+          if (sortedEntries.length < 100){
+                endStop = sortedEntries.length;
+          }
+          for (var i = 0; i < endStop; i++) {
+                let referer = (decodeURIComponent(sortedEntries[i][0])).replace(/(<([^>]+)>)/ig,"");
+                if (sortedEntries[i][0] == ""){
+                 //Root
+                 referer = `<span style="color: #b5b5b5;">(<i class="eye slash outline icon"></i> Unknown or Hidden)</span>`;
+                }
+                $("#stats_upstreamTable").append(`<tr>
+                      <td>${referer}</td>
+                      <td>${sortedEntries[i][1]}</td>
+                 </tr>`);
+          }
+    }
+
    function renderFileTypeGraph(requestURLs){
      //Create the device chart
      let fileExtensions = {};