Fixed #201 and #608

- Added domain / host name specific statistics
- Added upstream forward request count

src/mod/dynamicproxy/dpcore/dpcore.go

@@ -105,7 +105,6 @@ func NewDynamicProxyCore(target *url.URL, prepender string, dpcOptions *DpcoreOp
 	thisTransporter := http.DefaultTransport
 
 	//Hack the default transporter to handle more connections
-
 	optimalConcurrentConnection := 256
 	if dpcOptions.MaxConcurrentConnection > 0 {
 		optimalConcurrentConnection = dpcOptions.MaxConcurrentConnection
@@ -137,18 +136,6 @@ func NewDynamicProxyCore(target *url.URL, prepender string, dpcOptions *DpcoreOp
 	}
 }
 
-func singleJoiningSlash(a, b string) string {
-	aslash := strings.HasSuffix(a, "/")
-	bslash := strings.HasPrefix(b, "/")
-	switch {
-	case aslash && bslash:
-		return a + b[1:]
-	case !aslash && !bslash:
-		return a + "/" + b
-	}
-	return a + b
-}
-
 func joinURLPath(a, b *url.URL) (path, rawpath string) {
 	apath, bpath := a.EscapedPath(), b.EscapedPath()
 	aslash, bslash := strings.HasSuffix(apath, "/"), strings.HasPrefix(bpath, "/")

src/mod/dynamicproxy/proxyRequestHandler.go

@@ -320,6 +320,7 @@ func (router *Router) logRequest(r *http.Request, succ bool, statusCode int, for
 				UserAgent:                     r.UserAgent(),
 				RequestURL:                    r.Host + r.RequestURI,
 				Target:                        originalHostname,
+				Upstream:                      upstreamHostname,
 			}
 			router.Option.StatisticCollector.RecordRequest(requestInfo)
 		}()

src/mod/statistic/analytic/utils.go

@@ -36,6 +36,8 @@ func mergeDailySummaryExports(exports []*statistic.DailySummaryExport) *statisti
 		Referer:         make(map[string]int),
 		UserAgent:       make(map[string]int),
 		RequestURL:      make(map[string]int),
+		Downstreams:     make(map[string]int),
+		Upstreams:       make(map[string]int),
 	}
 
 	for _, export := range exports {
@@ -66,6 +68,14 @@ func mergeDailySummaryExports(exports []*statistic.DailySummaryExport) *statisti
 		for key, value := range export.RequestURL {
 			mergedExport.RequestURL[key] += value
 		}
+
+		for key, value := range export.Downstreams {
+			mergedExport.Downstreams[key] += value
+		}
+
+		for key, value := range export.Upstreams {
+			mergedExport.Upstreams[key] += value
+		}
 	}
 
 	return mergedExport

src/mod/statistic/statistic.go

@@ -24,12 +24,14 @@ type DailySummary struct {
 	ErrorRequest int64 //Invalid request of the day, including error or not found
 	ValidRequest int64 //Valid request of the day
 	//Type counters
-	ForwardTypes    *sync.Map //Map that hold the forward types
-	RequestOrigin   *sync.Map //Map that hold [country ISO code]: visitor counter
-	RequestClientIp *sync.Map //Map that hold all unique request IPs
-	Referer         *sync.Map //Map that store where the user was refered from
-	UserAgent       *sync.Map //Map that store the useragent of the request
-	RequestURL      *sync.Map //Request URL of the request object
+	ForwardTypes        *sync.Map //Map that hold the forward types
+	RequestOrigin       *sync.Map //Map that hold [country ISO code]: visitor counter
+	RequestClientIp     *sync.Map //Map that hold all unique request IPs
+	Referer             *sync.Map //Map that store where the user was refered from
+	UserAgent           *sync.Map //Map that store the useragent of the request
+	RequestURL          *sync.Map //Request URL of the request object
+	DownstreamHostnames *sync.Map //Request count of downstream hostname
+	UpstreamHostnames   *sync.Map //Forwarded request count of upstream hostname
 }
 
 type RequestInfo struct {
@@ -42,6 +44,7 @@ type RequestInfo struct {
 	UserAgent                     string //UserAgent of the downstream request
 	RequestURL                    string //Request URL
 	Target                        string //Target domain or hostname
+	Upstream                      string ////Upstream domain or hostname, if the request is forwarded to upstream
 }
 
 type CollectorOption struct {
@@ -233,6 +236,24 @@ func (c *Collector) RecordRequest(ri RequestInfo) {
 		} else {
 			c.DailySummary.RequestURL.Store(ri.RequestURL, ru.(int)+1)
 		}
+
+		//Record the downstream hostname
+		//This is the hostname that the user visited, not the target domain
+		ds, ok := c.DailySummary.DownstreamHostnames.Load(ri.Target)
+		if !ok {
+			c.DailySummary.DownstreamHostnames.Store(ri.Target, 1)
+		} else {
+			c.DailySummary.DownstreamHostnames.Store(ri.Target, ds.(int)+1)
+		}
+
+		//Record the upstream hostname
+		//This is the selected load balancer upstream hostname or ip
+		us, ok := c.DailySummary.UpstreamHostnames.Load(ri.Upstream)
+		if !ok {
+			c.DailySummary.UpstreamHostnames.Store(ri.Upstream, 1)
+		} else {
+			c.DailySummary.UpstreamHostnames.Store(ri.Upstream, us.(int)+1)
+		}
 	}()
 
 	//ADD MORE HERE IF NEEDED
@@ -271,15 +292,17 @@ func (c *Collector) ScheduleResetRealtimeStats() chan bool {
 
 func NewDailySummary() *DailySummary {
 	return &DailySummary{
-		TotalRequest:    0,
-		ErrorRequest:    0,
-		ValidRequest:    0,
-		ForwardTypes:    &sync.Map{},
-		RequestOrigin:   &sync.Map{},
-		RequestClientIp: &sync.Map{},
-		Referer:         &sync.Map{},
-		UserAgent:       &sync.Map{},
-		RequestURL:      &sync.Map{},
+		TotalRequest:        0,
+		ErrorRequest:        0,
+		ValidRequest:        0,
+		ForwardTypes:        &sync.Map{},
+		RequestOrigin:       &sync.Map{},
+		RequestClientIp:     &sync.Map{},
+		Referer:             &sync.Map{},
+		UserAgent:           &sync.Map{},
+		RequestURL:          &sync.Map{},
+		DownstreamHostnames: &sync.Map{},
+		UpstreamHostnames:   &sync.Map{},
 	}
 }
 

src/mod/statistic/structconv.go

@@ -13,6 +13,21 @@ type DailySummaryExport struct {
 	Referer         map[string]int
 	UserAgent       map[string]int
 	RequestURL      map[string]int
+	Downstreams     map[string]int
+	Upstreams       map[string]int
+}
+
+func SyncMapToMapStringInt(syncMap *sync.Map) map[string]int {
+	result := make(map[string]int)
+	syncMap.Range(func(key, value interface{}) bool {
+		strKey, okKey := key.(string)
+		intValue, okValue := value.(int)
+		if okKey && okValue {
+			result[strKey] = intValue
+		}
+		return true
+	})
+	return result
 }
 
 func DailySummaryToExport(summary DailySummary) DailySummaryExport {
@@ -26,77 +41,53 @@ func DailySummaryToExport(summary DailySummary) DailySummaryExport {
 		Referer:         make(map[string]int),
 		UserAgent:       make(map[string]int),
 		RequestURL:      make(map[string]int),
+		Downstreams:     make(map[string]int),
+		Upstreams:       make(map[string]int),
 	}
 
-	summary.ForwardTypes.Range(func(key, value interface{}) bool {
-		export.ForwardTypes[key.(string)] = value.(int)
-		return true
-	})
-
-	summary.RequestOrigin.Range(func(key, value interface{}) bool {
-		export.RequestOrigin[key.(string)] = value.(int)
-		return true
-	})
-
-	summary.RequestClientIp.Range(func(key, value interface{}) bool {
-		export.RequestClientIp[key.(string)] = value.(int)
-		return true
-	})
-
-	summary.Referer.Range(func(key, value interface{}) bool {
-		export.Referer[key.(string)] = value.(int)
-		return true
-	})
-
-	summary.UserAgent.Range(func(key, value interface{}) bool {
-		export.UserAgent[key.(string)] = value.(int)
-		return true
-	})
-
-	summary.RequestURL.Range(func(key, value interface{}) bool {
-		export.RequestURL[key.(string)] = value.(int)
-		return true
-	})
+	export.ForwardTypes = SyncMapToMapStringInt(summary.ForwardTypes)
+	export.RequestOrigin = SyncMapToMapStringInt(summary.RequestOrigin)
+	export.RequestClientIp = SyncMapToMapStringInt(summary.RequestClientIp)
+	export.Referer = SyncMapToMapStringInt(summary.Referer)
+	export.UserAgent = SyncMapToMapStringInt(summary.UserAgent)
+	export.RequestURL = SyncMapToMapStringInt(summary.RequestURL)
+	export.Downstreams = SyncMapToMapStringInt(summary.DownstreamHostnames)
+	export.Upstreams = SyncMapToMapStringInt(summary.UpstreamHostnames)
 
 	return export
 }
 
+func MapStringIntToSyncMap(m map[string]int) *sync.Map {
+	syncMap := &sync.Map{}
+	for k, v := range m {
+		syncMap.Store(k, v)
+	}
+	return syncMap
+}
+
 func DailySummaryExportToSummary(export DailySummaryExport) DailySummary {
 	summary := DailySummary{
-		TotalRequest:    export.TotalRequest,
-		ErrorRequest:    export.ErrorRequest,
-		ValidRequest:    export.ValidRequest,
-		ForwardTypes:    &sync.Map{},
-		RequestOrigin:   &sync.Map{},
-		RequestClientIp: &sync.Map{},
-		Referer:         &sync.Map{},
-		UserAgent:       &sync.Map{},
-		RequestURL:      &sync.Map{},
+		TotalRequest:        export.TotalRequest,
+		ErrorRequest:        export.ErrorRequest,
+		ValidRequest:        export.ValidRequest,
+		ForwardTypes:        &sync.Map{},
+		RequestOrigin:       &sync.Map{},
+		RequestClientIp:     &sync.Map{},
+		Referer:             &sync.Map{},
+		UserAgent:           &sync.Map{},
+		RequestURL:          &sync.Map{},
+		DownstreamHostnames: &sync.Map{},
+		UpstreamHostnames:   &sync.Map{},
 	}
 
-	for k, v := range export.ForwardTypes {
-		summary.ForwardTypes.Store(k, v)
-	}
-
-	for k, v := range export.RequestOrigin {
-		summary.RequestOrigin.Store(k, v)
-	}
-
-	for k, v := range export.RequestClientIp {
-		summary.RequestClientIp.Store(k, v)
-	}
-
-	for k, v := range export.Referer {
-		summary.Referer.Store(k, v)
-	}
-
-	for k, v := range export.UserAgent {
-		summary.UserAgent.Store(k, v)
-	}
-
-	for k, v := range export.RequestURL {
-		summary.RequestURL.Store(k, v)
-	}
+	summary.ForwardTypes = MapStringIntToSyncMap(export.ForwardTypes)
+	summary.RequestOrigin = MapStringIntToSyncMap(export.RequestOrigin)
+	summary.RequestClientIp = MapStringIntToSyncMap(export.RequestClientIp)
+	summary.Referer = MapStringIntToSyncMap(export.Referer)
+	summary.UserAgent = MapStringIntToSyncMap(export.UserAgent)
+	summary.RequestURL = MapStringIntToSyncMap(export.RequestURL)
+	summary.DownstreamHostnames = MapStringIntToSyncMap(export.Downstreams)
+	summary.UpstreamHostnames = MapStringIntToSyncMap(export.Upstreams)
 
 	return summary
 }