Updates v2.6 experimental build

+ Basic auth
+ Support TLS verification skip (for self signed certs)
+ Added trend analysis
+ Added referer and file type analysis
+ Added cert expire day display
+ Moved subdomain proxy logic to dpcore

src/mod/dynamicproxy/Server.go

@@ -12,9 +12,19 @@ import (
 	Server.go
 
 	Main server for dynamic proxy core
+
+	Routing Handler Priority (High to Low)
+	- Blacklist
+	- Whitelist
+	- Redirectable
+	- Subdomain Routing
+	- Vitrual Directory Routing
 */
 
 func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	/*
+		General Access Check
+	*/
 	//Check if this ip is in blacklist
 	clientIpAddr := geodb.GetRequesterIP(r)
 	if h.Parent.Option.GeodbStore.IsBlacklisted(clientIpAddr) {
@@ -30,6 +40,9 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	/*
+		Redirection Routing
+	*/
 	//Check if this is a redirection url
 	if h.Parent.Option.RedirectRuleTable.IsRedirectable(r) {
 		statusCode := h.Parent.Option.RedirectRuleTable.HandleRedirect(w, r)
@@ -53,21 +66,37 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		domainOnly = hostPath[0]
 	}
 
+	/*
+		Subdomain Routing
+	*/
 	if strings.Contains(r.Host, ".") {
 		//This might be a subdomain. See if there are any subdomain proxy router for this
-		//Remove the port if any
-
 		sep := h.Parent.getSubdomainProxyEndpointFromHostname(domainOnly)
 		if sep != nil {
+			if sep.RequireBasicAuth {
+				err := h.handleBasicAuthRouting(w, r, sep)
+				if err != nil {
+					return
+				}
+			}
 			h.subdomainRequest(w, r, sep)
 			return
 		}
 	}
 
+	/*
+		Virtual Directory Routing
+	*/
 	//Clean up the request URI
 	proxyingPath := strings.TrimSpace(r.RequestURI)
 	targetProxyEndpoint := h.Parent.getTargetProxyEndpointFromRequestURI(proxyingPath)
 	if targetProxyEndpoint != nil {
+		if targetProxyEndpoint.RequireBasicAuth {
+			err := h.handleBasicAuthRouting(w, r, targetProxyEndpoint)
+			if err != nil {
+				return
+			}
+		}
 		h.proxyRequest(w, r, targetProxyEndpoint)
 	} else if !strings.HasSuffix(proxyingPath, "/") {
 		potentialProxtEndpoint := h.Parent.getTargetProxyEndpointFromRequestURI(proxyingPath + "/")
@@ -75,11 +104,12 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if potentialProxtEndpoint != nil {
 			//Missing tailing slash. Redirect to target proxy endpoint
 			http.Redirect(w, r, r.RequestURI+"/", http.StatusTemporaryRedirect)
-			//h.proxyRequest(w, r, potentialProxtEndpoint)
 		} else {
+			//Passthrough the request to root
 			h.proxyRequest(w, r, h.Parent.Root)
 		}
 	} else {
+		//No routing rules found. Route to root.
 		h.proxyRequest(w, r, h.Parent.Root)
 	}
 }

src/mod/dynamicproxy/basicAuth.go

@@ -0,0 +1,47 @@
+package dynamicproxy
+
+import (
+	"errors"
+	"net/http"
+
+	"imuslab.com/zoraxy/mod/auth"
+)
+
+/*
+	BasicAuth.go
+
+	This file handles the basic auth on proxy endpoints
+	if RequireBasicAuth is set to true
+*/
+
+func (h *ProxyHandler) handleBasicAuthRouting(w http.ResponseWriter, r *http.Request, pe *ProxyEndpoint) error {
+	proxyType := "vdir-auth"
+	if pe.ProxyType == ProxyType_Subdomain {
+		proxyType = "subd-auth"
+	}
+	u, p, ok := r.BasicAuth()
+	if !ok {
+		w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+		w.WriteHeader(401)
+		return errors.New("unauthorized")
+	}
+
+	//Check for the credentials to see if there is one matching
+	hashedPassword := auth.Hash(p)
+	matchingFound := false
+	for _, cred := range pe.BasicAuthCredentials {
+		if u == cred.Username && hashedPassword == cred.PasswordHash {
+			matchingFound = true
+			break
+		}
+	}
+
+	if !matchingFound {
+		h.logRequest(r, false, 401, proxyType, pe.Domain)
+		w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+		w.WriteHeader(401)
+		return errors.New("unauthorized")
+	}
+
+	return nil
+}

src/mod/dynamicproxy/dpcore/dpcore.go

@@ -71,7 +71,7 @@ type requestCanceler interface {
 	CancelRequest(req *http.Request)
 }
 
-func NewDynamicProxyCore(target *url.URL, prepender string) *ReverseProxy {
+func NewDynamicProxyCore(target *url.URL, prepender string, ignoreTLSVerification bool) *ReverseProxy {
 	targetQuery := target.RawQuery
 	director := func(req *http.Request) {
 		req.URL.Scheme = target.Scheme
@@ -95,7 +95,12 @@ func NewDynamicProxyCore(target *url.URL, prepender string) *ReverseProxy {
 	thisTransporter.(*http.Transport).MaxIdleConnsPerHost = 3000
 	thisTransporter.(*http.Transport).IdleConnTimeout = 10 * time.Second
 	thisTransporter.(*http.Transport).MaxConnsPerHost = 0
-	thisTransporter.(*http.Transport).DisableCompression = true
+	//thisTransporter.(*http.Transport).DisableCompression = true
+
+	if ignoreTLSVerification {
+		//Ignore TLS certificate validation error
+		thisTransporter.(*http.Transport).TLSClientConfig.InsecureSkipVerify = true
+	}
 
 	return &ReverseProxy{
 		Director:  director,
@@ -278,9 +283,6 @@ func addXForwardedForHeader(req *http.Request) {
 
 func (p *ReverseProxy) ProxyHTTP(rw http.ResponseWriter, req *http.Request, rrr *ResponseRewriteRuleSet) error {
 	transport := p.Transport
-	if transport == nil {
-		transport = http.DefaultTransport
-	}
 
 	outreq := new(http.Request)
 	// Shallow copies of maps, like header

src/mod/dynamicproxy/dynamicproxy.go

@@ -5,7 +5,6 @@ import (
 	"crypto/tls"
 	"errors"
 	"log"
-	"net"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -14,57 +13,11 @@ import (
 	"time"
 
 	"imuslab.com/zoraxy/mod/dynamicproxy/dpcore"
-	"imuslab.com/zoraxy/mod/dynamicproxy/redirection"
-	"imuslab.com/zoraxy/mod/geodb"
-	"imuslab.com/zoraxy/mod/reverseproxy"
-	"imuslab.com/zoraxy/mod/statistic"
-	"imuslab.com/zoraxy/mod/tlscert"
 )
 
 /*
-Zoraxy Dynamic Proxy
+	Zoraxy Dynamic Proxy
 */
-type RouterOption struct {
-	Port               int
-	UseTls             bool
-	ForceHttpsRedirect bool
-	TlsManager         *tlscert.Manager
-	RedirectRuleTable  *redirection.RuleTable
-	GeodbStore         *geodb.Store
-	StatisticCollector *statistic.Collector
-}
-
-type Router struct {
-	Option            *RouterOption
-	ProxyEndpoints    *sync.Map
-	SubdomainEndpoint *sync.Map
-	Running           bool
-	Root              *ProxyEndpoint
-	mux               http.Handler
-	server            *http.Server
-	tlsListener       net.Listener
-	routingRules      []*RoutingRule
-
-	tlsRedirectStop chan bool
-}
-
-type ProxyEndpoint struct {
-	Root       string
-	Domain     string
-	RequireTLS bool
-	Proxy      *dpcore.ReverseProxy `json:"-"`
-}
-
-type SubdomainEndpoint struct {
-	MatchingDomain string
-	Domain         string
-	RequireTLS     bool
-	Proxy          *reverseproxy.ReverseProxy `json:"-"`
-}
-
-type ProxyHandler struct {
-	Parent *Router
-}
 
 func NewDynamicProxy(option RouterOption) (*Router, error) {
 	proxyMap := sync.Map{}
@@ -250,8 +203,8 @@ func (router *Router) IsProxiedSubdomain(r *http.Request) bool {
 /*
 Add an URL into a custom proxy services
 */
-func (router *Router) AddVirtualDirectoryProxyService(rootname string, domain string, requireTLS bool) error {
-
+func (router *Router) AddVirtualDirectoryProxyService(options *VdirOptions) error {
+	domain := options.Domain
 	if domain[len(domain)-1:] == "/" {
 		domain = domain[:len(domain)-1]
 	}
@@ -263,7 +216,7 @@ func (router *Router) AddVirtualDirectoryProxyService(rootname string, domain st
 	*/
 
 	webProxyEndpoint := domain
-	if requireTLS {
+	if options.RequireTLS {
 		webProxyEndpoint = "https://" + webProxyEndpoint
 	} else {
 		webProxyEndpoint = "http://" + webProxyEndpoint
@@ -274,18 +227,22 @@ func (router *Router) AddVirtualDirectoryProxyService(rootname string, domain st
 		return err
 	}
 
-	proxy := dpcore.NewDynamicProxyCore(path, rootname)
+	proxy := dpcore.NewDynamicProxyCore(path, options.RootName, options.SkipCertValidations)
 
 	endpointObject := ProxyEndpoint{
-		Root:       rootname,
-		Domain:     domain,
-		RequireTLS: requireTLS,
-		Proxy:      proxy,
+		ProxyType:            ProxyType_Vdir,
+		RootOrMatchingDomain: options.RootName,
+		Domain:               domain,
+		RequireTLS:           options.RequireTLS,
+		SkipCertValidations:  options.SkipCertValidations,
+		RequireBasicAuth:     options.RequireBasicAuth,
+		BasicAuthCredentials: options.BasicAuthCredentials,
+		Proxy:                proxy,
 	}
 
-	router.ProxyEndpoints.Store(rootname, &endpointObject)
+	router.ProxyEndpoints.Store(options.RootName, &endpointObject)
 
-	log.Println("Adding Proxy Rule: ", rootname+" to "+domain)
+	log.Println("Adding Proxy Rule: ", options.RootName+" to "+domain)
 	return nil
 }
 
@@ -307,13 +264,14 @@ func (router *Router) RemoveProxy(ptype string, key string) error {
 /*
 Add an default router for the proxy server
 */
-func (router *Router) SetRootProxy(proxyLocation string, requireTLS bool) error {
+func (router *Router) SetRootProxy(options *RootOptions) error {
+	proxyLocation := options.ProxyLocation
 	if proxyLocation[len(proxyLocation)-1:] == "/" {
 		proxyLocation = proxyLocation[:len(proxyLocation)-1]
 	}
 
 	webProxyEndpoint := proxyLocation
-	if requireTLS {
+	if options.RequireTLS {
 		webProxyEndpoint = "https://" + webProxyEndpoint
 	} else {
 		webProxyEndpoint = "http://" + webProxyEndpoint
@@ -324,13 +282,17 @@ func (router *Router) SetRootProxy(proxyLocation string, requireTLS bool) error
 		return err
 	}
 
-	proxy := dpcore.NewDynamicProxyCore(path, "")
+	proxy := dpcore.NewDynamicProxyCore(path, "", options.SkipCertValidations)
 
 	rootEndpoint := ProxyEndpoint{
-		Root:       "/",
-		Domain:     proxyLocation,
-		RequireTLS: requireTLS,
-		Proxy:      proxy,
+		ProxyType:            ProxyType_Vdir,
+		RootOrMatchingDomain: "/",
+		Domain:               proxyLocation,
+		RequireTLS:           options.RequireTLS,
+		SkipCertValidations:  options.SkipCertValidations,
+		RequireBasicAuth:     options.RequireBasicAuth,
+		BasicAuthCredentials: options.BasicAuthCredentials,
+		Proxy:                proxy,
 	}
 
 	router.Root = &rootEndpoint
@@ -338,14 +300,14 @@ func (router *Router) SetRootProxy(proxyLocation string, requireTLS bool) error
 }
 
 // Helpers to export the syncmap for easier processing
-func (r *Router) GetSDProxyEndpointsAsMap() map[string]*SubdomainEndpoint {
-	m := make(map[string]*SubdomainEndpoint)
+func (r *Router) GetSDProxyEndpointsAsMap() map[string]*ProxyEndpoint {
+	m := make(map[string]*ProxyEndpoint)
 	r.SubdomainEndpoint.Range(func(key, value interface{}) bool {
 		k, ok := key.(string)
 		if !ok {
 			return true
 		}
-		v, ok := value.(*SubdomainEndpoint)
+		v, ok := value.(*ProxyEndpoint)
 		if !ok {
 			return true
 		}

src/mod/dynamicproxy/proxyRequestHandler.go

@@ -28,23 +28,34 @@ func (router *Router) getTargetProxyEndpointFromRequestURI(requestURI string) *P
 	return targetProxyEndpoint
 }
 
-func (router *Router) getSubdomainProxyEndpointFromHostname(hostname string) *SubdomainEndpoint {
-	var targetSubdomainEndpoint *SubdomainEndpoint = nil
+func (router *Router) getSubdomainProxyEndpointFromHostname(hostname string) *ProxyEndpoint {
+	var targetSubdomainEndpoint *ProxyEndpoint = nil
 	ep, ok := router.SubdomainEndpoint.Load(hostname)
 	if ok {
-		targetSubdomainEndpoint = ep.(*SubdomainEndpoint)
+		targetSubdomainEndpoint = ep.(*ProxyEndpoint)
 	}
 
 	return targetSubdomainEndpoint
 }
 
+// Clearn URL Path (without the http:// part) replaces // in a URL to /
+func (router *Router) clearnURL(targetUrlOPath string) string {
+	return strings.ReplaceAll(targetUrlOPath, "//", "/")
+}
+
+// Rewrite URL rewrite the prefix part of a virtual directory URL with /
 func (router *Router) rewriteURL(rooturl string, requestURL string) string {
 	rewrittenURL := requestURL
 	rewrittenURL = strings.TrimPrefix(rewrittenURL, strings.TrimSuffix(rooturl, "/"))
+
+	if strings.Contains(rewrittenURL, "//") {
+		rewrittenURL = router.clearnURL(rewrittenURL)
+	}
 	return rewrittenURL
 }
 
-func (h *ProxyHandler) subdomainRequest(w http.ResponseWriter, r *http.Request, target *SubdomainEndpoint) {
+// Handle subdomain request
+func (h *ProxyHandler) subdomainRequest(w http.ResponseWriter, r *http.Request, target *ProxyEndpoint) {
 	r.Header.Set("X-Forwarded-Host", r.Host)
 	requestURL := r.URL.String()
 	if r.Header["Upgrade"] != nil && strings.ToLower(r.Header["Upgrade"][0]) == "websocket" {
@@ -69,8 +80,20 @@ func (h *ProxyHandler) subdomainRequest(w http.ResponseWriter, r *http.Request,
 		return
 	}
 
-	r.Host = r.URL.Host
-	err := target.Proxy.ServeHTTP(w, r)
+	originalHostHeader := r.Host
+	if r.URL != nil {
+		r.Host = r.URL.Host
+	} else {
+		//Fallback when the upstream proxy screw something up in the header
+		r.URL, _ = url.Parse(originalHostHeader)
+	}
+
+	err := target.Proxy.ServeHTTP(w, r, &dpcore.ResponseRewriteRuleSet{
+		ProxyDomain:  target.Domain,
+		OriginalHost: originalHostHeader,
+		UseTLS:       target.RequireTLS,
+		PathPrefix:   "",
+	})
 	var dnsError *net.DNSError
 	if err != nil {
 		if errors.As(err, &dnsError) {
@@ -87,8 +110,9 @@ func (h *ProxyHandler) subdomainRequest(w http.ResponseWriter, r *http.Request,
 	h.logRequest(r, true, 200, "subdomain-http", target.Domain)
 }
 
+// Handle vdir type request
 func (h *ProxyHandler) proxyRequest(w http.ResponseWriter, r *http.Request, target *ProxyEndpoint) {
-	rewriteURL := h.Parent.rewriteURL(target.Root, r.RequestURI)
+	rewriteURL := h.Parent.rewriteURL(target.RootOrMatchingDomain, r.RequestURI)
 	r.URL, _ = url.Parse(rewriteURL)
 
 	r.Header.Set("X-Forwarded-Host", r.Host)
@@ -110,12 +134,18 @@ func (h *ProxyHandler) proxyRequest(w http.ResponseWriter, r *http.Request, targ
 	}
 
 	originalHostHeader := r.Host
-	r.Host = r.URL.Host
+	if r.URL != nil {
+		r.Host = r.URL.Host
+	} else {
+		//Fallback when the upstream proxy screw something up in the header
+		r.URL, _ = url.Parse(originalHostHeader)
+	}
+
 	err := target.Proxy.ServeHTTP(w, r, &dpcore.ResponseRewriteRuleSet{
 		ProxyDomain:  target.Domain,
 		OriginalHost: originalHostHeader,
 		UseTLS:       target.RequireTLS,
-		PathPrefix:   target.Root,
+		PathPrefix:   target.RootOrMatchingDomain,
 	})
 
 	var dnsError *net.DNSError

src/mod/dynamicproxy/subdomain.go

@@ -4,7 +4,7 @@ import (
 	"log"
 	"net/url"
 
-	"imuslab.com/zoraxy/mod/reverseproxy"
+	"imuslab.com/zoraxy/mod/dynamicproxy/dpcore"
 )
 
 /*
@@ -12,13 +12,14 @@ import (
 
 */
 
-func (router *Router) AddSubdomainRoutingService(hostnameWithSubdomain string, domain string, requireTLS bool) error {
+func (router *Router) AddSubdomainRoutingService(options *SubdOptions) error {
+	domain := options.Domain
 	if domain[len(domain)-1:] == "/" {
 		domain = domain[:len(domain)-1]
 	}
 
 	webProxyEndpoint := domain
-	if requireTLS {
+	if options.RequireTLS {
 		webProxyEndpoint = "https://" + webProxyEndpoint
 	} else {
 		webProxyEndpoint = "http://" + webProxyEndpoint
@@ -30,15 +31,18 @@ func (router *Router) AddSubdomainRoutingService(hostnameWithSubdomain string, d
 		return err
 	}
 
-	proxy := reverseproxy.NewReverseProxy(path)
+	proxy := dpcore.NewDynamicProxyCore(path, "", options.SkipCertValidations)
 
-	router.SubdomainEndpoint.Store(hostnameWithSubdomain, &SubdomainEndpoint{
-		MatchingDomain: hostnameWithSubdomain,
-		Domain:         domain,
-		RequireTLS:     requireTLS,
-		Proxy:          proxy,
+	router.SubdomainEndpoint.Store(options.MatchingDomain, &ProxyEndpoint{
+		RootOrMatchingDomain: options.MatchingDomain,
+		Domain:               domain,
+		RequireTLS:           options.RequireTLS,
+		Proxy:                proxy,
+		SkipCertValidations:  options.SkipCertValidations,
+		RequireBasicAuth:     options.RequireBasicAuth,
+		BasicAuthCredentials: options.BasicAuthCredentials,
 	})
 
-	log.Println("Adding Subdomain Rule: ", hostnameWithSubdomain+" to "+domain)
+	log.Println("Adding Subdomain Rule: ", options.MatchingDomain+" to "+domain)
 	return nil
 }

src/mod/dynamicproxy/typedef.go

@@ -0,0 +1,112 @@
+package dynamicproxy
+
+import (
+	"net"
+	"net/http"
+	"sync"
+
+	"imuslab.com/zoraxy/mod/dynamicproxy/dpcore"
+	"imuslab.com/zoraxy/mod/dynamicproxy/redirection"
+	"imuslab.com/zoraxy/mod/geodb"
+	"imuslab.com/zoraxy/mod/statistic"
+	"imuslab.com/zoraxy/mod/tlscert"
+)
+
+const (
+	ProxyType_Subdomain = 0
+	ProxyType_Vdir      = 1
+)
+
+type ProxyHandler struct {
+	Parent *Router
+}
+
+type RouterOption struct {
+	Port               int
+	UseTls             bool
+	ForceHttpsRedirect bool
+	TlsManager         *tlscert.Manager
+	RedirectRuleTable  *redirection.RuleTable
+	GeodbStore         *geodb.Store
+	StatisticCollector *statistic.Collector
+}
+
+type Router struct {
+	Option            *RouterOption
+	ProxyEndpoints    *sync.Map
+	SubdomainEndpoint *sync.Map
+	Running           bool
+	Root              *ProxyEndpoint
+	mux               http.Handler
+	server            *http.Server
+	tlsListener       net.Listener
+	routingRules      []*RoutingRule
+
+	tlsRedirectStop chan bool
+}
+
+// Auth credential for basic auth on certain endpoints
+type BasicAuthCredentials struct {
+	Username     string
+	PasswordHash string
+}
+
+// Auth credential for basic auth on certain endpoints
+type BasicAuthUnhashedCredentials struct {
+	Username string
+	Password string
+}
+
+// A proxy endpoint record
+type ProxyEndpoint struct {
+	ProxyType            int                     //The type of this proxy, see const def
+	RootOrMatchingDomain string                  //Root for vdir or Matching domain for subd
+	Domain               string                  //Domain or IP to proxy to
+	RequireTLS           bool                    //Target domain require TLS
+	SkipCertValidations  bool                    //Set to true to accept self signed certs
+	RequireBasicAuth     bool                    //Set to true to request basic auth before proxy
+	BasicAuthCredentials []*BasicAuthCredentials `json:"-"`
+	Proxy                *dpcore.ReverseProxy    `json:"-"`
+}
+
+type RootOptions struct {
+	ProxyLocation        string
+	RequireTLS           bool
+	SkipCertValidations  bool
+	RequireBasicAuth     bool
+	BasicAuthCredentials []*BasicAuthCredentials
+}
+
+type VdirOptions struct {
+	RootName             string
+	Domain               string
+	RequireTLS           bool
+	SkipCertValidations  bool
+	RequireBasicAuth     bool
+	BasicAuthCredentials []*BasicAuthCredentials
+}
+
+type SubdOptions struct {
+	MatchingDomain       string
+	Domain               string
+	RequireTLS           bool
+	SkipCertValidations  bool
+	RequireBasicAuth     bool
+	BasicAuthCredentials []*BasicAuthCredentials
+}
+
+/*
+type ProxyEndpoint struct {
+	Root string
+	Domain         string
+	RequireTLS     bool
+	Proxy          *reverseproxy.ReverseProxy `json:"-"`
+}
+
+type SubdomainEndpoint struct {
+	MatchingDomain string
+	Domain         string
+	RequireTLS     bool
+	Proxy          *reverseproxy.ReverseProxy `json:"-"`
+}
+*/