Updated a lot of stuffs

+ Added comments for whitelist
+ Added automatic cert pick for multi-host certs (SNI)
+ Renamed .crt to .pem for cert store
+ Added best-fit selection for wildcard matching rules
+ Added x-proxy-by header
+ Added X-real-Ip header
+ Added Development Mode (Cache-Control: no-store)
+ Updated utm timeout to 10 seconds instead of 90

src/mod/dynamicproxy/Server.go

@@ -1,8 +1,6 @@
 package dynamicproxy
 
 import (
-	_ "embed"
-	"errors"
 	"net/http"
 	"net/url"
 	"os"
@@ -25,11 +23,6 @@ import (
 	- Vitrual Directory Routing
 */
 
-var (
-	//go:embed tld.json
-	rawTldMap []byte
-)
-
 func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	/*
 		Special Routing Rules, bypass most of the limitations
@@ -52,10 +45,12 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	//Inject debug headers
+	w.Header().Set("x-proxy-by", "zoraxy/"+h.Parent.Option.HostVersion)
+
 	/*
 		General Access Check
 	*/
-
 	respWritten := h.handleAccessRouting(w, r)
 	if respWritten {
 		return
@@ -81,6 +76,7 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	/*
 		Host Routing
 	*/
+
 	sep := h.Parent.getProxyEndpointFromHostname(domainOnly)
 	if sep != nil && !sep.Disabled {
 		if sep.RequireBasicAuth {
@@ -235,44 +231,3 @@ func (h *ProxyHandler) handleAccessRouting(w http.ResponseWriter, r *http.Reques
 
 	return false
 }
-
-// Return if the given host is already topped (e.g. example.com or example.co.uk) instead of
-// a host with subdomain (e.g. test.example.com)
-func (h *ProxyHandler) isTopLevelRedirectableDomain(requestHost string) bool {
-	parts := strings.Split(requestHost, ".")
-	if len(parts) > 2 {
-		//Cases where strange tld is used like .co.uk or .com.hk
-		_, ok := h.Parent.tldMap[strings.Join(parts[1:], ".")]
-		if ok {
-			//Already topped
-			return true
-		}
-	} else {
-		//Already topped
-		return true
-	}
-
-	return false
-}
-
-// GetTopLevelRedirectableDomain returns the toppest level of domain
-// that is redirectable. E.g. a.b.c.example.co.uk will return example.co.uk
-func (h *ProxyHandler) getTopLevelRedirectableDomain(unsetSubdomainHost string) (string, error) {
-	parts := strings.Split(unsetSubdomainHost, ".")
-	if h.isTopLevelRedirectableDomain(unsetSubdomainHost) {
-		//Already topped
-		return "", errors.New("already at top level domain")
-	}
-
-	for i := 0; i < len(parts); i++ {
-		possibleTld := parts[i:]
-		_, ok := h.Parent.tldMap[strings.Join(possibleTld, ".")]
-		if ok {
-			//This is tld length
-			tld := strings.Join(parts[i-1:], ".")
-			return "//" + tld, nil
-		}
-	}
-
-	return "", errors.New("unsupported top level domain given")
-}

src/mod/dynamicproxy/dpcore/dpcore.go

@@ -60,6 +60,7 @@ type ResponseRewriteRuleSet struct {
 	ProxyDomain  string
 	OriginalHost string
 	UseTLS       bool
+	NoCache      bool
 	PathPrefix   string //Vdir prefix for root, / will be rewrite to this
 }
 
@@ -243,7 +244,7 @@ func (p *ReverseProxy) logf(format string, args ...interface{}) {
 	}
 }
 
-func removeHeaders(header http.Header) {
+func removeHeaders(header http.Header, noCache bool) {
 	// Remove hop-by-hop headers listed in the "Connection" header.
 	if c := header.Get("Connection"); c != "" {
 		for _, f := range strings.Split(c, ",") {
@@ -260,9 +261,16 @@ func removeHeaders(header http.Header) {
 		}
 	}
 
-	if header.Get("A-Upgrade") != "" {
-		header.Set("Upgrade", header.Get("A-Upgrade"))
-		header.Del("A-Upgrade")
+	//Restore the Upgrade header if any
+	if header.Get("Zr-Origin-Upgrade") != "" {
+		header.Set("Upgrade", header.Get("Zr-Origin-Upgrade"))
+		header.Del("Zr-Origin-Upgrade")
+	}
+
+	//Disable cache if nocache is set
+	if noCache {
+		header.Del("Cache-Control")
+		header.Set("Cache-Control", "no-store")
 	}
 }
 
@@ -281,6 +289,11 @@ func addXForwardedForHeader(req *http.Request) {
 			req.Header.Set("X-Forwarded-Proto", "http")
 		}
 
+		if req.Header.Get("X-Real-Ip") == "" {
+			//Not exists. Fill it in with client IP
+			req.Header.Set("X-Real-Ip", clientIP)
+		}
+
 	}
 }
 
@@ -323,7 +336,7 @@ func (p *ReverseProxy) ProxyHTTP(rw http.ResponseWriter, req *http.Request, rrr
 	copyHeader(outreq.Header, req.Header)
 
 	// Remove hop-by-hop headers listed in the "Connection" header, Remove hop-by-hop headers.
-	removeHeaders(outreq.Header)
+	removeHeaders(outreq.Header, rrr.NoCache)
 
 	// Add X-Forwarded-For Header.
 	addXForwardedForHeader(outreq)
@@ -339,7 +352,7 @@ func (p *ReverseProxy) ProxyHTTP(rw http.ResponseWriter, req *http.Request, rrr
 	}
 
 	// Remove hop-by-hop headers listed in the "Connection" header of the response, Remove hop-by-hop headers.
-	removeHeaders(res.Header)
+	removeHeaders(res.Header, rrr.NoCache)
 
 	if p.ModifyResponse != nil {
 		if err := p.ModifyResponse(res); err != nil {

src/mod/dynamicproxy/dynamicproxy.go

@@ -35,12 +35,6 @@ func NewDynamicProxy(option RouterOption) (*Router, error) {
 		Parent: &thisRouter,
 	}
 
-	//Prase the tld map for tld redirection in main router
-	//See Server.go declarations
-	if len(rawTldMap) > 0 {
-		json.Unmarshal(rawTldMap, &thisRouter.tldMap)
-	}
-
 	return &thisRouter, nil
 }
 
@@ -74,12 +68,12 @@ func (router *Router) UpdateHttpToHttpsRedirectSetting(useRedirect bool) {
 func (router *Router) StartProxyService() error {
 	//Create a new server object
 	if router.server != nil {
-		return errors.New("Reverse proxy server already running")
+		return errors.New("reverse proxy server already running")
 	}
 
 	//Check if root route is set
 	if router.Root == nil {
-		return errors.New("Reverse proxy router root not set")
+		return errors.New("reverse proxy router root not set")
 	}
 
 	minVersion := tls.VersionTLS10
@@ -92,16 +86,6 @@ func (router *Router) StartProxyService() error {
 	}
 
 	if router.Option.UseTls {
-		/*
-			//Serve with TLS mode
-			ln, err := tls.Listen("tcp", ":"+strconv.Itoa(router.Option.Port), config)
-			if err != nil {
-				log.Println(err)
-				router.Running = false
-				return err
-			}
-			router.tlsListener = ln
-		*/
 		router.server = &http.Server{
 			Addr:      ":" + strconv.Itoa(router.Option.Port),
 			Handler:   router.mux,
@@ -216,7 +200,7 @@ func (router *Router) StartProxyService() error {
 
 func (router *Router) StopProxyService() error {
 	if router.server == nil {
-		return errors.New("Reverse proxy server already stopped")
+		return errors.New("reverse proxy server already stopped")
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
@@ -251,6 +235,7 @@ func (router *Router) Restart() error {
 			return err
 		}
 
+		time.Sleep(300 * time.Millisecond)
 		// Start the server
 		err = router.StartProxyService()
 		if err != nil {

src/mod/dynamicproxy/endpoints.go

@@ -7,7 +7,13 @@ import (
 )
 
 /*
-	Endpoint Functions
+	endpoint.go
+	author: tobychui
+
+	This script handle the proxy endpoint object actions
+	so proxyEndpoint can be handled like a proper oop object
+
+	Most of the functions are implemented in dynamicproxy.go
 */
 
 // Get virtual directory handler from given URI
@@ -87,3 +93,16 @@ func (ep *ProxyEndpoint) Clone() *ProxyEndpoint {
 	json.Unmarshal(js, &clonedProxyEndpoint)
 	return &clonedProxyEndpoint
 }
+
+// Remove this proxy endpoint from running proxy endpoint list
+func (ep *ProxyEndpoint) Remove() error {
+	ep.parent.ProxyEndpoints.Delete(ep.RootOrMatchingDomain)
+	return nil
+}
+
+// Write changes to runtime without respawning the proxy handler
+// use prepare -> remove -> add if you change anything in the endpoint
+// that effects the proxy routing src / dest
+func (ep *ProxyEndpoint) UpdateToRuntime() {
+	ep.parent.ProxyEndpoints.Store(ep.RootOrMatchingDomain, ep)
+}

src/mod/dynamicproxy/proxyRequestHandler.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"imuslab.com/zoraxy/mod/dynamicproxy/dpcore"
@@ -37,6 +38,7 @@ func (router *Router) getProxyEndpointFromHostname(hostname string) *ProxyEndpoi
 	}
 
 	//No hit. Try with wildcard
+	matchProxyEndpoints := []*ProxyEndpoint{}
 	router.ProxyEndpoints.Range(func(k, v interface{}) bool {
 		ep := v.(*ProxyEndpoint)
 		match, err := filepath.Match(ep.RootOrMatchingDomain, hostname)
@@ -45,12 +47,24 @@ func (router *Router) getProxyEndpointFromHostname(hostname string) *ProxyEndpoi
 			return true
 		}
 		if match {
-			targetSubdomainEndpoint = ep
-			return false
+			//targetSubdomainEndpoint = ep
+			matchProxyEndpoints = append(matchProxyEndpoints, ep)
+			return true
 		}
 		return true
 	})
 
+	if len(matchProxyEndpoints) == 1 {
+		//Only 1 match
+		return matchProxyEndpoints[0]
+	} else if len(matchProxyEndpoints) > 1 {
+		//More than one match. Get the best match one
+		sort.Slice(matchProxyEndpoints, func(i, j int) bool {
+			return matchProxyEndpoints[i].RootOrMatchingDomain < matchProxyEndpoints[j].RootOrMatchingDomain
+		})
+		return matchProxyEndpoints[0]
+	}
+
 	return targetSubdomainEndpoint
 }
 
@@ -77,7 +91,7 @@ func (h *ProxyHandler) hostRequest(w http.ResponseWriter, r *http.Request, targe
 	requestURL := r.URL.String()
 	if r.Header["Upgrade"] != nil && strings.ToLower(r.Header["Upgrade"][0]) == "websocket" {
 		//Handle WebSocket request. Forward the custom Upgrade header and rewrite origin
-		r.Header.Set("A-Upgrade", "websocket")
+		r.Header.Set("Zr-Origin-Upgrade", "websocket")
 		wsRedirectionEndpoint := target.Domain
 		if wsRedirectionEndpoint[len(wsRedirectionEndpoint)-1:] != "/" {
 			//Append / to the end of the redirection endpoint if not exists
@@ -109,6 +123,7 @@ func (h *ProxyHandler) hostRequest(w http.ResponseWriter, r *http.Request, targe
 		ProxyDomain:  target.Domain,
 		OriginalHost: originalHostHeader,
 		UseTLS:       target.RequireTLS,
+		NoCache:      h.Parent.Option.NoCache,
 		PathPrefix:   "",
 	})
 
@@ -137,7 +152,7 @@ func (h *ProxyHandler) vdirRequest(w http.ResponseWriter, r *http.Request, targe
 	r.Header.Set("X-Forwarded-Server", "zoraxy-"+h.Parent.Option.HostUUID)
 	if r.Header["Upgrade"] != nil && strings.ToLower(r.Header["Upgrade"][0]) == "websocket" {
 		//Handle WebSocket request. Forward the custom Upgrade header and rewrite origin
-		r.Header.Set("A-Upgrade", "websocket")
+		r.Header.Set("Zr-Origin-Upgrade", "websocket")
 		wsRedirectionEndpoint := target.Domain
 		if wsRedirectionEndpoint[len(wsRedirectionEndpoint)-1:] != "/" {
 			wsRedirectionEndpoint = wsRedirectionEndpoint + "/"

src/mod/dynamicproxy/router.go

@@ -97,3 +97,13 @@ func (router *Router) SetProxyRouteAsRoot(endpoint *ProxyEndpoint) error {
 	router.Root = endpoint
 	return nil
 }
+
+// ProxyEndpoint remove provide global access by key
+func (router *Router) RemoveProxyEndpointByRootname(rootnameOrMatchingDomain string) error {
+	targetEpt, err := router.LoadProxy(rootnameOrMatchingDomain)
+	if err != nil {
+		return err
+	}
+
+	return targetEpt.Remove()
+}

src/mod/dynamicproxy/typedef.go

@@ -25,9 +25,11 @@ type ProxyHandler struct {
 
 type RouterOption struct {
 	HostUUID           string //The UUID of Zoraxy, use for heading mod
+	HostVersion        string //The version of Zoraxy, use for heading mod
 	Port               int    //Incoming port
 	UseTls             bool   //Use TLS to serve incoming requsts
 	ForceTLSLatest     bool   //Force TLS1.2 or above
+	NoCache            bool   //Force set Cache-Control: no-store
 	ListenOnPort80     bool   //Enable port 80 http listener
 	ForceHttpsRedirect bool   //Force redirection of http to https endpoint
 	TlsManager         *tlscert.Manager