Added load balancer module

- Added load balancer module wip
- Updated geoipv4
- Reduced uptime timeout to 5 sec
- Optimized rate limit implementation
- Fixed minor UI bug in stream proxy

src/main.go

@@ -17,6 +17,7 @@ import (
 	"imuslab.com/zoraxy/mod/auth"
 	"imuslab.com/zoraxy/mod/database"
 	"imuslab.com/zoraxy/mod/dockerux"
+	"imuslab.com/zoraxy/mod/dynamicproxy/loadbalance"
 	"imuslab.com/zoraxy/mod/dynamicproxy/redirection"
 	"imuslab.com/zoraxy/mod/email"
 	"imuslab.com/zoraxy/mod/forwardproxy"
@@ -68,24 +69,25 @@ var (
 	/*
 		Handler Modules
 	*/
-	sysdb              *database.Database      //System database
+	sysdb              *database.Database        //System database
-	authAgent          *auth.AuthAgent         //Authentication agent
+	authAgent          *auth.AuthAgent           //Authentication agent
-	tlsCertManager     *tlscert.Manager        //TLS / SSL management
+	tlsCertManager     *tlscert.Manager          //TLS / SSL management
-	redirectTable      *redirection.RuleTable  //Handle special redirection rule sets
+	redirectTable      *redirection.RuleTable    //Handle special redirection rule sets
-	pathRuleHandler    *pathrule.Handler       //Handle specific path blocking or custom headers
+	loadbalancer       *loadbalance.RouteManager //Load balancer manager to get routing targets from proxy rules
-	geodbStore         *geodb.Store            //GeoIP database, for resolving IP into country code
+	pathRuleHandler    *pathrule.Handler         //Handle specific path blocking or custom headers
-	accessController   *access.Controller      //Access controller, handle black list and white list
+	geodbStore         *geodb.Store              //GeoIP database, for resolving IP into country code
-	netstatBuffers     *netstat.NetStatBuffers //Realtime graph buffers
+	accessController   *access.Controller        //Access controller, handle black list and white list
-	statisticCollector *statistic.Collector    //Collecting statistic from visitors
+	netstatBuffers     *netstat.NetStatBuffers   //Realtime graph buffers
-	uptimeMonitor      *uptime.Monitor         //Uptime monitor service worker
+	statisticCollector *statistic.Collector      //Collecting statistic from visitors
-	mdnsScanner        *mdns.MDNSHost          //mDNS discovery services
+	uptimeMonitor      *uptime.Monitor           //Uptime monitor service worker
-	ganManager         *ganserv.NetworkManager //Global Area Network Manager
+	mdnsScanner        *mdns.MDNSHost            //mDNS discovery services
-	webSshManager      *sshprox.Manager        //Web SSH connection service
+	ganManager         *ganserv.NetworkManager   //Global Area Network Manager
-	streamProxyManager *streamproxy.Manager    //Stream Proxy Manager for TCP / UDP forwarding
+	webSshManager      *sshprox.Manager          //Web SSH connection service
-	acmeHandler        *acme.ACMEHandler       //Handler for ACME Certificate renew
+	streamProxyManager *streamproxy.Manager      //Stream Proxy Manager for TCP / UDP forwarding
-	acmeAutoRenewer    *acme.AutoRenewer       //Handler for ACME auto renew ticking
+	acmeHandler        *acme.ACMEHandler         //Handler for ACME Certificate renew
-	staticWebServer    *webserv.WebServer      //Static web server for hosting simple stuffs
+	acmeAutoRenewer    *acme.AutoRenewer         //Handler for ACME auto renew ticking
-	forwardProxy       *forwardproxy.Handler   //HTTP Forward proxy, basically VPN for web browser
+	staticWebServer    *webserv.WebServer        //Static web server for hosting simple stuffs
+	forwardProxy       *forwardproxy.Handler     //HTTP Forward proxy, basically VPN for web browser
 
 	//Helper modules
 	EmailSender       *email.Sender         //Email sender that handle email sending

src/mod/dynamicproxy/customHeader.go

@@ -17,7 +17,7 @@ import (
 // return upstream header and downstream header key-value pairs
 // if the header is expected to be deleted, the value will be set to empty string
 func (ept *ProxyEndpoint) SplitInboundOutboundHeaders() ([][]string, [][]string) {
-	if len(ept.UserDefinedHeaders) == 0 {
+	if len(ept.UserDefinedHeaders) == 0 && ept.HSTSMaxAge == 0 && !ept.EnablePermissionPolicyHeader {
 		//Early return if there are no defined headers
 		return [][]string{}, [][]string{}
 	}

src/mod/dynamicproxy/loadbalance/loadbalance.go

@@ -0,0 +1,60 @@
+package loadbalance
+
+import (
+	"imuslab.com/zoraxy/mod/geodb"
+	"imuslab.com/zoraxy/mod/info/logger"
+	"imuslab.com/zoraxy/mod/uptime"
+)
+
+/*
+	Load Balancer
+
+	Handleing load balance request for upstream destinations
+*/
+
+type BalancePolicy int
+
+const (
+	BalancePolicy_RoundRobin BalancePolicy = 0 //Round robin, will ignore upstream if down
+	BalancePolicy_Fallback   BalancePolicy = 1 //Fallback only. Will only switch to next node if the first one failed
+	BalancePolicy_Random     BalancePolicy = 2 //Random, randomly pick one from the list that is online
+	BalancePolicy_GeoRegion  BalancePolicy = 3 //Use the one defined for this geo-location, when down, pick the next avaible node
+)
+
+type LoadBalanceRule struct {
+	Upstreams         []string      //Reverse proxy upstream servers
+	LoadBalancePolicy BalancePolicy //Policy in deciding which target IP to proxy
+	UseRegionLock     bool          //If this is enabled with BalancePolicy_Geo, when the main site failed, it will not pick another node
+	UseStickySession  bool          //Use sticky session, if you are serving EU countries, make sure to add the "Do you want cookie" warning
+
+	parent *RouteManager
+}
+
+type Options struct {
+	Geodb         *geodb.Store    //GeoIP resolver for checking incoming request origin country
+	UptimeMonitor *uptime.Monitor //For checking if the target is online, this might be nil when the module starts
+}
+
+type RouteManager struct {
+	Options Options
+	Logger  *logger.Logger
+}
+
+// Create a new load balance route manager
+func NewRouteManager(options *Options, logger *logger.Logger) *RouteManager {
+	newManager := RouteManager{
+		Options: *options,
+		Logger:  logger,
+	}
+	logger.PrintAndLog("INFO", "Load Balance Route Manager started", nil)
+	return &newManager
+}
+
+func (b *LoadBalanceRule) GetProxyTargetIP() {
+
+}
+
+// Print debug message
+func (m *RouteManager) debugPrint(message string, err error) {
+	m.Logger.PrintAndLog("LoadBalancer", message, err)
+}

src/mod/dynamicproxy/redirection/handler.go

@@ -1,7 +1,7 @@
 package redirection
 
 import (
-	"log"
+	"errors"
 	"net/http"
 	"strings"
 )
@@ -52,7 +52,7 @@ func (t *RuleTable) HandleRedirect(w http.ResponseWriter, r *http.Request) int {
 		//Invalid usage
 		w.WriteHeader(http.StatusInternalServerError)
 		w.Write([]byte("500 - Internal Server Error"))
-		log.Println("Target request URL do not have matching redirect rule. Check with IsRedirectable before calling HandleRedirect!")
+		t.log("Target request URL do not have matching redirect rule. Check with IsRedirectable before calling HandleRedirect!", errors.New("invalid usage"))
 		return 500
 	}
 }

src/mod/dynamicproxy/redirection/redirection.go

@@ -30,11 +30,12 @@ type RedirectRules struct {
 	StatusCode       int    //Status Code for redirection
 }
 
-func NewRuleTable(configPath string, allowRegex bool) (*RuleTable, error) {
+func NewRuleTable(configPath string, allowRegex bool, logger *logger.Logger) (*RuleTable, error) {
 	thisRuleTable := RuleTable{
 		rules:      sync.Map{},
 		configPath: configPath,
 		AllowRegex: allowRegex,
+		Logger:     logger,
 	}
 	//Load all the rules from the config path
 	if !utils.FileExists(configPath) {
@@ -67,7 +68,7 @@ func NewRuleTable(configPath string, allowRegex bool) (*RuleTable, error) {
 
 	//Map the rules into the sync map
 	for _, rule := range rules {
-		log.Println("Redirection rule added: " + rule.RedirectURL + " -> " + rule.TargetURL)
+		thisRuleTable.log("Redirection rule added: "+rule.RedirectURL+" -> "+rule.TargetURL, nil)
 		thisRuleTable.rules.Store(rule.RedirectURL, rule)
 	}
 
@@ -92,7 +93,7 @@ func (t *RuleTable) AddRedirectRule(redirectURL string, destURL string, forwardP
 	// Create a new file for writing the JSON data
 	file, err := os.Create(filepath)
 	if err != nil {
-		log.Printf("Error creating file %s: %s", filepath, err)
+		t.log("Error creating file "+filepath, err)
 		return err
 	}
 	defer file.Close()
@@ -100,7 +101,7 @@ func (t *RuleTable) AddRedirectRule(redirectURL string, destURL string, forwardP
 	// Encode the RedirectRules object to JSON and write it to the file
 	err = json.NewEncoder(file).Encode(newRule)
 	if err != nil {
-		log.Printf("Error encoding JSON to file %s: %s", filepath, err)
+		t.log("Error encoding JSON to file "+filepath, err)
 		return err
 	}
 
@@ -125,7 +126,7 @@ func (t *RuleTable) DeleteRedirectRule(redirectURL string) error {
 
 	// Delete the file
 	if err := os.Remove(filepath); err != nil {
-		log.Printf("Error deleting file %s: %s", filepath, err)
+		t.log("Error deleting file "+filepath, err)
 		return err
 	}
 

src/mod/uptime/uptime.go

@@ -227,7 +227,7 @@ func getWebsiteStatus(url string) (int, error) {
 
 	client := http.Client{
 		Jar:     jar,
-		Timeout: 10 * time.Second,
+		Timeout: 5 * time.Second,
 	}
 
 	req, _ := http.NewRequest("GET", url, nil)

src/reverseproxy.go

@@ -144,6 +144,10 @@ func ReverseProxtInit() {
 			Interval:        300, //5 minutes
 			MaxRecordsStore: 288, //1 day
 		})
+
+		//Pass the pointer of this uptime monitor into the load balancer
+		loadbalancer.Options.UptimeMonitor = uptimeMonitor
+
 		SystemWideLogger.Println("Uptime Monitor background service started")
 	}()
 }
@@ -221,7 +225,7 @@ func ReverseProxyHandleAddEndpoint(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	//Require basic auth?
+	// Require basic auth?
 	rba, _ := utils.PostPara(r, "bauth")
 	if rba == "" {
 		rba = "false"
@@ -230,23 +234,26 @@ func ReverseProxyHandleAddEndpoint(w http.ResponseWriter, r *http.Request) {
 	requireBasicAuth := (rba == "true")
 
 	// Require Rate Limiting?
-	rl, _ := utils.PostPara(r, "rate")
+	requireRateLimit := false
-	if rl == "" {
+	proxyRateLimit := 1000
-		rl = "false"
+
-	}
+	requireRateLimit, err = utils.PostBool(r, "rate")
-	requireRateLimit := (rl == "true")
-	rlnum, _ := utils.PostPara(r, "ratenum")
-	if rlnum == "" {
-		rlnum = "0"
-	}
-	proxyRateLimit, err := strconv.ParseInt(rlnum, 10, 64)
 	if err != nil {
-		utils.SendErrorResponse(w, "invalid rate limit number")
+		requireRateLimit = false
-		return
 	}
-	if proxyRateLimit <= 0 {
+	if requireRateLimit {
-		utils.SendErrorResponse(w, "rate limit number must be greater than 0")
+		proxyRateLimit, err = utils.PostInt(r, "ratenum")
-		return
+		if err != nil {
+			proxyRateLimit = 0
+		}
+		if err != nil {
+			utils.SendErrorResponse(w, "invalid rate limit number")
+			return
+		}
+		if proxyRateLimit <= 0 {
+			utils.SendErrorResponse(w, "rate limit number must be greater than 0")
+			return
+		}
 	}
 
 	// Bypass WebSocket Origin Check
@@ -331,7 +338,7 @@ func ReverseProxyHandleAddEndpoint(w http.ResponseWriter, r *http.Request) {
 			DefaultSiteValue:        "",
 			// Rate Limit
 			RequireRateLimit: requireRateLimit,
-			RateLimit:        proxyRateLimit,
+			RateLimit:        int64(proxyRateLimit),
 		}
 
 		preparedEndpoint, err := dynamicProxyRouter.PrepareProxyRoute(&thisProxyEndpoint)

src/start.go

@@ -14,6 +14,7 @@ import (
 	"imuslab.com/zoraxy/mod/auth"
 	"imuslab.com/zoraxy/mod/database"
 	"imuslab.com/zoraxy/mod/dockerux"
+	"imuslab.com/zoraxy/mod/dynamicproxy/loadbalance"
 	"imuslab.com/zoraxy/mod/dynamicproxy/redirection"
 	"imuslab.com/zoraxy/mod/forwardproxy"
 	"imuslab.com/zoraxy/mod/ganserv"
@@ -75,15 +76,22 @@ func startupSequence() {
 		panic(err)
 	}
 
+	//Create a system wide logger
+	l, err := logger.NewLogger("zr", "./log", *logOutputToFile)
+	if err == nil {
+		SystemWideLogger = l
+	} else {
+		panic(err)
+	}
+
 	//Create a redirection rule table
 	db.NewTable("redirect")
 	redirectAllowRegexp := false
 	db.Read("redirect", "regex", &redirectAllowRegexp)
-	redirectTable, err = redirection.NewRuleTable("./conf/redirect", redirectAllowRegexp)
+	redirectTable, err = redirection.NewRuleTable("./conf/redirect", redirectAllowRegexp, SystemWideLogger)
 	if err != nil {
 		panic(err)
 	}
-	redirectTable.Logger = SystemWideLogger
 
 	//Create a geodb store
 	geodbStore, err = geodb.NewGeoDb(sysdb, &geodb.StoreOptions{
@@ -94,6 +102,11 @@ func startupSequence() {
 		panic(err)
 	}
 
+	//Create a load balance route manager
+	loadbalancer = loadbalance.NewRouteManager(&loadbalance.Options{
+		Geodb: geodbStore,
+	}, SystemWideLogger)
+
 	//Create the access controller
 	accessController, err = access.NewAccessController(&access.Options{
 		Database:     sysdb,
@@ -112,14 +125,6 @@ func startupSequence() {
 		panic(err)
 	}
 
-	//Create a system wide logger
-	l, err := logger.NewLogger("zr", "./log", *logOutputToFile)
-	if err == nil {
-		SystemWideLogger = l
-	} else {
-		panic(err)
-	}
-
 	//Start the static web server
 	staticWebServer = webserv.NewWebServer(&webserv.WebServerOptions{
 		Sysdb:                  sysdb,

src/web/components/httprp.html

@@ -246,7 +246,7 @@
 
                 input = `
                     <div class="ui mini fluid input">
-                        <input type="text" class="Domain" value="${domain}">
+                        <input type="text" class="Domain" onchange="cleanProxyTargetValue(this)" value="${domain}">
                     </div>
                     <div class="ui checkbox" style="margin-top: 0.6em;">
                         <input type="checkbox" class="RequireTLS" ${tls}>
@@ -434,6 +434,21 @@
             }
         })
     }
+
+    //Clearn the proxy target value, make sure user do not enter http:// or https://
+    //and auto select TLS checkbox if https:// exists
+    function cleanProxyTargetValue(input){
+        let targetDomain = $(input).val().trim();
+        if (targetDomain.startsWith("http://")){
+            targetDomain = targetDomain.substr(7);
+            $(input).val(targetDomain);
+            $("#httpProxyList input.RequireTLS").parent().checkbox("set unchecked");
+        }else if (targetDomain.startsWith("https://")){
+            targetDomain = targetDomain.substr(8);
+            $(input).val(targetDomain);
+            $("#httpProxyList input.RequireTLS").parent().checkbox("set checked");
+        }
+    }
     
     /* button events */
     function editBasicAuthCredentials(uuid){

src/web/components/streamprox.html

@@ -123,6 +123,7 @@
             $('#streamProxyForm input, #streamProxyForm select').val('');
             $('#streamProxyForm select').dropdown('clear');
             $("#streamProxyForm input[name=timeout]").val(10);
+            $("#streamProxyForm .toggle.checkbox").checkbox("set unchecked");
         }
 
         function cancelStreamProxyEdit(event=undefined) {
@@ -305,6 +306,7 @@
                     }
                     initProxyConfigList();
                     cancelStreamProxyEdit();
+                    clearStreamProxyAddEditForm();
                     
                 },
                 error: function() {

src/web/hosterror.html

@@ -14,7 +14,7 @@
         <script src="https://cdnjs.cloudflare.com/ajax/libs/fomantic-ui/2.9.2/semantic.min.js"></script>
         <title>404 - Host Not Found</title>
         <style>
-            h1, h2, h3, h4, h5, p, a, span{
+            h1, h2, h3, h4, h5, p, a, span, .ui.list .item{
                 font-family: 'Noto Sans TC', sans-serif;
                 font-weight: 300;
                 color: rgb(88, 88, 88)
@@ -22,9 +22,6 @@
 
             .diagram{
                 background-color: #ebebeb;
-                box-shadow: 
-                    inset 0px 11px 8px -10px #CCC,
-                    inset 0px -11px 8px -10px #CCC; 
                 padding-bottom: 2em;
             }
 
@@ -135,7 +132,7 @@
                         <p>Please try again in a few minutes</p>
                         <h5 style="font-weight: 500;">If you are the owner of this website:</h5>
                         <div class="ui bulleted list">
-                            <div class="item">Check if the target web server is online</div>
+                            <div class="item">Check if the proxy rules that match this hostname exists</div>
                             <div class="item">Visit the Reverse Proxy management interface to correct any setting errors</div>
                         </div>
                     </div>