Added load balancer module

- Added load balancer module wip
- Updated geoipv4
- Reduced uptime timeout to 5 sec
- Optimized rate limit implementation
- Fixed minor UI bug in stream proxy
This commit is contained in:
Toby Chui 2024-06-19 00:38:48 +08:00
parent 704980d4f8
commit 973d0b3372
13 changed files with 3013 additions and 2207 deletions

View File

@ -17,6 +17,7 @@ import (
"imuslab.com/zoraxy/mod/auth"
"imuslab.com/zoraxy/mod/database"
"imuslab.com/zoraxy/mod/dockerux"
"imuslab.com/zoraxy/mod/dynamicproxy/loadbalance"
"imuslab.com/zoraxy/mod/dynamicproxy/redirection"
"imuslab.com/zoraxy/mod/email"
"imuslab.com/zoraxy/mod/forwardproxy"
@ -68,24 +69,25 @@ var (
/*
Handler Modules
*/
sysdb *database.Database //System database
authAgent *auth.AuthAgent //Authentication agent
tlsCertManager *tlscert.Manager //TLS / SSL management
redirectTable *redirection.RuleTable //Handle special redirection rule sets
pathRuleHandler *pathrule.Handler //Handle specific path blocking or custom headers
geodbStore *geodb.Store //GeoIP database, for resolving IP into country code
accessController *access.Controller //Access controller, handle black list and white list
netstatBuffers *netstat.NetStatBuffers //Realtime graph buffers
statisticCollector *statistic.Collector //Collecting statistic from visitors
uptimeMonitor *uptime.Monitor //Uptime monitor service worker
mdnsScanner *mdns.MDNSHost //mDNS discovery services
ganManager *ganserv.NetworkManager //Global Area Network Manager
webSshManager *sshprox.Manager //Web SSH connection service
streamProxyManager *streamproxy.Manager //Stream Proxy Manager for TCP / UDP forwarding
acmeHandler *acme.ACMEHandler //Handler for ACME Certificate renew
acmeAutoRenewer *acme.AutoRenewer //Handler for ACME auto renew ticking
staticWebServer *webserv.WebServer //Static web server for hosting simple stuffs
forwardProxy *forwardproxy.Handler //HTTP Forward proxy, basically VPN for web browser
sysdb *database.Database //System database
authAgent *auth.AuthAgent //Authentication agent
tlsCertManager *tlscert.Manager //TLS / SSL management
redirectTable *redirection.RuleTable //Handle special redirection rule sets
loadbalancer *loadbalance.RouteManager //Load balancer manager to get routing targets from proxy rules
pathRuleHandler *pathrule.Handler //Handle specific path blocking or custom headers
geodbStore *geodb.Store //GeoIP database, for resolving IP into country code
accessController *access.Controller //Access controller, handle black list and white list
netstatBuffers *netstat.NetStatBuffers //Realtime graph buffers
statisticCollector *statistic.Collector //Collecting statistic from visitors
uptimeMonitor *uptime.Monitor //Uptime monitor service worker
mdnsScanner *mdns.MDNSHost //mDNS discovery services
ganManager *ganserv.NetworkManager //Global Area Network Manager
webSshManager *sshprox.Manager //Web SSH connection service
streamProxyManager *streamproxy.Manager //Stream Proxy Manager for TCP / UDP forwarding
acmeHandler *acme.ACMEHandler //Handler for ACME Certificate renew
acmeAutoRenewer *acme.AutoRenewer //Handler for ACME auto renew ticking
staticWebServer *webserv.WebServer //Static web server for hosting simple stuffs
forwardProxy *forwardproxy.Handler //HTTP Forward proxy, basically VPN for web browser
//Helper modules
EmailSender *email.Sender //Email sender that handle email sending

View File

@ -17,7 +17,7 @@ import (
// return upstream header and downstream header key-value pairs
// if the header is expected to be deleted, the value will be set to empty string
func (ept *ProxyEndpoint) SplitInboundOutboundHeaders() ([][]string, [][]string) {
if len(ept.UserDefinedHeaders) == 0 {
if len(ept.UserDefinedHeaders) == 0 && ept.HSTSMaxAge == 0 && !ept.EnablePermissionPolicyHeader {
//Early return if there are no defined headers
return [][]string{}, [][]string{}
}

View File

@ -0,0 +1,60 @@
package loadbalance
import (
"imuslab.com/zoraxy/mod/geodb"
"imuslab.com/zoraxy/mod/info/logger"
"imuslab.com/zoraxy/mod/uptime"
)
/*
Load Balancer
Handleing load balance request for upstream destinations
*/
type BalancePolicy int
const (
BalancePolicy_RoundRobin BalancePolicy = 0 //Round robin, will ignore upstream if down
BalancePolicy_Fallback BalancePolicy = 1 //Fallback only. Will only switch to next node if the first one failed
BalancePolicy_Random BalancePolicy = 2 //Random, randomly pick one from the list that is online
BalancePolicy_GeoRegion BalancePolicy = 3 //Use the one defined for this geo-location, when down, pick the next avaible node
)
type LoadBalanceRule struct {
Upstreams []string //Reverse proxy upstream servers
LoadBalancePolicy BalancePolicy //Policy in deciding which target IP to proxy
UseRegionLock bool //If this is enabled with BalancePolicy_Geo, when the main site failed, it will not pick another node
UseStickySession bool //Use sticky session, if you are serving EU countries, make sure to add the "Do you want cookie" warning
parent *RouteManager
}
type Options struct {
Geodb *geodb.Store //GeoIP resolver for checking incoming request origin country
UptimeMonitor *uptime.Monitor //For checking if the target is online, this might be nil when the module starts
}
type RouteManager struct {
Options Options
Logger *logger.Logger
}
// Create a new load balance route manager
func NewRouteManager(options *Options, logger *logger.Logger) *RouteManager {
newManager := RouteManager{
Options: *options,
Logger: logger,
}
logger.PrintAndLog("INFO", "Load Balance Route Manager started", nil)
return &newManager
}
func (b *LoadBalanceRule) GetProxyTargetIP() {
}
// Print debug message
func (m *RouteManager) debugPrint(message string, err error) {
m.Logger.PrintAndLog("LoadBalancer", message, err)
}

View File

@ -1,7 +1,7 @@
package redirection
import (
"log"
"errors"
"net/http"
"strings"
)
@ -52,7 +52,7 @@ func (t *RuleTable) HandleRedirect(w http.ResponseWriter, r *http.Request) int {
//Invalid usage
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("500 - Internal Server Error"))
log.Println("Target request URL do not have matching redirect rule. Check with IsRedirectable before calling HandleRedirect!")
t.log("Target request URL do not have matching redirect rule. Check with IsRedirectable before calling HandleRedirect!", errors.New("invalid usage"))
return 500
}
}

View File

@ -30,11 +30,12 @@ type RedirectRules struct {
StatusCode int //Status Code for redirection
}
func NewRuleTable(configPath string, allowRegex bool) (*RuleTable, error) {
func NewRuleTable(configPath string, allowRegex bool, logger *logger.Logger) (*RuleTable, error) {
thisRuleTable := RuleTable{
rules: sync.Map{},
configPath: configPath,
AllowRegex: allowRegex,
Logger: logger,
}
//Load all the rules from the config path
if !utils.FileExists(configPath) {
@ -67,7 +68,7 @@ func NewRuleTable(configPath string, allowRegex bool) (*RuleTable, error) {
//Map the rules into the sync map
for _, rule := range rules {
log.Println("Redirection rule added: " + rule.RedirectURL + " -> " + rule.TargetURL)
thisRuleTable.log("Redirection rule added: "+rule.RedirectURL+" -> "+rule.TargetURL, nil)
thisRuleTable.rules.Store(rule.RedirectURL, rule)
}
@ -92,7 +93,7 @@ func (t *RuleTable) AddRedirectRule(redirectURL string, destURL string, forwardP
// Create a new file for writing the JSON data
file, err := os.Create(filepath)
if err != nil {
log.Printf("Error creating file %s: %s", filepath, err)
t.log("Error creating file "+filepath, err)
return err
}
defer file.Close()
@ -100,7 +101,7 @@ func (t *RuleTable) AddRedirectRule(redirectURL string, destURL string, forwardP
// Encode the RedirectRules object to JSON and write it to the file
err = json.NewEncoder(file).Encode(newRule)
if err != nil {
log.Printf("Error encoding JSON to file %s: %s", filepath, err)
t.log("Error encoding JSON to file "+filepath, err)
return err
}
@ -125,7 +126,7 @@ func (t *RuleTable) DeleteRedirectRule(redirectURL string) error {
// Delete the file
if err := os.Remove(filepath); err != nil {
log.Printf("Error deleting file %s: %s", filepath, err)
t.log("Error deleting file "+filepath, err)
return err
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -227,7 +227,7 @@ func getWebsiteStatus(url string) (int, error) {
client := http.Client{
Jar: jar,
Timeout: 10 * time.Second,
Timeout: 5 * time.Second,
}
req, _ := http.NewRequest("GET", url, nil)

View File

@ -144,6 +144,10 @@ func ReverseProxtInit() {
Interval: 300, //5 minutes
MaxRecordsStore: 288, //1 day
})
//Pass the pointer of this uptime monitor into the load balancer
loadbalancer.Options.UptimeMonitor = uptimeMonitor
SystemWideLogger.Println("Uptime Monitor background service started")
}()
}
@ -221,7 +225,7 @@ func ReverseProxyHandleAddEndpoint(w http.ResponseWriter, r *http.Request) {
return
}
//Require basic auth?
// Require basic auth?
rba, _ := utils.PostPara(r, "bauth")
if rba == "" {
rba = "false"
@ -230,23 +234,26 @@ func ReverseProxyHandleAddEndpoint(w http.ResponseWriter, r *http.Request) {
requireBasicAuth := (rba == "true")
// Require Rate Limiting?
rl, _ := utils.PostPara(r, "rate")
if rl == "" {
rl = "false"
}
requireRateLimit := (rl == "true")
rlnum, _ := utils.PostPara(r, "ratenum")
if rlnum == "" {
rlnum = "0"
}
proxyRateLimit, err := strconv.ParseInt(rlnum, 10, 64)
requireRateLimit := false
proxyRateLimit := 1000
requireRateLimit, err = utils.PostBool(r, "rate")
if err != nil {
utils.SendErrorResponse(w, "invalid rate limit number")
return
requireRateLimit = false
}
if proxyRateLimit <= 0 {
utils.SendErrorResponse(w, "rate limit number must be greater than 0")
return
if requireRateLimit {
proxyRateLimit, err = utils.PostInt(r, "ratenum")
if err != nil {
proxyRateLimit = 0
}
if err != nil {
utils.SendErrorResponse(w, "invalid rate limit number")
return
}
if proxyRateLimit <= 0 {
utils.SendErrorResponse(w, "rate limit number must be greater than 0")
return
}
}
// Bypass WebSocket Origin Check
@ -331,7 +338,7 @@ func ReverseProxyHandleAddEndpoint(w http.ResponseWriter, r *http.Request) {
DefaultSiteValue: "",
// Rate Limit
RequireRateLimit: requireRateLimit,
RateLimit: proxyRateLimit,
RateLimit: int64(proxyRateLimit),
}
preparedEndpoint, err := dynamicProxyRouter.PrepareProxyRoute(&thisProxyEndpoint)

View File

@ -14,6 +14,7 @@ import (
"imuslab.com/zoraxy/mod/auth"
"imuslab.com/zoraxy/mod/database"
"imuslab.com/zoraxy/mod/dockerux"
"imuslab.com/zoraxy/mod/dynamicproxy/loadbalance"
"imuslab.com/zoraxy/mod/dynamicproxy/redirection"
"imuslab.com/zoraxy/mod/forwardproxy"
"imuslab.com/zoraxy/mod/ganserv"
@ -75,15 +76,22 @@ func startupSequence() {
panic(err)
}
//Create a system wide logger
l, err := logger.NewLogger("zr", "./log", *logOutputToFile)
if err == nil {
SystemWideLogger = l
} else {
panic(err)
}
//Create a redirection rule table
db.NewTable("redirect")
redirectAllowRegexp := false
db.Read("redirect", "regex", &redirectAllowRegexp)
redirectTable, err = redirection.NewRuleTable("./conf/redirect", redirectAllowRegexp)
redirectTable, err = redirection.NewRuleTable("./conf/redirect", redirectAllowRegexp, SystemWideLogger)
if err != nil {
panic(err)
}
redirectTable.Logger = SystemWideLogger
//Create a geodb store
geodbStore, err = geodb.NewGeoDb(sysdb, &geodb.StoreOptions{
@ -94,6 +102,11 @@ func startupSequence() {
panic(err)
}
//Create a load balance route manager
loadbalancer = loadbalance.NewRouteManager(&loadbalance.Options{
Geodb: geodbStore,
}, SystemWideLogger)
//Create the access controller
accessController, err = access.NewAccessController(&access.Options{
Database: sysdb,
@ -112,14 +125,6 @@ func startupSequence() {
panic(err)
}
//Create a system wide logger
l, err := logger.NewLogger("zr", "./log", *logOutputToFile)
if err == nil {
SystemWideLogger = l
} else {
panic(err)
}
//Start the static web server
staticWebServer = webserv.NewWebServer(&webserv.WebServerOptions{
Sysdb: sysdb,

View File

@ -246,7 +246,7 @@
input = `
<div class="ui mini fluid input">
<input type="text" class="Domain" value="${domain}">
<input type="text" class="Domain" onchange="cleanProxyTargetValue(this)" value="${domain}">
</div>
<div class="ui checkbox" style="margin-top: 0.6em;">
<input type="checkbox" class="RequireTLS" ${tls}>
@ -434,6 +434,21 @@
}
})
}
//Clearn the proxy target value, make sure user do not enter http:// or https://
//and auto select TLS checkbox if https:// exists
function cleanProxyTargetValue(input){
let targetDomain = $(input).val().trim();
if (targetDomain.startsWith("http://")){
targetDomain = targetDomain.substr(7);
$(input).val(targetDomain);
$("#httpProxyList input.RequireTLS").parent().checkbox("set unchecked");
}else if (targetDomain.startsWith("https://")){
targetDomain = targetDomain.substr(8);
$(input).val(targetDomain);
$("#httpProxyList input.RequireTLS").parent().checkbox("set checked");
}
}
/* button events */
function editBasicAuthCredentials(uuid){

View File

@ -123,6 +123,7 @@
$('#streamProxyForm input, #streamProxyForm select').val('');
$('#streamProxyForm select').dropdown('clear');
$("#streamProxyForm input[name=timeout]").val(10);
$("#streamProxyForm .toggle.checkbox").checkbox("set unchecked");
}
function cancelStreamProxyEdit(event=undefined) {
@ -305,6 +306,7 @@
}
initProxyConfigList();
cancelStreamProxyEdit();
clearStreamProxyAddEditForm();
},
error: function() {

View File

@ -14,7 +14,7 @@
<script src="https://cdnjs.cloudflare.com/ajax/libs/fomantic-ui/2.9.2/semantic.min.js"></script>
<title>404 - Host Not Found</title>
<style>
h1, h2, h3, h4, h5, p, a, span{
h1, h2, h3, h4, h5, p, a, span, .ui.list .item{
font-family: 'Noto Sans TC', sans-serif;
font-weight: 300;
color: rgb(88, 88, 88)
@ -22,9 +22,6 @@
.diagram{
background-color: #ebebeb;
box-shadow:
inset 0px 11px 8px -10px #CCC,
inset 0px -11px 8px -10px #CCC;
padding-bottom: 2em;
}
@ -135,7 +132,7 @@
<p>Please try again in a few minutes</p>
<h5 style="font-weight: 500;">If you are the owner of this website:</h5>
<div class="ui bulleted list">
<div class="item">Check if the target web server is online</div>
<div class="item">Check if the proxy rules that match this hostname exists</div>
<div class="item">Visit the Reverse Proxy management interface to correct any setting errors</div>
</div>
</div>