From 44c1e60fb82171fa9783a1e0f5163918e11088c9 Mon Sep 17 00:00:00 2001 From: Toby Chui Date: Sat, 15 Apr 2023 23:29:47 +0800 Subject: [PATCH] Added rule handles to dynamic proxy core --- src/mod/dynamicproxy/Server.go | 17 +- src/mod/dynamicproxy/dynamicproxy.go | 37 +++++ src/mod/dynamicproxy/special.go | 85 ++++++++++ src/mod/statistic/handler.go | 9 -- src/mod/statistic/statistic.go | 10 +- src/mod/statistic/structconv.go | 66 ++++++++ src/mod/uptime/uptime.go | 227 +++++++++++++++++++++++++++ src/mod/utils/utils.go | 55 ------- 8 files changed, 437 insertions(+), 69 deletions(-) create mode 100644 src/mod/dynamicproxy/special.go create mode 100644 src/mod/statistic/structconv.go create mode 100644 src/mod/uptime/uptime.go diff --git a/src/mod/dynamicproxy/Server.go b/src/mod/dynamicproxy/Server.go index 9b9522e..7f2b1b4 100644 --- a/src/mod/dynamicproxy/Server.go +++ b/src/mod/dynamicproxy/Server.go @@ -37,6 +37,15 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + //Check if there are external routing rule matches. + //If yes, route them via external rr + matchedRoutingRule := h.Parent.GetMatchingRoutingRule(r) + if matchedRoutingRule != nil { + //Matching routing rule found. Let the sub-router handle it + matchedRoutingRule.Route(w, r) + return + } + //Extract request host to see if it is virtual directory or subdomain domainOnly := r.Host if strings.Contains(r.Host, ":") { @@ -57,10 +66,16 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { //Clean up the request URI proxyingPath := strings.TrimSpace(r.RequestURI) - targetProxyEndpoint := h.Parent.getTargetProxyEndpointFromRequestURI(proxyingPath) if targetProxyEndpoint != nil { h.proxyRequest(w, r, targetProxyEndpoint) + } else if !strings.HasSuffix(proxyingPath, "/") { + potentialProxtEndpoint := h.Parent.getTargetProxyEndpointFromRequestURI(proxyingPath + "/") + if potentialProxtEndpoint != nil { + h.proxyRequest(w, r, potentialProxtEndpoint) + } else { + h.proxyRequest(w, r, h.Parent.Root) + } } else { h.proxyRequest(w, r, h.Parent.Root) } diff --git a/src/mod/dynamicproxy/dynamicproxy.go b/src/mod/dynamicproxy/dynamicproxy.go index edd7df6..a0310c0 100644 --- a/src/mod/dynamicproxy/dynamicproxy.go +++ b/src/mod/dynamicproxy/dynamicproxy.go @@ -43,6 +43,7 @@ type Router struct { mux http.Handler server *http.Server tlsListener net.Listener + routingRules []*RoutingRule } type ProxyEndpoint struct { @@ -72,6 +73,7 @@ func NewDynamicProxy(option RouterOption) (*Router, error) { SubdomainEndpoint: &domainMap, Running: false, server: nil, + routingRules: []*RoutingRule{}, } thisRouter.mux = &ProxyHandler{ @@ -308,3 +310,38 @@ func (router *Router) SetRootProxy(proxyLocation string, requireTLS bool) error router.Root = &rootEndpoint return nil } + +//Helpers to export the syncmap for easier processing +func (r *Router) GetSDProxyEndpointsAsMap() map[string]*SubdomainEndpoint { + m := make(map[string]*SubdomainEndpoint) + r.SubdomainEndpoint.Range(func(key, value interface{}) bool { + k, ok := key.(string) + if !ok { + return true + } + v, ok := value.(*SubdomainEndpoint) + if !ok { + return true + } + m[k] = v + return true + }) + return m +} + +func (r *Router) GetVDProxyEndpointsAsMap() map[string]*ProxyEndpoint { + m := make(map[string]*ProxyEndpoint) + r.ProxyEndpoints.Range(func(key, value interface{}) bool { + k, ok := key.(string) + if !ok { + return true + } + v, ok := value.(*ProxyEndpoint) + if !ok { + return true + } + m[k] = v + return true + }) + return m +} diff --git a/src/mod/dynamicproxy/special.go b/src/mod/dynamicproxy/special.go new file mode 100644 index 0000000..c7d4707 --- /dev/null +++ b/src/mod/dynamicproxy/special.go @@ -0,0 +1,85 @@ +package dynamicproxy + +import ( + "errors" + "net/http" +) + +/* + Special.go + + This script handle special routing rules + by external modules +*/ + +type RoutingRule struct { + ID string + MatchRule func(r *http.Request) bool + RoutingHandler http.Handler + Enabled bool +} + +//Router functions +//Check if a routing rule exists given its id +func (router *Router) GetRoutingRuleById(rrid string) (*RoutingRule, error) { + for _, rr := range router.routingRules { + if rr.ID == rrid { + return rr, nil + } + } + + return nil, errors.New("routing rule with given id not found") +} + +//Add a routing rule to the router +func (router *Router) AddRoutingRules(rr *RoutingRule) error { + _, err := router.GetRoutingRuleById(rr.ID) + if err != nil { + //routing rule with given id already exists + return err + } + + router.routingRules = append(router.routingRules, rr) + return nil +} + +//Remove a routing rule from the router +func (router *Router) RemoveRoutingRule(rrid string) { + newRoutingRules := []*RoutingRule{} + for _, rr := range router.routingRules { + if rr.ID != rrid { + newRoutingRules = append(newRoutingRules, rr) + } + } + + router.routingRules = newRoutingRules +} + +//Get all routing rules +func (router *Router) GetAllRoutingRules() []*RoutingRule { + return router.routingRules +} + +//Get the matching routing rule that describe this request. +//Return nil if no routing rule is match +func (router *Router) GetMatchingRoutingRule(r *http.Request) *RoutingRule { + for _, thisRr := range router.routingRules { + if thisRr.IsMatch(r) { + return thisRr + } + } + return nil +} + +//Routing Rule functions +//Check if a request object match the +func (e *RoutingRule) IsMatch(r *http.Request) bool { + if !e.Enabled { + return false + } + return e.MatchRule(r) +} + +func (e *RoutingRule) Route(w http.ResponseWriter, r *http.Request) { + e.RoutingHandler.ServeHTTP(w, r) +} diff --git a/src/mod/statistic/handler.go b/src/mod/statistic/handler.go index 57e084b..85b0ca9 100644 --- a/src/mod/statistic/handler.go +++ b/src/mod/statistic/handler.go @@ -15,15 +15,6 @@ import ( */ func (c *Collector) HandleTodayStatLoad(w http.ResponseWriter, r *http.Request) { - type DailySummaryExport struct { - TotalRequest int64 //Total request of the day - ErrorRequest int64 //Invalid request of the day, including error or not found - ValidRequest int64 //Valid request of the day - - ForwardTypes map[string]int - RequestOrigin map[string]int - RequestClientIp map[string]int - } fast, err := utils.GetPara(r, "fast") if err != nil { diff --git a/src/mod/statistic/statistic.go b/src/mod/statistic/statistic.go index ed11968..bcb29b6 100644 --- a/src/mod/statistic/statistic.go +++ b/src/mod/statistic/statistic.go @@ -74,16 +74,18 @@ func (c *Collector) SaveSummaryOfDay() { //When it is called in 0:00am, make sure it is stored as yesterday key t := time.Now().Add(-30 * time.Second) summaryKey := t.Format("02_01_2006") - c.Option.Database.Write("stats", summaryKey, c.DailySummary) + saveData := DailySummaryToExport(*c.DailySummary) + c.Option.Database.Write("stats", summaryKey, saveData) } //Load the summary of a day given func (c *Collector) LoadSummaryOfDay(year int, month time.Month, day int) *DailySummary { date := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.Local) summaryKey := date.Format("02_01_2006") - var targetSummary = newDailySummary() - c.Option.Database.Read("stats", summaryKey, &targetSummary) - return targetSummary + targetSummaryExport := DailySummaryExport{} + c.Option.Database.Read("stats", summaryKey, &targetSummaryExport) + targetSummary := DailySummaryExportToSummary(targetSummaryExport) + return &targetSummary } //This function gives the current slot in the 288- 5 minutes interval of the day diff --git a/src/mod/statistic/structconv.go b/src/mod/statistic/structconv.go new file mode 100644 index 0000000..a208f57 --- /dev/null +++ b/src/mod/statistic/structconv.go @@ -0,0 +1,66 @@ +package statistic + +import "sync" + +type DailySummaryExport struct { + TotalRequest int64 //Total request of the day + ErrorRequest int64 //Invalid request of the day, including error or not found + ValidRequest int64 //Valid request of the day + + ForwardTypes map[string]int + RequestOrigin map[string]int + RequestClientIp map[string]int +} + +func DailySummaryToExport(summary DailySummary) DailySummaryExport { + export := DailySummaryExport{ + TotalRequest: summary.TotalRequest, + ErrorRequest: summary.ErrorRequest, + ValidRequest: summary.ValidRequest, + ForwardTypes: make(map[string]int), + RequestOrigin: make(map[string]int), + RequestClientIp: make(map[string]int), + } + + summary.ForwardTypes.Range(func(key, value interface{}) bool { + export.ForwardTypes[key.(string)] = value.(int) + return true + }) + + summary.RequestOrigin.Range(func(key, value interface{}) bool { + export.RequestOrigin[key.(string)] = value.(int) + return true + }) + + summary.RequestClientIp.Range(func(key, value interface{}) bool { + export.RequestClientIp[key.(string)] = value.(int) + return true + }) + + return export +} + +func DailySummaryExportToSummary(export DailySummaryExport) DailySummary { + summary := DailySummary{ + TotalRequest: export.TotalRequest, + ErrorRequest: export.ErrorRequest, + ValidRequest: export.ValidRequest, + ForwardTypes: &sync.Map{}, + RequestOrigin: &sync.Map{}, + RequestClientIp: &sync.Map{}, + } + + for k, v := range export.ForwardTypes { + summary.ForwardTypes.Store(k, v) + } + + for k, v := range export.RequestOrigin { + summary.RequestOrigin.Store(k, v) + } + + for k, v := range export.RequestClientIp { + summary.RequestClientIp.Store(k, v) + } + + return summary +} diff --git a/src/mod/uptime/uptime.go b/src/mod/uptime/uptime.go new file mode 100644 index 0000000..e4ee17f --- /dev/null +++ b/src/mod/uptime/uptime.go @@ -0,0 +1,227 @@ +package uptime + +import ( + "encoding/json" + "log" + "net/http" + "time" + + "imuslab.com/zoraxy/mod/utils" +) + +type Record struct { + Timestamp int64 + ID string + Name string + URL string + Protocol string + Online bool + StatusCode int + Latency int64 +} + +type Target struct { + ID string + Name string + URL string + Protocol string +} + +type Config struct { + Targets []*Target + Interval int + MaxRecordsStore int +} + +type Monitor struct { + Config *Config + OnlineStatusLog map[string][]*Record +} + +// Default configs +var exampleTarget = Target{ + ID: "example", + Name: "Example", + URL: "example.com", + Protocol: "https", +} + +//Create a new uptime monitor +func NewUptimeMonitor(config *Config) (*Monitor, error) { + //Create new monitor object + thisMonitor := Monitor{ + Config: config, + OnlineStatusLog: map[string][]*Record{}, + } + //Start the endpoint listener + ticker := time.NewTicker(time.Duration(config.Interval) * time.Second) + done := make(chan bool) + + //Start the uptime check once first before entering loop + thisMonitor.ExecuteUptimeCheck() + + go func() { + for { + select { + case <-done: + return + case t := <-ticker.C: + log.Println("Uptime updated - ", t.Unix()) + thisMonitor.ExecuteUptimeCheck() + } + } + }() + + return &thisMonitor, nil +} + +func (m *Monitor) ExecuteUptimeCheck() { + for _, target := range m.Config.Targets { + //For each target to check online, do the following + var thisRecord Record + if target.Protocol == "http" || target.Protocol == "https" { + online, laterncy, statusCode := getWebsiteStatusWithLatency(target.URL) + thisRecord = Record{ + Timestamp: time.Now().Unix(), + ID: target.ID, + Name: target.Name, + URL: target.URL, + Protocol: target.Protocol, + Online: online, + StatusCode: statusCode, + Latency: laterncy, + } + + //fmt.Println(thisRecord) + + } else { + log.Println("Unknown protocol: " + target.Protocol + ". Skipping") + continue + } + + thisRecords, ok := m.OnlineStatusLog[target.ID] + if !ok { + //First record. Create the array + m.OnlineStatusLog[target.ID] = []*Record{&thisRecord} + } else { + //Append to the previous record + thisRecords = append(thisRecords, &thisRecord) + + //Check if the record is longer than the logged record. If yes, clear out the old records + if len(thisRecords) > m.Config.MaxRecordsStore { + thisRecords = thisRecords[1:] + } + + m.OnlineStatusLog[target.ID] = thisRecords + } + } + + //TODO: Write results to db +} + +func (m *Monitor) AddTargetToMonitor(target *Target) { + // Add target to Config + m.Config.Targets = append(m.Config.Targets, target) + + // Add target to OnlineStatusLog + m.OnlineStatusLog[target.ID] = []*Record{} +} + +func (m *Monitor) RemoveTargetFromMonitor(targetId string) { + // Remove target from Config + for i, target := range m.Config.Targets { + if target.ID == targetId { + m.Config.Targets = append(m.Config.Targets[:i], m.Config.Targets[i+1:]...) + break + } + } + + // Remove target from OnlineStatusLog + delete(m.OnlineStatusLog, targetId) +} + +//Scan the config target. If a target exists in m.OnlineStatusLog no longer +//exists in m.Monitor.Config.Targets, it remove it from the log as well. +func (m *Monitor) CleanRecords() { + // Create a set of IDs for all targets in the config + targetIDs := make(map[string]bool) + for _, target := range m.Config.Targets { + targetIDs[target.ID] = true + } + + // Iterate over all log entries and remove any that have a target ID that + // is not in the set of current target IDs + newStatusLog := m.OnlineStatusLog + for id, _ := range m.OnlineStatusLog { + _, idExistsInTargets := targetIDs[id] + if !idExistsInTargets { + delete(newStatusLog, id) + } + } + + m.OnlineStatusLog = newStatusLog +} + +/* + Web Interface Handler +*/ + +func (m *Monitor) HandleUptimeLogRead(w http.ResponseWriter, r *http.Request) { + id, _ := utils.GetPara(r, "id") + if id == "" { + js, _ := json.Marshal(m.OnlineStatusLog) + w.Header().Set("Content-Type", "application/json") + w.Write(js) + } else { + //Check if that id exists + log, ok := m.OnlineStatusLog[id] + if !ok { + http.NotFound(w, r) + return + } + + js, _ := json.MarshalIndent(log, "", " ") + w.Header().Set("Content-Type", "application/json") + w.Write(js) + } + +} + +/* + Utilities +*/ + +// Get website stauts with latency given URL, return is conn succ and its latency and status code +func getWebsiteStatusWithLatency(url string) (bool, int64, int) { + start := time.Now().UnixNano() / int64(time.Millisecond) + statusCode, err := getWebsiteStatus(url) + end := time.Now().UnixNano() / int64(time.Millisecond) + if err != nil { + log.Println(err.Error()) + return false, 0, 0 + } else { + diff := end - start + succ := false + if statusCode >= 200 && statusCode < 300 { + //OK + succ = true + } else if statusCode >= 300 && statusCode < 400 { + //Redirection code + succ = true + } else { + succ = false + } + + return succ, diff, statusCode + } + +} + +func getWebsiteStatus(url string) (int, error) { + resp, err := http.Get(url) + if err != nil { + return 0, err + } + status_code := resp.StatusCode + return status_code, nil +} diff --git a/src/mod/utils/utils.go b/src/mod/utils/utils.go index 077524e..a604113 100644 --- a/src/mod/utils/utils.go +++ b/src/mod/utils/utils.go @@ -1,16 +1,13 @@ package utils import ( - "archive/tar" "bufio" - "compress/gzip" "encoding/base64" "errors" "io" "log" "net/http" "os" - "path/filepath" "strings" "time" ) @@ -176,55 +173,3 @@ func StringInArrayIgnoreCase(arr []string, str string) bool { return StringInArray(smallArray, strings.ToLower(str)) } - -func ExtractTarGzipByStream(basedir string, gzipStream io.Reader, onErrorResumeNext bool) error { - uncompressedStream, err := gzip.NewReader(gzipStream) - if err != nil { - return err - } - - tarReader := tar.NewReader(uncompressedStream) - - for { - header, err := tarReader.Next() - - if err == io.EOF { - break - } - - if err != nil { - return err - } - - switch header.Typeflag { - case tar.TypeDir: - err := os.Mkdir(header.Name, 0755) - if err != nil { - if !onErrorResumeNext { - return err - } - - } - case tar.TypeReg: - outFile, err := os.Create(filepath.Join(basedir, header.Name)) - if err != nil { - if !onErrorResumeNext { - return err - } - } - _, err = io.Copy(outFile, tarReader) - if err != nil { - if !onErrorResumeNext { - return err - } - } - outFile.Close() - - default: - //Unknown filetype, continue - - } - - } - return nil -}