Added rule handles to dynamic proxy core

src/mod/dynamicproxy/Server.go

@@ -37,6 +37,15 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	//Check if there are external routing rule matches.
+	//If yes, route them via external rr
+	matchedRoutingRule := h.Parent.GetMatchingRoutingRule(r)
+	if matchedRoutingRule != nil {
+		//Matching routing rule found. Let the sub-router handle it
+		matchedRoutingRule.Route(w, r)
+		return
+	}
+
 	//Extract request host to see if it is virtual directory or subdomain
 	domainOnly := r.Host
 	if strings.Contains(r.Host, ":") {
@@ -57,10 +66,16 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	//Clean up the request URI
 	proxyingPath := strings.TrimSpace(r.RequestURI)
-
 	targetProxyEndpoint := h.Parent.getTargetProxyEndpointFromRequestURI(proxyingPath)
 	if targetProxyEndpoint != nil {
 		h.proxyRequest(w, r, targetProxyEndpoint)
+	} else if !strings.HasSuffix(proxyingPath, "/") {
+		potentialProxtEndpoint := h.Parent.getTargetProxyEndpointFromRequestURI(proxyingPath + "/")
+		if potentialProxtEndpoint != nil {
+			h.proxyRequest(w, r, potentialProxtEndpoint)
+		} else {
+			h.proxyRequest(w, r, h.Parent.Root)
+		}
 	} else {
 		h.proxyRequest(w, r, h.Parent.Root)
 	}

src/mod/dynamicproxy/dynamicproxy.go

@@ -43,6 +43,7 @@ type Router struct {
 	mux               http.Handler
 	server            *http.Server
 	tlsListener       net.Listener
+	routingRules      []*RoutingRule
 }
 
 type ProxyEndpoint struct {
@@ -72,6 +73,7 @@ func NewDynamicProxy(option RouterOption) (*Router, error) {
 		SubdomainEndpoint: &domainMap,
 		Running:           false,
 		server:            nil,
+		routingRules:      []*RoutingRule{},
 	}
 
 	thisRouter.mux = &ProxyHandler{
@@ -308,3 +310,38 @@ func (router *Router) SetRootProxy(proxyLocation string, requireTLS bool) error
 	router.Root = &rootEndpoint
 	return nil
 }
+
+//Helpers to export the syncmap for easier processing
+func (r *Router) GetSDProxyEndpointsAsMap() map[string]*SubdomainEndpoint {
+	m := make(map[string]*SubdomainEndpoint)
+	r.SubdomainEndpoint.Range(func(key, value interface{}) bool {
+		k, ok := key.(string)
+		if !ok {
+			return true
+		}
+		v, ok := value.(*SubdomainEndpoint)
+		if !ok {
+			return true
+		}
+		m[k] = v
+		return true
+	})
+	return m
+}
+
+func (r *Router) GetVDProxyEndpointsAsMap() map[string]*ProxyEndpoint {
+	m := make(map[string]*ProxyEndpoint)
+	r.ProxyEndpoints.Range(func(key, value interface{}) bool {
+		k, ok := key.(string)
+		if !ok {
+			return true
+		}
+		v, ok := value.(*ProxyEndpoint)
+		if !ok {
+			return true
+		}
+		m[k] = v
+		return true
+	})
+	return m
+}

src/mod/dynamicproxy/special.go

@@ -0,0 +1,85 @@
+package dynamicproxy
+
+import (
+	"errors"
+	"net/http"
+)
+
+/*
+	Special.go
+
+	This script handle special routing rules
+	by external modules
+*/
+
+type RoutingRule struct {
+	ID             string
+	MatchRule      func(r *http.Request) bool
+	RoutingHandler http.Handler
+	Enabled        bool
+}
+
+//Router functions
+//Check if a routing rule exists given its id
+func (router *Router) GetRoutingRuleById(rrid string) (*RoutingRule, error) {
+	for _, rr := range router.routingRules {
+		if rr.ID == rrid {
+			return rr, nil
+		}
+	}
+
+	return nil, errors.New("routing rule with given id not found")
+}
+
+//Add a routing rule to the router
+func (router *Router) AddRoutingRules(rr *RoutingRule) error {
+	_, err := router.GetRoutingRuleById(rr.ID)
+	if err != nil {
+		//routing rule with given id already exists
+		return err
+	}
+
+	router.routingRules = append(router.routingRules, rr)
+	return nil
+}
+
+//Remove a routing rule from the router
+func (router *Router) RemoveRoutingRule(rrid string) {
+	newRoutingRules := []*RoutingRule{}
+	for _, rr := range router.routingRules {
+		if rr.ID != rrid {
+			newRoutingRules = append(newRoutingRules, rr)
+		}
+	}
+
+	router.routingRules = newRoutingRules
+}
+
+//Get all routing rules
+func (router *Router) GetAllRoutingRules() []*RoutingRule {
+	return router.routingRules
+}
+
+//Get the matching routing rule that describe this request.
+//Return nil if no routing rule is match
+func (router *Router) GetMatchingRoutingRule(r *http.Request) *RoutingRule {
+	for _, thisRr := range router.routingRules {
+		if thisRr.IsMatch(r) {
+			return thisRr
+		}
+	}
+	return nil
+}
+
+//Routing Rule functions
+//Check if a request object match the
+func (e *RoutingRule) IsMatch(r *http.Request) bool {
+	if !e.Enabled {
+		return false
+	}
+	return e.MatchRule(r)
+}
+
+func (e *RoutingRule) Route(w http.ResponseWriter, r *http.Request) {
+	e.RoutingHandler.ServeHTTP(w, r)
+}

src/mod/statistic/handler.go

@@ -15,15 +15,6 @@ import (
 */
 
 func (c *Collector) HandleTodayStatLoad(w http.ResponseWriter, r *http.Request) {
-	type DailySummaryExport struct {
-		TotalRequest int64 //Total request of the day
-		ErrorRequest int64 //Invalid request of the day, including error or not found
-		ValidRequest int64 //Valid request of the day
-
-		ForwardTypes    map[string]int
-		RequestOrigin   map[string]int
-		RequestClientIp map[string]int
-	}
 
 	fast, err := utils.GetPara(r, "fast")
 	if err != nil {

src/mod/statistic/statistic.go

@@ -74,16 +74,18 @@ func (c *Collector) SaveSummaryOfDay() {
 	//When it is called in 0:00am, make sure it is stored as yesterday key
 	t := time.Now().Add(-30 * time.Second)
 	summaryKey := t.Format("02_01_2006")
-	c.Option.Database.Write("stats", summaryKey, c.DailySummary)
+	saveData := DailySummaryToExport(*c.DailySummary)
+	c.Option.Database.Write("stats", summaryKey, saveData)
 }
 
 //Load the summary of a day given
 func (c *Collector) LoadSummaryOfDay(year int, month time.Month, day int) *DailySummary {
 	date := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.Local)
 	summaryKey := date.Format("02_01_2006")
-	var targetSummary = newDailySummary()
-	c.Option.Database.Read("stats", summaryKey, &targetSummary)
-	return targetSummary
+	targetSummaryExport := DailySummaryExport{}
+	c.Option.Database.Read("stats", summaryKey, &targetSummaryExport)
+	targetSummary := DailySummaryExportToSummary(targetSummaryExport)
+	return &targetSummary
 }
 
 //This function gives the current slot in the 288- 5 minutes interval of the day

src/mod/statistic/structconv.go

@@ -0,0 +1,66 @@
+package statistic
+
+import "sync"
+
+type DailySummaryExport struct {
+	TotalRequest int64 //Total request of the day
+	ErrorRequest int64 //Invalid request of the day, including error or not found
+	ValidRequest int64 //Valid request of the day
+
+	ForwardTypes    map[string]int
+	RequestOrigin   map[string]int
+	RequestClientIp map[string]int
+}
+
+func DailySummaryToExport(summary DailySummary) DailySummaryExport {
+	export := DailySummaryExport{
+		TotalRequest:    summary.TotalRequest,
+		ErrorRequest:    summary.ErrorRequest,
+		ValidRequest:    summary.ValidRequest,
+		ForwardTypes:    make(map[string]int),
+		RequestOrigin:   make(map[string]int),
+		RequestClientIp: make(map[string]int),
+	}
+
+	summary.ForwardTypes.Range(func(key, value interface{}) bool {
+		export.ForwardTypes[key.(string)] = value.(int)
+		return true
+	})
+
+	summary.RequestOrigin.Range(func(key, value interface{}) bool {
+		export.RequestOrigin[key.(string)] = value.(int)
+		return true
+	})
+
+	summary.RequestClientIp.Range(func(key, value interface{}) bool {
+		export.RequestClientIp[key.(string)] = value.(int)
+		return true
+	})
+
+	return export
+}
+
+func DailySummaryExportToSummary(export DailySummaryExport) DailySummary {
+	summary := DailySummary{
+		TotalRequest:    export.TotalRequest,
+		ErrorRequest:    export.ErrorRequest,
+		ValidRequest:    export.ValidRequest,
+		ForwardTypes:    &sync.Map{},
+		RequestOrigin:   &sync.Map{},
+		RequestClientIp: &sync.Map{},
+	}
+
+	for k, v := range export.ForwardTypes {
+		summary.ForwardTypes.Store(k, v)
+	}
+
+	for k, v := range export.RequestOrigin {
+		summary.RequestOrigin.Store(k, v)
+	}
+
+	for k, v := range export.RequestClientIp {
+		summary.RequestClientIp.Store(k, v)
+	}
+
+	return summary
+}

src/mod/uptime/uptime.go

@@ -0,0 +1,227 @@
+package uptime
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+	"time"
+
+	"imuslab.com/zoraxy/mod/utils"
+)
+
+type Record struct {
+	Timestamp  int64
+	ID         string
+	Name       string
+	URL        string
+	Protocol   string
+	Online     bool
+	StatusCode int
+	Latency    int64
+}
+
+type Target struct {
+	ID       string
+	Name     string
+	URL      string
+	Protocol string
+}
+
+type Config struct {
+	Targets         []*Target
+	Interval        int
+	MaxRecordsStore int
+}
+
+type Monitor struct {
+	Config          *Config
+	OnlineStatusLog map[string][]*Record
+}
+
+// Default configs
+var exampleTarget = Target{
+	ID:       "example",
+	Name:     "Example",
+	URL:      "example.com",
+	Protocol: "https",
+}
+
+//Create a new uptime monitor
+func NewUptimeMonitor(config *Config) (*Monitor, error) {
+	//Create new monitor object
+	thisMonitor := Monitor{
+		Config:          config,
+		OnlineStatusLog: map[string][]*Record{},
+	}
+	//Start the endpoint listener
+	ticker := time.NewTicker(time.Duration(config.Interval) * time.Second)
+	done := make(chan bool)
+
+	//Start the uptime check once first before entering loop
+	thisMonitor.ExecuteUptimeCheck()
+
+	go func() {
+		for {
+			select {
+			case <-done:
+				return
+			case t := <-ticker.C:
+				log.Println("Uptime updated - ", t.Unix())
+				thisMonitor.ExecuteUptimeCheck()
+			}
+		}
+	}()
+
+	return &thisMonitor, nil
+}
+
+func (m *Monitor) ExecuteUptimeCheck() {
+	for _, target := range m.Config.Targets {
+		//For each target to check online, do the following
+		var thisRecord Record
+		if target.Protocol == "http" || target.Protocol == "https" {
+			online, laterncy, statusCode := getWebsiteStatusWithLatency(target.URL)
+			thisRecord = Record{
+				Timestamp:  time.Now().Unix(),
+				ID:         target.ID,
+				Name:       target.Name,
+				URL:        target.URL,
+				Protocol:   target.Protocol,
+				Online:     online,
+				StatusCode: statusCode,
+				Latency:    laterncy,
+			}
+
+			//fmt.Println(thisRecord)
+
+		} else {
+			log.Println("Unknown protocol: " + target.Protocol + ". Skipping")
+			continue
+		}
+
+		thisRecords, ok := m.OnlineStatusLog[target.ID]
+		if !ok {
+			//First record. Create the array
+			m.OnlineStatusLog[target.ID] = []*Record{&thisRecord}
+		} else {
+			//Append to the previous record
+			thisRecords = append(thisRecords, &thisRecord)
+
+			//Check if the record is longer than the logged record. If yes, clear out the old records
+			if len(thisRecords) > m.Config.MaxRecordsStore {
+				thisRecords = thisRecords[1:]
+			}
+
+			m.OnlineStatusLog[target.ID] = thisRecords
+		}
+	}
+
+	//TODO: Write results to db
+}
+
+func (m *Monitor) AddTargetToMonitor(target *Target) {
+	// Add target to Config
+	m.Config.Targets = append(m.Config.Targets, target)
+
+	// Add target to OnlineStatusLog
+	m.OnlineStatusLog[target.ID] = []*Record{}
+}
+
+func (m *Monitor) RemoveTargetFromMonitor(targetId string) {
+	// Remove target from Config
+	for i, target := range m.Config.Targets {
+		if target.ID == targetId {
+			m.Config.Targets = append(m.Config.Targets[:i], m.Config.Targets[i+1:]...)
+			break
+		}
+	}
+
+	// Remove target from OnlineStatusLog
+	delete(m.OnlineStatusLog, targetId)
+}
+
+//Scan the config target. If a target exists in m.OnlineStatusLog no longer
+//exists in m.Monitor.Config.Targets, it remove it from the log as well.
+func (m *Monitor) CleanRecords() {
+	// Create a set of IDs for all targets in the config
+	targetIDs := make(map[string]bool)
+	for _, target := range m.Config.Targets {
+		targetIDs[target.ID] = true
+	}
+
+	// Iterate over all log entries and remove any that have a target ID that
+	// is not in the set of current target IDs
+	newStatusLog := m.OnlineStatusLog
+	for id, _ := range m.OnlineStatusLog {
+		_, idExistsInTargets := targetIDs[id]
+		if !idExistsInTargets {
+			delete(newStatusLog, id)
+		}
+	}
+
+	m.OnlineStatusLog = newStatusLog
+}
+
+/*
+	Web Interface Handler
+*/
+
+func (m *Monitor) HandleUptimeLogRead(w http.ResponseWriter, r *http.Request) {
+	id, _ := utils.GetPara(r, "id")
+	if id == "" {
+		js, _ := json.Marshal(m.OnlineStatusLog)
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(js)
+	} else {
+		//Check if that id exists
+		log, ok := m.OnlineStatusLog[id]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+
+		js, _ := json.MarshalIndent(log, "", " ")
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(js)
+	}
+
+}
+
+/*
+	Utilities
+*/
+
+// Get website stauts with latency given URL, return is conn succ and its latency and status code
+func getWebsiteStatusWithLatency(url string) (bool, int64, int) {
+	start := time.Now().UnixNano() / int64(time.Millisecond)
+	statusCode, err := getWebsiteStatus(url)
+	end := time.Now().UnixNano() / int64(time.Millisecond)
+	if err != nil {
+		log.Println(err.Error())
+		return false, 0, 0
+	} else {
+		diff := end - start
+		succ := false
+		if statusCode >= 200 && statusCode < 300 {
+			//OK
+			succ = true
+		} else if statusCode >= 300 && statusCode < 400 {
+			//Redirection code
+			succ = true
+		} else {
+			succ = false
+		}
+
+		return succ, diff, statusCode
+	}
+
+}
+
+func getWebsiteStatus(url string) (int, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return 0, err
+	}
+	status_code := resp.StatusCode
+	return status_code, nil
+}

src/mod/utils/utils.go

@@ -1,16 +1,13 @@
 package utils
 
 import (
-	"archive/tar"
 	"bufio"
-	"compress/gzip"
 	"encoding/base64"
 	"errors"
 	"io"
 	"log"
 	"net/http"
 	"os"
-	"path/filepath"
 	"strings"
 	"time"
 )
@@ -176,55 +173,3 @@ func StringInArrayIgnoreCase(arr []string, str string) bool {
 
 	return StringInArray(smallArray, strings.ToLower(str))
 }
-
-func ExtractTarGzipByStream(basedir string, gzipStream io.Reader, onErrorResumeNext bool) error {
-	uncompressedStream, err := gzip.NewReader(gzipStream)
-	if err != nil {
-		return err
-	}
-
-	tarReader := tar.NewReader(uncompressedStream)
-
-	for {
-		header, err := tarReader.Next()
-
-		if err == io.EOF {
-			break
-		}
-
-		if err != nil {
-			return err
-		}
-
-		switch header.Typeflag {
-		case tar.TypeDir:
-			err := os.Mkdir(header.Name, 0755)
-			if err != nil {
-				if !onErrorResumeNext {
-					return err
-				}
-
-			}
-		case tar.TypeReg:
-			outFile, err := os.Create(filepath.Join(basedir, header.Name))
-			if err != nil {
-				if !onErrorResumeNext {
-					return err
-				}
-			}
-			_, err = io.Copy(outFile, tarReader)
-			if err != nil {
-				if !onErrorResumeNext {
-					return err
-				}
-			}
-			outFile.Close()
-
-		default:
-			//Unknown filetype, continue
-
-		}
-
-	}
-	return nil
-}