Updates v2.6.2

+ Added advance stats operation tab
+ Added statistic reset #13
+ Added statistic export to csv and json (please use json)
+ Make subdomain clickable (not vdir) #12
+ Added TCP Proxy
+ Updates SMTP setup UI to make it more straight forward to setup

src/mod/statistic/analytic/analytic.go

@@ -1,10 +1,9 @@
 package analytic
 
 import (
-	"encoding/json"
+	"errors"
 	"net/http"
 	"strings"
-	"time"
 
 	"imuslab.com/zoraxy/mod/database"
 	"imuslab.com/zoraxy/mod/statistic"
@@ -24,105 +23,49 @@ func NewDataLoader(db *database.Database, sc *statistic.Collector) *DataLoader {
 	}
 }
 
-func (d *DataLoader) HandleSummaryList(w http.ResponseWriter, r *http.Request) {
-	entries, err := d.Database.ListTable("stats")
+// GetAllStatisticSummaryInRange return all the statisics within the time frame. The second array is the key (dates) of the statistic
+func (d *DataLoader) GetAllStatisticSummaryInRange(start, end string) ([]*statistic.DailySummaryExport, []string, error) {
+	dailySummaries := []*statistic.DailySummaryExport{}
+	collectedDates := []string{}
+	//Generate all the dates in between the range
+	keys, err := generateDateRange(start, end)
 	if err != nil {
-		utils.SendErrorResponse(w, "unable to load data from database")
-		return
+		return dailySummaries, collectedDates, err
 	}
 
-	entryDates := []string{}
-	for _, keypairs := range entries {
-		entryDates = append(entryDates, string(keypairs[0]))
-	}
-
-	js, _ := json.MarshalIndent(entryDates, "", " ")
-	utils.SendJSONResponse(w, string(js))
-}
-
-func (d *DataLoader) HandleLoadTargetDaySummary(w http.ResponseWriter, r *http.Request) {
-	day, err := utils.GetPara(r, "id")
-	if err != nil {
-		utils.SendErrorResponse(w, "id cannot be empty")
-		return
-	}
-
-	if strings.Contains(day, "-") {
-		//Must be underscore
-		day = strings.ReplaceAll(day, "-", "_")
-	}
-
-	if !statistic.IsBeforeToday(day) {
-		utils.SendErrorResponse(w, "given date is in the future")
-		return
-	}
-
-	var targetDailySummary statistic.DailySummaryExport
-
-	if day == time.Now().Format("2006_01_02") {
-		targetDailySummary = *d.StatisticCollector.GetExportSummary()
-	} else {
-		//Not today data
-		err = d.Database.Read("stats", day, &targetDailySummary)
-		if err != nil {
-			utils.SendErrorResponse(w, "target day data not found")
-			return
+	//Load all the data from database
+	for _, key := range keys {
+		thisStat := statistic.DailySummaryExport{}
+		err = d.Database.Read("stats", key, &thisStat)
+		if err == nil {
+			dailySummaries = append(dailySummaries, &thisStat)
+			collectedDates = append(collectedDates, key)
 		}
 	}
 
-	js, _ := json.Marshal(targetDailySummary)
-	utils.SendJSONResponse(w, string(js))
+	return dailySummaries, collectedDates, nil
+
 }
 
-func (d *DataLoader) HandleLoadTargetRangeSummary(w http.ResponseWriter, r *http.Request) {
-	//Get the start date from POST para
+func (d *DataLoader) GetStartAndEndDatesFromRequest(r *http.Request) (string, string, error) {
+	// Get the start date from POST para
 	start, err := utils.GetPara(r, "start")
 	if err != nil {
-		utils.SendErrorResponse(w, "start date cannot be empty")
-		return
+		return "", "", errors.New("start date cannot be empty")
 	}
 	if strings.Contains(start, "-") {
 		//Must be underscore
 		start = strings.ReplaceAll(start, "-", "_")
 	}
-	//Get end date from POST para
+	// Get end date from POST para
 	end, err := utils.GetPara(r, "end")
 	if err != nil {
-		utils.SendErrorResponse(w, "emd date cannot be empty")
-		return
+		return "", "", errors.New("end date cannot be empty")
 	}
 	if strings.Contains(end, "-") {
 		//Must be underscore
 		end = strings.ReplaceAll(end, "-", "_")
 	}
 
-	//Generate all the dates in between the range
-	keys, err := generateDateRange(start, end)
-	if err != nil {
-		utils.SendErrorResponse(w, err.Error())
-		return
-	}
-
-	//Load all the data from database
-	dailySummaries := []*statistic.DailySummaryExport{}
-	for _, key := range keys {
-		thisStat := statistic.DailySummaryExport{}
-		err = d.Database.Read("stats", key, &thisStat)
-		if err == nil {
-			dailySummaries = append(dailySummaries, &thisStat)
-		}
-	}
-
-	//Merge the summaries into one
-	mergedSummary := mergeDailySummaryExports(dailySummaries)
-
-	js, _ := json.Marshal(struct {
-		Summary *statistic.DailySummaryExport
-		Records []*statistic.DailySummaryExport
-	}{
-		Summary: mergedSummary,
-		Records: dailySummaries,
-	})
-
-	utils.SendJSONResponse(w, string(js))
+	return start, end, nil
 }

src/mod/statistic/analytic/handlers.go

@@ -0,0 +1,218 @@
+package analytic
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"imuslab.com/zoraxy/mod/statistic"
+	"imuslab.com/zoraxy/mod/utils"
+)
+
+func (d *DataLoader) HandleSummaryList(w http.ResponseWriter, r *http.Request) {
+	entries, err := d.Database.ListTable("stats")
+	if err != nil {
+		utils.SendErrorResponse(w, "unable to load data from database")
+		return
+	}
+
+	entryDates := []string{}
+	for _, keypairs := range entries {
+		entryDates = append(entryDates, string(keypairs[0]))
+	}
+
+	js, _ := json.MarshalIndent(entryDates, "", " ")
+	utils.SendJSONResponse(w, string(js))
+}
+
+func (d *DataLoader) HandleLoadTargetDaySummary(w http.ResponseWriter, r *http.Request) {
+	day, err := utils.GetPara(r, "id")
+	if err != nil {
+		utils.SendErrorResponse(w, "id cannot be empty")
+		return
+	}
+
+	if strings.Contains(day, "-") {
+		//Must be underscore
+		day = strings.ReplaceAll(day, "-", "_")
+	}
+
+	if !statistic.IsBeforeToday(day) {
+		utils.SendErrorResponse(w, "given date is in the future")
+		return
+	}
+
+	var targetDailySummary statistic.DailySummaryExport
+
+	if day == time.Now().Format("2006_01_02") {
+		targetDailySummary = *d.StatisticCollector.GetExportSummary()
+	} else {
+		//Not today data
+		err = d.Database.Read("stats", day, &targetDailySummary)
+		if err != nil {
+			utils.SendErrorResponse(w, "target day data not found")
+			return
+		}
+	}
+
+	js, _ := json.Marshal(targetDailySummary)
+	utils.SendJSONResponse(w, string(js))
+}
+
+func (d *DataLoader) HandleLoadTargetRangeSummary(w http.ResponseWriter, r *http.Request) {
+	start, end, err := d.GetStartAndEndDatesFromRequest(r)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	dailySummaries, _, err := d.GetAllStatisticSummaryInRange(start, end)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	//Merge the summaries into one
+	mergedSummary := mergeDailySummaryExports(dailySummaries)
+
+	js, _ := json.Marshal(struct {
+		Summary *statistic.DailySummaryExport
+		Records []*statistic.DailySummaryExport
+	}{
+		Summary: mergedSummary,
+		Records: dailySummaries,
+	})
+
+	utils.SendJSONResponse(w, string(js))
+}
+
+// Handle exporting of a given range statistics
+func (d *DataLoader) HandleRangeExport(w http.ResponseWriter, r *http.Request) {
+	start, end, err := d.GetStartAndEndDatesFromRequest(r)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	dailySummaries, dates, err := d.GetAllStatisticSummaryInRange(start, end)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	format, err := utils.GetPara(r, "format")
+	if err != nil {
+		format = "json"
+	}
+
+	if format == "csv" {
+		// Create a buffer to store CSV content
+		var csvContent strings.Builder
+
+		// Create a CSV writer
+		writer := csv.NewWriter(&csvContent)
+
+		// Write the header row
+		header := []string{"Date", "TotalRequest", "ErrorRequest", "ValidRequest", "ForwardTypes", "RequestOrigin", "RequestClientIp", "Referer", "UserAgent", "RequestURL"}
+		err := writer.Write(header)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// Write each data row
+		for i, item := range dailySummaries {
+			row := []string{
+				dates[i],
+				strconv.FormatInt(item.TotalRequest, 10),
+				strconv.FormatInt(item.ErrorRequest, 10),
+				strconv.FormatInt(item.ValidRequest, 10),
+				// Convert map values to a comma-separated string
+				strings.Join(mapToStringSlice(item.ForwardTypes), ","),
+				strings.Join(mapToStringSlice(item.RequestOrigin), ","),
+				strings.Join(mapToStringSlice(item.RequestClientIp), ","),
+				strings.Join(mapToStringSlice(item.Referer), ","),
+				strings.Join(mapToStringSlice(item.UserAgent), ","),
+				strings.Join(mapToStringSlice(item.RequestURL), ","),
+			}
+			err = writer.Write(row)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				return
+			}
+		}
+
+		// Flush the CSV writer
+		writer.Flush()
+
+		// Check for any errors during writing
+		if err := writer.Error(); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// Set the response headers
+		w.Header().Set("Content-Type", "text/csv")
+		w.Header().Set("Content-Disposition", "attachment; filename=analytics_"+start+"_to_"+end+".csv")
+
+		// Write the CSV content to the response writer
+		_, err = w.Write([]byte(csvContent.String()))
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	} else if format == "json" {
+		type exportData struct {
+			Stats []*statistic.DailySummaryExport
+			Dates []string
+		}
+
+		results := exportData{
+			Stats: dailySummaries,
+			Dates: dates,
+		}
+
+		js, _ := json.MarshalIndent(results, "", " ")
+		w.Header().Set("Content-Disposition", "attachment; filename=analytics_"+start+"_to_"+end+".json")
+		utils.SendJSONResponse(w, string(js))
+	} else {
+		utils.SendErrorResponse(w, "Unsupported export format")
+	}
+}
+
+// Reset all the keys within the given time period
+func (d *DataLoader) HandleRangeReset(w http.ResponseWriter, r *http.Request) {
+	start, end, err := d.GetStartAndEndDatesFromRequest(r)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	if r.Method != http.MethodDelete {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	keys, err := generateDateRange(start, end)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	for _, key := range keys {
+		log.Println("DELETING statistics " + key)
+		d.Database.Delete("stats", key)
+
+		if isTodayDate(key) {
+			//It is today's date. Also reset statistic collector value
+			log.Println("RESETING today's in-memory statistics")
+			d.StatisticCollector.ResetSummaryOfDay()
+		}
+	}
+
+	utils.SendOK(w)
+}

src/mod/statistic/analytic/utils.go

@@ -70,3 +70,25 @@ func mergeDailySummaryExports(exports []*statistic.DailySummaryExport) *statisti
 
 	return mergedExport
 }
+
+func mapToStringSlice(m map[string]int) []string {
+	slice := make([]string, 0, len(m))
+	for k := range m {
+		slice = append(slice, k)
+	}
+	return slice
+}
+
+func isTodayDate(dateStr string) bool {
+	today := time.Now().Local().Format("2006-01-02")
+	inputDate, err := time.Parse("2006-01-02", dateStr)
+	if err != nil {
+		inputDate, err = time.Parse("2006_01_02", dateStr)
+		if err != nil {
+			fmt.Println("Invalid date format")
+			return false
+		}
+	}
+
+	return inputDate.Format("2006-01-02") == today
+}

src/mod/statistic/statistic.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/microcosm-cc/bluemonday"
 	"imuslab.com/zoraxy/mod/database"
 )
 
@@ -96,6 +97,11 @@ func (c *Collector) LoadSummaryOfDay(year int, month time.Month, day int) *Daily
 	return &targetSummary
 }
 
+// Reset today summary, for debug or restoring injections
+func (c *Collector) ResetSummaryOfDay() {
+	c.DailySummary = newDailySummary()
+}
+
 // This function gives the current slot in the 288- 5 minutes interval of the day
 func (c *Collector) GetCurrentRealtimeStatIntervalId() int {
 	now := time.Now()
@@ -160,11 +166,15 @@ func (c *Collector) RecordRequest(ri RequestInfo) {
 		}
 
 		//Record the referer
-		rf, ok := c.DailySummary.Referer.Load(ri.Referer)
+		p := bluemonday.StripTagsPolicy()
+		filteredReferer := p.Sanitize(
+			ri.Referer,
+		)
+		rf, ok := c.DailySummary.Referer.Load(filteredReferer)
 		if !ok {
-			c.DailySummary.Referer.Store(ri.Referer, 1)
+			c.DailySummary.Referer.Store(filteredReferer, 1)
 		} else {
-			c.DailySummary.Referer.Store(ri.Referer, rf.(int)+1)
+			c.DailySummary.Referer.Store(filteredReferer, rf.(int)+1)
 		}
 
 		//Record the UserAgent