v2 init commit

src/mod/statistic/analytic/analytic.go

@@ -0,0 +1,128 @@
+package analytic
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+	"time"
+
+	"imuslab.com/zoraxy/mod/database"
+	"imuslab.com/zoraxy/mod/statistic"
+	"imuslab.com/zoraxy/mod/utils"
+)
+
+type DataLoader struct {
+	Database           *database.Database
+	StatisticCollector *statistic.Collector
+}
+
+// Create a new data loader for loading statistic from database
+func NewDataLoader(db *database.Database, sc *statistic.Collector) *DataLoader {
+	return &DataLoader{
+		Database:           db,
+		StatisticCollector: sc,
+	}
+}
+
+func (d *DataLoader) HandleSummaryList(w http.ResponseWriter, r *http.Request) {
+	entries, err := d.Database.ListTable("stats")
+	if err != nil {
+		utils.SendErrorResponse(w, "unable to load data from database")
+		return
+	}
+
+	entryDates := []string{}
+	for _, keypairs := range entries {
+		entryDates = append(entryDates, string(keypairs[0]))
+	}
+
+	js, _ := json.MarshalIndent(entryDates, "", " ")
+	utils.SendJSONResponse(w, string(js))
+}
+
+func (d *DataLoader) HandleLoadTargetDaySummary(w http.ResponseWriter, r *http.Request) {
+	day, err := utils.GetPara(r, "id")
+	if err != nil {
+		utils.SendErrorResponse(w, "id cannot be empty")
+		return
+	}
+
+	if strings.Contains(day, "-") {
+		//Must be underscore
+		day = strings.ReplaceAll(day, "-", "_")
+	}
+
+	if !statistic.IsBeforeToday(day) {
+		utils.SendErrorResponse(w, "given date is in the future")
+		return
+	}
+
+	var targetDailySummary statistic.DailySummaryExport
+
+	if day == time.Now().Format("2006_01_02") {
+		targetDailySummary = *d.StatisticCollector.GetExportSummary()
+	} else {
+		//Not today data
+		err = d.Database.Read("stats", day, &targetDailySummary)
+		if err != nil {
+			utils.SendErrorResponse(w, "target day data not found")
+			return
+		}
+	}
+
+	js, _ := json.Marshal(targetDailySummary)
+	utils.SendJSONResponse(w, string(js))
+}
+
+func (d *DataLoader) HandleLoadTargetRangeSummary(w http.ResponseWriter, r *http.Request) {
+	//Get the start date from POST para
+	start, err := utils.GetPara(r, "start")
+	if err != nil {
+		utils.SendErrorResponse(w, "start date cannot be empty")
+		return
+	}
+	if strings.Contains(start, "-") {
+		//Must be underscore
+		start = strings.ReplaceAll(start, "-", "_")
+	}
+	//Get end date from POST para
+	end, err := utils.GetPara(r, "end")
+	if err != nil {
+		utils.SendErrorResponse(w, "emd date cannot be empty")
+		return
+	}
+	if strings.Contains(end, "-") {
+		//Must be underscore
+		end = strings.ReplaceAll(end, "-", "_")
+	}
+
+	//Generate all the dates in between the range
+	keys, err := generateDateRange(start, end)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	//Load all the data from database
+	dailySummaries := []*statistic.DailySummaryExport{}
+	for _, key := range keys {
+		thisStat := statistic.DailySummaryExport{}
+		err = d.Database.Read("stats", key, &thisStat)
+		if err == nil {
+			dailySummaries = append(dailySummaries, &thisStat)
+		}
+	}
+
+	//Merge the summaries into one
+	mergedSummary := mergeDailySummaryExports(dailySummaries)
+
+	js, _ := json.Marshal(struct {
+		Summary *statistic.DailySummaryExport
+		Records []*statistic.DailySummaryExport
+	}{
+		Summary: mergedSummary,
+		Records: dailySummaries,
+	})
+
+	utils.SendJSONResponse(w, string(js))
+}

src/mod/statistic/analytic/utils.go

@@ -0,0 +1,72 @@
+package analytic
+
+import (
+	"fmt"
+	"time"
+
+	"imuslab.com/zoraxy/mod/statistic"
+)
+
+// Generate all the record keys from a given start and end dates
+func generateDateRange(startDate, endDate string) ([]string, error) {
+	layout := "2006_01_02"
+	start, err := time.Parse(layout, startDate)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing start date: %v", err)
+	}
+
+	end, err := time.Parse(layout, endDate)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing end date: %v", err)
+	}
+
+	var dateRange []string
+	for d := start; !d.After(end); d = d.AddDate(0, 0, 1) {
+		dateRange = append(dateRange, d.Format(layout))
+	}
+
+	return dateRange, nil
+}
+
+func mergeDailySummaryExports(exports []*statistic.DailySummaryExport) *statistic.DailySummaryExport {
+	mergedExport := &statistic.DailySummaryExport{
+		ForwardTypes:    make(map[string]int),
+		RequestOrigin:   make(map[string]int),
+		RequestClientIp: make(map[string]int),
+		Referer:         make(map[string]int),
+		UserAgent:       make(map[string]int),
+		RequestURL:      make(map[string]int),
+	}
+
+	for _, export := range exports {
+		mergedExport.TotalRequest += export.TotalRequest
+		mergedExport.ErrorRequest += export.ErrorRequest
+		mergedExport.ValidRequest += export.ValidRequest
+
+		for key, value := range export.ForwardTypes {
+			mergedExport.ForwardTypes[key] += value
+		}
+
+		for key, value := range export.RequestOrigin {
+			mergedExport.RequestOrigin[key] += value
+		}
+
+		for key, value := range export.RequestClientIp {
+			mergedExport.RequestClientIp[key] += value
+		}
+
+		for key, value := range export.Referer {
+			mergedExport.Referer[key] += value
+		}
+
+		for key, value := range export.UserAgent {
+			mergedExport.UserAgent[key] += value
+		}
+
+		for key, value := range export.RequestURL {
+			mergedExport.RequestURL[key] += value
+		}
+	}
+
+	return mergedExport
+}

src/mod/statistic/handler.go

@@ -0,0 +1,40 @@
+package statistic
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"imuslab.com/zoraxy/mod/utils"
+)
+
+/*
+	Handler.go
+
+	This script handles incoming request for loading the statistic of the day
+
+*/
+
+func (c *Collector) HandleTodayStatLoad(w http.ResponseWriter, r *http.Request) {
+
+	fast, err := utils.GetPara(r, "fast")
+	if err != nil {
+		fast = "false"
+	}
+	d := c.DailySummary
+	if fast == "true" {
+		//Only return the counter
+		exported := DailySummaryExport{
+			TotalRequest: d.TotalRequest,
+			ErrorRequest: d.ErrorRequest,
+			ValidRequest: d.ValidRequest,
+		}
+		js, _ := json.Marshal(exported)
+		utils.SendJSONResponse(w, string(js))
+	} else {
+		//Return everything
+		exported := c.GetExportSummary()
+		js, _ := json.Marshal(exported)
+		utils.SendJSONResponse(w, string(js))
+	}
+
+}

src/mod/statistic/statistic.go

@@ -0,0 +1,237 @@
+package statistic
+
+import (
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"imuslab.com/zoraxy/mod/database"
+)
+
+/*
+	Statistic Package
+
+	This packet is designed to collection information
+	and store them for future analysis
+*/
+
+// Faststat, a interval summary for all collected data and avoid
+// looping through every data everytime a overview is needed
+type DailySummary struct {
+	TotalRequest int64 //Total request of the day
+	ErrorRequest int64 //Invalid request of the day, including error or not found
+	ValidRequest int64 //Valid request of the day
+	//Type counters
+	ForwardTypes    *sync.Map //Map that hold the forward types
+	RequestOrigin   *sync.Map //Map that hold [country ISO code]: visitor counter
+	RequestClientIp *sync.Map //Map that hold all unique request IPs
+	Referer         *sync.Map //Map that store where the user was refered from
+	UserAgent       *sync.Map //Map that store the useragent of the request
+	RequestURL      *sync.Map //Request URL of the request object
+}
+
+type RequestInfo struct {
+	IpAddr                        string
+	RequestOriginalCountryISOCode string
+	Succ                          bool
+	StatusCode                    int
+	ForwardType                   string
+	Referer                       string
+	UserAgent                     string
+	RequestURL                    string
+	Target                        string
+}
+
+type CollectorOption struct {
+	Database *database.Database
+}
+
+type Collector struct {
+	rtdataStopChan chan bool
+	DailySummary   *DailySummary
+	Option         *CollectorOption
+}
+
+func NewStatisticCollector(option CollectorOption) (*Collector, error) {
+	option.Database.NewTable("stats")
+
+	//Create the collector object
+	thisCollector := Collector{
+		DailySummary: newDailySummary(),
+		Option:       &option,
+	}
+
+	//Load the stat if exists for today
+	//This will exists if the program was forcefully restarted
+	year, month, day := time.Now().Date()
+	summary := thisCollector.LoadSummaryOfDay(year, month, day)
+	if summary != nil {
+		thisCollector.DailySummary = summary
+	}
+
+	//Schedule the realtime statistic clearing at midnight everyday
+	rtstatStopChan := thisCollector.ScheduleResetRealtimeStats()
+	thisCollector.rtdataStopChan = rtstatStopChan
+
+	return &thisCollector, nil
+}
+
+// Write the current in-memory summary to database file
+func (c *Collector) SaveSummaryOfDay() {
+	//When it is called in 0:00am, make sure it is stored as yesterday key
+	t := time.Now().Add(-30 * time.Second)
+	summaryKey := t.Format("2006_01_02")
+	saveData := DailySummaryToExport(*c.DailySummary)
+	c.Option.Database.Write("stats", summaryKey, saveData)
+}
+
+// Load the summary of a day given
+func (c *Collector) LoadSummaryOfDay(year int, month time.Month, day int) *DailySummary {
+	date := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.Local)
+	summaryKey := date.Format("2006_01_02")
+	targetSummaryExport := DailySummaryExport{}
+	c.Option.Database.Read("stats", summaryKey, &targetSummaryExport)
+	targetSummary := DailySummaryExportToSummary(targetSummaryExport)
+	return &targetSummary
+}
+
+// This function gives the current slot in the 288- 5 minutes interval of the day
+func (c *Collector) GetCurrentRealtimeStatIntervalId() int {
+	now := time.Now()
+	startOfDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).Unix()
+	secondsSinceStartOfDay := now.Unix() - startOfDay
+	interval := secondsSinceStartOfDay / (5 * 60)
+	return int(interval)
+}
+
+func (c *Collector) Close() {
+	//Stop the ticker
+	c.rtdataStopChan <- true
+
+	//Write the buffered data into database
+	c.SaveSummaryOfDay()
+
+}
+
+// Main function to record all the inbound traffics
+// Note that this function run in go routine and might have concurrent R/W issue
+// Please make sure there is no racing paramters in this function
+func (c *Collector) RecordRequest(ri RequestInfo) {
+	go func() {
+		c.DailySummary.TotalRequest++
+		if ri.Succ {
+			c.DailySummary.ValidRequest++
+		} else {
+			c.DailySummary.ErrorRequest++
+		}
+
+		//Store the request info into correct types of maps
+		ft, ok := c.DailySummary.ForwardTypes.Load(ri.ForwardType)
+		if !ok {
+			c.DailySummary.ForwardTypes.Store(ri.ForwardType, 1)
+		} else {
+			c.DailySummary.ForwardTypes.Store(ri.ForwardType, ft.(int)+1)
+		}
+
+		originISO := strings.ToLower(ri.RequestOriginalCountryISOCode)
+		fo, ok := c.DailySummary.RequestOrigin.Load(originISO)
+		if !ok {
+			c.DailySummary.RequestOrigin.Store(originISO, 1)
+		} else {
+			c.DailySummary.RequestOrigin.Store(originISO, fo.(int)+1)
+		}
+
+		//Filter out CF forwarded requests
+		if strings.Contains(ri.IpAddr, ",") {
+			ips := strings.Split(strings.TrimSpace(ri.IpAddr), ",")
+			if len(ips) >= 1 {
+				ri.IpAddr = ips[0]
+			}
+		}
+
+		fi, ok := c.DailySummary.RequestClientIp.Load(ri.IpAddr)
+		if !ok {
+			c.DailySummary.RequestClientIp.Store(ri.IpAddr, 1)
+		} else {
+			c.DailySummary.RequestClientIp.Store(ri.IpAddr, fi.(int)+1)
+		}
+
+		//Record the referer
+		rf, ok := c.DailySummary.Referer.Load(ri.Referer)
+		if !ok {
+			c.DailySummary.Referer.Store(ri.Referer, 1)
+		} else {
+			c.DailySummary.Referer.Store(ri.Referer, rf.(int)+1)
+		}
+
+		//Record the UserAgent
+		ua, ok := c.DailySummary.UserAgent.Load(ri.UserAgent)
+		if !ok {
+			c.DailySummary.UserAgent.Store(ri.UserAgent, 1)
+		} else {
+			c.DailySummary.UserAgent.Store(ri.UserAgent, ua.(int)+1)
+		}
+
+		//ADD MORE HERE IF NEEDED
+
+		//Record request URL, if it is a page
+		ext := filepath.Ext(ri.RequestURL)
+
+		if ext != "" && !isWebPageExtension(ext) {
+			return
+		}
+
+		ru, ok := c.DailySummary.RequestURL.Load(ri.RequestURL)
+		if !ok {
+			c.DailySummary.RequestURL.Store(ri.RequestURL, 1)
+		} else {
+			c.DailySummary.RequestURL.Store(ri.RequestURL, ru.(int)+1)
+		}
+	}()
+}
+
+// nightly task
+func (c *Collector) ScheduleResetRealtimeStats() chan bool {
+	doneCh := make(chan bool)
+
+	go func() {
+		defer close(doneCh)
+
+		for {
+			// calculate duration until next midnight
+			now := time.Now()
+
+			// Get midnight of the next day in the local time zone
+			midnight := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, now.Location())
+
+			// Calculate the duration until midnight
+			duration := midnight.Sub(now)
+			select {
+			case <-time.After(duration):
+				// store daily summary to database and reset summary
+				c.SaveSummaryOfDay()
+				c.DailySummary = newDailySummary()
+			case <-doneCh:
+				// stop the routine
+				return
+			}
+		}
+	}()
+
+	return doneCh
+}
+
+func newDailySummary() *DailySummary {
+	return &DailySummary{
+		TotalRequest:    0,
+		ErrorRequest:    0,
+		ValidRequest:    0,
+		ForwardTypes:    &sync.Map{},
+		RequestOrigin:   &sync.Map{},
+		RequestClientIp: &sync.Map{},
+		Referer:         &sync.Map{},
+		UserAgent:       &sync.Map{},
+		RequestURL:      &sync.Map{},
+	}
+}

src/mod/statistic/structconv.go

@@ -0,0 +1,108 @@
+package statistic
+
+import "sync"
+
+type DailySummaryExport struct {
+	TotalRequest int64 //Total request of the day
+	ErrorRequest int64 //Invalid request of the day, including error or not found
+	ValidRequest int64 //Valid request of the day
+
+	ForwardTypes    map[string]int
+	RequestOrigin   map[string]int
+	RequestClientIp map[string]int
+	Referer         map[string]int
+	UserAgent       map[string]int
+	RequestURL      map[string]int
+}
+
+func DailySummaryToExport(summary DailySummary) DailySummaryExport {
+	export := DailySummaryExport{
+		TotalRequest:    summary.TotalRequest,
+		ErrorRequest:    summary.ErrorRequest,
+		ValidRequest:    summary.ValidRequest,
+		ForwardTypes:    make(map[string]int),
+		RequestOrigin:   make(map[string]int),
+		RequestClientIp: make(map[string]int),
+		Referer:         make(map[string]int),
+		UserAgent:       make(map[string]int),
+		RequestURL:      make(map[string]int),
+	}
+
+	summary.ForwardTypes.Range(func(key, value interface{}) bool {
+		export.ForwardTypes[key.(string)] = value.(int)
+		return true
+	})
+
+	summary.RequestOrigin.Range(func(key, value interface{}) bool {
+		export.RequestOrigin[key.(string)] = value.(int)
+		return true
+	})
+
+	summary.RequestClientIp.Range(func(key, value interface{}) bool {
+		export.RequestClientIp[key.(string)] = value.(int)
+		return true
+	})
+
+	summary.Referer.Range(func(key, value interface{}) bool {
+		export.Referer[key.(string)] = value.(int)
+		return true
+	})
+
+	summary.UserAgent.Range(func(key, value interface{}) bool {
+		export.UserAgent[key.(string)] = value.(int)
+		return true
+	})
+
+	summary.RequestURL.Range(func(key, value interface{}) bool {
+		export.RequestURL[key.(string)] = value.(int)
+		return true
+	})
+
+	return export
+}
+
+func DailySummaryExportToSummary(export DailySummaryExport) DailySummary {
+	summary := DailySummary{
+		TotalRequest:    export.TotalRequest,
+		ErrorRequest:    export.ErrorRequest,
+		ValidRequest:    export.ValidRequest,
+		ForwardTypes:    &sync.Map{},
+		RequestOrigin:   &sync.Map{},
+		RequestClientIp: &sync.Map{},
+		Referer:         &sync.Map{},
+		UserAgent:       &sync.Map{},
+		RequestURL:      &sync.Map{},
+	}
+
+	for k, v := range export.ForwardTypes {
+		summary.ForwardTypes.Store(k, v)
+	}
+
+	for k, v := range export.RequestOrigin {
+		summary.RequestOrigin.Store(k, v)
+	}
+
+	for k, v := range export.RequestClientIp {
+		summary.RequestClientIp.Store(k, v)
+	}
+
+	for k, v := range export.Referer {
+		summary.Referer.Store(k, v)
+	}
+
+	for k, v := range export.UserAgent {
+		summary.UserAgent.Store(k, v)
+	}
+
+	for k, v := range export.RequestURL {
+		summary.RequestURL.Store(k, v)
+	}
+
+	return summary
+}
+
+// External object function call
+func (c *Collector) GetExportSummary() *DailySummaryExport {
+	exportFormatDailySummary := DailySummaryToExport(*c.DailySummary)
+	return &exportFormatDailySummary
+}

src/mod/statistic/utils.go

@@ -0,0 +1,28 @@
+package statistic
+
+import (
+	"fmt"
+	"time"
+)
+
+func isWebPageExtension(ext string) bool {
+	webPageExts := []string{".html", ".htm", ".php", ".jsp", ".aspx", ".js", ".jsx"}
+	for _, e := range webPageExts {
+		if e == ext {
+			return true
+		}
+	}
+	return false
+}
+
+func IsBeforeToday(dateString string) bool {
+	layout := "2006_01_02"
+	date, err := time.Parse(layout, dateString)
+	if err != nil {
+		fmt.Println("Error parsing date:", err)
+		return false
+	}
+
+	today := time.Now().UTC().Truncate(24 * time.Hour)
+	return date.Before(today) || dateString == time.Now().Format(layout)
+}