Merge pull request #246 from tobychui/3.0.9

Update 3.0.9

CHANGELOG.md

@@ -1,3 +1,17 @@
+# v3.0.8 15 Jul 2024
+
++ Added apache style logging mechanism (and build-in log viewer) [#218](https://github.com/tobychui/zoraxy/issues/218)
++ Fixed keep alive flushing issues [#235](https://github.com/tobychui/zoraxy/issues/235)
++ Added multi-upstream supports [#100](https://github.com/tobychui/zoraxy/issues/100)
++ Added stick session load balancer
++ Added weighted random load balancer
++ Added domain cleaning logic to domain / IP input fields
++ Added HSTS "include subdomain" auto injector
++ Added work-in-progress SSO / Oauth Server UI
++ Fixed uptime monitor not updating on proxy rule change bug
++ Optimized UI for create new proxy rule
++ Removed service expose proxy feature
+
 # v3.0.7 20 Jun 2024
 
 + Fixed redirection enable bug [#199](https://github.com/tobychui/zoraxy/issues/199)

README.md

@@ -21,12 +21,14 @@ A general purpose HTTP reverse proxy and forwarding tool. Now written in Go!
   - DNS Challenge for Let's Encrypt and [these DNS providers](https://go-acme.github.io/lego/dns/)
 - Blacklist / Whitelist by country or IP address (single IP, CIDR or wildcard for beginners)
 - Global Area Network Controller Web UI (ZeroTier not included)
-- TCP Tunneling / Proxy
+- Stream Proxy (TCP & UDP)
 - Integrated Up-time Monitor
 - Web-SSH Terminal
 - Utilities
   - CIDR IP converters
   - mDNS Scanner
+  - Wake-On-Lan
+  - Debug Forward Proxy
   - IP Scanner
 - Others
   - Basic single-admin management mode
@@ -96,12 +98,12 @@ See the [/docker](https://github.com/tobychui/zoraxy/tree/main/docker) folder fo
 Usage of zoraxy:
   -autorenew int
         ACME auto TLS/SSL certificate renew check interval (seconds) (default 86400)
+  -cfgupgrade
+        Enable auto config upgrade if breaking change is detected (default true)
   -docker
         Run Zoraxy in docker compatibility mode
   -fastgeoip
         Enable high speed geoip lookup, require 1GB extra memory (Not recommend for low end devices)
-  -log
-        Log terminal output to file (default true)
   -mdns
         Enable mDNS scanner and transponder (default true)
   -mdnsname string

docker/Dockerfile

@@ -31,7 +31,6 @@ WORKDIR /opt/zoraxy/config/
 
 ENV AUTORENEW="86400"
 ENV FASTGEOIP="false"
-ENV LOG="true"
 ENV MDNS="true"
 ENV MDNSNAME="''"
 ENV NOAUTH="false"
@@ -43,6 +42,6 @@ ENV WEBROOT="./www"
 ENV ZTAUTH="''"
 ENV ZTPORT="9993"
 
-ENTRYPOINT "zoraxy" "-docker=true" "-autorenew=${AUTORENEW}" "-fastgeoip=${FASTGEOIP}" "-log=${LOG}" "-mdns=${MDNS}" "-mdnsname=${MDNSNAME}" "-noauth=${NOAUTH}" "-port=:${PORT}" "-sshlb=${SSHLB}" "-version=${VERSION}" "-webfm=${WEBFM}" "-webroot=${WEBROOT}" "-ztauth=${ZTAUTH}" "-ztport=${ZTPORT}"
+ENTRYPOINT "zoraxy" "-docker=true" "-autorenew=${AUTORENEW}" "-fastgeoip=${FASTGEOIP}" "-mdns=${MDNS}" "-mdnsname=${MDNSNAME}" "-noauth=${NOAUTH}" "-port=:${PORT}" "-sshlb=${SSHLB}" "-version=${VERSION}" "-webfm=${WEBFM}" "-webroot=${WEBROOT}" "-ztauth=${ZTAUTH}" "-ztport=${ZTPORT}"
 
 HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 CMD nc -vz 127.0.0.1 $PORT || exit 1

src/api.go

@@ -87,6 +87,7 @@ func initAPIs() {
 	authRouter.HandleFunc("/api/cert/tls", handleToggleTLSProxy)
 	authRouter.HandleFunc("/api/cert/tlsRequireLatest", handleSetTlsRequireLatest)
 	authRouter.HandleFunc("/api/cert/upload", handleCertUpload)
+	authRouter.HandleFunc("/api/cert/download", handleCertDownload)
 	authRouter.HandleFunc("/api/cert/list", handleListCertificate)
 	authRouter.HandleFunc("/api/cert/listdomains", handleListDomains)
 	authRouter.HandleFunc("/api/cert/checkDefault", handleDefaultCertCheck)
@@ -127,7 +128,7 @@ func initAPIs() {
 	//Statistic & uptime monitoring API
 	authRouter.HandleFunc("/api/stats/summary", statisticCollector.HandleTodayStatLoad)
 	authRouter.HandleFunc("/api/stats/countries", HandleCountryDistrSummary)
-	authRouter.HandleFunc("/api/stats/netstat", netstat.HandleGetNetworkInterfaceStats)
+	authRouter.HandleFunc("/api/stats/netstat", netstatBuffers.HandleGetNetworkInterfaceStats)
 	authRouter.HandleFunc("/api/stats/netstatgraph", netstatBuffers.HandleGetBufferedNetworkInterfaceStats)
 	authRouter.HandleFunc("/api/stats/listnic", netstat.HandleListNetworkInterfaces)
 	authRouter.HandleFunc("/api/utm/list", HandleUptimeMonitorListing)

src/cert.go

@@ -233,6 +233,51 @@ func handleSetTlsRequireLatest(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// Handle download of the selected certificate
+func handleCertDownload(w http.ResponseWriter, r *http.Request) {
+	// get the certificate name
+	certname, err := utils.GetPara(r, "certname")
+	if err != nil {
+		utils.SendErrorResponse(w, "invalid certname given")
+		return
+	}
+	certname = filepath.Base(certname) //prevent path escape
+
+	// check if the cert exists
+	pubKey := filepath.Join(filepath.Join("./conf/certs"), certname+".key")
+	priKey := filepath.Join(filepath.Join("./conf/certs"), certname+".pem")
+
+	if utils.FileExists(pubKey) && utils.FileExists(priKey) {
+		//Zip them and serve them via http download
+		seeking, _ := utils.GetBool(r, "seek")
+		if seeking {
+			//This request only check if the key exists. Do not provide download
+			utils.SendOK(w)
+			return
+		}
+
+		//Serve both file in zip
+		zipTmpFolder := "./tmp/download"
+		os.MkdirAll(zipTmpFolder, 0775)
+		zipFileName := filepath.Join(zipTmpFolder, certname+".zip")
+		err := utils.ZipFiles(zipFileName, pubKey, priKey)
+		if err != nil {
+			http.Error(w, "Failed to create zip file", http.StatusInternalServerError)
+			return
+		}
+		defer os.Remove(zipFileName) // Clean up the zip file after serving
+
+		// Serve the zip file
+		w.Header().Set("Content-Disposition", "attachment; filename=\""+certname+"_export.zip\"")
+		w.Header().Set("Content-Type", "application/zip")
+		http.ServeFile(w, r, zipFileName)
+	} else {
+		//Not both key exists
+		utils.SendErrorResponse(w, "invalid key-pairs: private key or public key not found in key store")
+		return
+	}
+}
+
 // Handle upload of the certificate
 func handleCertUpload(w http.ResponseWriter, r *http.Request) {
 	// check if request method is POST

src/main.go

@@ -57,7 +57,7 @@ var enableAutoUpdate = flag.Bool("cfgupgrade", true, "Enable auto config upgrade
 
 var (
 	name        = "Zoraxy"
-	version     = "3.0.8"
+	version     = "3.0.9"
 	nodeUUID    = "generic" //System uuid, in uuidv4 format
 	development = false     //Set this to false to use embedded web fs
 	bootTime    = time.Now().Unix()

src/mod/acme/acme_dns.go

@@ -1,11 +1,6 @@
 package acme
 
 import (
-	"errors"
-	"log"
-	"os"
-	"strings"
-
 	"github.com/go-acme/lego/v4/challenge"
 	"imuslab.com/zoraxy/mod/acme/acmedns"
 )
@@ -29,7 +24,7 @@ func GetDnsChallengeProviderByName(dnsProvider string, dnsCredentials string) (c
 /*
 	Original implementation of DNS ACME using OS.Env as payload
 */
-
+/*
 func setCredentialsIntoEnvironmentVariables(credentials map[string]string) {
 	for key, value := range credentials {
 		err := os.Setenv(key, value)
@@ -41,6 +36,7 @@ func setCredentialsIntoEnvironmentVariables(credentials map[string]string) {
 	}
 }
 
+
 func extractDnsCredentials(input string) (map[string]string, error) {
 	result := make(map[string]string)
 
@@ -70,3 +66,5 @@ func extractDnsCredentials(input string) (map[string]string, error) {
 
 	return result, nil
 }
+
+*/

src/mod/acme/acmedns/acmedns.go

@@ -6,6 +6,7 @@ package acmedns
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 
 	"github.com/go-acme/lego/v4/challenge"
 	"github.com/go-acme/lego/v4/providers/dns/alidns"
@@ -660,6 +661,7 @@ func GetDNSProviderByJsonConfig(name string, js string)(challenge.Provider, erro
 			if err != nil {
 				return nil, err
 			}
+			cfg.PropagationTimeout = 1200 * time.Second
 			return netcup.NewDNSProviderConfig(cfg)
 	case "netlify":
 		cfg := netlify.NewDefaultConfig()

src/mod/dynamicproxy/redirection/redirection.go

@@ -195,6 +195,6 @@ func (t *RuleTable) log(message string, err error) {
 			log.Println("[Redirect] " + message + ": " + err.Error())
 		}
 	} else {
-		t.Logger.PrintAndLog("Redirect", message, err)
+		t.Logger.PrintAndLog("redirect", message, err)
 	}
 }

src/mod/info/logviewer/logviewer.go

@@ -3,7 +3,6 @@ package logviewer
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io/fs"
 	"net/http"
 	"os"
@@ -105,7 +104,6 @@ func (v *Viewer) LoadLogFile(filename string) (string, error) {
 	filename = filepath.ToSlash(filename)
 	filename = strings.ReplaceAll(filename, "../", "")
 	logFilepath := filepath.Join(v.option.RootFolder, filename)
-	fmt.Println(logFilepath)
 	if utils.FileExists(logFilepath) {
 		//Load it
 		content, err := os.ReadFile(logFilepath)

src/mod/netstat/netstat.go

@@ -3,8 +3,6 @@ package netstat
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
-	"log"
 	"net/http"
 	"os"
 	"os/exec"
@@ -14,6 +12,7 @@ import (
 	"strings"
 	"time"
 
+	"imuslab.com/zoraxy/mod/info/logger"
 	"imuslab.com/zoraxy/mod/utils"
 )
 
@@ -35,10 +34,11 @@ type NetStatBuffers struct {
 	Stats           []*FlowStat  //Statistic of the flow
 	StopChan        chan bool    //Channel to stop the ticker
 	EventTicker     *time.Ticker //Ticker for event logging
+	logger          *logger.Logger
 }
 
 // Get a new network statistic buffers
-func NewNetStatBuffer(recordCount int) (*NetStatBuffers, error) {
+func NewNetStatBuffer(recordCount int, systemWideLogger *logger.Logger) (*NetStatBuffers, error) {
 	//Flood fill the stats with 0
 	initialStats := []*FlowStat{}
 	for i := 0; i < recordCount; i++ {
@@ -65,21 +65,22 @@ func NewNetStatBuffer(recordCount int) (*NetStatBuffers, error) {
 		Stats:           initialStats,
 		StopChan:        stopCh,
 		EventTicker:     ticker,
+		logger:          systemWideLogger,
 	}
 
 	//Get the initial measurements of netstats
-	rx, tx, err := GetNetworkInterfaceStats()
+	rx, tx, err := thisNetBuffer.GetNetworkInterfaceStats()
 	if err != nil {
-		log.Println("Unable to get NIC stats: ", err.Error())
+		systemWideLogger.PrintAndLog("netstat", "Unable to get NIC stats: ", err)
 	}
 
 	retryCount := 0
 	for rx == 0 && tx == 0 && retryCount < 10 {
 		//Strange. Retry
-		log.Println("NIC stats return all 0. Retrying...")
+		systemWideLogger.PrintAndLog("netstat", "NIC stats return all 0. Retrying...", nil)
-		rx, tx, err = GetNetworkInterfaceStats()
+		rx, tx, err = thisNetBuffer.GetNetworkInterfaceStats()
 		if err != nil {
-			log.Println("Unable to get NIC stats: ", err.Error())
+			systemWideLogger.PrintAndLog("netstat", "Unable to get NIC stats: ", err)
 		}
 		retryCount++
 	}
@@ -94,20 +95,20 @@ func NewNetStatBuffer(recordCount int) (*NetStatBuffers, error) {
 		for {
 			select {
 			case <-n.StopChan:
-				fmt.Println("- Netstats listener stopped")
+				systemWideLogger.PrintAndLog("netstat", "Netstats listener stopped", nil)
 				return
 
 			case <-ticker.C:
 				if n.PreviousStat.RX == 0 && n.PreviousStat.TX == 0 {
 					//Initiation state is still not done. Ignore request
-					log.Println("No initial states. Waiting")
+					systemWideLogger.PrintAndLog("netstat", "No initial states. Waiting", nil)
 					return
 				}
 				// Get the latest network interface stats
-				rx, tx, err := GetNetworkInterfaceStats()
+				rx, tx, err := thisNetBuffer.GetNetworkInterfaceStats()
 				if err != nil {
 					// Log the error, but don't stop the buffer
-					log.Printf("Failed to get network interface stats: %v", err)
+					systemWideLogger.PrintAndLog("netstat", "Failed to get network interface stats", err)
 					continue
 				}
 
@@ -173,8 +174,8 @@ func (n *NetStatBuffers) Close() {
 	n.EventTicker.Stop()
 }
 
-func HandleGetNetworkInterfaceStats(w http.ResponseWriter, r *http.Request) {
+func (n *NetStatBuffers) HandleGetNetworkInterfaceStats(w http.ResponseWriter, r *http.Request) {
-	rx, tx, err := GetNetworkInterfaceStats()
+	rx, tx, err := n.GetNetworkInterfaceStats()
 	if err != nil {
 		utils.SendErrorResponse(w, err.Error())
 		return
@@ -193,7 +194,7 @@ func HandleGetNetworkInterfaceStats(w http.ResponseWriter, r *http.Request) {
 }
 
 // Get network interface stats, return accumulated rx bits, tx bits and error if any
-func GetNetworkInterfaceStats() (int64, int64, error) {
+func (n *NetStatBuffers) GetNetworkInterfaceStats() (int64, int64, error) {
 	if runtime.GOOS == "windows" {
 		//Windows wmic sometime freeze and not respond.
 		//The safer way is to make a bypass mechanism
@@ -262,7 +263,7 @@ func GetNetworkInterfaceStats() (int64, int64, error) {
 		result = <-callbackChan
 		cmd = nil
 		if result.Err != nil {
-			log.Println("Unable to extract NIC info from wmic: " + result.Err.Error())
+			n.logger.PrintAndLog("netstat", "Unable to extract NIC info from wmic", result.Err)
 		}
 		return result.RX, result.TX, result.Err
 	} else if runtime.GOOS == "linux" {

src/mod/update/update.go

@@ -9,11 +9,11 @@ package update
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"strconv"
 	"strings"
 
-	v308 "imuslab.com/zoraxy/mod/update/v308"
 	"imuslab.com/zoraxy/mod/utils"
 )
 
@@ -22,6 +22,13 @@ import (
 // This function support cross versions updates (e.g. 307 -> 310)
 func RunConfigUpdate(fromVersion int, toVersion int) {
 	versionFile := "./conf/version"
+	isFirstTimeInit, _ := isFirstTimeInitialize("./conf/proxy/")
+	if isFirstTimeInit {
+		//Create version file and exit
+		os.MkdirAll("./conf/", 0775)
+		os.WriteFile(versionFile, []byte(strconv.Itoa(toVersion)), 0775)
+		return
+	}
 	if fromVersion == 0 {
 		//Run auto previous version detection
 		fromVersion = 307
@@ -65,12 +72,37 @@ func GetVersionIntFromVersionNumber(version string) int {
 	return versionInt
 }
 
-func runUpdateRoutineWithVersion(fromVersion int, toVersion int) {
+// Check if the folder "./conf/proxy/" exists and contains files
-	if fromVersion == 307 && toVersion == 308 {
+func isFirstTimeInitialize(path string) (bool, error) {
-		//Updating from v3.0.7 to v3.0.8
+	// Check if the folder exists
-		err := v308.UpdateFrom307To308()
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) {
+		// The folder does not exist
+		return true, nil
+	}
 	if err != nil {
-			panic(err)
+		// Some other error occurred
+		return false, err
 	}
+
+	// Check if it is a directory
+	if !info.IsDir() {
+		// The path is not a directory
+		return false, fmt.Errorf("%s is not a directory", path)
 	}
+
+	// Read the directory contents
+	files, err := ioutil.ReadDir(path)
+	if err != nil {
+		return false, err
+	}
+
+	// Check if the directory is empty
+	if len(files) == 0 {
+		// The folder exists but is empty
+		return true, nil
+	}
+
+	// The folder exists and contains files
+	return false, nil
 }

src/mod/update/updatelogic.go

@@ -0,0 +1,16 @@
+package update
+
+import v308 "imuslab.com/zoraxy/mod/update/v308"
+
+// Updater Core logic
+func runUpdateRoutineWithVersion(fromVersion int, toVersion int) {
+	if fromVersion == 307 && toVersion == 308 {
+		//Updating from v3.0.7 to v3.0.8
+		err := v308.UpdateFrom307To308()
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	//ADD MORE VERSIONS HERE
+}

src/mod/utils/conv.go

@@ -1,6 +1,10 @@
 package utils
 
 import (
+	"archive/zip"
+	"io"
+	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 )
@@ -50,3 +54,52 @@ func ReplaceSpecialCharacters(filename string) string {
 
 	return filename
 }
+
+/* Zip File Handler */
+// zipFiles compresses multiple files into a single zip archive file
+func ZipFiles(filename string, files ...string) error {
+	newZipFile, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer newZipFile.Close()
+
+	zipWriter := zip.NewWriter(newZipFile)
+	defer zipWriter.Close()
+
+	for _, file := range files {
+		if err := addFileToZip(zipWriter, file); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// addFileToZip adds an individual file to a zip archive
+func addFileToZip(zipWriter *zip.Writer, filename string) error {
+	fileToZip, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer fileToZip.Close()
+
+	info, err := fileToZip.Stat()
+	if err != nil {
+		return err
+	}
+
+	header, err := zip.FileInfoHeader(info)
+	if err != nil {
+		return err
+	}
+
+	header.Name = filepath.Base(filename)
+	header.Method = zip.Deflate
+
+	writer, err := zipWriter.CreateHeader(header)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(writer, fileToZip)
+	return err
+}

src/mod/utils/utils.go

@@ -49,6 +49,24 @@ func GetPara(r *http.Request, key string) (string, error) {
 	}
 }
 
+// Get GET paramter as boolean, accept 1 or true
+func GetBool(r *http.Request, key string) (bool, error) {
+	x, err := GetPara(r, key)
+	if err != nil {
+		return false, err
+	}
+
+	x = strings.TrimSpace(x)
+
+	if x == "1" || strings.ToLower(x) == "true" || strings.ToLower(x) == "on" {
+		return true, nil
+	} else if x == "0" || strings.ToLower(x) == "false" || strings.ToLower(x) == "off" {
+		return false, nil
+	}
+
+	return false, errors.New("invalid boolean given")
+}
+
 // Get POST paramter
 func PostPara(r *http.Request, key string) (string, error) {
 	r.ParseForm()

src/start.go

@@ -145,7 +145,7 @@ func startupSequence() {
 	staticWebServer.RestorePreviousState()
 
 	//Create a netstat buffer
-	netstatBuffers, err = netstat.NewNetStatBuffer(300)
+	netstatBuffers, err = netstat.NewNetStatBuffer(300, SystemWideLogger)
 	if err != nil {
 		SystemWideLogger.PrintAndLog("Network", "Failed to load network statistic info", err)
 		panic(err)

src/web/components/cert.html

@@ -358,10 +358,10 @@
                     let isExpired = entry.RemainingDays <= 0;
 
                     $("#certifiedDomainList").append(`<tr>
-                        <td>${entry.Domain}</td>
+                        <td><a style="cursor: pointer;" title="Download certificate" onclick="handleCertDownload('${entry.Domain}');">${entry.Domain}</a></td>
                         <td>${entry.LastModifiedDate}</td>
                         <td class="${isExpired?"expired":"valid"} certdate">${entry.ExpireDate} (${!isExpired?entry.RemainingDays+" days left":"Expired"})</td>
-                        <td><i class="${entry.UseDNS?"green check": "red times"} circle outline icon"></i></td>
+                        <td><i class="${entry.UseDNS?"green check": "red times"} icon"></i></td>
                         <td><button title="Renew Certificate" class="ui mini basic icon button renewButton" onclick="renewCertificate('${entry.Domain}', '${entry.UseDNS}', this);"><i class="ui green refresh icon"></i></button></td>
                         <td><button title="Delete key-pair" class="ui mini basic red icon button" onclick="deleteCertificate('${entry.Domain}');"><i class="ui red trash icon"></i></button></td>
                     </tr>`);
@@ -397,6 +397,19 @@
             initManagedDomainCertificateList();
         });
     }
+
+    function handleCertDownload(certName){
+        $.get("/api/cert/download?seek=true&certname=" + certName, function(data){
+            if (data.error != undefined){
+                //Error resolving certificate
+                msgbox(data.error, false);
+            }else{
+                //Continue to download  
+                window.open("/api/cert/download?certname=" + certName);
+            }
+        });
+    }
+
     //Handle domain keys upload
     function handleDomainKeysUpload(callback=undefined){
         let domain = $("#certdomain").val();

tools/dns_challenge_update/code-gen/extract.go

@@ -28,6 +28,7 @@ var defTemplate string = `package acmedns
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 
 	"github.com/go-acme/lego/v4/challenge"
 {{imports}}
@@ -282,6 +283,18 @@ func main() {
 		}
 		return ` + providerName + `.NewDNSProviderConfig(cfg)`
 
+		//Add fixed for Netcup timeout
+		if strings.ToLower(providerName) == "netcup" {
+			codeSegment = `
+		case "` + providerName + `":
+			cfg := ` + providerName + `.NewDefaultConfig()
+			err := json.Unmarshal([]byte(js), &cfg)
+			if err != nil {
+				return nil, err
+			}
+			cfg.PropagationTimeout = 1200 * time.Second
+			return ` + providerName + `.NewDNSProviderConfig(cfg)`
+		}
 		generatedConvertcode += codeSegment
 		importList += `	"github.com/go-acme/lego/v4/providers/dns/` + providerName + "\"\n"
 	}

tools/dns_challenge_update/code-gen/update.sh

@@ -21,6 +21,7 @@ go run ./extract.go
 go run ./extract.go -- "win7"
 
 echo "Cleaning up lego"
+sleep 2
 # Comment the line below if you dont want to pull everytime update
 # This is to help go compiler to not load all the lego source file when compile
 #rm -rf ./lego/