Merge pull request #439 from tobychui/v3.1.5

Fixed hostname case sensitive bug
Fixed ACME table too wide css bug
Fixed HSTS toggle button bug
Fixed slow GeoIP resolve mode concurrent r/w bug
Added close connection as default site option
Added experimental authelia support
Added custom header support to websocket
Added levelDB as database implementation (not currently used)
Added external GeoIP db loading support
Restructured a lot of modules

.github/workflows/docker.yml

@@ -0,0 +1,43 @@
+name: Build and push Docker image
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  setup-build-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.release.tag_name }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      
+      - name: Setup building file structure
+        run: |
+          cp -lr $GITHUB_WORKSPACE/src/ $GITHUB_WORKSPACE/docker/
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./docker
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: |
+            zoraxydocker/zoraxy:latest
+            zoraxydocker/zoraxy:${{ github.event.release.tag_name }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+

.github/workflows/main.yml

@@ -1,41 +0,0 @@
-name: Image Publisher
-
-on:
-  release:
-    types: [ published ]
-
-jobs:
-  setup-build-push:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.release.tag_name }}
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to Docker & GHCR
-        run: |
-          echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
-      
-      - name: Setup building file structure
-        run: |
-          cp -r $GITHUB_WORKSPACE/src/ $GITHUB_WORKSPACE/docker/
-
-      - name: Build the image
-        run: |
-          cd $GITHUB_WORKSPACE/docker/
-          docker buildx create --name mainbuilder --driver docker-container --platform linux/amd64,linux/arm64 --use
-          
-          docker buildx build --push \
-            --build-arg VERSION=${{ github.event.release.tag_name }} \
-            --provenance=false \
-            --platform linux/amd64,linux/arm64 \
-            --tag zoraxydocker/zoraxy:${{ github.event.release.tag_name }} \
-            --tag zoraxydocker/zoraxy:latest \
-            .

.gitignore

@@ -30,7 +30,13 @@ src/certs/*
 src/rules/*
 src/README.md
 docker/ContainerTester.sh
-docker/ImagePublisher.sh
+docker/docker-compose.yaml
 src/mod/acme/test/stackoverflow.pem
 /tools/dns_challenge_update/code-gen/acmedns
 /tools/dns_challenge_update/code-gen/lego
+src/tmp/localhost.key
+src/tmp/localhost.pem
+src/www/html/index.html
+src/sys.uuid
+src/zoraxy
+src/log/

CHANGELOG.md

@@ -1,3 +1,132 @@
+# v3.1.4 24 Nov 2024
+
++ **Added Dark Theme Mode** [#390](https://github.com/tobychui/zoraxy/issues/390) [#82](https://github.com/tobychui/zoraxy/issues/82)
++ Added an auto sniffer for self-signed certificates
++ Added robots.txt to the project
++ Introduced an EU wrapper in the front-end for automatic registration of 26 countries [#378](https://github.com/tobychui/zoraxy/issues/378)
++ Moved all hard-coded values to a dedicated def.go file
++ Fixed a panic issue occurring on unsupported platform exits
++ Integrated fixes for SSH proxy and Docker snippet updates [#330](https://github.com/tobychui/zoraxy/issues/330) [#348](https://github.com/tobychui/zoraxy/issues/348)
++ **Changed the default listening port to 443 and enable TLS by default**
++ Optimized GeoIP database slow-search mode CPU usage
+
+
+# v3.1.3 12 Nov 2024
+
++ Fixed a critical security bug [CVE-2024-52010](https://github.com/advisories/GHSA-7hpf-g48v-hw3j)
+
+# v3.1.2 03 Nov 2024
+
++ Added auto start port 80 listener on acme certificate generator
++ Added polling interval and propagation timeout option in ACME module [#300](https://github.com/tobychui/zoraxy/issues/300)
++ Added support for custom header variables [#318](https://github.com/tobychui/zoraxy/issues/318)
++ Added support for X-Remote-User 
++ Added port scanner [#342](https://github.com/tobychui/zoraxy/issues/342)
++ Optimized code base for stream proxy and config file storage [#320](https://github.com/tobychui/zoraxy/issues/320)
++ Removed sorting on cert list
++ Fixed request certificate button bug 
++ Fixed cert auto renew logic [#316](https://github.com/tobychui/zoraxy/issues/316)
++ Fixed unable to remove new stream proxy bug
++ Fixed many other minor bugs  [#328](https://github.com/tobychui/zoraxy/issues/328) [#297](https://github.com/tobychui/zoraxy/issues/297)
++ Added more code to SSO system (disabled in release)
+
+
+# v3.1.1. 09 Sep 2024
+
++ Updated country name in access list [#287](https://github.com/tobychui/zoraxy/issues/287)
++ Added tour for basic operations
++ Updated acme log to system wide logger implementation
++ Fixed path traversal in file manager [#274](https://github.com/tobychui/zoraxy/issues/274)
++ Removed Proxmox debug code
++ Fixed trie tree implementations
+
+**Thanks to all contributors**
+
++ Fix existing containers list in docker popup [7brend7](https://github.com/tobychui/zoraxy/issues?q=is%3Apr+author%3A7brend7)
++ Fix network I/O chart not rendering [JokerQyou](https://github.com/tobychui/zoraxy/issues?q=is%3Apr+author%3AJokerQyou)
++ Fix typo remvoeClass to removeClass [Aahmadsyamim](https://github.com/tobychui/zoraxy/issues?q=is%3Apr+author%3Aahmadsyamim)
++ Updated weighted random upstream implementation [bouroo](https://github.com/tobychui/zoraxy/issues?q=is%3Apr+author%3Abouroo)
+
+# v3.1.0 31 Jul 2024
+
++ Updated log viewer with filter and auto refresh [#243](https://github.com/tobychui/zoraxy/issues/243)
++ Fixed csrf vulnerability [#267](https://github.com/tobychui/zoraxy/issues/267)
++ Fixed promox issue
++ Fixed status code bug in upstream log [#254](https://github.com/tobychui/zoraxy/issues/254)
++ Added host overwrite and hop-by-hop header remover
++ Added early renew days settings [#256](https://github.com/tobychui/zoraxy/issues/256)
++ Updated make file to force no CGO in cicd process
++ Fixed bug in updater
++ Fixed wildcard certificate renew bug [#249](https://github.com/tobychui/zoraxy/issues/249)
++ Added certificate download function [#227](https://github.com/tobychui/zoraxy/issues/227)
+
+# v3.0.9 16 Jul 2024
+
++ Added certificate download [#227](https://github.com/tobychui/zoraxy/issues/227)
++ Updated netcup timeout value [#231](https://github.com/tobychui/zoraxy/issues/231)
++ Updated geoip db
++ Removed debug print from log viewer
++ Upgraded netstat log printing to new log formatter
++ Improved update module implementation
+
+# v3.0.8 15 Jul 2024
+
++ Added apache style logging mechanism (and build-in log viewer) [#218](https://github.com/tobychui/zoraxy/issues/218)
++ Fixed keep alive flushing issues [#235](https://github.com/tobychui/zoraxy/issues/235)
++ Added multi-upstream supports [#100](https://github.com/tobychui/zoraxy/issues/100)
++ Added stick session load balancer
++ Added weighted random load balancer
++ Added domain cleaning logic to domain / IP input fields
++ Added HSTS "include subdomain" auto injector
++ Added work-in-progress SSO / Oauth Server UI
++ Fixed uptime monitor not updating on proxy rule change bug
++ Optimized UI for create new proxy rule
++ Removed service expose proxy feature
+
+# v3.0.7 20 Jun 2024
+
++ Fixed redirection enable bug [#199](https://github.com/tobychui/zoraxy/issues/199)
++ Fixed header tool user agent rewrite sequence
++ Optimized rate limit UI
++ Added HSTS and Permission Policy Editor [#163](https://github.com/tobychui/zoraxy/issues/163)
++ Docker UX optimization start parameter `-docker`
++ Docker container selector implementation for conditional compilations for Windows
+
+From contributors:
+
++ Add Rate Limits Limits to Zoraxy fixes [185](https://github.com/tobychui/zoraxy/issues/185) by [Kirari04](https://github.com/Kirari04)
++ Add docker containers list to set rule by [7brend7](https://github.com/7brend7) [PR202](https://github.com/tobychui/zoraxy/pull/202)
+
+# v3.0.6 10 Jun 2024
+
++ Added fastly_client_ip to X-Real-IP auto rewrite
++ Added atomic accumulator to TCP proxy
++ Added white logo for future dark theme
++ Added multi selection for white / blacklist [#176](https://github.com/tobychui/zoraxy/issues/176)
++ Moved custom header rewrite to dpcore
++ Restructure dpcore header rewrite sequence
++ Added advance custom header settings (zoraxy to upstream and zoraxy to downstream mode)
++ Added header remove feature
++ Removed password requirement for SMTP [#162](https://github.com/tobychui/zoraxy/issues/162) [#80](https://github.com/tobychui/zoraxy/issues/80) 
++ Restructured TCP proxy into Stream Proxy (Support both TCP and UDP) [#147](https://github.com/tobychui/zoraxy/issues/147)
++ Added stream proxy auto start [#169](https://github.com/tobychui/zoraxy/issues/169)
++ Optimized UX for reminding user to click Apply after port change
++ Added version number to footer [#160](https://github.com/tobychui/zoraxy/issues/160)
+
+From contributors:
+
++ Fixed missing / unnecessary error check [PR187](https://github.com/tobychui/zoraxy/pull/187) by [Kirari04](https://github.com/Kirari04)
+
+# v3.0.5 May 26 2024
+
+
++ Optimized uptime monitor error message [#121](https://github.com/tobychui/zoraxy/issues/121)
++ Optimized detection logic for internal proxy target and header rewrite condition for HTTP_HOST [#164](https://github.com/tobychui/zoraxy/issues/164)
++ Fixed ovh DNS challenge provider form generator bug [#161](https://github.com/tobychui/zoraxy/issues/161)
++ Added permission policy module (not enabled)
++ Added single-use cookiejar to uptime monitor request client to handle cookie issues on some poorly written back-end server [#149](https://github.com/tobychui/zoraxy/issues/149)
+
+
 # v3.0.4 May 18 2024
 
 ## This release tidied up the contribution by [Teifun2](https://github.com/Teifun2) and added a new way to generate DNS challenge based certificate (e.g. wildcards) from Let's Encrypt without changing any environment variables. This also fixes a few previous ACME module EAB settings bug related to concurrent save.

README.md

@@ -4,7 +4,6 @@
 
 A general purpose HTTP reverse proxy and forwarding tool. Now written in Go!
 
-
 ### Features
 
 - Simple to use interface with detail in-system instructions
@@ -21,29 +20,39 @@ A general purpose HTTP reverse proxy and forwarding tool. Now written in Go!
   - DNS Challenge for Let's Encrypt and [these DNS providers](https://go-acme.github.io/lego/dns/)
 - Blacklist / Whitelist by country or IP address (single IP, CIDR or wildcard for beginners)
 - Global Area Network Controller Web UI (ZeroTier not included)
-- TCP Tunneling / Proxy
+- Stream Proxy (TCP & UDP)
 - Integrated Up-time Monitor
 - Web-SSH Terminal
 - Utilities
   - CIDR IP converters
   - mDNS Scanner
+  - Wake-On-Lan
+  - Debug Forward Proxy
   - IP Scanner
 - Others
   - Basic single-admin management mode
   - External permission management system for easy system integration
   - SMTP config for password reset
+  - Dark Theme Mode
 
 ## Downloads
 
 [Windows](https://github.com/tobychui/zoraxy/releases/latest/download/zoraxy_windows_amd64.exe)
-/[Linux (amd64)](https://github.com/tobychui/zoraxy/releases/latest/download/zoraxy_linux_amd64)
-/[Linux (arm64)](https://github.com/tobychui/zoraxy/releases/latest/download/zoraxy_linux_arm64)
+/ [Linux (amd64)](https://github.com/tobychui/zoraxy/releases/latest/download/zoraxy_linux_amd64)
+/ [Linux (arm64)](https://github.com/tobychui/zoraxy/releases/latest/download/zoraxy_linux_arm64)
 
-For other systems or architectures, please see [Release](https://github.com/tobychui/zoraxy/releases/latest/) 
+For other systems or architectures, please see [Releases](https://github.com/tobychui/zoraxy/releases/latest/) 
+
+## Getting Started
+
+[Installing Zoraxy Reverse Proxy: Your Gateway to Efficient Web Routing](https://geekscircuit.com/installing-zoraxy-reverse-proxy-your-gateway-to-efficient-web-routing/)
+
+Thank you for the well written and easy to follow tutorial by Reddit user [itsvmn](https://www.reddit.com/user/itsvmn/)! 
+If you have no background in setting up reverse proxy or web routing, you should check this out before you start setting up your Zoraxy. 
 
 ## Build from Source
 
-Requires Go 1.22 or higher
+Requires Go 1.23 or higher
 
 ```bash
 git clone https://github.com/tobychui/zoraxy
@@ -56,7 +65,7 @@ sudo ./zoraxy -port=:8000
 
 ## Usage
 
-Zoraxy provides basic authentication system for standalone mode. To use it in standalone mode, follow the instructionss below for your desired deployment platform.
+Zoraxy provides basic authentication system for standalone mode. To use it in standalone mode, follow the instructions below for your desired deployment platform.
 
 ### Standalone Mode
 
@@ -84,16 +93,20 @@ The installation method is same as Linux. For other ARM SBCs, please refer to yo
 
 See the [/docker](https://github.com/tobychui/zoraxy/tree/main/docker) folder for more details.
 
-### Start Paramters
+### Start Parameters
 
 ```
 Usage of zoraxy:
   -autorenew int
         ACME auto TLS/SSL certificate renew check interval (seconds) (default 86400)
+  -cfgupgrade
+        Enable auto config upgrade if breaking change is detected (default true)
+  -docker
+        Run Zoraxy in docker compatibility mode
+  -earlyrenew int
+        Number of days to early renew a soon expiring certificate (days) (default 30)
   -fastgeoip
         Enable high speed geoip lookup, require 1GB extra memory (Not recommend for low end devices)
-  -log
-        Log terminal output to file (default true)
   -mdns
         Enable mDNS scanner and transponder (default true)
   -mdnsname string
@@ -124,7 +137,8 @@ If you already have an upstream reverse proxy server in place with permission ma
 ./zoraxy -noauth=true
 ```
 
-*Note: For security reaons, you should only enable no-auth if you are running Zoraxy in a trusted environment or with another authentication management proxy in front.*
+> [!WARNING]
+> For security reasons, you should only enable no-auth if you are running Zoraxy in a trusted environment or with another authentication management proxy in front.*
 
 ## Screenshots
 
@@ -144,7 +158,7 @@ This project also compatible with [ZeroTier](https://www.zerotier.com/). However
 
 To use Zoraxy with ZeroTier, assuming you already have a valid license, install ZeroTier on your host and then run Zoraxy in sudo mode (or Run As Administrator if you are on Windows). The program will automatically grab the authtoken in the correct location on your host.
 
-If you prefer not to run Zoraxy in sudo mode or you have some weird installation profile, you can also pass in the ZeroTier auth token using the following flags::
+If you prefer not to run Zoraxy in sudo mode or you have some weird installation profile, you can also pass in the ZeroTier auth token using the following flags:
 
 ```bash
 ./zoraxy -ztauth="your_zerotier_authtoken" -ztport=9993
@@ -165,7 +179,7 @@ Web SSH currently only supports Linux based OSes. The following platforms are su
 
 ### Loopback Connection
 
-Loopback web SSH connection, by default, is disabled. This means that if you are trying to connect to an address like 127.0.0.1 or localhost, the system will reject your connection for security reasons. To enable loopback for testing or development purpose, use the following flags to override the loopback checking:
+Loopback web SSH connections, by default, are disabled. This means that if you are trying to connect to an address like 127.0.0.1 or localhost, the system will reject your connection for security reasons. To enable loopback for testing or development purpose, use the following flags to override the loopback checking:
 
 ```bash
 ./zoraxy -sshlb=true
@@ -182,4 +196,3 @@ If you like the project and want to support us, please consider a donation. You
 
 This project is open-sourced under AGPL. I open-sourced this project so everyone can check for security issues and benefit all users. **This software is intended to be free of charge. If you have acquired this software from a third-party seller, the authors of this repository bears no responsibility for any technical difficulties assistance or support.**
 
-

docker/Dockerfile

@@ -1,17 +1,8 @@
-FROM docker.io/golang:alpine
-# VERSION comes from the main.yml workflow --build-arg
-ARG VERSION
-
-RUN apk add --no-cache bash netcat-openbsd sudo
+FROM docker.io/golang:alpine AS build-zoraxy
 
 RUN mkdir -p /opt/zoraxy/source/ &&\
-    mkdir -p /opt/zoraxy/config/ &&\
     mkdir -p /usr/local/bin/
 
-RUN chmod -R 770 /opt/zoraxy/
-
-VOLUME [ "/opt/zoraxy/config/" ]
-
 # If you build it yourself, you will need to add the src directory into the docker directory.
 COPY ./src/ /opt/zoraxy/source/
 
@@ -19,17 +10,59 @@ WORKDIR /opt/zoraxy/source/
 
 RUN go mod tidy &&\
     go build -o /usr/local/bin/zoraxy &&\
-    rm -r /opt/zoraxy/source/
+    chmod 755 /usr/local/bin/zoraxy
 
-RUN chmod 755 /usr/local/bin/zoraxy &&\
-    chmod +x /usr/local/bin/zoraxy
+FROM docker.io/ubuntu:latest AS build-zerotier
+
+RUN mkdir -p /opt/zerotier/source/ &&\
+    mkdir -p /usr/local/bin/
+
+WORKDIR /opt/zerotier/source/
+
+RUN apt-get update -y &&\
+    apt-get install -y curl jq build-essential pkg-config clang cargo libssl-dev
+
+RUN curl -Lo ZeroTierOne.tar.gz https://codeload.github.com/zerotier/ZeroTierOne/tar.gz/refs/tags/1.10.6 &&\
+    tar -xzvf ZeroTierOne.tar.gz &&\
+    cd ZeroTierOne-* &&\
+    make &&\
+    mv ./zerotier-one /usr/local/bin/zerotier-one &&\
+    chmod 755 /usr/local/bin/zerotier-one
+
+FROM docker.io/ubuntu:latest
+
+RUN apt-get update -y &&\
+    apt-get install -y bash sudo netcat-openbsd libssl-dev ca-certificates
+
+COPY --chmod=700 ./entrypoint.sh /opt/zoraxy/
+COPY --from=build-zoraxy /usr/local/bin/zoraxy /usr/local/bin/zoraxy
+COPY --from=build-zerotier /usr/local/bin/zerotier-one /usr/local/bin/zerotier-one
 
 WORKDIR /opt/zoraxy/config/
 
-ENV VERSION=$VERSION
-ENV ARGS="-noauth=false"
+ENV ZEROTIER="false"
 
-ENTRYPOINT "zoraxy" "-port=:8000" "${ARGS}"
+ENV AUTORENEW="86400"
+ENV CFGUPGRADE="true"
+ENV DB="auto"
+ENV DOCKER="true"
+ENV EARLYRENEW="30"
+ENV FASTGEOIP="false"
+ENV MDNS="true"
+ENV MDNSNAME="''"
+ENV NOAUTH="false"
+ENV PORT="8000"
+ENV SSHLB="false"
+ENV UPDATE_GEOIP="false"
+ENV VERSION="false"
+ENV WEBFM="true"
+ENV WEBROOT="./www"
+ENV ZTAUTH=""
+ENV ZTPORT="9993"
 
-HEALTHCHECK --interval=5s --timeout=5s --retries=2 CMD nc -vz 127.0.0.1 8000 || exit 1
+VOLUME [ "/opt/zoraxy/config/" ]
+
+ENTRYPOINT [ "/opt/zoraxy/entrypoint.sh" ]
+
+HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 CMD nc -vz 127.0.0.1 $PORT || exit 1
 

docker/README.md

@@ -1,65 +1,104 @@
-# [zoraxy](https://github.com/tobychui/zoraxy/) </br>
+# Zoraxy Docker
 
 [![Repo](https://img.shields.io/badge/Docker-Repo-007EC6?labelColor-555555&color-007EC6&logo=docker&logoColor=fff&style=flat-square)](https://hub.docker.com/r/zoraxydocker/zoraxy)
 [![Version](https://img.shields.io/docker/v/zoraxydocker/zoraxy/latest?labelColor-555555&color-007EC6&style=flat-square)](https://hub.docker.com/r/zoraxydocker/zoraxy)
 [![Size](https://img.shields.io/docker/image-size/zoraxydocker/zoraxy/latest?sort=semver&labelColor-555555&color-007EC6&style=flat-square)](https://hub.docker.com/r/zoraxydocker/zoraxy)
 [![Pulls](https://img.shields.io/docker/pulls/zoraxydocker/zoraxy?labelColor-555555&color-007EC6&style=flat-square)](https://hub.docker.com/r/zoraxydocker/zoraxy)
 
-## Setup: </br>
-Although not required, it is recommended to give Zoraxy a dedicated location on the host to mount the container. That way, the host/user can access them whenever needed. A volume will be created automatically within Docker if a location is not specified. </br>
+## Usage
 
-You may also need to portforward your 80/443 to allow http and https traffic. If you are accessing the interface from outside of the local network, you may also need to forward your management port. If you know how to do this, great! If not, find the manufacturer of your router and search on how to do that. There are too many to be listed here. </br>
+If you are attempting to access your service from outside your network, make sure to forward ports 80 and 443 to the Zoraxy host to allow web traffic. If you know how to do this, great! If not, find the manufacturer of your router and search on how to do that. There are too many to be listed here. Read more about it from [whatismyip](https://www.whatismyip.com/port-forwarding/).
+
+In the examples below, make sure to update `/path/to/zoraxy/config/`. If a path is not provided, a Docker volume will be created at the location but it is recommended to store the data at a defined host location or a named Docker volume.
+
+Once setup, access the webui at `http://<host-ip>:8000` to configure Zoraxy. Change the port in the URL if you changed the management port.
+
+### Docker Run
 
-### Using Docker run </br>
 ```
-docker run -d --name (container name) -p (ports) -v (path to storage directory):/opt/zoraxy/data/ -e ARGS='(your arguments)' zoraxydocker/zoraxy:latest
+docker run -d \
+  --name zoraxy \
+  --restart unless-stopped \
+  -p 80:80 \
+  -p 443:443 \
+  -p 8000:8000 \
+  -v /path/to/zoraxy/config/:/opt/zoraxy/config/ \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -v /etc/localtime:/etc/localtime \
+  -e FASTGEOIP="true" \
+  zoraxydocker/zoraxy:latest
 ```
 
-### Using Docker Compose </br>
+### Docker Compose
+
 ```yml
-version: '3.3'
 services:
-  zoraxy-docker:
-    image: zoraxydocker/zoraxy:latest
-    container_name: (container name)
-    ports:
-      - 80:80
-      - 443:443
-      - (external):8000
-    volumes:
-      - (path to storage directory):/opt/zoraxy/config/
-    environment:
-      ARGS: '(your arguments)'
-```
-
-| Operator | Need | Details |
-|:-|:-|:-|
-| `-d` | Yes | will run the container in the background. |
-| `--name (container name)` | No | Sets the name of the container to the following word. You can change this to whatever you want. |
-| `-p (ports)` | Yes | Depending on how your network is setup, you may need to portforward 80, 443, and the management port. |
-| `-v (path to storage directory):/opt/zoraxy/config/` | Recommend | Sets the folder that holds your files. This should be the place you just chose. By default, it will create a Docker volume for the files for persistency but they will not be accessible. |
-| `-e ARGS='(your arguments)'` | No | Sets the arguments to run Zoraxy with. Enter them as you would normally. By default, it is ran with `-noauth=false` but <b>you cannot change the management port.</b> This is required for the healthcheck to work. |
-| `zoraxydocker/zoraxy:latest` | Yes | The repository on Docker hub. By default, it is the latest version that is published. |
-
-## Examples: </br>
-### Docker Run </br>
-```
-docker run -d --name zoraxy -p 80:80 -p 443:443 -p 8005:8000/tcp -v /home/docker/Containers/Zoraxy:/opt/zoraxy/config/ -e ARGS='-noauth=false' zoraxydocker/zoraxy:latest
-```
-
-### Docker Compose </br>
-```yml
-version: '3.3'
-services:
-  zoraxy-docker:
+  zoraxy:
     image: zoraxydocker/zoraxy:latest
     container_name: zoraxy
+    restart: unless-stopped
     ports:
       - 80:80
       - 443:443
-      - 8005:8000/tcp
+      - 8000:8000
     volumes:
-      - /home/docker/Containers/Zoraxy:/opt/zoraxy/config/
+      - /path/to/zoraxy/config/:/opt/zoraxy/config/
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /etc/localtime:/etc/localtime
     environment:
-      ARGS: '-noauth=false'
+      FASTGEOIP: "true"
 ```
+
+### Ports
+
+| Port | Details |
+|:-|:-|
+| `80` | HTTP traffic. |
+| `443` | HTTPS traffic. |
+| `8000` | Management interface. Can be changed with the `PORT` env. |
+
+### Volumes
+
+| Volume | Details |
+|:-|:-|
+| `/opt/zoraxy/config/` | Zoraxy configuration. |
+| `/var/run/docker.sock` | Docker socket. Used for additional functionality with Zoraxy. |
+| `/etc/localtime` | Localtime. Set to ensure the host and container are synchronized. |
+
+### Environment
+
+Variables are the same as those in [Start Parameters](https://github.com/tobychui/zoraxy?tab=readme-ov-file#start-paramters).
+
+| Variable | Default | Details |
+|:-|:-|:-|
+| `AUTORENEW` | `86400` (Integer) | ACME auto TLS/SSL certificate renew check interval. |
+| `CFGUPGRADE` | `true` (Boolean) | Enable auto config upgrade if breaking change is detected. |
+| `DB` | `auto` (String) | Database backend to use (leveldb, boltdb, auto) Note that fsdb will be used on unsupported platforms like RISCV (default "auto"). |
+| `DOCKER` | `true` (Boolean) | Run Zoraxy in docker compatibility mode. |
+| `EARLYRENEW` | `30` (Integer) | Number of days to early renew a soon expiring certificate. |
+| `FASTGEOIP` | `false`  (Boolean) | Enable high speed geoip lookup, require 1GB extra memory (Not recommend for low end devices). |
+| `MDNS` | `true` (Boolean) | Enable mDNS scanner and transponder. |
+| `MDNSNAME` | `''` (String) | mDNS name, leave empty to use default (zoraxy_{node-uuid}.local). |
+| `NOAUTH` | `false` (Boolean) | Disable authentication for management interface. |
+| `PORT` | `8000` (Integer) | Management web interface listening port |
+| `SSHLB` | `false` (Boolean) | Allow loopback web ssh connection (DANGER). |
+| `UPDATE_GEOIP` | `false` (Boolean) | Download the latest GeoIP data and exit. |
+| `VERSION` | `false` (Boolean) | Show version of this server. |
+| `WEBFM` | `true` (Boolean) | Enable web file manager for static web server root folder. |
+| `WEBROOT` | `./www` (String) | Static web server root folder. Only allow change in start parameters. |
+| `ZEROTIER` | `false` (Boolean) | Enable ZeroTier functionality for GAN. |
+| `ZTAUTH` | `""` (String) | ZeroTier authtoken for the local node. |
+| `ZTPORT` | `9993` (Integer) | ZeroTier controller API port. |
+
+> [!IMPORTANT]
+> Contrary to the Zoraxy README, Docker usage of the port flag should NOT include the colon. Ex: `-e PORT="8000"` for Docker run and `PORT: "8000"` for Docker compose.
+
+### Building
+
+To build the Docker image:
+  - Check out the repository/branch.
+  - Copy the Zoraxy `src/` directory into the `docker/` (here) directory.
+  - Run the build command with `docker build -t zoraxy_build .`
+  - You can now use the image `zoraxy_build`
+    - If you wish to change the image name, then modify`zoraxy_build` in the previous step and then build again.
+

docker/entrypoint.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+update-ca-certificates
+echo "CA certificates updated."
+
+zoraxy -update_geoip=true
+echo "Updated GeoIP data."
+
+if [ "$ZEROTIER" = "true" ]; then
+  if [ ! -d "/opt/zoraxy/config/zerotier/" ]; then
+    mkdir -p /opt/zoraxy/config/zerotier/
+  fi
+  ln -s /opt/zoraxy/config/zerotier/ /var/lib/zerotier-one
+  zerotier-one -d
+  echo "ZeroTier daemon started."
+fi
+
+echo "Starting Zoraxy..."
+exec zoraxy \
+  -autorenew="$AUTORENEW" \
+  -cfgupgrade="$CFGUPGRADE" \
+  -db="$DB" \
+  -docker="$DOCKER" \
+  -earlyrenew="$EARLYRENEW" \
+  -fastgeoip="$FASTGEOIP" \
+  -mdns="$MDNS" \
+  -mdnsname="$MDNSNAME" \
+  -noauth="$NOAUTH" \
+  -port=:"$PORT" \
+  -sshlb="$SSHLB" \
+  -update_geoip="$UPDATE_GEOIP" \
+  -version="$VERSION" \
+  -webfm="$WEBFM" \
+  -webroot="$WEBROOT" \
+  -ztauth="$ZTAUTH" \
+  -ztport="$ZTPORT"
+

docs/CNAME

@@ -1 +1 @@
-zoraxy.arozos.com
+zoraxy.aroz.org

docs/index.html

@@ -12,19 +12,19 @@
     <meta name="description" content="A reverse proxy server and cluster network gateway for noobs">
 
     <!-- Facebook Meta Tags -->
-    <meta property="og:url" content="https://zoraxy.arozos.com/">
+    <meta property="og:url" content="https://zoraxy.aroz.org/">
     <meta property="og:type" content="website">
     <meta property="og:title" content="Cluster Proxy Gateway | Zoraxy">
     <meta property="og:description" content="A reverse proxy server and cluster network gateway for noobs">
-    <meta property="og:image" content="https://zoraxy.arozos.com/img/og.png">
+    <meta property="og:image" content="https://zoraxy.aroz.org/img/og.png">
 
     <!-- Twitter Meta Tags -->
     <meta name="twitter:card" content="summary_large_image">
-    <meta property="twitter:domain" content="arozos.com">
-    <meta property="twitter:url" content="https://zoraxy.arozos.com/">
+    <meta property="twitter:domain" content="aroz.org">
+    <meta property="twitter:url" content="https://zoraxy.aroz.org/">
     <meta name="twitter:title" content="Cluster Proxy Gateway | Zoraxy">
     <meta name="twitter:description" content="A reverse proxy server and cluster network gateway for noobs">
-    <meta name="twitter:image" content="https://zoraxy.arozos.com/img/og.png">
+    <meta name="twitter:image" content="https://zoraxy.aroz.org/img/og.png">
 
     <!-- Favicons -->
     <link href="favicon.png" rel="icon">
@@ -80,7 +80,7 @@
             <div class="bannerHeaderWrapper">
               <h1 class="bannerHeader">Zoraxy</h1>
 			  <div class="ui divider"></div><br>
-              <p class="bannerSubheader">All in one homelab network routing solution</p>
+              <p class="bannerSubheader">Beyond Reverse Proxy: Your Ultimate Homelab Network Tool</p>
             </div>
             <br><br>
             <a class="ui basic big button" style="background-color: white;" href="#features"><i class="ui blue arrow down icon"></i> Learn More</a>

example/README.md

@@ -15,4 +15,12 @@ The templates folder contains the template for overriding the build in error or
 
 To use the template, copy and paste the `wwww` folder to the same directory as zoraxy executable (aka the src/ file if you `go build` with the current folder tree). 
 
+
+
+### Other Templates
+
+There are a few pre-built templates that works with Zoraxy where you can find in the `other-templates` folder. Copy the folder into `www` and rename the folder to `templates` to active them. 
+
+
+
 It is worth mentioning that the uwu icons for not-found and access-denied are created by @SAWARATSUKI

example/other-templates/templates_cf/notfound.html

@@ -0,0 +1,154 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <meta name="apple-mobile-web-app-capable" content="yes" />
+        <meta name="viewport" content="user-scalable=no, width=device-width, initial-scale=1, maximum-scale=1"/>
+        <meta charset="UTF-8">
+        <meta name="theme-color" content="#4b75ff">
+        <link rel="icon" type="image/png" href="img/small_icon.png"/>
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/fomantic-ui/2.9.2/semantic.min.css">
+        <link rel="preconnect" href="https://fonts.googleapis.com">
+        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+        <link href="https://fonts.googleapis.com/css2?family=Noto+Sans+TC:wght@300;400;500;700;900&display=swap" rel="stylesheet">
+        <script src="https://code.jquery.com/jquery-3.6.4.min.js"></script>
+        <script src="https://cdnjs.cloudflare.com/ajax/libs/fomantic-ui/2.9.2/semantic.min.js"></script>
+        <title>404 - Host Not Found</title>
+        <style>
+            h1, h2, h3, h4, h5, p, a, span, .ui.list .item{
+                font-family: 'Noto Sans TC', sans-serif;
+                font-weight: 300;
+                color: rgb(88, 88, 88)
+            }
+
+            .diagram{
+                background-color: #ebebeb;
+                padding-bottom: 2em;
+            }
+
+            .diagramHeader{
+                margin-top: 0.2em;
+            }
+
+            @media (max-width:512px) { 
+                .widescreenOnly{
+                    display: none !important;
+                    
+                }
+
+                .four.wide.column:not(.widescreenOnly){
+                    width: 50% !important;
+                }
+
+                .ui.grid{
+                    justify-content: center !important;
+                }
+            }
+        </style>
+    </head>
+    <body>
+        <div>
+            <br><br>
+            <div class="ui container">
+                <h1 style="font-size: 4rem;">Error 404</h1>
+                <p style="font-size: 2rem; margin-bottom: 0.4em;">Target Host Not Found</p>
+                <small id="timestamp"></small>
+            </div>
+            <br><br>
+        </div>
+        <div class="diagram">
+            <div class="ui text container">
+                <div class="ui grid">
+                    <div class="four wide column widescreenOnly" align="center">
+                        <svg version="1.1" id="client_svg" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+                            width="100%" viewBox="0 0 200 200" enable-background="new 0 0 200 200" xml:space="preserve">
+                        <path fill="#C9CACA" d="M184.795,143.037c0,9.941-8.059,18-18,18H33.494c-9.941,0-18-8.059-18-18V44.952c0-9.941,8.059-18,18-18
+                            h133.301c9.941,0,18,8.059,18,18V143.037z"/>
+                        <circle fill="#FFFFFF" cx="37.39" cy="50.88" r="6.998"/>
+                        <circle fill="#FFFFFF" cx="54.115" cy="50.88" r="6.998"/>
+                        <path fill="#FFFFFF" d="M167.188,50.88c0,3.865-3.133,6.998-6.998,6.998H72.379c-3.865,0-6.998-3.133-6.998-6.998l0,0
+                            c0-3.865,3.133-6.998,6.998-6.998h87.811C164.055,43.882,167.188,47.015,167.188,50.88L167.188,50.88z"/>
+                        <rect x="31.296" y="66.907" fill="#FFFFFF" width="132.279" height="77.878"/>
+                        <circle fill="#9BCA3E" cx="96.754" cy="144.785" r="37.574"/>
+                        <polyline fill="none" stroke="#FFFFFF" stroke-width="8" stroke-miterlimit="10" points="108.497,133.047 93.373,153.814 
+                            82.989,143.204 "/>
+                        </svg>
+                        <small>You</small>
+                        <h2 class="diagramHeader">Browser</h2>
+                        <p style="font-weight: 500; color: #9bca3e;">Working</p>
+                    </div>  
+                    <div class="two wide column widescreenOnly" style="margin-top: 8em; text-align: center;">
+                        <i class="ui big grey exchange alternate icon" style="color:rgb(167, 167, 167) !important;"></i>
+                    </div>
+                    <div class="four wide column widescreenOnly" align="center">
+                        <svg version="1.1" id="cloud_svg" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+                            width="100%" viewBox="0 0 200 200" enable-background="new 0 0 200 200" xml:space="preserve">
+                            <ellipse fill="#9FA0A0" cx="46.979" cy="108.234" rx="25.399" ry="25.139"/>
+                            <circle fill="#9FA0A0" cx="109.407" cy="100.066" r="50.314"/>
+                            <circle fill="#9FA0A0" cx="22.733" cy="129.949" r="19.798"/>
+                            <circle fill="#9FA0A0" cx="172.635" cy="125.337" r="24.785"/>
+                            <path fill="#9FA0A0" d="M193.514,133.318c0,9.28-7.522,16.803-16.803,16.803H28.223c-9.281,0-16.803-7.522-16.803-16.803l0,0
+                                c0-9.28,7.522-16.804,16.803-16.804h148.488C185.991,116.515,193.514,124.038,193.514,133.318L193.514,133.318z"/>
+                            <circle fill="#9BCA3D" cx="100" cy="149.572" r="38.267"/>
+                            <polyline fill="none" stroke="#FFFFFF" stroke-width="8" stroke-miterlimit="10" points="113.408,136.402 95.954,160.369 
+                                83.971,148.123 "/>
+                        </svg>
+
+                        <small>Gateway Node</small>
+                        <h2 class="diagramHeader">Reverse Proxy</h2>
+                        <p style="font-weight: 500; color: #9bca3e;">Working</p>
+                    </div>  
+                    <div class="two wide column widescreenOnly" style="margin-top: 8em; text-align: center;">
+                        <i class="ui big grey exchange alternate icon" style="color:rgb(167, 167, 167) !important;"></i>
+                    </div>
+                    <div class="four wide column" align="center">
+                        <svg version="1.1" id="host_svg" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+                            width="100%" viewBox="0 0 200 200" enable-background="new 0 0 200 200" xml:space="preserve">
+                        <path fill="#999999" d="M168.484,113.413c0,9.941,3.317,46.324-6.624,46.324H35.359c-9.941,0-5.873-39.118-5.715-46.324
+                            l17.053-50.909c1.928-9.879,8.059-18,18-18h69.419c9.941,0,15.464,7.746,18,18L168.484,113.413z"/>
+                        <rect x="38.068" y="118.152" fill="#FFFFFF" width="122.573" height="34.312"/>
+                        <circle fill="#BD2426" cx="141.566" cy="135.873" r="8.014"/>
+                        <circle fill="#BD2426" cx="99.354" cy="152.464" r="36.343"/>
+                        <line fill="none" stroke="#FFFFFF" stroke-width="6" stroke-miterlimit="10" x1="90.5" y1="144.125" x2="107.594" y2="161.946"/>
+                        <line fill="none" stroke="#FFFFFF" stroke-width="6" stroke-miterlimit="10" x1="90.5" y1="161.946" x2="107.594" y2="144.79"/>
+                        </svg>
+                        <small id="host"></small>
+                        <h2 class="diagramHeader">Host</h2>
+                        <p style="font-weight: 500; color: #bd2426;">Not Found</p>
+                    </div>
+                  </div>
+            </div>
+        </div>
+        <div>
+            <br>
+            <div class="ui container">
+                <div class="ui stackable grid">
+                    <div class="eight wide column">
+                        <h1>What happend?</h1>
+                        <p>The reverse proxy target domain is not found.<br>For more information, see the error message on the reverse proxy terminal.</p>
+                    </div>
+                    <div class="eight wide column">
+                        <h1>What can I do?</h1>
+                        <h5 style="font-weight: 500;">If you are a visitor of this website: </h5>
+                        <p>Please try again in a few minutes</p>
+                        <h5 style="font-weight: 500;">If you are the owner of this website:</h5>
+                        <div class="ui bulleted list">
+                            <div class="item">Check if the proxy rules that match this hostname exists</div>
+                            <div class="item">Visit the Reverse Proxy management interface to correct any setting errors</div>
+                        </div>
+                    </div>
+                  </div>
+            </div>
+            <br>
+        </div>
+        <div class="ui divider"></div>
+        <div class="ui container" style="color: grey; font-size: 90%">
+            <p>Powered by Zoraxy</p>
+        </div>
+        <br><br>
+
+        <script>
+            $("#timestamp").text(new Date());
+            $("#host").text(location.href);
+        </script>
+    </body>
+</html>

src/Makefile

@@ -19,7 +19,7 @@ clean:
 
 $(PLATFORMS):
 	@echo "Building $(os)/$(arch)"
-	GOROOT_FINAL=Git/ GOOS=$(os) GOARCH=$(arch) $(if $(filter linux/arm,$(os)/$(arch)),GOARM=6,) go build -o './dist/zoraxy_$(os)_$(arch)'  -ldflags "-s -w" -trimpath
+	GOROOT_FINAL=Git/ GOOS=$(os) GOARCH=$(arch) $(if $(filter linux/arm,$(os)/$(arch)),GOARM=6,) CGO_ENABLED="0" go build -o './dist/zoraxy_$(os)_$(arch)'  -ldflags "-s -w" -trimpath
 #	GOROOT_FINAL=Git/ GOOS=$(os) GOARCH=$(arch) GOARM=6 go build -o './dist/zoraxy_$(os)_$(arch)'  -ldflags "-s -w" -trimpath
 
 

src/accesslist.go

@@ -230,7 +230,17 @@ func handleCountryBlacklistAdd(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	rule.AddCountryCodeToBlackList(countryCode, comment)
+	//Check if the country code contains comma, if yes, split it
+	if strings.Contains(countryCode, ",") {
+		codes := strings.Split(countryCode, ",")
+		for _, code := range codes {
+			code = strings.TrimSpace(code)
+			rule.AddCountryCodeToBlackList(code, comment)
+		}
+	} else {
+		countryCode = strings.TrimSpace(countryCode)
+		rule.AddCountryCodeToBlackList(countryCode, comment)
+	}
 
 	utils.SendOK(w)
 }
@@ -254,7 +264,17 @@ func handleCountryBlacklistRemove(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	rule.RemoveCountryCodeFromBlackList(countryCode)
+	//Check if the country code contains comma, if yes, split it
+	if strings.Contains(countryCode, ",") {
+		codes := strings.Split(countryCode, ",")
+		for _, code := range codes {
+			code = strings.TrimSpace(code)
+			rule.RemoveCountryCodeFromBlackList(code)
+		}
+	} else {
+		countryCode = strings.TrimSpace(countryCode)
+		rule.RemoveCountryCodeFromBlackList(countryCode)
+	}
 
 	utils.SendOK(w)
 }
@@ -397,7 +417,17 @@ func handleCountryWhitelistAdd(w http.ResponseWriter, r *http.Request) {
 	p := bluemonday.StrictPolicy()
 	comment = p.Sanitize(comment)
 
-	rule.AddCountryCodeToWhitelist(countryCode, comment)
+	//Check if the country code contains comma, if yes, split it
+	if strings.Contains(countryCode, ",") {
+		codes := strings.Split(countryCode, ",")
+		for _, code := range codes {
+			code = strings.TrimSpace(code)
+			rule.AddCountryCodeToWhitelist(code, comment)
+		}
+	} else {
+		countryCode = strings.TrimSpace(countryCode)
+		rule.AddCountryCodeToWhitelist(countryCode, comment)
+	}
 
 	utils.SendOK(w)
 }
@@ -420,7 +450,17 @@ func handleCountryWhitelistRemove(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	rule.RemoveCountryCodeFromWhitelist(countryCode)
+	//Check if the country code contains comma, if yes, split it
+	if strings.Contains(countryCode, ",") {
+		codes := strings.Split(countryCode, ",")
+		for _, code := range codes {
+			code = strings.TrimSpace(code)
+			rule.RemoveCountryCodeFromWhitelist(code)
+		}
+	} else {
+		countryCode = strings.TrimSpace(countryCode)
+		rule.RemoveCountryCodeFromWhitelist(countryCode)
+	}
 
 	utils.SendOK(w)
 }

src/acme.go

@@ -38,7 +38,21 @@ func initACME() *acme.ACMEHandler {
 		port = getRandomPort(30000)
 	}
 
-	return acme.NewACME("https://acme-v02.api.letsencrypt.org/directory", strconv.Itoa(port), sysdb)
+	return acme.NewACME("https://acme-v02.api.letsencrypt.org/directory", strconv.Itoa(port), sysdb, SystemWideLogger)
+}
+
+// Restart ACME handler and auto renewer
+func restartACMEHandler() {
+	SystemWideLogger.Println("Restarting ACME handler")
+	//Clos the current handler and auto renewer
+	acmeHandler.Close()
+	acmeAutoRenewer.Close()
+	acmeDeregisterSpecialRoutingRule()
+
+	//Reinit the handler with a new random port
+	acmeHandler = initACME()
+
+	acmeRegisterSpecialRoutingRule()
 }
 
 // create the special routing rule for ACME
@@ -82,12 +96,29 @@ func acmeRegisterSpecialRoutingRule() {
 	}
 }
 
+// remove the special routing rule for ACME
+func acmeDeregisterSpecialRoutingRule() {
+	SystemWideLogger.Println("Removing ACME routing rule")
+	dynamicProxyRouter.RemoveRoutingRule("acme-autorenew")
+}
+
 // This function check if the renew setup is satisfied. If not, toggle them automatically
 func AcmeCheckAndHandleRenewCertificate(w http.ResponseWriter, r *http.Request) {
 	isForceHttpsRedirectEnabledOriginally := false
+	requireRestorePort80 := false
 	dnsPara, _ := utils.PostBool(r, "dns")
 	if !dnsPara {
+
 		if dynamicProxyRouter.Option.Port == 443 {
+			//Check if port 80 is enabled
+			if !dynamicProxyRouter.Option.ListenOnPort80 {
+				//Enable port 80 temporarily
+				SystemWideLogger.PrintAndLog("ACME", "Temporarily enabling port 80 listener to handle ACME request ", nil)
+				dynamicProxyRouter.UpdatePort80ListenerState(true)
+				requireRestorePort80 = true
+				time.Sleep(2 * time.Second)
+			}
+
 			//Enable port 80 to 443 redirect
 			if !dynamicProxyRouter.Option.ForceHttpsRedirect {
 				SystemWideLogger.Println("Temporary enabling HTTP to HTTPS redirect for ACME certificate renew requests")
@@ -107,8 +138,8 @@ func AcmeCheckAndHandleRenewCertificate(w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	//Add a 3 second delay to make sure everything is settle down
-	time.Sleep(3 * time.Second)
+	//Add a 2 second delay to make sure everything is settle down
+	time.Sleep(2 * time.Second)
 
 	// Pass over to the acmeHandler to deal with the communication
 	acmeHandler.HandleRenewCertificate(w, r)
@@ -117,13 +148,17 @@ func AcmeCheckAndHandleRenewCertificate(w http.ResponseWriter, r *http.Request)
 	tlsCertManager.UpdateLoadedCertList()
 
 	//Restore original settings
-	if dynamicProxyRouter.Option.Port == 443 && !dnsPara {
-		if !isForceHttpsRedirectEnabledOriginally {
-			//Default is off. Turn the redirection off
-			SystemWideLogger.PrintAndLog("ACME", "Restoring HTTP to HTTPS redirect settings", nil)
-			dynamicProxyRouter.UpdateHttpToHttpsRedirectSetting(false)
-		}
+	if requireRestorePort80 {
+		//Restore port 80 listener
+		SystemWideLogger.PrintAndLog("ACME", "Restoring previous port 80 listener settings", nil)
+		dynamicProxyRouter.UpdatePort80ListenerState(false)
 	}
+	if !isForceHttpsRedirectEnabledOriginally {
+		//Default is off. Turn the redirection off
+		SystemWideLogger.PrintAndLog("ACME", "Restoring HTTP to HTTPS redirect settings", nil)
+		dynamicProxyRouter.UpdateHttpToHttpsRedirectSetting(false)
+	}
+
 }
 
 // HandleACMEPreferredCA return the user preferred / default CA for new subdomain auto creation

src/api.go

@@ -8,6 +8,8 @@ import (
 	"imuslab.com/zoraxy/mod/acme/acmedns"
 	"imuslab.com/zoraxy/mod/acme/acmewizard"
 	"imuslab.com/zoraxy/mod/auth"
+	"imuslab.com/zoraxy/mod/dynamicproxy/domainsniff"
+	"imuslab.com/zoraxy/mod/ipscan"
 	"imuslab.com/zoraxy/mod/netstat"
 	"imuslab.com/zoraxy/mod/netutils"
 	"imuslab.com/zoraxy/mod/utils"
@@ -17,34 +19,11 @@ import (
 	API.go
 
 	This file contains all the API called by the web management interface
-
 */
 
-var requireAuth = true
-
-func initAPIs() {
-
-	authRouter := auth.NewManagedHTTPRouter(auth.RouterOption{
-		AuthAgent:   authAgent,
-		RequireAuth: requireAuth,
-		DeniedHandler: func(w http.ResponseWriter, r *http.Request) {
-			http.Error(w, "401 - Unauthorized", http.StatusUnauthorized)
-		},
-	})
-
-	//Register the standard web services urls
-	fs := http.FileServer(http.FS(webres))
-	if development {
-		fs = http.FileServer(http.Dir("web/"))
-	}
-	//Add a layer of middleware for advance  control
-	advHandler := FSHandler(fs)
-	http.Handle("/", advHandler)
-
-	//Authentication APIs
-	registerAuthAPIs(requireAuth)
-
-	//Reverse proxy
+// Register the APIs for HTTP proxy management functions
+func RegisterHTTPProxyAPIs(authRouter *auth.RouterDef) {
+	/* Reverse Proxy Settings & Status */
 	authRouter.HandleFunc("/api/proxy/enable", ReverseProxyHandleOnOff)
 	authRouter.HandleFunc("/api/proxy/add", ReverseProxyHandleAddEndpoint)
 	authRouter.HandleFunc("/api/proxy/status", ReverseProxyStatus)
@@ -55,76 +34,113 @@ func initAPIs() {
 	authRouter.HandleFunc("/api/proxy/setAlias", ReverseProxyHandleAlias)
 	authRouter.HandleFunc("/api/proxy/del", DeleteProxyEndpoint)
 	authRouter.HandleFunc("/api/proxy/updateCredentials", UpdateProxyBasicAuthCredentials)
-	authRouter.HandleFunc("/api/proxy/tlscheck", HandleCheckSiteSupportTLS)
+	authRouter.HandleFunc("/api/proxy/tlscheck", domainsniff.HandleCheckSiteSupportTLS)
 	authRouter.HandleFunc("/api/proxy/setIncoming", HandleIncomingPortSet)
 	authRouter.HandleFunc("/api/proxy/useHttpsRedirect", HandleUpdateHttpsRedirect)
 	authRouter.HandleFunc("/api/proxy/listenPort80", HandleUpdatePort80Listener)
 	authRouter.HandleFunc("/api/proxy/requestIsProxied", HandleManagementProxyCheck)
 	authRouter.HandleFunc("/api/proxy/developmentMode", HandleDevelopmentModeChange)
-	//Reverse proxy virtual directory APIs
+	/* Reverse proxy upstream (load balance) */
+	authRouter.HandleFunc("/api/proxy/upstream/list", ReverseProxyUpstreamList)
+	authRouter.HandleFunc("/api/proxy/upstream/add", ReverseProxyUpstreamAdd)
+	authRouter.HandleFunc("/api/proxy/upstream/setPriority", ReverseProxyUpstreamSetPriority)
+	authRouter.HandleFunc("/api/proxy/upstream/update", ReverseProxyUpstreamUpdate)
+	authRouter.HandleFunc("/api/proxy/upstream/remove", ReverseProxyUpstreamDelete)
+	/* Reverse proxy virtual directory */
 	authRouter.HandleFunc("/api/proxy/vdir/list", ReverseProxyListVdir)
 	authRouter.HandleFunc("/api/proxy/vdir/add", ReverseProxyAddVdir)
 	authRouter.HandleFunc("/api/proxy/vdir/del", ReverseProxyDeleteVdir)
 	authRouter.HandleFunc("/api/proxy/vdir/edit", ReverseProxyEditVdir)
-	//Reverse proxy user define header apis
+	/* Reverse proxy user-defined header */
 	authRouter.HandleFunc("/api/proxy/header/list", HandleCustomHeaderList)
 	authRouter.HandleFunc("/api/proxy/header/add", HandleCustomHeaderAdd)
 	authRouter.HandleFunc("/api/proxy/header/remove", HandleCustomHeaderRemove)
-	//Reverse proxy auth related APIs
+	authRouter.HandleFunc("/api/proxy/header/handleHSTS", HandleHSTSState)
+	authRouter.HandleFunc("/api/proxy/header/handleHopByHop", HandleHopByHop)
+	authRouter.HandleFunc("/api/proxy/header/handleHostOverwrite", HandleHostOverwrite)
+	authRouter.HandleFunc("/api/proxy/header/handlePermissionPolicy", HandlePermissionPolicy)
+	/* Reverse proxy auth related */
 	authRouter.HandleFunc("/api/proxy/auth/exceptions/list", ListProxyBasicAuthExceptionPaths)
 	authRouter.HandleFunc("/api/proxy/auth/exceptions/add", AddProxyBasicAuthExceptionPaths)
 	authRouter.HandleFunc("/api/proxy/auth/exceptions/delete", RemoveProxyBasicAuthExceptionPaths)
+}
 
-	//TLS / SSL config
+// Register the APIs for TLS / SSL certificate management functions
+func RegisterTLSAPIs(authRouter *auth.RouterDef) {
 	authRouter.HandleFunc("/api/cert/tls", handleToggleTLSProxy)
 	authRouter.HandleFunc("/api/cert/tlsRequireLatest", handleSetTlsRequireLatest)
 	authRouter.HandleFunc("/api/cert/upload", handleCertUpload)
+	authRouter.HandleFunc("/api/cert/download", handleCertDownload)
 	authRouter.HandleFunc("/api/cert/list", handleListCertificate)
 	authRouter.HandleFunc("/api/cert/listdomains", handleListDomains)
 	authRouter.HandleFunc("/api/cert/checkDefault", handleDefaultCertCheck)
 	authRouter.HandleFunc("/api/cert/delete", handleCertRemove)
+}
 
-	//Redirection config
+// Register the APIs for Authentication handlers like Authelia and OAUTH2
+func RegisterAuthenticationHandlerAPIs(authRouter *auth.RouterDef) {
+	authRouter.HandleFunc("/api/sso/Authelia", autheliaRouter.HandleSetAutheliaURLAndHTTPS)
+}
+
+// Register the APIs for redirection rules management functions
+func RegisterRedirectionAPIs(authRouter *auth.RouterDef) {
 	authRouter.HandleFunc("/api/redirect/list", handleListRedirectionRules)
 	authRouter.HandleFunc("/api/redirect/add", handleAddRedirectionRule)
 	authRouter.HandleFunc("/api/redirect/delete", handleDeleteRedirectionRule)
 	authRouter.HandleFunc("/api/redirect/regex", handleToggleRedirectRegexpSupport)
+}
 
-	//Access Rules API
+// Register the APIs for access rules management functions
+func RegisterAccessRuleAPIs(authRouter *auth.RouterDef) {
+	/* Access Rules Settings & Status */
 	authRouter.HandleFunc("/api/access/list", handleListAccessRules)
 	authRouter.HandleFunc("/api/access/attach", handleAttachRuleToHost)
 	authRouter.HandleFunc("/api/access/create", handleCreateAccessRule)
 	authRouter.HandleFunc("/api/access/remove", handleRemoveAccessRule)
 	authRouter.HandleFunc("/api/access/update", handleUpadateAccessRule)
-	//Blacklist APIs
+	/* Blacklist */
 	authRouter.HandleFunc("/api/blacklist/list", handleListBlacklisted)
 	authRouter.HandleFunc("/api/blacklist/country/add", handleCountryBlacklistAdd)
 	authRouter.HandleFunc("/api/blacklist/country/remove", handleCountryBlacklistRemove)
 	authRouter.HandleFunc("/api/blacklist/ip/add", handleIpBlacklistAdd)
 	authRouter.HandleFunc("/api/blacklist/ip/remove", handleIpBlacklistRemove)
 	authRouter.HandleFunc("/api/blacklist/enable", handleBlacklistEnable)
-	//Whitelist APIs
+	/* Whitelist */
 	authRouter.HandleFunc("/api/whitelist/list", handleListWhitelisted)
 	authRouter.HandleFunc("/api/whitelist/country/add", handleCountryWhitelistAdd)
 	authRouter.HandleFunc("/api/whitelist/country/remove", handleCountryWhitelistRemove)
 	authRouter.HandleFunc("/api/whitelist/ip/add", handleIpWhitelistAdd)
 	authRouter.HandleFunc("/api/whitelist/ip/remove", handleIpWhitelistRemove)
 	authRouter.HandleFunc("/api/whitelist/enable", handleWhitelistEnable)
+}
 
-	//Path Blocker APIs
+// Register the APIs for path blocking rules management functions, WIP
+func RegisterPathRuleAPIs(authRouter *auth.RouterDef) {
 	authRouter.HandleFunc("/api/pathrule/add", pathRuleHandler.HandleAddBlockingPath)
 	authRouter.HandleFunc("/api/pathrule/list", pathRuleHandler.HandleListBlockingPath)
 	authRouter.HandleFunc("/api/pathrule/remove", pathRuleHandler.HandleRemoveBlockingPath)
+}
 
-	//Statistic & uptime monitoring API
+// Register the APIs statistic anlysis and uptime monitoring functions
+func RegisterStatisticalAPIs(authRouter *auth.RouterDef) {
+	/* Traffic Summary */
 	authRouter.HandleFunc("/api/stats/summary", statisticCollector.HandleTodayStatLoad)
 	authRouter.HandleFunc("/api/stats/countries", HandleCountryDistrSummary)
-	authRouter.HandleFunc("/api/stats/netstat", netstat.HandleGetNetworkInterfaceStats)
+	authRouter.HandleFunc("/api/stats/netstat", netstatBuffers.HandleGetNetworkInterfaceStats)
 	authRouter.HandleFunc("/api/stats/netstatgraph", netstatBuffers.HandleGetBufferedNetworkInterfaceStats)
 	authRouter.HandleFunc("/api/stats/listnic", netstat.HandleListNetworkInterfaces)
+	/* Zoraxy Analytic */
+	authRouter.HandleFunc("/api/analytic/list", AnalyticLoader.HandleSummaryList)
+	authRouter.HandleFunc("/api/analytic/load", AnalyticLoader.HandleLoadTargetDaySummary)
+	authRouter.HandleFunc("/api/analytic/loadRange", AnalyticLoader.HandleLoadTargetRangeSummary)
+	authRouter.HandleFunc("/api/analytic/exportRange", AnalyticLoader.HandleRangeExport)
+	authRouter.HandleFunc("/api/analytic/resetRange", AnalyticLoader.HandleRangeReset)
+	/* UpTime Monitor */
 	authRouter.HandleFunc("/api/utm/list", HandleUptimeMonitorListing)
+}
 
-	//Global Area Network APIs
+// Register the APIs for Global Area Network management functions, Will be moving to plugin soon
+func RegisterGANAPIs(authRouter *auth.RouterDef) {
 	authRouter.HandleFunc("/api/gan/network/info", ganManager.HandleGetNodeID)
 	authRouter.HandleFunc("/api/gan/network/add", ganManager.HandleAddNetwork)
 	authRouter.HandleFunc("/api/gan/network/remove", ganManager.HandleRemoveNetwork)
@@ -139,30 +155,70 @@ func initAPIs() {
 	authRouter.HandleFunc("/api/gan/members/name", ganManager.HandleMemberNaming)
 	authRouter.HandleFunc("/api/gan/members/authorize", ganManager.HandleMemberAuthorization)
 	authRouter.HandleFunc("/api/gan/members/delete", ganManager.HandleMemberDelete)
+}
 
-	//TCP Proxy
-	authRouter.HandleFunc("/api/tcpprox/config/add", tcpProxyManager.HandleAddProxyConfig)
-	authRouter.HandleFunc("/api/tcpprox/config/edit", tcpProxyManager.HandleEditProxyConfigs)
-	authRouter.HandleFunc("/api/tcpprox/config/list", tcpProxyManager.HandleListConfigs)
-	authRouter.HandleFunc("/api/tcpprox/config/start", tcpProxyManager.HandleStartProxy)
-	authRouter.HandleFunc("/api/tcpprox/config/stop", tcpProxyManager.HandleStopProxy)
-	authRouter.HandleFunc("/api/tcpprox/config/delete", tcpProxyManager.HandleRemoveProxy)
-	authRouter.HandleFunc("/api/tcpprox/config/status", tcpProxyManager.HandleGetProxyStatus)
-	authRouter.HandleFunc("/api/tcpprox/config/validate", tcpProxyManager.HandleConfigValidate)
+// Register the APIs for Stream (TCP / UDP) Proxy management functions
+func RegisterStreamProxyAPIs(authRouter *auth.RouterDef) {
+	authRouter.HandleFunc("/api/streamprox/config/add", streamProxyManager.HandleAddProxyConfig)
+	authRouter.HandleFunc("/api/streamprox/config/edit", streamProxyManager.HandleEditProxyConfigs)
+	authRouter.HandleFunc("/api/streamprox/config/list", streamProxyManager.HandleListConfigs)
+	authRouter.HandleFunc("/api/streamprox/config/start", streamProxyManager.HandleStartProxy)
+	authRouter.HandleFunc("/api/streamprox/config/stop", streamProxyManager.HandleStopProxy)
+	authRouter.HandleFunc("/api/streamprox/config/delete", streamProxyManager.HandleRemoveProxy)
+	authRouter.HandleFunc("/api/streamprox/config/status", streamProxyManager.HandleGetProxyStatus)
+}
 
-	//mDNS APIs
+// Register the APIs for mDNS service management functions
+func RegisterMDNSAPIs(authRouter *auth.RouterDef) {
 	authRouter.HandleFunc("/api/mdns/list", HandleMdnsListing)
 	authRouter.HandleFunc("/api/mdns/discover", HandleMdnsScanning)
+}
 
-	//Zoraxy Analytic
-	authRouter.HandleFunc("/api/analytic/list", AnalyticLoader.HandleSummaryList)
-	authRouter.HandleFunc("/api/analytic/load", AnalyticLoader.HandleLoadTargetDaySummary)
-	authRouter.HandleFunc("/api/analytic/loadRange", AnalyticLoader.HandleLoadTargetRangeSummary)
-	authRouter.HandleFunc("/api/analytic/exportRange", AnalyticLoader.HandleRangeExport)
-	authRouter.HandleFunc("/api/analytic/resetRange", AnalyticLoader.HandleRangeReset)
+// Register the APIs for ACME and Auto Renewer management functions
+func RegisterACMEAndAutoRenewerAPIs(authRouter *auth.RouterDef) {
+	/* ACME Core */
+	authRouter.HandleFunc("/api/acme/listExpiredDomains", acmeHandler.HandleGetExpiredDomains)
+	authRouter.HandleFunc("/api/acme/obtainCert", AcmeCheckAndHandleRenewCertificate)
+	/* Auto Renewer */
+	authRouter.HandleFunc("/api/acme/autoRenew/enable", acmeAutoRenewer.HandleAutoRenewEnable)
+	authRouter.HandleFunc("/api/acme/autoRenew/ca", HandleACMEPreferredCA)
+	authRouter.HandleFunc("/api/acme/autoRenew/email", acmeAutoRenewer.HandleACMEEmail)
+	authRouter.HandleFunc("/api/acme/autoRenew/setDomains", acmeAutoRenewer.HandleSetAutoRenewDomains)
+	authRouter.HandleFunc("/api/acme/autoRenew/setEAB", acmeAutoRenewer.HanldeSetEAB)
+	authRouter.HandleFunc("/api/acme/autoRenew/setDNS", acmeAutoRenewer.HandleSetDNS)
+	authRouter.HandleFunc("/api/acme/autoRenew/listDomains", acmeAutoRenewer.HandleLoadAutoRenewDomains)
+	authRouter.HandleFunc("/api/acme/autoRenew/renewPolicy", acmeAutoRenewer.HandleRenewPolicy)
+	authRouter.HandleFunc("/api/acme/autoRenew/renewNow", acmeAutoRenewer.HandleRenewNow)
+	authRouter.HandleFunc("/api/acme/dns/providers", acmedns.HandleServeProvidersJson)
+	/* ACME Wizard */
+	authRouter.HandleFunc("/api/acme/wizard", acmewizard.HandleGuidedStepCheck)
+}
 
-	//Network utilities
-	authRouter.HandleFunc("/api/tools/ipscan", HandleIpScan)
+// Register the APIs for Static Web Server management functions
+func RegisterStaticWebServerAPIs(authRouter *auth.RouterDef) {
+	/* Static Web Server Controls */
+	authRouter.HandleFunc("/api/webserv/status", staticWebServer.HandleGetStatus)
+	authRouter.HandleFunc("/api/webserv/start", staticWebServer.HandleStartServer)
+	authRouter.HandleFunc("/api/webserv/stop", staticWebServer.HandleStopServer)
+	authRouter.HandleFunc("/api/webserv/setPort", HandleStaticWebServerPortChange)
+	authRouter.HandleFunc("/api/webserv/setDirList", staticWebServer.SetEnableDirectoryListing)
+	/* File Manager */
+	if *allowWebFileManager {
+		authRouter.HandleFunc("/api/fs/list", staticWebServer.FileManager.HandleList)
+		authRouter.HandleFunc("/api/fs/upload", staticWebServer.FileManager.HandleUpload)
+		authRouter.HandleFunc("/api/fs/download", staticWebServer.FileManager.HandleDownload)
+		authRouter.HandleFunc("/api/fs/newFolder", staticWebServer.FileManager.HandleNewFolder)
+		authRouter.HandleFunc("/api/fs/copy", staticWebServer.FileManager.HandleFileCopy)
+		authRouter.HandleFunc("/api/fs/move", staticWebServer.FileManager.HandleFileMove)
+		authRouter.HandleFunc("/api/fs/properties", staticWebServer.FileManager.HandleFileProperties)
+		authRouter.HandleFunc("/api/fs/del", staticWebServer.FileManager.HandleFileDelete)
+	}
+}
+
+// Register the APIs for Network Utilities functions
+func RegisterNetworkUtilsAPIs(authRouter *auth.RouterDef) {
+	authRouter.HandleFunc("/api/tools/ipscan", ipscan.HandleIpScan)
+	authRouter.HandleFunc("/api/tools/portscan", ipscan.HandleScanPort)
 	authRouter.HandleFunc("/api/tools/traceroute", netutils.HandleTraceRoute)
 	authRouter.HandleFunc("/api/tools/ping", netutils.HandlePing)
 	authRouter.HandleFunc("/api/tools/whois", netutils.HandleWhois)
@@ -175,95 +231,42 @@ func initAPIs() {
 	authRouter.HandleFunc("/api/tools/smtp/test", HandleTestEmailSend)
 	authRouter.HandleFunc("/api/tools/fwdproxy/enable", forwardProxy.HandleToogle)
 	authRouter.HandleFunc("/api/tools/fwdproxy/port", forwardProxy.HandlePort)
-
-	//Account Reset
-	http.HandleFunc("/api/account/reset", HandleAdminAccountResetEmail)
-	http.HandleFunc("/api/account/new", HandleNewPasswordSetup)
-
-	//ACME & Auto Renewer
-	authRouter.HandleFunc("/api/acme/listExpiredDomains", acmeHandler.HandleGetExpiredDomains)
-	authRouter.HandleFunc("/api/acme/obtainCert", AcmeCheckAndHandleRenewCertificate)
-	authRouter.HandleFunc("/api/acme/autoRenew/enable", acmeAutoRenewer.HandleAutoRenewEnable)
-	authRouter.HandleFunc("/api/acme/autoRenew/ca", HandleACMEPreferredCA)
-	authRouter.HandleFunc("/api/acme/autoRenew/email", acmeAutoRenewer.HandleACMEEmail)
-	authRouter.HandleFunc("/api/acme/autoRenew/setDomains", acmeAutoRenewer.HandleSetAutoRenewDomains)
-	authRouter.HandleFunc("/api/acme/autoRenew/setEAB", acmeAutoRenewer.HanldeSetEAB)
-	authRouter.HandleFunc("/api/acme/autoRenew/setDNS", acmeAutoRenewer.HanldeSetDNS)
-	authRouter.HandleFunc("/api/acme/autoRenew/listDomains", acmeAutoRenewer.HandleLoadAutoRenewDomains)
-	authRouter.HandleFunc("/api/acme/autoRenew/renewPolicy", acmeAutoRenewer.HandleRenewPolicy)
-	authRouter.HandleFunc("/api/acme/autoRenew/renewNow", acmeAutoRenewer.HandleRenewNow)
-	authRouter.HandleFunc("/api/acme/dns/providers", acmedns.HandleServeProvidersJson)
-	authRouter.HandleFunc("/api/acme/wizard", acmewizard.HandleGuidedStepCheck) //ACME Wizard
-
-	//Static Web Server
-	authRouter.HandleFunc("/api/webserv/status", staticWebServer.HandleGetStatus)
-	authRouter.HandleFunc("/api/webserv/start", staticWebServer.HandleStartServer)
-	authRouter.HandleFunc("/api/webserv/stop", staticWebServer.HandleStopServer)
-	authRouter.HandleFunc("/api/webserv/setPort", HandleStaticWebServerPortChange)
-	authRouter.HandleFunc("/api/webserv/setDirList", staticWebServer.SetEnableDirectoryListing)
-	if *allowWebFileManager {
-		//Web Directory Manager file operation functions
-		authRouter.HandleFunc("/api/fs/list", staticWebServer.FileManager.HandleList)
-		authRouter.HandleFunc("/api/fs/upload", staticWebServer.FileManager.HandleUpload)
-		authRouter.HandleFunc("/api/fs/download", staticWebServer.FileManager.HandleDownload)
-		authRouter.HandleFunc("/api/fs/newFolder", staticWebServer.FileManager.HandleNewFolder)
-		authRouter.HandleFunc("/api/fs/copy", staticWebServer.FileManager.HandleFileCopy)
-		authRouter.HandleFunc("/api/fs/move", staticWebServer.FileManager.HandleFileMove)
-		authRouter.HandleFunc("/api/fs/properties", staticWebServer.FileManager.HandleFileProperties)
-		authRouter.HandleFunc("/api/fs/del", staticWebServer.FileManager.HandleFileDelete)
-	}
-
-	//Others
-	http.HandleFunc("/api/info/x", HandleZoraxyInfo)
-	authRouter.HandleFunc("/api/info/geoip", HandleGeoIpLookup)
-	authRouter.HandleFunc("/api/conf/export", ExportConfigAsZip)
-	authRouter.HandleFunc("/api/conf/import", ImportConfigFromZip)
-
-	//Debug
-	authRouter.HandleFunc("/api/info/pprof", pprof.Index)
-
-	//If you got APIs to add, append them here
 }
 
-// Function to renders Auth related APIs
-func registerAuthAPIs(requireAuth bool) {
-	//Auth APIs
-	http.HandleFunc("/api/auth/login", authAgent.HandleLogin)
-	http.HandleFunc("/api/auth/logout", authAgent.HandleLogout)
-	http.HandleFunc("/api/auth/checkLogin", func(w http.ResponseWriter, r *http.Request) {
+// Register the APIs for Auth functions, due to scoping issue some functions are defined here
+func RegisterAuthAPIs(requireAuth bool, targetMux *http.ServeMux) {
+	targetMux.HandleFunc("/api/auth/login", authAgent.HandleLogin)
+	targetMux.HandleFunc("/api/auth/logout", authAgent.HandleLogout)
+	targetMux.HandleFunc("/api/auth/checkLogin", func(w http.ResponseWriter, r *http.Request) {
 		if requireAuth {
 			authAgent.CheckLogin(w, r)
 		} else {
 			utils.SendJSONResponse(w, "true")
 		}
 	})
-	http.HandleFunc("/api/auth/username", func(w http.ResponseWriter, r *http.Request) {
+	targetMux.HandleFunc("/api/auth/username", func(w http.ResponseWriter, r *http.Request) {
 		username, err := authAgent.GetUserName(w, r)
 		if err != nil {
 			http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
 			return
 		}
-
 		js, _ := json.Marshal(username)
 		utils.SendJSONResponse(w, string(js))
 	})
-	http.HandleFunc("/api/auth/userCount", func(w http.ResponseWriter, r *http.Request) {
-		uc := authAgent.GetUserCounts()
-		js, _ := json.Marshal(uc)
+	targetMux.HandleFunc("/api/auth/userCount", func(w http.ResponseWriter, r *http.Request) {
+		js, _ := json.Marshal(authAgent.GetUserCounts())
 		utils.SendJSONResponse(w, string(js))
 	})
-	http.HandleFunc("/api/auth/register", func(w http.ResponseWriter, r *http.Request) {
+	targetMux.HandleFunc("/api/auth/register", func(w http.ResponseWriter, r *http.Request) {
 		if authAgent.GetUserCounts() == 0 {
 			//Allow register root admin
-			authAgent.HandleRegisterWithoutEmail(w, r, func(username, reserved string) {
-
-			})
+			authAgent.HandleRegisterWithoutEmail(w, r, func(username, reserved string) {})
 		} else {
 			//This function is disabled
 			utils.SendErrorResponse(w, "Root management account already exists")
 		}
 	})
-	http.HandleFunc("/api/auth/changePassword", func(w http.ResponseWriter, r *http.Request) {
+	targetMux.HandleFunc("/api/auth/changePassword", func(w http.ResponseWriter, r *http.Request) {
 		username, err := authAgent.GetUserName(w, r)
 		if err != nil {
 			http.Error(w, "401 - Unauthorized", http.StatusUnauthorized)
@@ -298,5 +301,60 @@ func registerAuthAPIs(requireAuth bool) {
 		authAgent.UnregisterUser(username)
 		authAgent.CreateUserAccount(username, newPassword, "")
 	})
-
+}
+
+/* Register all the APIs */
+func initAPIs(targetMux *http.ServeMux) {
+	authRouter := auth.NewManagedHTTPRouter(auth.RouterOption{
+		AuthAgent:   authAgent,
+		RequireAuth: requireAuth,
+		TargetMux:   targetMux,
+		DeniedHandler: func(w http.ResponseWriter, r *http.Request) {
+			http.Error(w, "401 - Unauthorized", http.StatusUnauthorized)
+		},
+	})
+
+	//Register the standard web services urls
+	fs := http.FileServer(http.FS(webres))
+	if DEVELOPMENT_BUILD {
+		fs = http.FileServer(http.Dir("web/"))
+	}
+	//Add a layer of middleware for advance control
+	advHandler := FSHandler(fs)
+	targetMux.Handle("/", advHandler)
+
+	//Register the APIs
+	RegisterAuthAPIs(requireAuth, targetMux)
+	RegisterHTTPProxyAPIs(authRouter)
+	RegisterTLSAPIs(authRouter)
+	RegisterAuthenticationHandlerAPIs(authRouter)
+	RegisterRedirectionAPIs(authRouter)
+	RegisterAccessRuleAPIs(authRouter)
+	RegisterPathRuleAPIs(authRouter)
+	RegisterStatisticalAPIs(authRouter)
+	RegisterGANAPIs(authRouter)
+	RegisterStreamProxyAPIs(authRouter)
+	RegisterMDNSAPIs(authRouter)
+	RegisterNetworkUtilsAPIs(authRouter)
+	RegisterACMEAndAutoRenewerAPIs(authRouter)
+	RegisterStaticWebServerAPIs(authRouter)
+
+	//Account Reset
+	targetMux.HandleFunc("/api/account/reset", HandleAdminAccountResetEmail)
+	targetMux.HandleFunc("/api/account/new", HandleNewPasswordSetup)
+
+	//Docker UX Optimizations
+	authRouter.HandleFunc("/api/docker/available", DockerUXOptimizer.HandleDockerAvailable)
+	authRouter.HandleFunc("/api/docker/containers", DockerUXOptimizer.HandleDockerContainersList)
+
+	//Others
+	targetMux.HandleFunc("/api/info/x", HandleZoraxyInfo)
+	authRouter.HandleFunc("/api/info/geoip", HandleGeoIpLookup)
+	authRouter.HandleFunc("/api/conf/export", ExportConfigAsZip)
+	authRouter.HandleFunc("/api/conf/import", ImportConfigFromZip)
+	authRouter.HandleFunc("/api/log/list", LogViewer.HandleListLog)
+	authRouter.HandleFunc("/api/log/read", LogViewer.HandleReadLog)
+
+	//Debug
+	authRouter.HandleFunc("/api/info/pprof", pprof.Index)
 }

src/cert.go

@@ -177,32 +177,36 @@ func handleListDomains(w http.ResponseWriter, r *http.Request) {
 
 // Handle front-end toggling TLS mode
 func handleToggleTLSProxy(w http.ResponseWriter, r *http.Request) {
-	currentTlsSetting := false
+	currentTlsSetting := true //Default to true
+	if dynamicProxyRouter.Option != nil {
+		currentTlsSetting = dynamicProxyRouter.Option.UseTls
+	}
 	if sysdb.KeyExists("settings", "usetls") {
 		sysdb.Read("settings", "usetls", &currentTlsSetting)
 	}
 
-	newState, err := utils.PostPara(r, "set")
-	if err != nil {
-		//No setting. Get the current status
+	if r.Method == http.MethodGet {
+		//Get the current status
 		js, _ := json.Marshal(currentTlsSetting)
 		utils.SendJSONResponse(w, string(js))
-	} else {
-		if newState == "true" {
+	} else if r.Method == http.MethodPost {
+		newState, err := utils.PostBool(r, "set")
+		if err != nil {
+			utils.SendErrorResponse(w, "new state not set or invalid")
+			return
+		}
+		if newState {
 			sysdb.Write("settings", "usetls", true)
 			SystemWideLogger.Println("Enabling TLS mode on reverse proxy")
 			dynamicProxyRouter.UpdateTLSSetting(true)
-		} else if newState == "false" {
+		} else {
 			sysdb.Write("settings", "usetls", false)
 			SystemWideLogger.Println("Disabling TLS mode on reverse proxy")
 			dynamicProxyRouter.UpdateTLSSetting(false)
-		} else {
-			utils.SendErrorResponse(w, "invalid state given. Only support true or false")
-			return
 		}
-
 		utils.SendOK(w)
-
+	} else {
+		http.Error(w, "405 - Method not allowed", http.StatusMethodNotAllowed)
 	}
 }
 
@@ -233,6 +237,51 @@ func handleSetTlsRequireLatest(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// Handle download of the selected certificate
+func handleCertDownload(w http.ResponseWriter, r *http.Request) {
+	// get the certificate name
+	certname, err := utils.GetPara(r, "certname")
+	if err != nil {
+		utils.SendErrorResponse(w, "invalid certname given")
+		return
+	}
+	certname = filepath.Base(certname) //prevent path escape
+
+	// check if the cert exists
+	pubKey := filepath.Join(filepath.Join("./conf/certs"), certname+".key")
+	priKey := filepath.Join(filepath.Join("./conf/certs"), certname+".pem")
+
+	if utils.FileExists(pubKey) && utils.FileExists(priKey) {
+		//Zip them and serve them via http download
+		seeking, _ := utils.GetBool(r, "seek")
+		if seeking {
+			//This request only check if the key exists. Do not provide download
+			utils.SendOK(w)
+			return
+		}
+
+		//Serve both file in zip
+		zipTmpFolder := "./tmp/download"
+		os.MkdirAll(zipTmpFolder, 0775)
+		zipFileName := filepath.Join(zipTmpFolder, certname+".zip")
+		err := utils.ZipFiles(zipFileName, pubKey, priKey)
+		if err != nil {
+			http.Error(w, "Failed to create zip file", http.StatusInternalServerError)
+			return
+		}
+		defer os.Remove(zipFileName) // Clean up the zip file after serving
+
+		// Serve the zip file
+		w.Header().Set("Content-Disposition", "attachment; filename=\""+certname+"_export.zip\"")
+		w.Header().Set("Content-Type", "application/zip")
+		http.ServeFile(w, r, zipFileName)
+	} else {
+		//Not both key exists
+		utils.SendErrorResponse(w, "invalid key-pairs: private key or public key not found in key store")
+		return
+	}
+}
+
 // Handle upload of the certificate
 func handleCertUpload(w http.ResponseWriter, r *http.Request) {
 	// check if request method is POST

src/config.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"imuslab.com/zoraxy/mod/dynamicproxy"
+	"imuslab.com/zoraxy/mod/dynamicproxy/loadbalance"
 	"imuslab.com/zoraxy/mod/utils"
 )
 
@@ -58,7 +59,7 @@ func LoadReverseProxyConfig(configFilepath string) error {
 		thisConfigEndpoint.RootOrMatchingDomain = "/"
 	}
 
-	if thisConfigEndpoint.ProxyType == dynamicproxy.ProxyType_Root {
+	if thisConfigEndpoint.ProxyType == dynamicproxy.ProxyTypeRoot {
 		//This is a root config file
 		rootProxyEndpoint, err := dynamicProxyRouter.PrepareProxyRoute(&thisConfigEndpoint)
 		if err != nil {
@@ -67,7 +68,7 @@ func LoadReverseProxyConfig(configFilepath string) error {
 
 		dynamicProxyRouter.SetProxyRouteAsRoot(rootProxyEndpoint)
 
-	} else if thisConfigEndpoint.ProxyType == dynamicproxy.ProxyType_Host {
+	} else if thisConfigEndpoint.ProxyType == dynamicproxy.ProxyTypeHost {
 		//This is a host config file
 		readyProxyEndpoint, err := dynamicProxyRouter.PrepareProxyRoute(&thisConfigEndpoint)
 		if err != nil {
@@ -79,7 +80,7 @@ func LoadReverseProxyConfig(configFilepath string) error {
 		return errors.New("not supported proxy type")
 	}
 
-	SystemWideLogger.PrintAndLog("Proxy", thisConfigEndpoint.RootOrMatchingDomain+" -> "+thisConfigEndpoint.Domain+" routing rule loaded", nil)
+	SystemWideLogger.PrintAndLog("proxy-config", thisConfigEndpoint.RootOrMatchingDomain+" -> "+loadbalance.GetUpstreamsAsString(thisConfigEndpoint.ActiveOrigins)+" routing rule loaded", nil)
 	return nil
 }
 
@@ -96,7 +97,7 @@ func filterProxyConfigFilename(filename string) string {
 func SaveReverseProxyConfig(endpoint *dynamicproxy.ProxyEndpoint) error {
 	//Get filename for saving
 	filename := filepath.Join("./conf/proxy/", endpoint.RootOrMatchingDomain+".config")
-	if endpoint.ProxyType == dynamicproxy.ProxyType_Root {
+	if endpoint.ProxyType == dynamicproxy.ProxyTypeRoot {
 		filename = "./conf/proxy/root.config"
 	}
 
@@ -128,20 +129,30 @@ func RemoveReverseProxyConfig(endpoint string) error {
 // Get the default root config that point to the internal static web server
 // this will be used if root config is not found (new deployment / missing root.config file)
 func GetDefaultRootConfig() (*dynamicproxy.ProxyEndpoint, error) {
-	//Default settings
-	rootProxyEndpoint, err := dynamicProxyRouter.PrepareProxyRoute(&dynamicproxy.ProxyEndpoint{
-		ProxyType:               dynamicproxy.ProxyType_Root,
-		RootOrMatchingDomain:    "/",
-		Domain:                  "127.0.0.1:" + staticWebServer.GetListeningPort(),
-		RequireTLS:              false,
-		BypassGlobalTLS:         false,
-		SkipCertValidations:     false,
-		VirtualDirectories:      []*dynamicproxy.VirtualDirectoryEndpoint{},
-		RequireBasicAuth:        false,
+	//Default Authentication Provider
+	defaultAuth := &dynamicproxy.AuthenticationProvider{
+		AuthMethod:              dynamicproxy.AuthMethodNone,
 		BasicAuthCredentials:    []*dynamicproxy.BasicAuthCredentials{},
 		BasicAuthExceptionRules: []*dynamicproxy.BasicAuthExceptionRule{},
-		DefaultSiteOption:       dynamicproxy.DefaultSite_InternalStaticWebServer,
-		DefaultSiteValue:        "",
+	}
+	//Default settings
+	rootProxyEndpoint, err := dynamicProxyRouter.PrepareProxyRoute(&dynamicproxy.ProxyEndpoint{
+		ProxyType:            dynamicproxy.ProxyTypeRoot,
+		RootOrMatchingDomain: "/",
+		ActiveOrigins: []*loadbalance.Upstream{
+			{
+				OriginIpOrDomain:    "127.0.0.1:" + staticWebServer.GetListeningPort(),
+				RequireTLS:          false,
+				SkipCertValidations: false,
+				Weight:              0,
+			},
+		},
+		InactiveOrigins:        []*loadbalance.Upstream{},
+		BypassGlobalTLS:        false,
+		VirtualDirectories:     []*dynamicproxy.VirtualDirectoryEndpoint{},
+		AuthenticationProvider: defaultAuth,
+		DefaultSiteOption:      dynamicproxy.DefaultSite_InternalStaticWebServer,
+		DefaultSiteValue:       "",
 	})
 	if err != nil {
 		return nil, err
@@ -160,7 +171,6 @@ func ExportConfigAsZip(w http.ResponseWriter, r *http.Request) {
 	if includeSysDBRaw == "true" {
 		//Include the system database in backup snapshot
 		//Temporary set it to read only
-		sysdb.ReadOnly = true
 		includeSysDB = true
 	}
 
@@ -234,8 +244,6 @@ func ExportConfigAsZip(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		//Restore sysdb state
-		sysdb.ReadOnly = false
 	}
 
 	if err != nil {

src/def.go

@@ -0,0 +1,144 @@
+package main
+
+/*
+	Type and flag definations
+
+	This file contains all the type and flag definations
+	Author: tobychui
+*/
+
+import (
+	"embed"
+	"flag"
+	"net/http"
+	"time"
+
+	"imuslab.com/zoraxy/mod/access"
+	"imuslab.com/zoraxy/mod/acme"
+	"imuslab.com/zoraxy/mod/auth"
+	"imuslab.com/zoraxy/mod/auth/sso/authelia"
+	"imuslab.com/zoraxy/mod/database"
+	"imuslab.com/zoraxy/mod/dockerux"
+	"imuslab.com/zoraxy/mod/dynamicproxy/loadbalance"
+	"imuslab.com/zoraxy/mod/dynamicproxy/redirection"
+	"imuslab.com/zoraxy/mod/email"
+	"imuslab.com/zoraxy/mod/forwardproxy"
+	"imuslab.com/zoraxy/mod/ganserv"
+	"imuslab.com/zoraxy/mod/geodb"
+	"imuslab.com/zoraxy/mod/info/logger"
+	"imuslab.com/zoraxy/mod/info/logviewer"
+	"imuslab.com/zoraxy/mod/mdns"
+	"imuslab.com/zoraxy/mod/netstat"
+	"imuslab.com/zoraxy/mod/pathrule"
+	"imuslab.com/zoraxy/mod/sshprox"
+	"imuslab.com/zoraxy/mod/statistic"
+	"imuslab.com/zoraxy/mod/statistic/analytic"
+	"imuslab.com/zoraxy/mod/streamproxy"
+	"imuslab.com/zoraxy/mod/tlscert"
+	"imuslab.com/zoraxy/mod/uptime"
+	"imuslab.com/zoraxy/mod/webserv"
+)
+
+const (
+	/* Build Constants */
+	SYSTEM_NAME       = "Zoraxy"
+	SYSTEM_VERSION    = "3.1.5"
+	DEVELOPMENT_BUILD = false /* Development: Set to false to use embedded web fs */
+
+	/* System Constants */
+	DATABASE_PATH              = "sys.db"
+	TMP_FOLDER                 = "./tmp"
+	WEBSERV_DEFAULT_PORT       = 5487
+	MDNS_HOSTNAME_PREFIX       = "zoraxy_" /* Follow by node UUID */
+	MDNS_IDENTIFY_DEVICE_TYPE  = "Network Gateway"
+	MDNS_IDENTIFY_DOMAIN       = "zoraxy.aroz.org"
+	MDNS_IDENTIFY_VENDOR       = "imuslab.com"
+	MDNS_SCAN_TIMEOUT          = 30 /* Seconds */
+	MDNS_SCAN_UPDATE_INTERVAL  = 15 /* Minutes */
+	GEODB_CACHE_CLEAR_INTERVAL = 15 /* Minutes */
+	ACME_AUTORENEW_CONFIG_PATH = "./conf/acme_conf.json"
+	CSRF_COOKIENAME            = "zoraxy_csrf"
+	LOG_PREFIX                 = "zr"
+	LOG_FOLDER                 = "./log"
+	LOG_EXTENSION              = ".log"
+
+	/* Configuration Folder Storage Path Constants */
+	CONF_HTTP_PROXY   = "./conf/proxy"
+	CONF_STREAM_PROXY = "./conf/streamproxy"
+	CONF_CERT_STORE   = "./conf/certs"
+	CONF_REDIRECTION  = "./conf/redirect"
+	CONF_ACCESS_RULE  = "./conf/access"
+	CONF_PATH_RULE    = "./conf/rules/pathrules"
+)
+
+/* System Startup Flags */
+var (
+	webUIPort                  = flag.String("port", ":8000", "Management web interface listening port")
+	databaseBackend            = flag.String("db", "auto", "Database backend to use (leveldb, boltdb, auto) Note that fsdb will be used on unsupported platforms like RISCV")
+	noauth                     = flag.Bool("noauth", false, "Disable authentication for management interface")
+	showver                    = flag.Bool("version", false, "Show version of this server")
+	allowSshLoopback           = flag.Bool("sshlb", false, "Allow loopback web ssh connection (DANGER)")
+	allowMdnsScanning          = flag.Bool("mdns", true, "Enable mDNS scanner and transponder")
+	mdnsName                   = flag.String("mdnsname", "", "mDNS name, leave empty to use default (zoraxy_{node-uuid}.local)")
+	ztAuthToken                = flag.String("ztauth", "", "ZeroTier authtoken for the local node")
+	ztAPIPort                  = flag.Int("ztport", 9993, "ZeroTier controller API port")
+	runningInDocker            = flag.Bool("docker", false, "Run Zoraxy in docker compatibility mode")
+	acmeAutoRenewInterval      = flag.Int("autorenew", 86400, "ACME auto TLS/SSL certificate renew check interval (seconds)")
+	acmeCertAutoRenewDays      = flag.Int("earlyrenew", 30, "Number of days to early renew a soon expiring certificate (days)")
+	enableHighSpeedGeoIPLookup = flag.Bool("fastgeoip", false, "Enable high speed geoip lookup, require 1GB extra memory (Not recommend for low end devices)")
+	staticWebServerRoot        = flag.String("webroot", "./www", "Static web server root folder. Only allow chnage in start paramters")
+	allowWebFileManager        = flag.Bool("webfm", true, "Enable web file manager for static web server root folder")
+	enableAutoUpdate           = flag.Bool("cfgupgrade", true, "Enable auto config upgrade if breaking change is detected")
+
+	/* Maintaince Function Flags */
+	geoDbUpdate = flag.Bool("update_geoip", false, "Download the latest GeoIP data and exit")
+)
+
+/* Global Variables and Handlers */
+var (
+	nodeUUID    = "generic" //System uuid in uuidv4 format, load from database on startup
+	bootTime    = time.Now().Unix()
+	requireAuth = true //Require authentication for webmin panel, override from flag
+
+	/*
+		Binary Embedding File System
+	*/
+	//go:embed web/*
+	webres embed.FS
+
+	/*
+		Handler Modules
+	*/
+	sysdb          *database.Database              //System database
+	authAgent      *auth.AuthAgent                 //Authentication agent
+	tlsCertManager *tlscert.Manager                //TLS / SSL management
+	redirectTable  *redirection.RuleTable          //Handle special redirection rule sets
+	webminPanelMux *http.ServeMux                  //Server mux for handling webmin panel APIs
+	csrfMiddleware func(http.Handler) http.Handler //CSRF protection middleware
+
+	pathRuleHandler    *pathrule.Handler         //Handle specific path blocking or custom headers
+	geodbStore         *geodb.Store              //GeoIP database, for resolving IP into country code
+	accessController   *access.Controller        //Access controller, handle black list and white list
+	netstatBuffers     *netstat.NetStatBuffers   //Realtime graph buffers
+	statisticCollector *statistic.Collector      //Collecting statistic from visitors
+	uptimeMonitor      *uptime.Monitor           //Uptime monitor service worker
+	mdnsScanner        *mdns.MDNSHost            //mDNS discovery services
+	ganManager         *ganserv.NetworkManager   //Global Area Network Manager
+	webSshManager      *sshprox.Manager          //Web SSH connection service
+	streamProxyManager *streamproxy.Manager      //Stream Proxy Manager for TCP / UDP forwarding
+	acmeHandler        *acme.ACMEHandler         //Handler for ACME Certificate renew
+	acmeAutoRenewer    *acme.AutoRenewer         //Handler for ACME auto renew ticking
+	staticWebServer    *webserv.WebServer        //Static web server for hosting simple stuffs
+	forwardProxy       *forwardproxy.Handler     //HTTP Forward proxy, basically VPN for web browser
+	loadBalancer       *loadbalance.RouteManager //Global scope loadbalancer, store the state of the lb routing
+
+	//Authentication Provider
+	autheliaRouter *authelia.AutheliaRouter //Authelia router for Authelia authentication
+
+	//Helper modules
+	EmailSender       *email.Sender         //Email sender that handle email sending
+	AnalyticLoader    *analytic.DataLoader  //Data loader for Zoraxy Analytic
+	DockerUXOptimizer *dockerux.UXOptimizer //Docker user experience optimizer, community contribution only
+	SystemWideLogger  *logger.Logger        //Logger for Zoraxy
+	LogViewer         *logviewer.Viewer     //Log viewer HTTP handlers
+)

src/emails.go

@@ -272,17 +272,14 @@ func HandleNewPasswordSetup(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	//Delete the user account
-	authAgent.UnregisterUser(username)
-
-	//Ok. Set the new password
-	err = authAgent.CreateUserAccount(username, newPassword, "")
-	if err != nil {
+	// Un register the user account
+	if err := authAgent.UnregisterUser(username); err != nil {
 		utils.SendErrorResponse(w, err.Error())
 		return
 	}
 
-	if err != nil {
+	//Ok. Set the new password
+	if err := authAgent.CreateUserAccount(username, newPassword, ""); err != nil {
 		utils.SendErrorResponse(w, err.Error())
 		return
 	}

src/go.mod

@@ -1,99 +1,129 @@
 module imuslab.com/zoraxy
 
-go 1.21
+go 1.22.0
 
 toolchain go1.22.2
 
 require (
 	github.com/boltdb/bolt v1.3.1
-	github.com/go-acme/lego/v4 v4.16.1
+	github.com/docker/docker v27.0.0+incompatible
+	github.com/go-acme/lego/v4 v4.19.2
 	github.com/go-ping/ping v1.1.0
+	github.com/go-session/session v3.1.2+incompatible
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/sessions v1.2.2
 	github.com/gorilla/websocket v1.5.1
 	github.com/grandcat/zeroconf v1.0.0
 	github.com/likexian/whois v1.15.1
 	github.com/microcosm-cc/bluemonday v1.0.26
-	golang.org/x/net v0.23.0
-	golang.org/x/sys v0.18.0
-	golang.org/x/text v0.14.0
+	golang.org/x/net v0.29.0
+	golang.org/x/sys v0.25.0
+	golang.org/x/text v0.18.0
 )
 
 require (
-	cloud.google.com/go/compute v1.20.1 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/auth v0.9.3 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.9.0 // indirect
+	github.com/benbjohnson/clock v1.3.0 // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.114 // indirect
+	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/syndtr/goleveldb v1.0.0 // indirect
+	github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 // indirect
+	github.com/tidwall/buntdb v1.1.2 // indirect
+	github.com/tidwall/gjson v1.12.1 // indirect
+	github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e // indirect
+	github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 // indirect
+	github.com/tjfoc/gmsm v1.4.1 // indirect
+	github.com/vultr/govultr/v3 v3.9.1 // indirect
+	go.mongodb.org/mongo-driver v1.12.0 // indirect
+)
+
+require (
+	cloud.google.com/go/compute/metadata v0.5.1 // indirect
 	github.com/AdamSLevy/jsonrpc2/v14 v14.1.0 // indirect
 	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.1.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.1.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns v1.2.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.2.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.29 // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect
-	github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 // indirect
-	github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 // indirect
+	github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 // indirect
-	github.com/akamai/AkamaiOPEN-edgegrid-golang v1.2.2 // indirect
-	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1755 // indirect
-	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.26.6 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.16.16 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/lightsail v1.34.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/route53 v1.37.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
-	github.com/aws/smithy-go v1.19.0 // indirect
+	github.com/aliyun/alibaba-cloud-sdk-go v1.63.15 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.30.5 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.33 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.32 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/lightsail v1.40.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/route53 v1.43.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 // indirect
+	github.com/aws/smithy-go v1.20.4 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/civo/civogo v0.3.11 // indirect
-	github.com/cloudflare/cloudflare-go v0.86.0 // indirect
-	github.com/cpu/goacmedns v0.1.1 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/deepmap/oapi-codegen v1.9.1 // indirect
+	github.com/cloudflare/cloudflare-go v0.104.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
-	github.com/dnsimple/dnsimple-go v1.2.0 // indirect
-	github.com/exoscale/egoscale v0.102.3 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/dnsimple/dnsimple-go v1.7.0 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-errors/errors v1.0.1 // indirect
-	github.com/go-jose/go-jose/v4 v4.0.1 // indirect
-	github.com/go-resty/resty/v2 v2.11.0 // indirect
-	github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
-	github.com/goccy/go-json v0.10.2 // indirect
-	github.com/gofrs/uuid v4.4.0+incompatible // indirect
+	github.com/go-jose/go-jose/v4 v4.0.4 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-oauth2/oauth2/v4 v4.5.2
+	github.com/go-resty/resty/v2 v2.13.1 // indirect
+	github.com/go-viper/mapstructure/v2 v2.1.0 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/gofrs/uuid v4.4.0+incompatible
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/s2a-go v0.1.4 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
-	github.com/googleapis/gax-go/v2 v2.11.0 // indirect
-	github.com/gophercloud/gophercloud v1.0.0 // indirect
-	github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect
+	github.com/google/s2a-go v0.1.8 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
+	github.com/googleapis/gax-go/v2 v2.13.0 // indirect
+	github.com/gophercloud/gophercloud v1.14.0 // indirect
+	github.com/gorilla/csrf v1.7.2
 	github.com/gorilla/css v1.0.1 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df // indirect
 	github.com/infobloxopen/infoblox-go-client v1.1.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -103,72 +133,80 @@ require (
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/labbsr0x/bindman-dns-webhook v1.0.2 // indirect
 	github.com/labbsr0x/goh v1.0.1 // indirect
-	github.com/linode/linodego v1.28.0 // indirect
+	github.com/linode/linodego v1.40.0 // indirect
 	github.com/liquidweb/liquidweb-cli v0.6.9 // indirect
 	github.com/liquidweb/liquidweb-go v1.6.4 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/miekg/dns v1.1.58 // indirect
+	github.com/miekg/dns v1.1.62 // indirect
 	github.com/mimuret/golang-iij-dpf v0.9.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04 // indirect
 	github.com/nrdcg/auroradns v1.1.0 // indirect
-	github.com/nrdcg/bunny-go v0.0.0-20230728143221-c9dda82568d9 // indirect
-	github.com/nrdcg/desec v0.7.0 // indirect
+	github.com/nrdcg/bunny-go v0.0.0-20240207213615-dde5bf4577a3 // indirect
+	github.com/nrdcg/desec v0.8.0 // indirect
 	github.com/nrdcg/dnspod-go v0.4.0 // indirect
 	github.com/nrdcg/freemyip v0.2.0 // indirect
 	github.com/nrdcg/goinwx v0.10.0 // indirect
 	github.com/nrdcg/mailinabox v0.2.0 // indirect
 	github.com/nrdcg/namesilo v0.2.1 // indirect
 	github.com/nrdcg/nodion v0.1.0 // indirect
-	github.com/nrdcg/porkbun v0.3.0 // indirect
+	github.com/nrdcg/porkbun v0.4.0 // indirect
 	github.com/nzdjb/go-metaname v1.0.0 // indirect
-	github.com/oracle/oci-go-sdk v24.3.0+incompatible // indirect
-	github.com/ovh/go-ovh v1.4.3 // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
+	github.com/ovh/go-ovh v1.6.0 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/pquerna/otp v1.4.0 // indirect
-	github.com/sacloud/api-client-go v0.2.8 // indirect
-	github.com/sacloud/go-http v0.1.6 // indirect
-	github.com/sacloud/iaas-api-go v1.11.1 // indirect
-	github.com/sacloud/packages-go v0.0.9 // indirect
-	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22 // indirect
+	github.com/sacloud/api-client-go v0.2.10 // indirect
+	github.com/sacloud/go-http v0.1.8 // indirect
+	github.com/sacloud/iaas-api-go v1.12.0 // indirect
+	github.com/sacloud/packages-go v0.0.10 // indirect
+	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 // indirect
-	github.com/softlayer/softlayer-go v1.1.3 // indirect
+	github.com/softlayer/softlayer-go v1.1.5 // indirect
 	github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect
-	github.com/spf13/cast v1.3.1 // indirect
-	github.com/stretchr/objx v0.5.1 // indirect
-	github.com/stretchr/testify v1.8.4 // indirect
-	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.490 // indirect
-	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dnspod v1.0.490 // indirect
-	github.com/transip/gotransip/v6 v6.23.0 // indirect
-	github.com/ultradns/ultradns-go-sdk v1.6.1-20231103022937-8589b6a // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
+	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.1002 // indirect
+	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dnspod v1.0.1002 // indirect
+	github.com/transip/gotransip/v6 v6.26.0 // indirect
+	github.com/ultradns/ultradns-go-sdk v1.7.0-20240913052650-970ca9a // indirect
 	github.com/vinyldns/go-vinyldns v0.9.16 // indirect
-	github.com/vultr/govultr/v2 v2.17.2 // indirect
-	github.com/yandex-cloud/go-genproto v0.0.0-20220805142335-27b56ddae16f // indirect
-	github.com/yandex-cloud/go-sdk v0.0.0-20220805164847-cf028e604997 // indirect
+	github.com/xlzd/gotp v0.1.0
+	github.com/yandex-cloud/go-genproto v0.0.0-20240911120709-1fa0cb6f47c2 // indirect
+	github.com/yandex-cloud/go-sdk v0.0.0-20240911121212-e4e74d0d02f5 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.uber.org/ratelimit v0.2.0 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
-	golang.org/x/mod v0.16.0 // indirect
-	golang.org/x/oauth2 v0.16.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.19.0 // indirect
-	google.golang.org/api v0.126.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/grpc v1.55.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
+	go.opentelemetry.io/otel v1.29.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect
+	go.opentelemetry.io/otel/metric v1.29.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.28.0 // indirect
+	go.opentelemetry.io/otel/trace v1.29.0 // indirect
+	go.uber.org/ratelimit v0.3.0 // indirect
+	golang.org/x/crypto v0.27.0 // indirect
+	golang.org/x/mod v0.21.0 // indirect
+	golang.org/x/oauth2 v0.23.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/time v0.6.0 // indirect
+	golang.org/x/tools v0.25.0 // indirect
+	google.golang.org/api v0.197.0 // indirect
+	google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
+	google.golang.org/grpc v1.66.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/ns1/ns1-go.v2 v2.7.13 // indirect
+	gopkg.in/ns1/ns1-go.v2 v2.12.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gotest.tools/v3 v3.5.1 // indirect
 )

src/main.go

@@ -1,7 +1,36 @@
 package main
 
+/*
+  ______
+ |___  /
+    / / ___  _ __ __ ___  ___   _
+   / / / _ \| '__/ _` \ \/ / | | |
+  / /_| (_) | | | (_| |>  <| |_| |
+ /_____\___/|_|  \__,_/_/\_\\__, |
+                             __/ |
+                            |___/
+
+Zoraxy - A general purpose HTTP reverse proxy and forwarding tool
+Author: tobychui
+License: AGPLv3
+
+--------------------------------------------
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, version 3 of the License or any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+*/
+
 import (
-	"embed"
 	"flag"
 	"fmt"
 	"log"
@@ -12,86 +41,13 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"imuslab.com/zoraxy/mod/access"
-	"imuslab.com/zoraxy/mod/acme"
-	"imuslab.com/zoraxy/mod/auth"
-	"imuslab.com/zoraxy/mod/database"
-	"imuslab.com/zoraxy/mod/dynamicproxy/redirection"
-	"imuslab.com/zoraxy/mod/email"
-	"imuslab.com/zoraxy/mod/forwardproxy"
-	"imuslab.com/zoraxy/mod/ganserv"
+	"github.com/gorilla/csrf"
 	"imuslab.com/zoraxy/mod/geodb"
-	"imuslab.com/zoraxy/mod/info/logger"
-	"imuslab.com/zoraxy/mod/mdns"
-	"imuslab.com/zoraxy/mod/netstat"
-	"imuslab.com/zoraxy/mod/pathrule"
-	"imuslab.com/zoraxy/mod/sshprox"
-	"imuslab.com/zoraxy/mod/statistic"
-	"imuslab.com/zoraxy/mod/statistic/analytic"
-	"imuslab.com/zoraxy/mod/tcpprox"
-	"imuslab.com/zoraxy/mod/tlscert"
-	"imuslab.com/zoraxy/mod/uptime"
+	"imuslab.com/zoraxy/mod/update"
 	"imuslab.com/zoraxy/mod/utils"
-	"imuslab.com/zoraxy/mod/webserv"
 )
 
-// General flags
-var webUIPort = flag.String("port", ":8000", "Management web interface listening port")
-var noauth = flag.Bool("noauth", false, "Disable authentication for management interface")
-var showver = flag.Bool("version", false, "Show version of this server")
-var allowSshLoopback = flag.Bool("sshlb", false, "Allow loopback web ssh connection (DANGER)")
-var allowMdnsScanning = flag.Bool("mdns", true, "Enable mDNS scanner and transponder")
-var mdnsName = flag.String("mdnsname", "", "mDNS name, leave empty to use default (zoraxy_{node-uuid}.local)")
-var ztAuthToken = flag.String("ztauth", "", "ZeroTier authtoken for the local node")
-var ztAPIPort = flag.Int("ztport", 9993, "ZeroTier controller API port")
-var acmeAutoRenewInterval = flag.Int("autorenew", 86400, "ACME auto TLS/SSL certificate renew check interval (seconds)")
-var enableHighSpeedGeoIPLookup = flag.Bool("fastgeoip", false, "Enable high speed geoip lookup, require 1GB extra memory (Not recommend for low end devices)")
-var staticWebServerRoot = flag.String("webroot", "./www", "Static web server root folder. Only allow chnage in start paramters")
-var allowWebFileManager = flag.Bool("webfm", true, "Enable web file manager for static web server root folder")
-var logOutputToFile = flag.Bool("log", true, "Log terminal output to file")
-
-var (
-	name        = "Zoraxy"
-	version     = "3.0.5"
-	nodeUUID    = "generic"
-	development = false //Set this to false to use embedded web fs
-	bootTime    = time.Now().Unix()
-
-	/*
-		Binary Embedding File System
-	*/
-	//go:embed web/*
-	webres embed.FS
-
-	/*
-		Handler Modules
-	*/
-	sysdb              *database.Database      //System database
-	authAgent          *auth.AuthAgent         //Authentication agent
-	tlsCertManager     *tlscert.Manager        //TLS / SSL management
-	redirectTable      *redirection.RuleTable  //Handle special redirection rule sets
-	pathRuleHandler    *pathrule.Handler       //Handle specific path blocking or custom headers
-	geodbStore         *geodb.Store            //GeoIP database, for resolving IP into country code
-	accessController   *access.Controller      //Access controller, handle black list and white list
-	netstatBuffers     *netstat.NetStatBuffers //Realtime graph buffers
-	statisticCollector *statistic.Collector    //Collecting statistic from visitors
-	uptimeMonitor      *uptime.Monitor         //Uptime monitor service worker
-	mdnsScanner        *mdns.MDNSHost          //mDNS discovery services
-	ganManager         *ganserv.NetworkManager //Global Area Network Manager
-	webSshManager      *sshprox.Manager        //Web SSH connection service
-	tcpProxyManager    *tcpprox.Manager        //TCP Proxy Manager
-	acmeHandler        *acme.ACMEHandler       //Handler for ACME Certificate renew
-	acmeAutoRenewer    *acme.AutoRenewer       //Handler for ACME auto renew ticking
-	staticWebServer    *webserv.WebServer      //Static web server for hosting simple stuffs
-	forwardProxy       *forwardproxy.Handler   //HTTP Forward proxy, basically VPN for web browser
-
-	//Helper modules
-	EmailSender      *email.Sender        //Email sender that handle email sending
-	AnalyticLoader   *analytic.DataLoader //Data loader for Zoraxy Analytic
-	SystemWideLogger *logger.Logger       //Logger for Zoraxy
-)
-
-// Kill signal handler. Do something before the system the core terminate.
+/* SIGTERM handler, do shutdown sequences before closing */
 func SetupCloseHandler() {
 	c := make(chan os.Signal, 2)
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
@@ -102,42 +58,30 @@ func SetupCloseHandler() {
 	}()
 }
 
-func ShutdownSeq() {
-	fmt.Println("- Shutting down " + name)
-	fmt.Println("- Closing GeoDB ")
-	geodbStore.Close()
-	fmt.Println("- Closing Netstats Listener")
-	netstatBuffers.Close()
-	fmt.Println("- Closing Statistic Collector")
-	statisticCollector.Close()
-	if mdnsTickerStop != nil {
-		fmt.Println("- Stopping mDNS Discoverer (might take a few minutes)")
-		// Stop the mdns service
-		mdnsTickerStop <- true
-	}
-
-	mdnsScanner.Close()
-	fmt.Println("- Closing Certificates Auto Renewer")
-	acmeAutoRenewer.Close()
-	//Remove the tmp folder
-	fmt.Println("- Cleaning up tmp files")
-	os.RemoveAll("./tmp")
-
-	fmt.Println("- Closing system wide logger")
-	SystemWideLogger.Close()
-
-	//Close database, final
-	fmt.Println("- Stopping system database")
-	sysdb.Close()
-}
-
 func main() {
 	//Parse startup flags
 	flag.Parse()
+
+	/* Maintaince Function Modes */
 	if *showver {
-		fmt.Println(name + " - Version " + version)
+		fmt.Println(SYSTEM_NAME + " - Version " + SYSTEM_VERSION)
 		os.Exit(0)
 	}
+	if *geoDbUpdate {
+		geodb.DownloadGeoDBUpdate("./conf/geodb")
+		os.Exit(0)
+	}
+
+	/* Main Zoraxy Routines */
+	if !utils.ValidateListeningAddress(*webUIPort) {
+		fmt.Println("Malformed -port (listening address) paramter. Do you mean -port=:" + *webUIPort + "?")
+		os.Exit(0)
+	}
+
+	if *enableAutoUpdate {
+		fmt.Println("Checking required config update")
+		update.RunConfigUpdate(0, update.GetVersionIntFromVersionNumber(SYSTEM_VERSION))
+	}
 
 	SetupCloseHandler()
 
@@ -154,12 +98,22 @@ func main() {
 	}
 	nodeUUID = string(uuidBytes)
 
-	//Startup all modules
+	//Create a new webmin mux and csrf middleware layer
+	webminPanelMux = http.NewServeMux()
+	csrfMiddleware = csrf.Protect(
+		[]byte(nodeUUID),
+		csrf.CookieName(CSRF_COOKIENAME),
+		csrf.Secure(false),
+		csrf.Path("/"),
+		csrf.SameSite(csrf.SameSiteLaxMode),
+	)
+
+	//Startup all modules, see start.go
 	startupSequence()
 
 	//Initiate management interface APIs
 	requireAuth = !(*noauth)
-	initAPIs()
+	initAPIs(webminPanelMux)
 
 	//Start the reverse proxy server in go routine
 	go func() {
@@ -171,11 +125,10 @@ func main() {
 	//Start the finalize sequences
 	finalSequence()
 
-	SystemWideLogger.Println("Zoraxy started. Visit control panel at http://localhost" + *webUIPort)
-	err = http.ListenAndServe(*webUIPort, nil)
+	SystemWideLogger.Println(SYSTEM_NAME + " started. Visit control panel at http://localhost" + *webUIPort)
+	err = http.ListenAndServe(*webUIPort, csrfMiddleware(webminPanelMux))
 
 	if err != nil {
 		log.Fatal(err)
 	}
-
 }

src/mod/acme/acme.go

@@ -11,7 +11,6 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
-	"log"
 	"net"
 	"net/http"
 	"os"
@@ -22,18 +21,29 @@ import (
 
 	"github.com/go-acme/lego/v4/certcrypto"
 	"github.com/go-acme/lego/v4/certificate"
+	"github.com/go-acme/lego/v4/challenge/dns01"
 	"github.com/go-acme/lego/v4/challenge/http01"
 	"github.com/go-acme/lego/v4/lego"
 	"github.com/go-acme/lego/v4/registration"
 	"imuslab.com/zoraxy/mod/database"
+	"imuslab.com/zoraxy/mod/info/logger"
 	"imuslab.com/zoraxy/mod/utils"
 )
 
+var defaultNameservers = []string{
+	"8.8.8.8:53", // Google DNS
+	"8.8.4.4:53", // Google DNS
+	"1.1.1.1:53", // Cloudflare DNS
+	"1.0.0.1:53", // Cloudflare DNS
+}
+
 type CertificateInfoJSON struct {
-	AcmeName string `json:"acme_name"`
-	AcmeUrl  string `json:"acme_url"`
-	SkipTLS  bool   `json:"skip_tls"`
-	UseDNS   bool   `json:"dns"`
+	AcmeName    string   `json:"acme_name"`  //ACME provider name
+	AcmeUrl     string   `json:"acme_url"`   //Custom ACME URL (if any)
+	SkipTLS     bool     `json:"skip_tls"`   //Skip TLS verification of upstream
+	UseDNS      bool     `json:"dns"`        //Use DNS challenge
+	PropTimeout int      `json:"prop_time"`  //Propagation timeout
+	DNSServers  []string `json:"dnsServers"` // DNS servers
 }
 
 // ACMEUser represents a user in the ACME system.
@@ -68,25 +78,38 @@ type ACMEHandler struct {
 	DefaultAcmeServer string
 	Port              string
 	Database          *database.Database
+	Logger            *logger.Logger
 }
 
 // NewACME creates a new ACMEHandler instance.
-func NewACME(acmeServer string, port string, database *database.Database) *ACMEHandler {
+func NewACME(defaultAcmeServer string, port string, database *database.Database, logger *logger.Logger) *ACMEHandler {
 	return &ACMEHandler{
-		DefaultAcmeServer: acmeServer,
+		DefaultAcmeServer: defaultAcmeServer,
 		Port:              port,
 		Database:          database,
+		Logger:            logger,
 	}
 }
 
+func (a *ACMEHandler) Logf(message string, err error) {
+	a.Logger.PrintAndLog("ACME", message, err)
+}
+
+// Close closes the ACMEHandler.
+// ACME Handler does not need to close anything
+// Function defined for future compatibility
+func (a *ACMEHandler) Close() error {
+	return nil
+}
+
 // ObtainCert obtains a certificate for the specified domains.
-func (a *ACMEHandler) ObtainCert(domains []string, certificateName string, email string, caName string, caUrl string, skipTLS bool, useDNS bool) (bool, error) {
-	log.Println("[ACME] Obtaining certificate...")
+func (a *ACMEHandler) ObtainCert(domains []string, certificateName string, email string, caName string, caUrl string, skipTLS bool, useDNS bool, propagationTimeout int, dnsServers string) (bool, error) {
+	a.Logf("Obtaining certificate for: "+strings.Join(domains, ", "), nil)
 
 	// generate private key
 	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {
-		log.Println(err)
+		a.Logf("Private key generation failed", err)
 		return false, err
 	}
 
@@ -102,7 +125,7 @@ func (a *ACMEHandler) ObtainCert(domains []string, certificateName string, email
 	// skip TLS verify if need
 	// Ref: https://github.com/go-acme/lego/blob/6af2c756ac73a9cb401621afca722d0f4112b1b8/lego/client_config.go#L74
 	if skipTLS {
-		log.Println("[INFO] Ignore TLS/SSL Verification Error for ACME Server")
+		a.Logf("Ignoring TLS/SSL Verification Error for ACME Server", nil)
 		config.HTTPClient.Transport = &http.Transport{
 			Proxy: http.ProxyFromEnvironment,
 			DialContext: (&net.Dialer{
@@ -129,16 +152,16 @@ func (a *ACMEHandler) ObtainCert(domains []string, certificateName string, email
 
 	// if not custom ACME url, load it from ca.json
 	if caName == "custom" {
-		log.Println("[INFO] Using Custom ACME " + caUrl + " for CA Directory URL")
+		a.Logf("Using Custom ACME "+caUrl+" for CA Directory URL", nil)
 	} else {
 		caLinkOverwrite, err := loadCAApiServerFromName(caName)
 		if err == nil {
 			config.CADirURL = caLinkOverwrite
-			log.Println("[INFO] Using " + caLinkOverwrite + " for CA Directory URL")
+			a.Logf("Using "+caLinkOverwrite+" for CA Directory URL", nil)
 		} else {
 			// (caName == "" || caUrl == "") will use default acme
 			config.CADirURL = a.DefaultAcmeServer
-			log.Println("[INFO] Using Default ACME " + a.DefaultAcmeServer + " for CA Directory URL")
+			a.Logf("Using Default ACME "+a.DefaultAcmeServer+" for CA Directory URL", nil)
 		}
 	}
 
@@ -146,50 +169,72 @@ func (a *ACMEHandler) ObtainCert(domains []string, certificateName string, email
 
 	client, err := lego.NewClient(config)
 	if err != nil {
-		log.Println(err)
+		a.Logf("Failed to spawn new ACME client from current config", err)
 		return false, err
 	}
 
+	// Load certificate info from JSON file
+	certInfo, err := LoadCertInfoJSON(fmt.Sprintf("./conf/certs/%s.json", certificateName))
+	if err == nil {
+		useDNS = certInfo.UseDNS
+		if dnsServers == "" && certInfo.DNSServers != nil && len(certInfo.DNSServers) > 0 {
+			dnsServers = strings.Join(certInfo.DNSServers, ",")
+		}
+		propagationTimeout = certInfo.PropTimeout
+	}
+
+	// Clean DNS servers
+	dnsNameservers := strings.Split(dnsServers, ",")
+	for i := range dnsNameservers {
+		dnsNameservers[i] = strings.TrimSpace(dnsNameservers[i])
+	}
+
 	// setup how to receive challenge
 	if useDNS {
 		if !a.Database.TableExists("acme") {
 			a.Database.NewTable("acme")
-			return false, errors.New("DNS Provider and DNS Credenital configuration required for ACME Provider (Error -1)")
+			return false, errors.New("DNS Provider and DNS Credential configuration required for ACME Provider (Error -1)")
 		}
 
 		if !a.Database.KeyExists("acme", certificateName+"_dns_provider") || !a.Database.KeyExists("acme", certificateName+"_dns_credentials") {
-			return false, errors.New("DNS Provider and DNS Credenital configuration required for ACME Provider (Error -2)")
+			return false, errors.New("DNS Provider and DNS Credential configuration required for ACME Provider (Error -2)")
 		}
 
 		var dnsCredentials string
 		err := a.Database.Read("acme", certificateName+"_dns_credentials", &dnsCredentials)
 		if err != nil {
-			log.Println(err)
+			a.Logf("Read DNS credential failed", err)
 			return false, err
 		}
 
 		var dnsProvider string
 		err = a.Database.Read("acme", certificateName+"_dns_provider", &dnsProvider)
 		if err != nil {
-			log.Println(err)
+			a.Logf("Read DNS Provider failed", err)
 			return false, err
 		}
 
-		provider, err := GetDnsChallengeProviderByName(dnsProvider, dnsCredentials)
+		provider, err := GetDnsChallengeProviderByName(dnsProvider, dnsCredentials, propagationTimeout)
 		if err != nil {
-			log.Println(err)
+			a.Logf("Unable to resolve DNS challenge provider", err)
 			return false, err
 		}
 
-		err = client.Challenge.SetDNS01Provider(provider)
+		if len(dnsNameservers) > 0 && dnsNameservers[0] != "" {
+			a.Logf("Using DNS servers: "+strings.Join(dnsNameservers, ", "), nil)
+			err = client.Challenge.SetDNS01Provider(provider, dns01.AddRecursiveNameservers(dnsNameservers))
+		} else {
+			// Use default DNS-01 nameservers if dnsServers is empty
+			err = client.Challenge.SetDNS01Provider(provider, dns01.AddRecursiveNameservers(defaultNameservers))
+		}
 		if err != nil {
-			log.Println(err)
+			a.Logf("Failed to resolve DNS01 Provider", err)
 			return false, err
 		}
 	} else {
 		err = client.Challenge.SetHTTP01Provider(http01.NewProviderServer("", a.Port))
 		if err != nil {
-			log.Println(err)
+			a.Logf("Failed to resolve HTTP01 Provider", err)
 			return false, err
 		}
 	}
@@ -205,7 +250,7 @@ func (a *ACMEHandler) ObtainCert(domains []string, certificateName string, email
 	var reg *registration.Resource
 	// New users will need to register
 	if client.GetExternalAccountRequired() {
-		log.Println("External Account Required for this ACME Provider.")
+		a.Logf("External Account Required for this ACME Provider", nil)
 		// IF KID and HmacEncoded is overidden
 
 		if !a.Database.TableExists("acme") {
@@ -220,20 +265,18 @@ func (a *ACMEHandler) ObtainCert(domains []string, certificateName string, email
 		var kid string
 		var hmacEncoded string
 		err := a.Database.Read("acme", config.CADirURL+"_kid", &kid)
-
 		if err != nil {
-			log.Println(err)
+			a.Logf("Failed to read kid from database", err)
 			return false, err
 		}
 
 		err = a.Database.Read("acme", config.CADirURL+"_hmacEncoded", &hmacEncoded)
-
 		if err != nil {
-			log.Println(err)
+			a.Logf("Failed to read HMAC from database", err)
 			return false, err
 		}
 
-		log.Println("EAB Credential retrieved.", kid, hmacEncoded)
+		a.Logf("EAB Credential retrieved: "+kid+" / "+hmacEncoded, nil)
 		if kid != "" && hmacEncoded != "" {
 			reg, err = client.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
 				TermsOfServiceAgreed: true,
@@ -242,14 +285,14 @@ func (a *ACMEHandler) ObtainCert(domains []string, certificateName string, email
 			})
 		}
 		if err != nil {
-			log.Println(err)
+			a.Logf("Register with external account binder failed", err)
 			return false, err
 		}
 		//return false, errors.New("External Account Required for this ACME Provider.")
 	} else {
 		reg, err = client.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: true})
 		if err != nil {
-			log.Println(err)
+			a.Logf("Unable to register client", err)
 			return false, err
 		}
 	}
@@ -262,7 +305,7 @@ func (a *ACMEHandler) ObtainCert(domains []string, certificateName string, email
 	}
 	certificates, err := client.Certificate.Obtain(request)
 	if err != nil {
-		log.Println(err)
+		a.Logf("Obtain certificate failed", err)
 		return false, err
 	}
 
@@ -270,32 +313,34 @@ func (a *ACMEHandler) ObtainCert(domains []string, certificateName string, email
 	// private key, and a certificate URL.
 	err = os.WriteFile("./conf/certs/"+certificateName+".pem", certificates.Certificate, 0777)
 	if err != nil {
-		log.Println(err)
+		a.Logf("Failed to write public key to disk", err)
 		return false, err
 	}
 	err = os.WriteFile("./conf/certs/"+certificateName+".key", certificates.PrivateKey, 0777)
 	if err != nil {
-		log.Println(err)
+		a.Logf("Failed to write private key to disk", err)
 		return false, err
 	}
 
 	// Save certificate's ACME info for renew usage
-	certInfo := &CertificateInfoJSON{
-		AcmeName: caName,
-		AcmeUrl:  caUrl,
-		SkipTLS:  skipTLS,
-		UseDNS:   useDNS,
+	certInfo = &CertificateInfoJSON{
+		AcmeName:    caName,
+		AcmeUrl:     caUrl,
+		SkipTLS:     skipTLS,
+		UseDNS:      useDNS,
+		PropTimeout: propagationTimeout,
+		DNSServers:  dnsNameservers,
 	}
 
 	certInfoBytes, err := json.Marshal(certInfo)
 	if err != nil {
-		log.Println(err)
+		a.Logf("Marshal certificate renew config failed", err)
 		return false, err
 	}
 
 	err = os.WriteFile("./conf/certs/"+certificateName+".json", certInfoBytes, 0777)
 	if err != nil {
-		log.Println(err)
+		a.Logf("Failed to write certificate renew config to file", err)
 		return false, err
 	}
 
@@ -313,7 +358,7 @@ func (a *ACMEHandler) CheckCertificate() []string {
 	expiredCerts := []string{}
 
 	if err != nil {
-		log.Println(err)
+		a.Logf("Failed to load certificate folder", err)
 		return []string{}
 	}
 
@@ -410,14 +455,14 @@ func (a *ACMEHandler) HandleRenewCertificate(w http.ResponseWriter, r *http.Requ
 
 	ca, err := utils.PostPara(r, "ca")
 	if err != nil {
-		log.Println("[INFO] CA not set. Using default")
+		a.Logf("CA not set. Using default", nil)
 		ca, caUrl = "", ""
 	}
 
 	if ca == "custom" {
 		caUrl, err = utils.PostPara(r, "caURL")
 		if err != nil {
-			log.Println("[INFO] Custom CA set but no URL provide, Using default")
+			a.Logf("Custom CA set but no URL provide, Using default", nil)
 			ca, caUrl = "", ""
 		}
 	}
@@ -448,7 +493,44 @@ func (a *ACMEHandler) HandleRenewCertificate(w http.ResponseWriter, r *http.Requ
 	}
 
 	domains := strings.Split(domainPara, ",")
-	result, err := a.ObtainCert(domains, filename, email, ca, caUrl, skipTLS, dns)
+
+	// Default propagation timeout is 300 seconds
+	propagationTimeout := 300
+	if dns {
+		ppgTimeout, err := utils.PostPara(r, "ppgTimeout")
+		if err == nil {
+			propagationTimeout, err = strconv.Atoi(ppgTimeout)
+			if err != nil {
+				utils.SendErrorResponse(w, "Invalid propagation timeout value")
+				return
+			}
+			if propagationTimeout < 60 {
+				//Minimum propagation timeout is 60 seconds
+				propagationTimeout = 60
+			}
+		}
+	}
+
+	//Clean spaces in front or behind each domain
+	cleanedDomains := []string{}
+	for _, domain := range domains {
+		cleanedDomains = append(cleanedDomains, strings.TrimSpace(domain))
+	}
+
+	// Extract DNS servers from the request
+	var dnsServers []string
+	dnsServersPara, err := utils.PostPara(r, "dnsServers")
+	if err == nil && dnsServersPara != "" {
+		dnsServers = strings.Split(dnsServersPara, ",")
+		for i := range dnsServers {
+			dnsServers[i] = strings.TrimSpace(dnsServers[i])
+		}
+	}
+
+	// Convert DNS servers slice to a single string
+	dnsServersString := strings.Join(dnsServers, ",")
+
+	result, err := a.ObtainCert(cleanedDomains, filename, email, ca, caUrl, skipTLS, dns, propagationTimeout, dnsServersString)
 	if err != nil {
 		utils.SendErrorResponse(w, jsonEscape(err.Error()))
 		return
@@ -460,7 +542,7 @@ func (a *ACMEHandler) HandleRenewCertificate(w http.ResponseWriter, r *http.Requ
 func jsonEscape(i string) string {
 	b, err := json.Marshal(i)
 	if err != nil {
-		log.Println("Unable to escape json data: " + err.Error())
+		//log.Println("Unable to escape json data: " + err.Error())
 		return i
 	}
 	s := string(b)
@@ -491,5 +573,10 @@ func LoadCertInfoJSON(filename string) (*CertificateInfoJSON, error) {
 		return nil, err
 	}
 
+	// Clean DNS servers
+	for i := range certInfo.DNSServers {
+		certInfo.DNSServers[i] = strings.TrimSpace(certInfo.DNSServers[i])
+	}
+
 	return certInfo, nil
 }

src/mod/acme/acme_dns.go

@@ -1,72 +1,56 @@
 package acme
 
 import (
-	"errors"
-	"log"
-	"os"
-	"strings"
+	"encoding/json"
+	"strconv"
 
 	"github.com/go-acme/lego/v4/challenge"
 	"imuslab.com/zoraxy/mod/acme/acmedns"
 )
 
-func GetDnsChallengeProviderByName(dnsProvider string, dnsCredentials string) (challenge.Provider, error) {
-
-	//Original Implementation
-	/*credentials, err := extractDnsCredentials(dnsCredentials)
+// Preprocessor function to get DNS challenge provider by name
+func GetDnsChallengeProviderByName(dnsProvider string, dnsCredentials string, ppgTimeout int) (challenge.Provider, error) {
+	//Unpack the dnsCredentials (json string) to map
+	var dnsCredentialsMap map[string]interface{}
+	err := json.Unmarshal([]byte(dnsCredentials), &dnsCredentialsMap)
 	if err != nil {
 		return nil, err
 	}
-	setCredentialsIntoEnvironmentVariables(credentials)
 
-	provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
-	*/
-
-	//New implementation using acmedns CICD pipeline generated datatype
-	return acmedns.GetDNSProviderByJsonConfig(dnsProvider, dnsCredentials)
-}
-
-/*
-	Original implementation of DNS ACME using OS.Env as payload
-*/
-
-func setCredentialsIntoEnvironmentVariables(credentials map[string]string) {
-	for key, value := range credentials {
-		err := os.Setenv(key, value)
-		if err != nil {
-			log.Println("[ERR] Failed to set environment variable %s: %v", key, err)
-		} else {
-			log.Println("[INFO] Environment variable %s set successfully", key)
-		}
-	}
-}
-
-func extractDnsCredentials(input string) (map[string]string, error) {
-	result := make(map[string]string)
-
-	// Split the input string by newline character
-	lines := strings.Split(input, "\n")
-
-	// Iterate over each line
-	for _, line := range lines {
-		// Split the line by "=" character
-		//use SpliyN to make sure not to split the value if the value is base64
-		parts := strings.SplitN(line, "=", 1)
-
-		// Check if the line is in the correct format
-		if len(parts) == 2 {
-			key := strings.TrimSpace(parts[0])
-			value := strings.TrimSpace(parts[1])
-
-			// Add the key-value pair to the map
-			result[key] = value
-
-			if value == "" || key == "" {
-				//invalid config
-				return result, errors.New("DNS credential extract failed")
-			}
+	//Clear the PollingInterval and PropagationTimeout field and conert to int
+	userDefinedPollingInterval := 2
+	if dnsCredentialsMap["PollingInterval"] != nil {
+		userDefinedPollingIntervalRaw := dnsCredentialsMap["PollingInterval"].(string)
+		delete(dnsCredentialsMap, "PollingInterval")
+		convertedPollingInterval, err := strconv.Atoi(userDefinedPollingIntervalRaw)
+		if err == nil {
+			userDefinedPollingInterval = convertedPollingInterval
 		}
 	}
 
-	return result, nil
+	userDefinedPropagationTimeout := ppgTimeout
+	if dnsCredentialsMap["PropagationTimeout"] != nil {
+		userDefinedPropagationTimeoutRaw := dnsCredentialsMap["PropagationTimeout"].(string)
+		delete(dnsCredentialsMap, "PropagationTimeout")
+		convertedPropagationTimeout, err := strconv.Atoi(userDefinedPropagationTimeoutRaw)
+		if err == nil {
+			//Overwrite the default propagation timeout if it is requeted from UI
+			userDefinedPropagationTimeout = convertedPropagationTimeout
+		}
+	}
+
+	//Restructure dnsCredentials string from map
+	dnsCredentialsBytes, err := json.Marshal(dnsCredentialsMap)
+	if err != nil {
+		return nil, err
+	}
+	dnsCredentials = string(dnsCredentialsBytes)
+
+	//Using acmedns CICD pipeline generated datatype to optain the DNS provider
+	return acmedns.GetDNSProviderByJsonConfig(
+		dnsProvider,
+		dnsCredentials,
+		int64(userDefinedPropagationTimeout),
+		int64(userDefinedPollingInterval),
+	)
 }

src/mod/acme/acmewizard/acmewizard.go

@@ -75,6 +75,15 @@ func HandleGuidedStepCheck(w http.ResponseWriter, r *http.Request) {
 		httpServerReachable := isHTTPServerAvailable(domain)
 		js, _ := json.Marshal(httpServerReachable)
 		utils.SendJSONResponse(w, string(js))
+	} else if stepNo == 10 {
+		//Resolve public Ip address for tour
+		publicIp, err := getPublicIPAddress()
+		if err != nil {
+			utils.SendErrorResponse(w, err.Error())
+			return
+		}
+		js, _ := json.Marshal(publicIp)
+		utils.SendJSONResponse(w, string(js))
 	} else {
 		utils.SendErrorResponse(w, "invalid step number")
 	}

src/mod/acme/autorenew.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"log"
 	"net/http"
 	"net/mail"
 	"os"
@@ -12,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"imuslab.com/zoraxy/mod/info/logger"
 	"imuslab.com/zoraxy/mod/utils"
 )
 
@@ -26,6 +26,7 @@ type AutoRenewConfig struct {
 	Email        string   //Email for acme
 	RenewAll     bool     //Renew all or selective renew with the slice below
 	FilesToRenew []string //If RenewAll is false, renew these certificate files
+	DNSServers   string   // DNS servers
 }
 
 type AutoRenewer struct {
@@ -34,7 +35,9 @@ type AutoRenewer struct {
 	AcmeHandler       *ACMEHandler
 	RenewerConfig     *AutoRenewConfig
 	RenewTickInterval int64
+	EarlyRenewDays    int //How many days before cert expire to renew certificate
 	TickerstopChan    chan bool
+	Logger            *logger.Logger //System wide logger
 }
 
 type ExpiredCerts struct {
@@ -44,11 +47,15 @@ type ExpiredCerts struct {
 
 // Create an auto renew agent, require config filepath and auto scan & renew interval (seconds)
 // Set renew check interval to 0 for auto (1 day)
-func NewAutoRenewer(config string, certFolder string, renewCheckInterval int64, AcmeHandler *ACMEHandler) (*AutoRenewer, error) {
+func NewAutoRenewer(config string, certFolder string, renewCheckInterval int64, earlyRenewDays int, AcmeHandler *ACMEHandler, logger *logger.Logger) (*AutoRenewer, error) {
 	if renewCheckInterval == 0 {
 		renewCheckInterval = 86400 //1 day
 	}
 
+	if earlyRenewDays == 0 {
+		earlyRenewDays = 30
+	}
+
 	//Load the config file. If not found, create one
 	if !utils.FileExists(config) {
 		//Create one
@@ -82,8 +89,12 @@ func NewAutoRenewer(config string, certFolder string, renewCheckInterval int64,
 		AcmeHandler:       AcmeHandler,
 		RenewerConfig:     &renewerConfig,
 		RenewTickInterval: renewCheckInterval,
+		EarlyRenewDays:    earlyRenewDays,
+		Logger:            logger,
 	}
 
+	thisRenewer.Logf("ACME early renew set to "+fmt.Sprint(earlyRenewDays)+" days and check interval set to "+fmt.Sprint(renewCheckInterval)+" seconds", nil)
+
 	if thisRenewer.RenewerConfig.Enabled {
 		//Start the renew ticker
 		thisRenewer.StartAutoRenewTicker()
@@ -95,6 +106,10 @@ func NewAutoRenewer(config string, certFolder string, renewCheckInterval int64,
 	return &thisRenewer, nil
 }
 
+func (a *AutoRenewer) Logf(message string, err error) {
+	a.Logger.PrintAndLog("cert-renew", message, err)
+}
+
 func (a *AutoRenewer) StartAutoRenewTicker() {
 	//Stop the previous ticker if still running
 	if a.TickerstopChan != nil {
@@ -113,7 +128,7 @@ func (a *AutoRenewer) StartAutoRenewTicker() {
 			case <-done:
 				return
 			case <-ticker.C:
-				log.Println("Check and renew certificates in progress")
+				a.Logf("Check and renew certificates in progress", nil)
 				a.CheckAndRenewCertificates()
 			}
 		}
@@ -135,7 +150,7 @@ func (a *AutoRenewer) StopAutoRenewTicker() {
 // opr = setSelected -> Enter a list of file names (or matching rules) for auto renew
 // opr = setAuto -> Set to use auto detect certificates and renew
 func (a *AutoRenewer) HandleSetAutoRenewDomains(w http.ResponseWriter, r *http.Request) {
-	opr, err := utils.GetPara(r, "opr")
+	opr, err := utils.PostPara(r, "opr")
 	if err != nil {
 		utils.SendErrorResponse(w, "Operation not set")
 		return
@@ -165,6 +180,8 @@ func (a *AutoRenewer) HandleSetAutoRenewDomains(w http.ResponseWriter, r *http.R
 		a.RenewerConfig.RenewAll = true
 		a.saveRenewConfigToFile()
 		utils.SendOK(w)
+	} else {
+		utils.SendErrorResponse(w, "invalid operation given")
 	}
 
 }
@@ -208,42 +225,52 @@ func (a *AutoRenewer) HandleRenewNow(w http.ResponseWriter, r *http.Request) {
 	utils.SendJSONResponse(w, string(js))
 }
 
+// HandleAutoRenewEnable get and set the auto renew enable state
 func (a *AutoRenewer) HandleAutoRenewEnable(w http.ResponseWriter, r *http.Request) {
-	val, err := utils.PostPara(r, "enable")
-	if err != nil {
+	if r.Method == http.MethodGet {
 		js, _ := json.Marshal(a.RenewerConfig.Enabled)
 		utils.SendJSONResponse(w, string(js))
-	} else {
-		if val == "true" {
+	} else if r.Method == http.MethodPost {
+		val, err := utils.PostBool(r, "enable")
+		if err != nil {
+			utils.SendErrorResponse(w, "invalid or empty enable state")
+		}
+		if val {
 			//Check if the email is not empty
 			if a.RenewerConfig.Email == "" {
 				utils.SendErrorResponse(w, "Email is not set")
 				return
 			}
-
 			a.RenewerConfig.Enabled = true
 			a.saveRenewConfigToFile()
-			log.Println("[ACME] ACME auto renew enabled")
+			a.Logf("ACME auto renew enabled", nil)
 			a.StartAutoRenewTicker()
 		} else {
 			a.RenewerConfig.Enabled = false
 			a.saveRenewConfigToFile()
-			log.Println("[ACME] ACME auto renew disabled")
+			a.Logf("ACME auto renew disabled", nil)
 			a.StopAutoRenewTicker()
 		}
+	} else {
+		http.Error(w, "405 - Method not allowed", http.StatusMethodNotAllowed)
 	}
+
 }
 
 func (a *AutoRenewer) HandleACMEEmail(w http.ResponseWriter, r *http.Request) {
-
-	email, err := utils.PostPara(r, "set")
-	if err != nil {
+	if r.Method == http.MethodGet {
 		//Return the current email to user
 		js, _ := json.Marshal(a.RenewerConfig.Email)
 		utils.SendJSONResponse(w, string(js))
-	} else {
+	} else if r.Method == http.MethodPost {
+		email, err := utils.PostPara(r, "set")
+		if err != nil {
+			utils.SendErrorResponse(w, "invalid or empty email given")
+			return
+		}
+
 		//Check if the email is valid
-		_, err := mail.ParseAddress(email)
+		_, err = mail.ParseAddress(email)
 		if err != nil {
 			utils.SendErrorResponse(w, err.Error())
 			return
@@ -252,8 +279,11 @@ func (a *AutoRenewer) HandleACMEEmail(w http.ResponseWriter, r *http.Request) {
 		//Set the new config
 		a.RenewerConfig.Email = email
 		a.saveRenewConfigToFile()
-	}
 
+		utils.SendOK(w)
+	} else {
+		http.Error(w, "405 - Method not allowed", http.StatusMethodNotAllowed)
+	}
 }
 
 // Check and renew certificates. This check all the certificates in the
@@ -263,7 +293,7 @@ func (a *AutoRenewer) CheckAndRenewCertificates() ([]string, error) {
 	certFolder := a.CertFolder
 	files, err := os.ReadDir(certFolder)
 	if err != nil {
-		log.Println("Unable to renew certificates: " + err.Error())
+		a.Logf("Read certificate store failed", err)
 		return []string{}, err
 	}
 
@@ -277,13 +307,12 @@ func (a *AutoRenewer) CheckAndRenewCertificates() ([]string, error) {
 				if err != nil {
 					continue
 				}
-				if CertExpireSoon(certBytes) || CertIsExpired(certBytes) {
+				if CertExpireSoon(certBytes, a.EarlyRenewDays) || CertIsExpired(certBytes) {
 					//This cert is expired
-
 					DNSName, err := ExtractDomains(certBytes)
 					if err != nil {
 						//Maybe self signed. Ignore this
-						log.Println("Encounted error when trying to resolve DNS name for cert " + file.Name())
+						a.Logf("Encounted error when trying to resolve DNS name for cert "+file.Name(), err)
 						continue
 					}
 
@@ -305,13 +334,12 @@ func (a *AutoRenewer) CheckAndRenewCertificates() ([]string, error) {
 				if err != nil {
 					continue
 				}
-				if CertExpireSoon(certBytes) || CertIsExpired(certBytes) {
+				if CertExpireSoon(certBytes, a.EarlyRenewDays) || CertIsExpired(certBytes) {
 					//This cert is expired
-
 					DNSName, err := ExtractDomains(certBytes)
 					if err != nil {
 						//Maybe self signed. Ignore this
-						log.Println("Encounted error when trying to resolve DNS name for cert " + file.Name())
+						a.Logf("Encounted error when trying to resolve DNS name for cert "+file.Name(), err)
 						continue
 					}
 
@@ -327,6 +355,7 @@ func (a *AutoRenewer) CheckAndRenewCertificates() ([]string, error) {
 	return a.renewExpiredDomains(expiredCertList)
 }
 
+// Close the auto renewer
 func (a *AutoRenewer) Close() {
 	if a.TickerstopChan != nil {
 		a.TickerstopChan <- true
@@ -338,7 +367,7 @@ func (a *AutoRenewer) Close() {
 func (a *AutoRenewer) renewExpiredDomains(certs []*ExpiredCerts) ([]string, error) {
 	renewedCertFiles := []string{}
 	for _, expiredCert := range certs {
-		log.Println("Renewing " + expiredCert.Filepath + " (Might take a few minutes)")
+		a.Logf("Renewing "+expiredCert.Filepath+" (Might take a few minutes)", nil)
 		fileName := filepath.Base(expiredCert.Filepath)
 		certName := fileName[:len(fileName)-len(filepath.Ext(fileName))]
 
@@ -346,21 +375,33 @@ func (a *AutoRenewer) renewExpiredDomains(certs []*ExpiredCerts) ([]string, erro
 		certInfoFilename := fmt.Sprintf("%s/%s.json", filepath.Dir(expiredCert.Filepath), certName)
 		certInfo, err := LoadCertInfoJSON(certInfoFilename)
 		if err != nil {
-			log.Printf("Renew %s certificate error, can't get the ACME detail for cert: %v, trying org section as ca", certName, err)
+			a.Logf("Renew "+certName+"certificate error, can't get the ACME detail for certificate, trying org section as ca", err)
 
 			if CAName, extractErr := ExtractIssuerNameFromPEM(expiredCert.Filepath); extractErr != nil {
-				log.Printf("extract issuer name for cert error: %v, using default ca", extractErr)
+				a.Logf("Extract issuer name for cert error, using default ca", err)
 				certInfo = &CertificateInfoJSON{}
 			} else {
 				certInfo = &CertificateInfoJSON{AcmeName: CAName}
 			}
 		}
 
-		_, err = a.AcmeHandler.ObtainCert(expiredCert.Domains, certName, a.RenewerConfig.Email, certInfo.AcmeName, certInfo.AcmeUrl, certInfo.SkipTLS, certInfo.UseDNS)
+		//For upgrading config from older version of Zoraxy which don't have timeout
+		if certInfo.PropTimeout == 0 {
+			//Set default timeout
+			certInfo.PropTimeout = 300
+		}
+
+		// Extract DNS servers from the certificate info if available
+		var dnsServers string
+		if len(certInfo.DNSServers) > 0 {
+			dnsServers = strings.Join(certInfo.DNSServers, ",")
+		}
+
+		_, err = a.AcmeHandler.ObtainCert(expiredCert.Domains, certName, a.RenewerConfig.Email, certInfo.AcmeName, certInfo.AcmeUrl, certInfo.SkipTLS, certInfo.UseDNS, certInfo.PropTimeout, dnsServers)
 		if err != nil {
-			log.Println("Renew " + fileName + "(" + strings.Join(expiredCert.Domains, ",") + ") failed: " + err.Error())
+			a.Logf("Renew "+fileName+"("+strings.Join(expiredCert.Domains, ",")+") failed", err)
 		} else {
-			log.Println("Successfully renewed " + filepath.Base(expiredCert.Filepath))
+			a.Logf("Successfully renewed "+filepath.Base(expiredCert.Filepath), nil)
 			renewedCertFiles = append(renewedCertFiles, filepath.Base(expiredCert.Filepath))
 		}
 	}
@@ -406,7 +447,7 @@ func (a *AutoRenewer) HanldeSetEAB(w http.ResponseWriter, r *http.Request) {
 }
 
 // Handle update auto renew DNS configuration
-func (a *AutoRenewer) HanldeSetDNS(w http.ResponseWriter, r *http.Request) {
+func (a *AutoRenewer) HandleSetDNS(w http.ResponseWriter, r *http.Request) {
 	dnsProvider, err := utils.PostPara(r, "dnsProvider")
 	if err != nil {
 		utils.SendErrorResponse(w, "dnsProvider not set")
@@ -425,12 +466,18 @@ func (a *AutoRenewer) HanldeSetDNS(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	dnsServers, err := utils.PostPara(r, "dnsServers")
+	if err != nil {
+		dnsServers = ""
+	}
+
 	if !a.AcmeHandler.Database.TableExists("acme") {
 		a.AcmeHandler.Database.NewTable("acme")
 	}
 
 	a.AcmeHandler.Database.Write("acme", filename+"_dns_provider", dnsProvider)
 	a.AcmeHandler.Database.Write("acme", filename+"_dns_credentials", dnsCredentials)
+	a.AcmeHandler.Database.Write("acme", filename+"_dns_servers", dnsServers)
 
 	utils.SendOK(w)
 

src/mod/acme/ca.go

@@ -3,7 +3,7 @@ package acme
 /*
 	CA.go
 
-	This script load CA defination from embedded ca.json
+	This script load CA definition from embedded ca.json
 */
 import (
 	_ "embed"
@@ -13,7 +13,7 @@ import (
 	"strings"
 )
 
-// CA Defination, load from embeded json when startup
+// CA definition, load from embeded json when startup
 type CaDef struct {
 	Production map[string]string
 	Test       map[string]string

src/mod/acme/utils.go

@@ -5,14 +5,14 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"time"
 )
 
 // Get the issuer name from pem file
 func ExtractIssuerNameFromPEM(pemFilePath string) (string, error) {
 	// Read the PEM file
-	pemData, err := ioutil.ReadFile(pemFilePath)
+	pemData, err := os.ReadFile(pemFilePath)
 	if err != nil {
 		return "", err
 	}
@@ -81,13 +81,14 @@ func CertIsExpired(certBytes []byte) bool {
 	return false
 }
 
-func CertExpireSoon(certBytes []byte) bool {
+// CertExpireSoon check if the given cert bytes will expires within the given number of days from now
+func CertExpireSoon(certBytes []byte, numberOfDays int) bool {
 	block, _ := pem.Decode(certBytes)
 	if block != nil {
 		cert, err := x509.ParseCertificate(block.Bytes)
 		if err == nil {
 			expirationDate := cert.NotAfter
-			threshold := 14 * 24 * time.Hour // 14 days
+			threshold := time.Duration(numberOfDays) * 24 * time.Hour
 
 			timeRemaining := time.Until(expirationDate)
 			if timeRemaining <= threshold {

src/mod/auth/auth.go

@@ -14,10 +14,10 @@ import (
 	"strings"
 
 	"encoding/hex"
-	"log"
 
 	"github.com/gorilla/sessions"
 	db "imuslab.com/zoraxy/mod/database"
+	"imuslab.com/zoraxy/mod/info/logger"
 	"imuslab.com/zoraxy/mod/utils"
 )
 
@@ -27,6 +27,7 @@ type AuthAgent struct {
 	SessionStore            *sessions.CookieStore
 	Database                *db.Database
 	LoginRedirectionHandler func(http.ResponseWriter, *http.Request)
+	Logger                  *logger.Logger
 }
 
 type AuthEndpoints struct {
@@ -37,12 +38,12 @@ type AuthEndpoints struct {
 	Autologin     string
 }
 
-//Constructor
-func NewAuthenticationAgent(sessionName string, key []byte, sysdb *db.Database, allowReg bool, loginRedirectionHandler func(http.ResponseWriter, *http.Request)) *AuthAgent {
+// Constructor
+func NewAuthenticationAgent(sessionName string, key []byte, sysdb *db.Database, allowReg bool, systemLogger *logger.Logger, loginRedirectionHandler func(http.ResponseWriter, *http.Request)) *AuthAgent {
 	store := sessions.NewCookieStore(key)
 	err := sysdb.NewTable("auth")
 	if err != nil {
-		log.Println("Failed to create auth database. Terminating.")
+		systemLogger.Println("Failed to create auth database. Terminating.")
 		panic(err)
 	}
 
@@ -52,13 +53,14 @@ func NewAuthenticationAgent(sessionName string, key []byte, sysdb *db.Database,
 		SessionStore:            store,
 		Database:                sysdb,
 		LoginRedirectionHandler: loginRedirectionHandler,
+		Logger:                  systemLogger,
 	}
 
 	//Return the authAgent
 	return &newAuthAgent
 }
 
-func GetSessionKey(sysdb *db.Database) (string, error) {
+func GetSessionKey(sysdb *db.Database, logger *logger.Logger) (string, error) {
 	sysdb.NewTable("auth")
 	sessionKey := ""
 	if !sysdb.KeyExists("auth", "sessionkey") {
@@ -66,9 +68,9 @@ func GetSessionKey(sysdb *db.Database) (string, error) {
 		rand.Read(key)
 		sessionKey = string(key)
 		sysdb.Write("auth", "sessionkey", sessionKey)
-		log.Println("[Auth] New authentication session key generated")
+		logger.PrintAndLog("auth", "New authentication session key generated", nil)
 	} else {
-		log.Println("[Auth] Authentication session key loaded from database")
+		logger.PrintAndLog("auth", "Authentication session key loaded from database", nil)
 		err := sysdb.Read("auth", "sessionkey", &sessionKey)
 		if err != nil {
 			return "", errors.New("database read error. Is the database file corrupted?")
@@ -77,7 +79,7 @@ func GetSessionKey(sysdb *db.Database) (string, error) {
 	return sessionKey, nil
 }
 
-//This function will handle an http request and redirect to the given login address if not logged in
+// This function will handle an http request and redirect to the given login address if not logged in
 func (a *AuthAgent) HandleCheckAuth(w http.ResponseWriter, r *http.Request, handler func(http.ResponseWriter, *http.Request)) {
 	if a.CheckAuth(r) {
 		//User already logged in
@@ -88,14 +90,14 @@ func (a *AuthAgent) HandleCheckAuth(w http.ResponseWriter, r *http.Request, hand
 	}
 }
 
-//Handle login request, require POST username and password
+// Handle login request, require POST username and password
 func (a *AuthAgent) HandleLogin(w http.ResponseWriter, r *http.Request) {
 
 	//Get username from request using POST mode
 	username, err := utils.PostPara(r, "username")
 	if err != nil {
 		//Username not defined
-		log.Println("[Auth] " + r.RemoteAddr + " trying to login with username: " + username)
+		a.Logger.PrintAndLog("auth", r.RemoteAddr+" trying to login with username: "+username, nil)
 		utils.SendErrorResponse(w, "Username not defined or empty.")
 		return
 	}
@@ -124,11 +126,11 @@ func (a *AuthAgent) HandleLogin(w http.ResponseWriter, r *http.Request) {
 		a.LoginUserByRequest(w, r, username, rememberme)
 
 		//Print the login message to console
-		log.Println(username + " logged in.")
+		a.Logger.PrintAndLog("auth", username+" logged in.", nil)
 		utils.SendOK(w)
 	} else {
 		//Password incorrect
-		log.Println(username + " login request rejected: " + rejectionReason)
+		a.Logger.PrintAndLog("auth", username+" login request rejected: "+rejectionReason, nil)
 
 		utils.SendErrorResponse(w, rejectionReason)
 		return
@@ -140,14 +142,14 @@ func (a *AuthAgent) ValidateUsernameAndPassword(username string, password string
 	return succ
 }
 
-//validate the username and password, return reasons if the auth failed
+// validate the username and password, return reasons if the auth failed
 func (a *AuthAgent) ValidateUsernameAndPasswordWithReason(username string, password string) (bool, string) {
 	hashedPassword := Hash(password)
 	var passwordInDB string
 	err := a.Database.Read("auth", "passhash/"+username, &passwordInDB)
 	if err != nil {
 		//User not found or db exception
-		log.Println("[Auth] " + username + " login with incorrect password")
+		a.Logger.PrintAndLog("auth", username+" login with incorrect password", nil)
 		return false, "Invalid username or password"
 	}
 
@@ -158,7 +160,7 @@ func (a *AuthAgent) ValidateUsernameAndPasswordWithReason(username string, passw
 	}
 }
 
-//Login the user by creating a valid session for this user
+// Login the user by creating a valid session for this user
 func (a *AuthAgent) LoginUserByRequest(w http.ResponseWriter, r *http.Request, username string, rememberme bool) {
 	session, _ := a.SessionStore.Get(r, a.SessionName)
 
@@ -181,11 +183,15 @@ func (a *AuthAgent) LoginUserByRequest(w http.ResponseWriter, r *http.Request, u
 	session.Save(r, w)
 }
 
-//Handle logout, reply OK after logged out. WILL NOT DO REDIRECTION
+// Handle logout, reply OK after logged out. WILL NOT DO REDIRECTION
 func (a *AuthAgent) HandleLogout(w http.ResponseWriter, r *http.Request) {
 	username, err := a.GetUserName(w, r)
+	if err != nil {
+		utils.SendErrorResponse(w, "user not logged in")
+		return
+	}
 	if username != "" {
-		log.Println(username + " logged out.")
+		a.Logger.PrintAndLog("auth", username+" logged out", nil)
 	}
 	// Revoke users authentication
 	err = a.Logout(w, r)
@@ -194,7 +200,7 @@ func (a *AuthAgent) HandleLogout(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	w.Write([]byte("OK"))
+	utils.SendOK(w)
 }
 
 func (a *AuthAgent) Logout(w http.ResponseWriter, r *http.Request) error {
@@ -204,11 +210,11 @@ func (a *AuthAgent) Logout(w http.ResponseWriter, r *http.Request) error {
 	}
 	session.Values["authenticated"] = false
 	session.Values["username"] = nil
-	session.Save(r, w)
-	return nil
+	session.Options.MaxAge = -1
+	return session.Save(r, w)
 }
 
-//Get the current session username from request
+// Get the current session username from request
 func (a *AuthAgent) GetUserName(w http.ResponseWriter, r *http.Request) (string, error) {
 	if a.CheckAuth(r) {
 		//This user has logged in.
@@ -220,7 +226,7 @@ func (a *AuthAgent) GetUserName(w http.ResponseWriter, r *http.Request) (string,
 	}
 }
 
-//Get the current session user email from request
+// Get the current session user email from request
 func (a *AuthAgent) GetUserEmail(w http.ResponseWriter, r *http.Request) (string, error) {
 	if a.CheckAuth(r) {
 		//This user has logged in.
@@ -239,7 +245,7 @@ func (a *AuthAgent) GetUserEmail(w http.ResponseWriter, r *http.Request) (string
 	}
 }
 
-//Check if the user has logged in, return true / false in JSON
+// Check if the user has logged in, return true / false in JSON
 func (a *AuthAgent) CheckLogin(w http.ResponseWriter, r *http.Request) {
 	if a.CheckAuth(r) {
 		utils.SendJSONResponse(w, "true")
@@ -248,7 +254,7 @@ func (a *AuthAgent) CheckLogin(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-//Handle new user register. Require POST username, password, group.
+// Handle new user register. Require POST username, password, group.
 func (a *AuthAgent) HandleRegister(w http.ResponseWriter, r *http.Request, callback func(string, string)) {
 	//Get username from request
 	newusername, err := utils.PostPara(r, "username")
@@ -291,10 +297,10 @@ func (a *AuthAgent) HandleRegister(w http.ResponseWriter, r *http.Request, callb
 
 	//Return to the client with OK
 	utils.SendOK(w)
-	log.Println("[Auth] New user " + newusername + " added to system.")
+	a.Logger.PrintAndLog("auth", "New user "+newusername+" added to system.", nil)
 }
 
-//Handle new user register without confirmation email. Require POST username, password, group.
+// Handle new user register without confirmation email. Require POST username, password, group.
 func (a *AuthAgent) HandleRegisterWithoutEmail(w http.ResponseWriter, r *http.Request, callback func(string, string)) {
 	//Get username from request
 	newusername, err := utils.PostPara(r, "username")
@@ -324,15 +330,16 @@ func (a *AuthAgent) HandleRegisterWithoutEmail(w http.ResponseWriter, r *http.Re
 
 	//Return to the client with OK
 	utils.SendOK(w)
-	log.Println("[Auth] Admin account created: " + newusername)
+	a.Logger.PrintAndLog("auth", "Admin account created: "+newusername, nil)
 }
 
-//Check authentication from request header's session value
+// Check authentication from request header's session value
 func (a *AuthAgent) CheckAuth(r *http.Request) bool {
 	session, err := a.SessionStore.Get(r, a.SessionName)
 	if err != nil {
 		return false
 	}
+
 	// Check if user is authenticated
 	if auth, ok := session.Values["authenticated"].(bool); !ok || !auth {
 		return false
@@ -340,8 +347,8 @@ func (a *AuthAgent) CheckAuth(r *http.Request) bool {
 	return true
 }
 
-//Handle de-register of users. Require POST username.
-//THIS FUNCTION WILL NOT CHECK FOR PERMISSION. PLEASE USE WITH PERMISSION HANDLER
+// Handle de-register of users. Require POST username.
+// THIS FUNCTION WILL NOT CHECK FOR PERMISSION. PLEASE USE WITH PERMISSION HANDLER
 func (a *AuthAgent) HandleUnregister(w http.ResponseWriter, r *http.Request) {
 	//Check if the user is logged in
 	if !a.CheckAuth(r) {
@@ -365,7 +372,7 @@ func (a *AuthAgent) HandleUnregister(w http.ResponseWriter, r *http.Request) {
 
 	//Return to the client with OK
 	utils.SendOK(w)
-	log.Println("[Auth] User " + username + " has been removed from the system.")
+	a.Logger.PrintAndLog("auth", "User "+username+" has been removed from the system", nil)
 }
 
 func (a *AuthAgent) UnregisterUser(username string) error {
@@ -381,7 +388,7 @@ func (a *AuthAgent) UnregisterUser(username string) error {
 	return nil
 }
 
-//Get the number of users in the system
+// Get the number of users in the system
 func (a *AuthAgent) GetUserCounts() int {
 	entries, _ := a.Database.ListTable("auth")
 	usercount := 0
@@ -393,12 +400,12 @@ func (a *AuthAgent) GetUserCounts() int {
 	}
 
 	if usercount == 0 {
-		log.Println("There are no user in the database.")
+		a.Logger.PrintAndLog("auth", "There are no user in the database", nil)
 	}
 	return usercount
 }
 
-//List all username within the system
+// List all username within the system
 func (a *AuthAgent) ListUsers() []string {
 	entries, _ := a.Database.ListTable("auth")
 	results := []string{}
@@ -411,7 +418,7 @@ func (a *AuthAgent) ListUsers() []string {
 	return results
 }
 
-//Check if the given username exists
+// Check if the given username exists
 func (a *AuthAgent) UserExists(username string) bool {
 	userpasswordhash := ""
 	err := a.Database.Read("auth", "passhash/"+username, &userpasswordhash)
@@ -421,7 +428,7 @@ func (a *AuthAgent) UserExists(username string) bool {
 	return true
 }
 
-//Update the session expire time given the request header.
+// Update the session expire time given the request header.
 func (a *AuthAgent) UpdateSessionExpireTime(w http.ResponseWriter, r *http.Request) bool {
 	session, _ := a.SessionStore.Get(r, a.SessionName)
 	if session.Values["authenticated"].(bool) {
@@ -446,7 +453,7 @@ func (a *AuthAgent) UpdateSessionExpireTime(w http.ResponseWriter, r *http.Reque
 	}
 }
 
-//Create user account
+// Create user account
 func (a *AuthAgent) CreateUserAccount(newusername string, password string, email string) error {
 	//Check user already exists
 	if a.UserExists(newusername) {
@@ -470,7 +477,7 @@ func (a *AuthAgent) CreateUserAccount(newusername string, password string, email
 	return nil
 }
 
-//Hash the given raw string into sha512 hash
+// Hash the given raw string into sha512 hash
 func Hash(raw string) string {
 	h := sha512.New()
 	h.Write([]byte(raw))

src/mod/auth/router.go

@@ -2,7 +2,7 @@ package auth
 
 import (
 	"errors"
-	"log"
+	"fmt"
 	"net/http"
 )
 
@@ -10,7 +10,7 @@ type RouterOption struct {
 	AuthAgent     *AuthAgent
 	RequireAuth   bool                                     //This router require authentication
 	DeniedHandler func(http.ResponseWriter, *http.Request) //Things to do when request is rejected
-
+	TargetMux     *http.ServeMux
 }
 
 type RouterDef struct {
@@ -28,24 +28,38 @@ func NewManagedHTTPRouter(option RouterOption) *RouterDef {
 func (router *RouterDef) HandleFunc(endpoint string, handler func(http.ResponseWriter, *http.Request)) error {
 	//Check if the endpoint already registered
 	if _, exist := router.endpoints[endpoint]; exist {
-		log.Println("WARNING! Duplicated registering of web endpoint: " + endpoint)
+		fmt.Println("WARNING! Duplicated registering of web endpoint: " + endpoint)
 		return errors.New("endpoint register duplicated")
 	}
 
 	authAgent := router.option.AuthAgent
 
 	//OK. Register handler
-	http.HandleFunc(endpoint, func(w http.ResponseWriter, r *http.Request) {
-		//Check authentication of the user
-		if router.option.RequireAuth {
-			authAgent.HandleCheckAuth(w, r, func(w http.ResponseWriter, r *http.Request) {
+	if router.option.TargetMux == nil {
+		http.HandleFunc(endpoint, func(w http.ResponseWriter, r *http.Request) {
+			//Check authentication of the user
+			if router.option.RequireAuth {
+				authAgent.HandleCheckAuth(w, r, func(w http.ResponseWriter, r *http.Request) {
+					handler(w, r)
+				})
+			} else {
 				handler(w, r)
-			})
-		} else {
-			handler(w, r)
-		}
+			}
 
-	})
+		})
+	} else {
+		router.option.TargetMux.HandleFunc(endpoint, func(w http.ResponseWriter, r *http.Request) {
+			//Check authentication of the user
+			if router.option.RequireAuth {
+				authAgent.HandleCheckAuth(w, r, func(w http.ResponseWriter, r *http.Request) {
+					handler(w, r)
+				})
+			} else {
+				handler(w, r)
+			}
+
+		})
+	}
 
 	router.endpoints[endpoint] = handler
 

src/mod/auth/sso/authelia/authelia.go

@@ -0,0 +1,136 @@
+package authelia
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"imuslab.com/zoraxy/mod/database"
+	"imuslab.com/zoraxy/mod/info/logger"
+	"imuslab.com/zoraxy/mod/utils"
+)
+
+type AutheliaRouterOptions struct {
+	UseHTTPS    bool   //If the Authelia server is using HTTPS
+	AutheliaURL string //The URL of the Authelia server
+	Logger      *logger.Logger
+	Database    *database.Database
+}
+
+type AutheliaRouter struct {
+	options *AutheliaRouterOptions
+}
+
+// NewAutheliaRouter creates a new AutheliaRouter object
+func NewAutheliaRouter(options *AutheliaRouterOptions) *AutheliaRouter {
+	options.Database.NewTable("authelia")
+
+	//Read settings from database, if exists
+	options.Database.Read("authelia", "autheliaURL", &options.AutheliaURL)
+	options.Database.Read("authelia", "useHTTPS", &options.UseHTTPS)
+
+	return &AutheliaRouter{
+		options: options,
+	}
+}
+
+// HandleSetAutheliaURLAndHTTPS is the internal handler for setting the Authelia URL and HTTPS
+func (ar *AutheliaRouter) HandleSetAutheliaURLAndHTTPS(w http.ResponseWriter, r *http.Request) {
+	if r.Method == http.MethodGet {
+		//Return the current settings
+		js, _ := json.Marshal(map[string]interface{}{
+			"useHTTPS":    ar.options.UseHTTPS,
+			"autheliaURL": ar.options.AutheliaURL,
+		})
+
+		utils.SendJSONResponse(w, string(js))
+		return
+	} else if r.Method == http.MethodPost {
+		//Update the settings
+		autheliaURL, err := utils.PostPara(r, "autheliaURL")
+		if err != nil {
+			utils.SendErrorResponse(w, "autheliaURL not found")
+			return
+		}
+
+		useHTTPS, err := utils.PostBool(r, "useHTTPS")
+		if err != nil {
+			useHTTPS = false
+		}
+
+		//Write changes to runtime
+		ar.options.AutheliaURL = autheliaURL
+		ar.options.UseHTTPS = useHTTPS
+
+		//Write changes to database
+		ar.options.Database.Write("authelia", "autheliaURL", autheliaURL)
+		ar.options.Database.Write("authelia", "useHTTPS", useHTTPS)
+
+		utils.SendOK(w)
+	} else {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+}
+
+// handleAutheliaAuth is the internal handler for Authelia authentication
+// Set useHTTPS to true if your authelia server is using HTTPS
+// Set autheliaURL to the URL of the Authelia server, e.g. authelia.example.com
+func (ar *AutheliaRouter) HandleAutheliaAuth(w http.ResponseWriter, r *http.Request) error {
+	client := &http.Client{}
+
+	if ar.options.AutheliaURL == "" {
+		ar.options.Logger.PrintAndLog("Authelia", "Authelia URL not set", nil)
+		w.WriteHeader(500)
+		w.Write([]byte("500 - Internal Server Error"))
+		return errors.New("authelia URL not set")
+	}
+	protocol := "http"
+	if ar.options.UseHTTPS {
+		protocol = "https"
+	}
+
+	autheliaBaseURL := protocol + "://" + ar.options.AutheliaURL
+	//Remove tailing slash if any
+	if autheliaBaseURL[len(autheliaBaseURL)-1] == '/' {
+		autheliaBaseURL = autheliaBaseURL[:len(autheliaBaseURL)-1]
+	}
+
+	//Make a request to Authelia to verify the request
+	req, err := http.NewRequest("POST", autheliaBaseURL+"/api/verify", nil)
+	if err != nil {
+		ar.options.Logger.PrintAndLog("Authelia", "Unable to create request", err)
+		w.WriteHeader(401)
+		return errors.New("unauthorized")
+	}
+
+	scheme := "http"
+	if r.TLS != nil {
+		scheme = "https"
+	}
+	req.Header.Add("X-Original-URL", fmt.Sprintf("%s://%s", scheme, r.Host))
+
+	// Copy cookies from the incoming request
+	for _, cookie := range r.Cookies() {
+		req.AddCookie(cookie)
+	}
+
+	// Making the verification request
+	resp, err := client.Do(req)
+	if err != nil {
+		ar.options.Logger.PrintAndLog("Authelia", "Unable to verify", err)
+		w.WriteHeader(401)
+		return errors.New("unauthorized")
+	}
+
+	if resp.StatusCode != 200 {
+		redirectURL := autheliaBaseURL + "/?rd=" + url.QueryEscape(scheme+"://"+r.Host+r.URL.String()) + "&rm=" + r.Method
+		http.Redirect(w, r, redirectURL, http.StatusSeeOther)
+		return errors.New("unauthorized")
+	}
+
+	return nil
+}

src/mod/database/database.go

@@ -9,17 +9,39 @@ package database
 */
 
 import (
-	"sync"
+	"log"
+	"runtime"
+
+	"imuslab.com/zoraxy/mod/database/dbinc"
 )
 
 type Database struct {
-	Db       interface{} //This will be nil on openwrt and *bolt.DB in the rest of the systems
-	Tables   sync.Map
-	ReadOnly bool
+	Db          interface{} //This will be nil on openwrt, leveldb.DB on x64 platforms or bolt.DB on other platforms
+	BackendType dbinc.BackendType
+	Backend     dbinc.Backend
 }
 
-func NewDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
-	return newDatabase(dbfile, readOnlyMode)
+func NewDatabase(dbfile string, backendType dbinc.BackendType) (*Database, error) {
+	if runtime.GOARCH == "riscv64" {
+		log.Println("RISCV hardware detected, ignoring the backend type and using FS emulated database")
+	}
+	return newDatabase(dbfile, backendType)
+}
+
+// Get the recommended backend type for the current system
+func GetRecommendedBackendType() dbinc.BackendType {
+	//Check if the system is running on RISCV hardware
+	if runtime.GOARCH == "riscv64" {
+		//RISCV hardware, currently only support FS emulated database
+		return dbinc.BackendFSOnly
+	} else if runtime.GOOS == "windows" || (runtime.GOOS == "linux" && runtime.GOARCH == "amd64") {
+		//Powerful hardware
+		return dbinc.BackendBoltDB
+		//return dbinc.BackendLevelDB
+	}
+
+	//Default to BoltDB, the safest option
+	return dbinc.BackendBoltDB
 }
 
 /*
@@ -29,39 +51,33 @@ func NewDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
 	err := sysdb.DropTable("MyTable")
 */
 
-func (d *Database) UpdateReadWriteMode(readOnly bool) {
-	d.ReadOnly = readOnly
-}
-
-//Dump the whole db into a log file
-func (d *Database) Dump(filename string) ([]string, error) {
-	return d.dump(filename)
-}
-
-//Create a new table
+// Create a new table
 func (d *Database) NewTable(tableName string) error {
 	return d.newTable(tableName)
 }
 
-//Check is table exists
+// Check is table exists
 func (d *Database) TableExists(tableName string) bool {
 	return d.tableExists(tableName)
 }
 
-//Drop the given table
+// Drop the given table
 func (d *Database) DropTable(tableName string) error {
 	return d.dropTable(tableName)
 }
 
 /*
-	Write to database with given tablename and key. Example Usage:
+Write to database with given tablename and key. Example Usage:
+
 	type demo struct{
 		content string
 	}
+
 	thisDemo := demo{
 		content: "Hello World",
 	}
-	err := sysdb.Write("MyTable", "username/message",thisDemo);
+
+err := sysdb.Write("MyTable", "username/message",thisDemo);
 */
 func (d *Database) Write(tableName string, key string, value interface{}) error {
 	return d.write(tableName, key, value)
@@ -81,14 +97,21 @@ func (d *Database) Read(tableName string, key string, assignee interface{}) erro
 	return d.read(tableName, key, assignee)
 }
 
+/*
+Check if a key exists in the database table given tablename and key
+
+	if sysdb.KeyExists("MyTable", "username/message"){
+		log.Println("Key exists")
+	}
+*/
 func (d *Database) KeyExists(tableName string, key string) bool {
 	return d.keyExists(tableName, key)
 }
 
 /*
-	Delete a value from the database table given tablename and key
+Delete a value from the database table given tablename and key
 
-	err := sysdb.Delete("MyTable", "username/message");
+err := sysdb.Delete("MyTable", "username/message");
 */
 func (d *Database) Delete(tableName string, key string) error {
 	return d.delete(tableName, key)
@@ -115,6 +138,9 @@ func (d *Database) ListTable(tableName string) ([][][]byte, error) {
 	return d.listTable(tableName)
 }
 
+/*
+Close the database connection
+*/
 func (d *Database) Close() {
 	d.close()
 }

src/mod/database/database_core.go

@@ -4,183 +4,67 @@
 package database
 
 import (
-	"encoding/json"
 	"errors"
-	"log"
-	"sync"
 
-	"github.com/boltdb/bolt"
+	"imuslab.com/zoraxy/mod/database/dbbolt"
+	"imuslab.com/zoraxy/mod/database/dbinc"
+	"imuslab.com/zoraxy/mod/database/dbleveldb"
 )
 
-func newDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
-	db, err := bolt.Open(dbfile, 0600, nil)
-	if err != nil {
-		return nil, err
+func newDatabase(dbfile string, backendType dbinc.BackendType) (*Database, error) {
+	if backendType == dbinc.BackendFSOnly {
+		return nil, errors.New("Unsupported backend type for this platform")
 	}
 
-	tableMap := sync.Map{}
-	//Build the table list from database
-	err = db.View(func(tx *bolt.Tx) error {
-		return tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
-			tableMap.Store(string(name), "")
-			return nil
-		})
-	})
+	if backendType == dbinc.BackendLevelDB {
+		db, err := dbleveldb.NewDB(dbfile)
+		return &Database{
+			Db:          nil,
+			BackendType: backendType,
+			Backend:     db,
+		}, err
+	}
 
+	db, err := dbbolt.NewBoltDatabase(dbfile)
 	return &Database{
-		Db:       db,
-		Tables:   tableMap,
-		ReadOnly: readOnlyMode,
+		Db:          nil,
+		BackendType: backendType,
+		Backend:     db,
 	}, err
 }
 
-//Dump the whole db into a log file
-func (d *Database) dump(filename string) ([]string, error) {
-	results := []string{}
-
-	d.Tables.Range(func(tableName, v interface{}) bool {
-		entries, err := d.ListTable(tableName.(string))
-		if err != nil {
-			log.Println("Reading table " + tableName.(string) + " failed: " + err.Error())
-			return false
-		}
-		for _, keypairs := range entries {
-			results = append(results, string(keypairs[0])+":"+string(keypairs[1])+"\n")
-		}
-		return true
-	})
-
-	return results, nil
-}
-
-//Create a new table
 func (d *Database) newTable(tableName string) error {
-	if d.ReadOnly == true {
-		return errors.New("Operation rejected in ReadOnly mode")
-	}
-
-	err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte(tableName))
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-
-	d.Tables.Store(tableName, "")
-	return err
+	return d.Backend.NewTable(tableName)
 }
 
-//Check is table exists
 func (d *Database) tableExists(tableName string) bool {
-	if _, ok := d.Tables.Load(tableName); ok {
-		return true
-	}
-	return false
+	return d.Backend.TableExists(tableName)
 }
 
-//Drop the given table
 func (d *Database) dropTable(tableName string) error {
-	if d.ReadOnly == true {
-		return errors.New("Operation rejected in ReadOnly mode")
-	}
-
-	err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
-		err := tx.DeleteBucket([]byte(tableName))
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	return err
+	return d.Backend.DropTable(tableName)
 }
 
-//Write to table
 func (d *Database) write(tableName string, key string, value interface{}) error {
-	if d.ReadOnly {
-		return errors.New("Operation rejected in ReadOnly mode")
-	}
-
-	jsonString, err := json.Marshal(value)
-	if err != nil {
-		return err
-	}
-	err = d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte(tableName))
-		if err != nil {
-			return err
-		}
-		b := tx.Bucket([]byte(tableName))
-		err = b.Put([]byte(key), jsonString)
-		return err
-	})
-	return err
+	return d.Backend.Write(tableName, key, value)
 }
 
 func (d *Database) read(tableName string, key string, assignee interface{}) error {
-	err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
-		b := tx.Bucket([]byte(tableName))
-		v := b.Get([]byte(key))
-		json.Unmarshal(v, &assignee)
-		return nil
-	})
-	return err
+	return d.Backend.Read(tableName, key, assignee)
 }
 
 func (d *Database) keyExists(tableName string, key string) bool {
-	resultIsNil := false
-	if !d.TableExists(tableName) {
-		//Table not exists. Do not proceed accessing key
-		log.Println("[DB] ERROR: Requesting key from table that didn't exist!!!")
-		return false
-	}
-	err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
-		b := tx.Bucket([]byte(tableName))
-		v := b.Get([]byte(key))
-		if v == nil {
-			resultIsNil = true
-		}
-		return nil
-	})
-
-	if err != nil {
-		return false
-	} else {
-		if resultIsNil {
-			return false
-		} else {
-			return true
-		}
-	}
+	return d.Backend.KeyExists(tableName, key)
 }
 
 func (d *Database) delete(tableName string, key string) error {
-	if d.ReadOnly {
-		return errors.New("Operation rejected in ReadOnly mode")
-	}
-
-	err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
-		tx.Bucket([]byte(tableName)).Delete([]byte(key))
-		return nil
-	})
-
-	return err
+	return d.Backend.Delete(tableName, key)
 }
 
 func (d *Database) listTable(tableName string) ([][][]byte, error) {
-	var results [][][]byte
-	err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
-		b := tx.Bucket([]byte(tableName))
-		c := b.Cursor()
-
-		for k, v := c.First(); k != nil; k, v = c.Next() {
-			results = append(results, [][]byte{k, v})
-		}
-		return nil
-	})
-	return results, err
+	return d.Backend.ListTable(tableName)
 }
 
 func (d *Database) close() {
-	d.Db.(*bolt.DB).Close()
+	d.Backend.Close()
 }

src/mod/database/database_openwrt.go

@@ -10,10 +10,19 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"sync"
+
+	"imuslab.com/zoraxy/mod/database/dbinc"
 )
 
-func newDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
+/*
+	OpenWRT or RISCV backend
+
+	For OpenWRT or RISCV platform, we will use the filesystem as the database backend
+	as boltdb or leveldb is not supported on these platforms, including boltDB and LevelDB
+	in conditional compilation will create a build error on these platforms
+*/
+
+func newDatabase(dbfile string, backendType dbinc.BackendType) (*Database, error) {
 	dbRootPath := filepath.ToSlash(filepath.Clean(dbfile))
 	dbRootPath = "fsdb/" + dbRootPath
 	err := os.MkdirAll(dbRootPath, 0755)
@@ -21,24 +30,11 @@ func newDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
 		return nil, err
 	}
 
-	tableMap := sync.Map{}
-	//build the table list from file system
-	files, err := filepath.Glob(filepath.Join(dbRootPath, "/*"))
-	if err != nil {
-		return nil, err
-	}
-
-	for _, file := range files {
-		if isDirectory(file) {
-			tableMap.Store(filepath.Base(file), "")
-		}
-	}
-
 	log.Println("Filesystem Emulated Key-value Database Service Started: " + dbRootPath)
 	return &Database{
-		Db:       dbRootPath,
-		Tables:   tableMap,
-		ReadOnly: readOnlyMode,
+		Db:          dbRootPath,
+		BackendType: dbinc.BackendFSOnly,
+		Backend:     nil,
 	}, nil
 }
 
@@ -61,9 +57,7 @@ func (d *Database) dump(filename string) ([]string, error) {
 }
 
 func (d *Database) newTable(tableName string) error {
-	if d.ReadOnly {
-		return errors.New("Operation rejected in ReadOnly mode")
-	}
+
 	tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
 	if !fileExists(tablePath) {
 		return os.MkdirAll(tablePath, 0755)
@@ -85,9 +79,7 @@ func (d *Database) tableExists(tableName string) bool {
 }
 
 func (d *Database) dropTable(tableName string) error {
-	if d.ReadOnly {
-		return errors.New("Operation rejected in ReadOnly mode")
-	}
+
 	tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
 	if d.tableExists(tableName) {
 		return os.RemoveAll(tablePath)
@@ -98,9 +90,7 @@ func (d *Database) dropTable(tableName string) error {
 }
 
 func (d *Database) write(tableName string, key string, value interface{}) error {
-	if d.ReadOnly {
-		return errors.New("Operation rejected in ReadOnly mode")
-	}
+
 	tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
 	js, err := json.Marshal(value)
 	if err != nil {
@@ -138,9 +128,7 @@ func (d *Database) keyExists(tableName string, key string) bool {
 }
 
 func (d *Database) delete(tableName string, key string) error {
-	if d.ReadOnly {
-		return errors.New("Operation rejected in ReadOnly mode")
-	}
+
 	if !d.keyExists(tableName, key) {
 		return errors.New("key not exists")
 	}

src/mod/database/dbbolt/dbbolt.go

@@ -0,0 +1,141 @@
+package dbbolt
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/boltdb/bolt"
+)
+
+type Database struct {
+	Db interface{} //This is the bolt database object
+}
+
+func NewBoltDatabase(dbfile string) (*Database, error) {
+	db, err := bolt.Open(dbfile, 0600, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Database{
+		Db: db,
+	}, err
+}
+
+// Create a new table
+func (d *Database) NewTable(tableName string) error {
+	err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(tableName))
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	return err
+}
+
+// Check is table exists
+func (d *Database) TableExists(tableName string) bool {
+	return d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(tableName))
+		if b == nil {
+			return errors.New("table not exists")
+		}
+		return nil
+	}) == nil
+}
+
+// Drop the given table
+func (d *Database) DropTable(tableName string) error {
+	err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
+		err := tx.DeleteBucket([]byte(tableName))
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	return err
+}
+
+// Write to table
+func (d *Database) Write(tableName string, key string, value interface{}) error {
+	jsonString, err := json.Marshal(value)
+	if err != nil {
+		return err
+	}
+	err = d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(tableName))
+		if err != nil {
+			return err
+		}
+		b := tx.Bucket([]byte(tableName))
+		err = b.Put([]byte(key), jsonString)
+		return err
+	})
+	return err
+}
+
+func (d *Database) Read(tableName string, key string, assignee interface{}) error {
+	err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(tableName))
+		v := b.Get([]byte(key))
+		json.Unmarshal(v, &assignee)
+		return nil
+	})
+	return err
+}
+
+func (d *Database) KeyExists(tableName string, key string) bool {
+	resultIsNil := false
+	if !d.TableExists(tableName) {
+		//Table not exists. Do not proceed accessing key
+		//log.Println("[DB] ERROR: Requesting key from table that didn't exist!!!")
+		return false
+	}
+	err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(tableName))
+		v := b.Get([]byte(key))
+		if v == nil {
+			resultIsNil = true
+		}
+		return nil
+	})
+
+	if err != nil {
+		return false
+	} else {
+		if resultIsNil {
+			return false
+		} else {
+			return true
+		}
+	}
+}
+
+func (d *Database) Delete(tableName string, key string) error {
+	err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
+		tx.Bucket([]byte(tableName)).Delete([]byte(key))
+		return nil
+	})
+
+	return err
+}
+
+func (d *Database) ListTable(tableName string) ([][][]byte, error) {
+	var results [][][]byte
+	err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(tableName))
+		c := b.Cursor()
+
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			results = append(results, [][]byte{k, v})
+		}
+		return nil
+	})
+	return results, err
+}
+
+func (d *Database) Close() {
+	d.Db.(*bolt.DB).Close()
+}

src/mod/database/dbbolt/dbbolt_test.go

@@ -0,0 +1,67 @@
+package dbbolt_test
+
+import (
+	"os"
+	"testing"
+
+	"imuslab.com/zoraxy/mod/database/dbbolt"
+)
+
+func TestNewBoltDatabase(t *testing.T) {
+	dbfile := "test.db"
+	defer os.Remove(dbfile)
+
+	db, err := dbbolt.NewBoltDatabase(dbfile)
+	if err != nil {
+		t.Fatalf("Failed to create new Bolt database: %v", err)
+	}
+	defer db.Close()
+
+	if db.Db == nil {
+		t.Fatalf("Expected non-nil database object")
+	}
+}
+
+func TestNewTable(t *testing.T) {
+	dbfile := "test.db"
+	defer os.Remove(dbfile)
+
+	db, err := dbbolt.NewBoltDatabase(dbfile)
+	if err != nil {
+		t.Fatalf("Failed to create new Bolt database: %v", err)
+	}
+	defer db.Close()
+
+	err = db.NewTable("testTable")
+	if err != nil {
+		t.Fatalf("Failed to create new table: %v", err)
+	}
+}
+
+func TestTableExists(t *testing.T) {
+	dbfile := "test.db"
+	defer os.Remove(dbfile)
+
+	db, err := dbbolt.NewBoltDatabase(dbfile)
+	if err != nil {
+		t.Fatalf("Failed to create new Bolt database: %v", err)
+	}
+	defer db.Close()
+
+	tableName := "testTable"
+	err = db.NewTable(tableName)
+	if err != nil {
+		t.Fatalf("Failed to create new table: %v", err)
+	}
+
+	exists := db.TableExists(tableName)
+	if !exists {
+		t.Fatalf("Expected table %s to exist", tableName)
+	}
+
+	nonExistentTable := "nonExistentTable"
+	exists = db.TableExists(nonExistentTable)
+	if exists {
+		t.Fatalf("Expected table %s to not exist", nonExistentTable)
+	}
+}

src/mod/database/dbinc/dbinc.go

@@ -0,0 +1,39 @@
+package dbinc
+
+/*
+	dbinc is the interface for all database backend
+*/
+type BackendType int
+
+const (
+	BackendBoltDB  BackendType = iota //Default backend
+	BackendFSOnly                     //OpenWRT or RISCV backend
+	BackendLevelDB                    //LevelDB backend
+
+	BackEndAuto = BackendBoltDB
+)
+
+type Backend interface {
+	NewTable(tableName string) error
+	TableExists(tableName string) bool
+	DropTable(tableName string) error
+	Write(tableName string, key string, value interface{}) error
+	Read(tableName string, key string, assignee interface{}) error
+	KeyExists(tableName string, key string) bool
+	Delete(tableName string, key string) error
+	ListTable(tableName string) ([][][]byte, error)
+	Close()
+}
+
+func (b BackendType) String() string {
+	switch b {
+	case BackendBoltDB:
+		return "BoltDB"
+	case BackendFSOnly:
+		return "File System Emulated Key-Value Store"
+	case BackendLevelDB:
+		return "LevelDB"
+	default:
+		return "Unknown"
+	}
+}

src/mod/database/dbleveldb/dbleveldb.go

@@ -0,0 +1,152 @@
+package dbleveldb
+
+import (
+	"encoding/json"
+	"log"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+	"imuslab.com/zoraxy/mod/database/dbinc"
+)
+
+// Ensure the DB struct implements the Backend interface
+var _ dbinc.Backend = (*DB)(nil)
+
+type DB struct {
+	db               *leveldb.DB
+	Table            sync.Map      //For emulating table creation
+	batch            leveldb.Batch //Batch write
+	writeFlushTicker *time.Ticker  //Ticker for flushing data into disk
+	writeFlushStop   chan bool     //Stop channel for write flush ticker
+}
+
+func NewDB(path string) (*DB, error) {
+	//If the path is not a directory (e.g. /tmp/dbfile.db), convert the filename to directory
+	if filepath.Ext(path) != "" {
+		path = strings.ReplaceAll(path, ".", "_")
+	}
+
+	db, err := leveldb.OpenFile(path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	thisDB := &DB{
+		db:    db,
+		Table: sync.Map{},
+		batch: leveldb.Batch{},
+	}
+
+	//Create a ticker to flush data into disk every 1 seconds
+	writeFlushTicker := time.NewTicker(1 * time.Second)
+	writeFlushStop := make(chan bool)
+	go func() {
+		for {
+			select {
+			case <-writeFlushTicker.C:
+				if thisDB.batch.Len() == 0 {
+					//No flushing needed
+					continue
+				}
+				err = db.Write(&thisDB.batch, nil)
+				if err != nil {
+					log.Println("[LevelDB] Failed to flush data into disk: ", err)
+				}
+				thisDB.batch.Reset()
+			case <-writeFlushStop:
+				return
+			}
+		}
+	}()
+
+	thisDB.writeFlushTicker = writeFlushTicker
+	thisDB.writeFlushStop = writeFlushStop
+
+	return thisDB, nil
+}
+
+func (d *DB) NewTable(tableName string) error {
+	//Create a table entry in the sync.Map
+	d.Table.Store(tableName, true)
+	return nil
+}
+
+func (d *DB) TableExists(tableName string) bool {
+	_, ok := d.Table.Load(tableName)
+	return ok
+}
+
+func (d *DB) DropTable(tableName string) error {
+	d.Table.Delete(tableName)
+	iter := d.db.NewIterator(nil, nil)
+	defer iter.Release()
+
+	for iter.Next() {
+		key := iter.Key()
+		if filepath.Dir(string(key)) == tableName {
+			err := d.db.Delete(key, nil)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (d *DB) Write(tableName string, key string, value interface{}) error {
+	data, err := json.Marshal(value)
+	if err != nil {
+		return err
+	}
+	d.batch.Put([]byte(filepath.ToSlash(filepath.Join(tableName, key))), data)
+	return nil
+}
+
+func (d *DB) Read(tableName string, key string, assignee interface{}) error {
+	data, err := d.db.Get([]byte(filepath.ToSlash(filepath.Join(tableName, key))), nil)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(data, assignee)
+}
+
+func (d *DB) KeyExists(tableName string, key string) bool {
+	_, err := d.db.Get([]byte(filepath.ToSlash(filepath.Join(tableName, key))), nil)
+	return err == nil
+}
+
+func (d *DB) Delete(tableName string, key string) error {
+	return d.db.Delete([]byte(filepath.ToSlash(filepath.Join(tableName, key))), nil)
+}
+
+func (d *DB) ListTable(tableName string) ([][][]byte, error) {
+	iter := d.db.NewIterator(util.BytesPrefix([]byte(tableName+"/")), nil)
+	defer iter.Release()
+
+	var result [][][]byte
+	for iter.Next() {
+		key := iter.Key()
+		//The key contains the table name as prefix. Trim it before returning
+		value := iter.Value()
+		result = append(result, [][]byte{[]byte(strings.TrimPrefix(string(key), tableName+"/")), value})
+	}
+
+	err := iter.Error()
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (d *DB) Close() {
+	//Write the remaining data in batch back into disk
+	d.writeFlushStop <- true
+	d.writeFlushTicker.Stop()
+	d.db.Write(&d.batch, nil)
+	d.db.Close()
+}

src/mod/database/dbleveldb/dbleveldb_test.go

@@ -0,0 +1,141 @@
+package dbleveldb_test
+
+import (
+	"os"
+	"testing"
+
+	"imuslab.com/zoraxy/mod/database/dbleveldb"
+)
+
+func TestNewDB(t *testing.T) {
+	path := "/tmp/testdb"
+	defer os.RemoveAll(path)
+
+	db, err := dbleveldb.NewDB(path)
+	if err != nil {
+		t.Fatalf("Failed to create new DB: %v", err)
+	}
+	defer db.Close()
+}
+
+func TestNewTable(t *testing.T) {
+	path := "/tmp/testdb"
+	defer os.RemoveAll(path)
+
+	db, err := dbleveldb.NewDB(path)
+	if err != nil {
+		t.Fatalf("Failed to create new DB: %v", err)
+	}
+	defer db.Close()
+
+	err = db.NewTable("testTable")
+	if err != nil {
+		t.Fatalf("Failed to create new table: %v", err)
+	}
+}
+
+func TestTableExists(t *testing.T) {
+	path := "/tmp/testdb"
+	defer os.RemoveAll(path)
+
+	db, err := dbleveldb.NewDB(path)
+	if err != nil {
+		t.Fatalf("Failed to create new DB: %v", err)
+	}
+	defer db.Close()
+
+	db.NewTable("testTable")
+	if !db.TableExists("testTable") {
+		t.Fatalf("Table should exist")
+	}
+}
+
+func TestDropTable(t *testing.T) {
+	path := "/tmp/testdb"
+	defer os.RemoveAll(path)
+
+	db, err := dbleveldb.NewDB(path)
+	if err != nil {
+		t.Fatalf("Failed to create new DB: %v", err)
+	}
+	defer db.Close()
+
+	db.NewTable("testTable")
+	err = db.DropTable("testTable")
+	if err != nil {
+		t.Fatalf("Failed to drop table: %v", err)
+	}
+
+	if db.TableExists("testTable") {
+		t.Fatalf("Table should not exist")
+	}
+}
+
+func TestWriteAndRead(t *testing.T) {
+	path := "/tmp/testdb"
+	defer os.RemoveAll(path)
+
+	db, err := dbleveldb.NewDB(path)
+	if err != nil {
+		t.Fatalf("Failed to create new DB: %v", err)
+	}
+	defer db.Close()
+
+	db.NewTable("testTable")
+	err = db.Write("testTable", "testKey", "testValue")
+	if err != nil {
+		t.Fatalf("Failed to write to table: %v", err)
+	}
+
+	var value string
+	err = db.Read("testTable", "testKey", &value)
+	if err != nil {
+		t.Fatalf("Failed to read from table: %v", err)
+	}
+
+	if value != "testValue" {
+		t.Fatalf("Expected 'testValue', got '%v'", value)
+	}
+}
+func TestListTable(t *testing.T) {
+	path := "/tmp/testdb"
+	defer os.RemoveAll(path)
+
+	db, err := dbleveldb.NewDB(path)
+	if err != nil {
+		t.Fatalf("Failed to create new DB: %v", err)
+	}
+	defer db.Close()
+
+	db.NewTable("testTable")
+	err = db.Write("testTable", "testKey1", "testValue1")
+	if err != nil {
+		t.Fatalf("Failed to write to table: %v", err)
+	}
+	err = db.Write("testTable", "testKey2", "testValue2")
+	if err != nil {
+		t.Fatalf("Failed to write to table: %v", err)
+	}
+
+	result, err := db.ListTable("testTable")
+	if err != nil {
+		t.Fatalf("Failed to list table: %v", err)
+	}
+
+	if len(result) != 2 {
+		t.Fatalf("Expected 2 entries, got %v", len(result))
+	}
+
+	expected := map[string]string{
+		"testTable/testKey1": "\"testValue1\"",
+		"testTable/testKey2": "\"testValue2\"",
+	}
+
+	for _, entry := range result {
+		key := string(entry[0])
+		value := string(entry[1])
+		if expected[key] != value {
+			t.Fatalf("Expected value '%v' for key '%v', got '%v'", expected[key], key, value)
+		}
+	}
+}

src/mod/dockerux/docker.go

@@ -0,0 +1,57 @@
+//go:build !windows
+// +build !windows
+
+package dockerux
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
+	"imuslab.com/zoraxy/mod/utils"
+)
+
+func (d *UXOptimizer) HandleDockerAvailable(w http.ResponseWriter, r *http.Request) {
+	js, _ := json.Marshal(d.RunninInDocker)
+	utils.SendJSONResponse(w, string(js))
+}
+
+func (d *UXOptimizer) HandleDockerContainersList(w http.ResponseWriter, r *http.Request) {
+	apiClient, err := client.NewClientWithOpts(client.WithVersion("1.43"))
+	if err != nil {
+		d.SystemWideLogger.PrintAndLog("Docker", "Unable to create new docker client", err)
+		utils.SendErrorResponse(w, "Docker client initiation failed")
+		return
+	}
+	defer apiClient.Close()
+
+	containers, err := apiClient.ContainerList(context.Background(), container.ListOptions{All: true})
+	if err != nil {
+		d.SystemWideLogger.PrintAndLog("Docker", "List docker container failed", err)
+		utils.SendErrorResponse(w, "List docker container failed")
+		return
+	}
+
+	networks, err := apiClient.NetworkList(context.Background(), types.NetworkListOptions{})
+	if err != nil {
+		d.SystemWideLogger.PrintAndLog("Docker", "List docker network failed", err)
+		utils.SendErrorResponse(w, "List docker network failed")
+		return
+	}
+
+	result := map[string]interface{}{
+		"network":    networks,
+		"containers": containers,
+	}
+
+	js, err := json.Marshal(result)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	utils.SendJSONResponse(w, string(js))
+}

src/mod/dockerux/docker_windows.go

@@ -0,0 +1,32 @@
+//go:build windows
+// +build windows
+
+package dockerux
+
+/*
+
+	Windows docker UX optimizer dummy
+
+	This is a dummy module for Windows as docker features
+	is useless on Windows and create a larger binary size
+
+	docker on Windows build are trimmed to reduce binary size
+	and make it compatibile with Windows 7
+*/
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"imuslab.com/zoraxy/mod/utils"
+)
+
+// Windows build not support docker
+func (d *UXOptimizer) HandleDockerAvailable(w http.ResponseWriter, r *http.Request) {
+	js, _ := json.Marshal(d.RunninInDocker)
+	utils.SendJSONResponse(w, string(js))
+}
+
+func (d *UXOptimizer) HandleDockerContainersList(w http.ResponseWriter, r *http.Request) {
+	utils.SendErrorResponse(w, "Platform not supported")
+}

src/mod/dockerux/dockerux.go

@@ -0,0 +1,24 @@
+package dockerux
+
+import "imuslab.com/zoraxy/mod/info/logger"
+
+/*
+	Docker Optimizer
+
+	This script add support for optimizing docker user experience
+	Note that this module are community contribute only. For bug
+	report, please directly tag the Pull Request author.
+*/
+
+type UXOptimizer struct {
+	RunninInDocker   bool
+	SystemWideLogger *logger.Logger
+}
+
+//Create a new docker optimizer
+func NewDockerOptimizer(IsRunningInDocker bool, logger *logger.Logger) *UXOptimizer {
+	return &UXOptimizer{
+		RunninInDocker:   IsRunningInDocker,
+		SystemWideLogger: logger,
+	}
+}

src/mod/dynamicproxy/Server.go

@@ -14,18 +14,25 @@ import (
 	Main server for dynamic proxy core
 
 	Routing Handler Priority (High to Low)
-	- Blacklist
-	- Whitelist
+	- Special Routing Rule (e.g. acme)
 	- Redirectable
 	- Subdomain Routing
-	- Vitrual Directory Routing
+		- Access Router
+			- Blacklist
+			- Whitelist
+		- Rate Limitor
+		- SSO Auth
+		- Basic Auth
+		- Vitrual Directory Proxy
+		- Subdomain Proxy
+	- Root router (default site router)
 */
 
 func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	/*
 		Special Routing Rules, bypass most of the limitations
 	*/
-	//Check if there are external routing rule matches.
+	//Check if there are external routing rule (rr) matches.
 	//If yes, route them via external rr
 	matchedRoutingRule := h.Parent.GetMatchingRoutingRule(r)
 	if matchedRoutingRule != nil {
@@ -34,16 +41,13 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	//Inject headers
-	w.Header().Set("x-proxy-by", "zoraxy/"+h.Parent.Option.HostVersion)
-
 	/*
 		Redirection Routing
 	*/
 	//Check if this is a redirection url
 	if h.Parent.Option.RedirectRuleTable.IsRedirectable(r) {
 		statusCode := h.Parent.Option.RedirectRuleTable.HandleRedirect(w, r)
-		h.logRequest(r, statusCode != 500, statusCode, "redirect", "")
+		h.Parent.logRequest(r, statusCode != 500, statusCode, "redirect", "")
 		return
 	}
 
@@ -70,14 +74,22 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		//Validate basic auth
-		if sep.RequireBasicAuth {
-			err := h.handleBasicAuthRouting(w, r, sep)
+		// Rate Limit
+		if sep.RequireRateLimit {
+			err := h.handleRateLimitRouting(w, r, sep)
 			if err != nil {
+				h.Parent.Option.Logger.LogHTTPRequest(r, "host", 307)
 				return
 			}
 		}
 
+		//Validate basic auth
+		respWritten := handleAuthProviderRouting(sep, w, r, h)
+		if respWritten {
+			//Request handled by subroute
+			return
+		}
+
 		//Check if any virtual directory rules matches
 		proxyingPath := strings.TrimSpace(r.RequestURI)
 		targetProxyEndpoint := sep.GetVirtualDirectoryHandlerFromRequestURI(proxyingPath)
@@ -85,11 +97,12 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			//Virtual directory routing rule found. Route via vdir mode
 			h.vdirRequest(w, r, targetProxyEndpoint)
 			return
-		} else if !strings.HasSuffix(proxyingPath, "/") && sep.ProxyType != ProxyType_Root {
+		} else if !strings.HasSuffix(proxyingPath, "/") && sep.ProxyType != ProxyTypeRoot {
 			potentialProxtEndpoint := sep.GetVirtualDirectoryHandlerFromRequestURI(proxyingPath + "/")
 			if potentialProxtEndpoint != nil && !potentialProxtEndpoint.Disabled {
 				//Missing tailing slash. Redirect to target proxy endpoint
 				http.Redirect(w, r, r.RequestURI+"/", http.StatusTemporaryRedirect)
+				h.Parent.Option.Logger.LogHTTPRequest(r, "redirect", 307)
 				return
 			}
 		}
@@ -129,7 +142,7 @@ func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 /*
 handleRootRouting
 
-This function handle root routing situations where there are no subdomain
+This function handle root routing (aka default sites) situations where there are no subdomain
 , vdir or special routing rule matches the requested URI.
 
 Once entered this routing segment, the root routing options will take over
@@ -149,7 +162,6 @@ func (h *ProxyHandler) handleRootRouting(w http.ResponseWriter, r *http.Request)
 		fallthrough
 	case DefaultSite_ReverseProxy:
 		//They both share the same behavior
-
 		//Check if any virtual directory rules matches
 		proxyingPath := strings.TrimSpace(r.RequestURI)
 		targetProxyEndpoint := proot.GetVirtualDirectoryHandlerFromRequestURI(proxyingPath)
@@ -157,7 +169,7 @@ func (h *ProxyHandler) handleRootRouting(w http.ResponseWriter, r *http.Request)
 			//Virtual directory routing rule found. Route via vdir mode
 			h.vdirRequest(w, r, targetProxyEndpoint)
 			return
-		} else if !strings.HasSuffix(proxyingPath, "/") && proot.ProxyType != ProxyType_Root {
+		} else if !strings.HasSuffix(proxyingPath, "/") && proot.ProxyType != ProxyTypeRoot {
 			potentialProxtEndpoint := proot.GetVirtualDirectoryHandlerFromRequestURI(proxyingPath + "/")
 			if potentialProxtEndpoint != nil && !targetProxyEndpoint.Disabled {
 				//Missing tailing slash. Redirect to target proxy endpoint
@@ -174,8 +186,13 @@ func (h *ProxyHandler) handleRootRouting(w http.ResponseWriter, r *http.Request)
 			redirectTarget = "about:blank"
 		}
 
+		//Check if the default site values start with http or https
+		if !strings.HasPrefix(redirectTarget, "http://") && !strings.HasPrefix(redirectTarget, "https://") {
+			redirectTarget = "http://" + redirectTarget
+		}
+
 		//Check if it is an infinite loopback redirect
-		parsedURL, err := url.Parse(proot.DefaultSiteValue)
+		parsedURL, err := url.Parse(redirectTarget)
 		if err != nil {
 			//Error when parsing target. Send to root
 			h.hostRequest(w, r, h.Parent.Root)
@@ -183,12 +200,12 @@ func (h *ProxyHandler) handleRootRouting(w http.ResponseWriter, r *http.Request)
 		}
 		hostname := parsedURL.Hostname()
 		if hostname == domainOnly {
-			h.logRequest(r, false, 500, "root-redirect", domainOnly)
+			h.Parent.logRequest(r, false, 500, "root-redirect", domainOnly)
 			http.Error(w, "Loopback redirects due to invalid settings", 500)
 			return
 		}
 
-		h.logRequest(r, false, 307, "root-redirect", domainOnly)
+		h.Parent.logRequest(r, false, 307, "root-redirect", domainOnly)
 		http.Redirect(w, r, redirectTarget, http.StatusTemporaryRedirect)
 	case DefaultSite_NotFoundPage:
 		//Serve the not found page, use template if exists
@@ -200,5 +217,25 @@ func (h *ProxyHandler) handleRootRouting(w http.ResponseWriter, r *http.Request)
 		} else {
 			w.Write(template)
 		}
+	case DefaultSite_NoResponse:
+		//No response. Just close the connection
+		h.Parent.logRequest(r, false, 444, "root-noresponse", domainOnly)
+		hijacker, ok := w.(http.Hijacker)
+		if !ok {
+			return
+		}
+		conn, _, err := hijacker.Hijack()
+		if err != nil {
+			return
+		}
+		conn.Close()
+	case DefaultSite_TeaPot:
+		//I'm a teapot
+		h.Parent.logRequest(r, false, 418, "root-teapot", domainOnly)
+		http.Error(w, "I'm a teapot", http.StatusTeapot)
+	default:
+		//Unknown routing option. Send empty response
+		h.Parent.logRequest(r, false, 544, "root-unknown", domainOnly)
+		http.Error(w, "544 - No Route Defined", 544)
 	}
 }

src/mod/dynamicproxy/access.go

@@ -1,7 +1,6 @@
 package dynamicproxy
 
 import (
-	"log"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -16,7 +15,7 @@ func (h *ProxyHandler) handleAccessRouting(ruleID string, w http.ResponseWriter,
 	accessRule, err := h.Parent.Option.AccessController.GetAccessRuleByID(ruleID)
 	if err != nil {
 		//Unable to load access rule. Target rule not found?
-		log.Println("[Proxy] Unable to load access rule: " + ruleID)
+		h.Parent.Option.Logger.PrintAndLog("proxy-access", "Unable to load access rule: "+ruleID, err)
 		w.WriteHeader(http.StatusInternalServerError)
 		w.Write([]byte("500 - Internal Server Error"))
 		return true
@@ -24,7 +23,7 @@ func (h *ProxyHandler) handleAccessRouting(ruleID string, w http.ResponseWriter,
 
 	isBlocked, blockedReason := accessRequestBlocked(accessRule, h.Parent.Option.WebDirectory, w, r)
 	if isBlocked {
-		h.logRequest(r, false, 403, blockedReason, "")
+		h.Parent.logRequest(r, false, 403, blockedReason, "")
 	}
 	return isBlocked
 }

src/mod/dynamicproxy/authProviders.go

@@ -0,0 +1,108 @@
+package dynamicproxy
+
+import (
+	"errors"
+	"net/http"
+	"strings"
+
+	"imuslab.com/zoraxy/mod/auth"
+)
+
+/*
+	authProviders.go
+
+	This script handle authentication providers
+*/
+
+/*
+Central Authentication Provider Router
+
+This function will route the request to the correct authentication provider
+if the return value is true, do not continue to the next handler
+
+handleAuthProviderRouting takes in 4 parameters:
+- sep: the ProxyEndpoint object
+- w: the http.ResponseWriter object
+- r: the http.Request object
+- h: the ProxyHandler object
+
+and return a boolean indicate if the request is written to http.ResponseWriter
+- true: the request is handled, do not write to http.ResponseWriter
+- false: the request is not handled (usually means auth ok), continue to the next handler
+*/
+func handleAuthProviderRouting(sep *ProxyEndpoint, w http.ResponseWriter, r *http.Request, h *ProxyHandler) bool {
+	if sep.AuthenticationProvider.AuthMethod == AuthMethodBasic {
+		err := h.handleBasicAuthRouting(w, r, sep)
+		if err != nil {
+			h.Parent.Option.Logger.LogHTTPRequest(r, "host", 401)
+			return true
+		}
+	} else if sep.AuthenticationProvider.AuthMethod == AuthMethodAuthelia {
+		err := h.handleAutheliaAuth(w, r)
+		if err != nil {
+			h.Parent.Option.Logger.LogHTTPRequest(r, "host", 401)
+			return true
+		}
+	}
+
+	//No authentication provider, do not need to handle
+	return false
+}
+
+/* Basic Auth */
+func (h *ProxyHandler) handleBasicAuthRouting(w http.ResponseWriter, r *http.Request, pe *ProxyEndpoint) error {
+	err := handleBasicAuth(w, r, pe)
+	if err != nil {
+		h.Parent.logRequest(r, false, 401, "host", r.URL.Hostname())
+	}
+	return err
+}
+
+// Handle basic auth logic
+// do not write to http.ResponseWriter if err return is not nil (already handled by this function)
+func handleBasicAuth(w http.ResponseWriter, r *http.Request, pe *ProxyEndpoint) error {
+	if len(pe.AuthenticationProvider.BasicAuthExceptionRules) > 0 {
+		//Check if the current path matches the exception rules
+		for _, exceptionRule := range pe.AuthenticationProvider.BasicAuthExceptionRules {
+			if strings.HasPrefix(r.RequestURI, exceptionRule.PathPrefix) {
+				//This path is excluded from basic auth
+				return nil
+			}
+		}
+	}
+
+	u, p, ok := r.BasicAuth()
+	if !ok {
+		w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+		w.WriteHeader(401)
+		return errors.New("unauthorized")
+	}
+
+	//Check for the credentials to see if there is one matching
+	hashedPassword := auth.Hash(p)
+	matchingFound := false
+	for _, cred := range pe.AuthenticationProvider.BasicAuthCredentials {
+		if u == cred.Username && hashedPassword == cred.PasswordHash {
+			matchingFound = true
+
+			//Set the X-Remote-User header
+			r.Header.Set("X-Remote-User", u)
+			break
+		}
+	}
+
+	if !matchingFound {
+		w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+		w.WriteHeader(401)
+		return errors.New("unauthorized")
+	}
+
+	return nil
+}
+
+/* Authelia */
+
+// Handle authelia auth routing
+func (h *ProxyHandler) handleAutheliaAuth(w http.ResponseWriter, r *http.Request) error {
+	return h.Parent.Option.AutheliaRouter.HandleAutheliaAuth(w, r)
+}

src/mod/dynamicproxy/basicAuth.go

@@ -1,63 +0,0 @@
-package dynamicproxy
-
-import (
-	"errors"
-	"net/http"
-	"strings"
-
-	"imuslab.com/zoraxy/mod/auth"
-)
-
-/*
-	BasicAuth.go
-
-	This file handles the basic auth on proxy endpoints
-	if RequireBasicAuth is set to true
-*/
-
-func (h *ProxyHandler) handleBasicAuthRouting(w http.ResponseWriter, r *http.Request, pe *ProxyEndpoint) error {
-	err := handleBasicAuth(w, r, pe)
-	if err != nil {
-		h.logRequest(r, false, 401, "host", pe.Domain)
-	}
-	return err
-}
-
-// Handle basic auth logic
-// do not write to http.ResponseWriter if err return is not nil (already handled by this function)
-func handleBasicAuth(w http.ResponseWriter, r *http.Request, pe *ProxyEndpoint) error {
-	if len(pe.BasicAuthExceptionRules) > 0 {
-		//Check if the current path matches the exception rules
-		for _, exceptionRule := range pe.BasicAuthExceptionRules {
-			if strings.HasPrefix(r.RequestURI, exceptionRule.PathPrefix) {
-				//This path is excluded from basic auth
-				return nil
-			}
-		}
-	}
-
-	u, p, ok := r.BasicAuth()
-	if !ok {
-		w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
-		w.WriteHeader(401)
-		return errors.New("unauthorized")
-	}
-
-	//Check for the credentials to see if there is one matching
-	hashedPassword := auth.Hash(p)
-	matchingFound := false
-	for _, cred := range pe.BasicAuthCredentials {
-		if u == cred.Username && hashedPassword == cred.PasswordHash {
-			matchingFound = true
-			break
-		}
-	}
-
-	if !matchingFound {
-		w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
-		w.WriteHeader(401)
-		return errors.New("unauthorized")
-	}
-
-	return nil
-}

src/mod/dynamicproxy/customHeader.go

@@ -0,0 +1,12 @@
+package dynamicproxy
+
+/*
+	CustomHeader.go
+
+	This script handle parsing and injecting custom headers
+	into the dpcore routing logic
+
+	Updates: 2024-10-26
+	Contents from this file has been moved to rewrite/rewrite.go
+	This file is kept for contributors to understand the structure
+*/

src/mod/dynamicproxy/domainsniff/domainsniff.go

@@ -1,11 +1,26 @@
 package domainsniff
 
+/*
+	Domainsniff
+
+	This package contain codes that perform project / domain specific behavior in Zoraxy
+	If you want Zoraxy to handle a particular domain or open source project in a special way,
+	you can add the checking logic here.
+
+*/
 import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
 	"net"
+	"net/http"
+	"strings"
 	"time"
+
+	"imuslab.com/zoraxy/mod/utils"
 )
 
-//Check if the domain is reachable and return err if not reachable
+// Check if the domain is reachable and return err if not reachable
 func DomainReachableWithError(domain string) error {
 	timeout := 1 * time.Second
 	conn, err := net.DialTimeout("tcp", domain, timeout)
@@ -17,7 +32,115 @@ func DomainReachableWithError(domain string) error {
 	return nil
 }
 
-//Check if domain reachable
+// Check if a domain have TLS but it is self-signed or expired
+// Return false if sniff error
+func DomainIsSelfSigned(domain string) bool {
+	//Extract the domain from URl in case the user input the full URL
+	host, port, err := net.SplitHostPort(domain)
+	if err != nil {
+		host = domain
+	} else {
+		domain = host + ":" + port
+	}
+	if !strings.Contains(domain, ":") {
+		domain = domain + ":443"
+	}
+
+	//Get the certificate
+	conn, err := net.Dial("tcp", domain)
+	if err != nil {
+		return false
+	}
+	defer conn.Close()
+
+	//Connect with TLS using secure verify
+	tlsConn := tls.Client(conn, nil)
+	err = tlsConn.Handshake()
+	if err == nil {
+		//This is a valid certificate
+		fmt.Println()
+		return false
+	}
+
+	//Connect with TLS using insecure skip verify
+	config := &tls.Config{
+		InsecureSkipVerify: true,
+	}
+	tlsConn = tls.Client(conn, config)
+	err = tlsConn.Handshake()
+	//If the handshake is successful, this is a self-signed certificate
+	return err == nil
+}
+
+// Check if domain reachable
 func DomainReachable(domain string) bool {
 	return DomainReachableWithError(domain) == nil
 }
+
+// Check if domain is served by a web server using HTTPS
+func DomainUsesTLS(targetURL string) bool {
+	//Check if the site support https
+	httpsUrl := fmt.Sprintf("https://%s", targetURL)
+	httpUrl := fmt.Sprintf("http://%s", targetURL)
+
+	client := http.Client{Timeout: 5 * time.Second}
+
+	resp, err := client.Head(httpsUrl)
+	if err == nil && resp.StatusCode == http.StatusOK {
+		return true
+	}
+
+	resp, err = client.Head(httpUrl)
+	if err == nil && resp.StatusCode == http.StatusOK {
+		return false
+	}
+
+	//If the site is not reachable, return false
+	return false
+}
+
+/*
+	Request Handlers
+*/
+//Check if site support TLS
+//Pass in ?selfsignchk=true to also check for self-signed certificate
+func HandleCheckSiteSupportTLS(w http.ResponseWriter, r *http.Request) {
+	targetURL, err := utils.PostPara(r, "url")
+	if err != nil {
+		utils.SendErrorResponse(w, "invalid url given")
+		return
+	}
+
+	//If the selfsign flag is set, also chec for self-signed certificate
+	_, err = utils.PostBool(r, "selfsignchk")
+	if err == nil {
+		//Return the https and selfsign status
+		type result struct {
+			Protocol string `json:"protocol"`
+			SelfSign bool   `json:"selfsign"`
+		}
+
+		scanResult := result{Protocol: "http", SelfSign: false}
+
+		if DomainUsesTLS(targetURL) {
+			scanResult.Protocol = "https"
+			if DomainIsSelfSigned(targetURL) {
+				scanResult.SelfSign = true
+			}
+		}
+
+		js, _ := json.Marshal(scanResult)
+		utils.SendJSONResponse(w, string(js))
+		return
+	}
+
+	if DomainUsesTLS(targetURL) {
+		js, _ := json.Marshal("https")
+		utils.SendJSONResponse(w, string(js))
+		return
+	} else {
+		js, _ := json.Marshal("http")
+		utils.SendJSONResponse(w, string(js))
+		return
+	}
+}

src/mod/dynamicproxy/domainsniff/proxmox.go

@@ -0,0 +1,21 @@
+package domainsniff
+
+import "net/http"
+
+/*
+	Promox API sniffer
+
+	This handler sniff proxmox API endpoint and
+	adjust the request accordingly to fix shits
+	in the proxmox API server
+*/
+
+func IsProxmox(r *http.Request) bool {
+	// Check if any of the cookies is named PVEAuthCookie
+	for _, cookie := range r.Cookies() {
+		if cookie.Name == "PVEAuthCookie" {
+			return true
+		}
+	}
+	return false
+}

src/mod/dynamicproxy/dpcore/dpcore.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"imuslab.com/zoraxy/mod/dynamicproxy/domainsniff"
 	"imuslab.com/zoraxy/mod/dynamicproxy/permissionpolicy"
 )
 
@@ -50,18 +51,30 @@ type ReverseProxy struct {
 	ModifyResponse func(*http.Response) error
 
 	//Prepender is an optional prepend text for URL rewrite
-	//
 	Prepender string
 
 	Verbal bool
+
+	//Appended by Zoraxy project
+
 }
 
 type ResponseRewriteRuleSet struct {
-	ProxyDomain  string
-	OriginalHost string
-	UseTLS       bool
-	NoCache      bool
-	PathPrefix   string //Vdir prefix for root, / will be rewrite to this
+	/* Basic Rewrite Rulesets */
+	ProxyDomain       string
+	OriginalHost      string
+	UseTLS            bool
+	NoCache           bool
+	PathPrefix        string //Vdir prefix for root, / will be rewrite to this
+	UpstreamHeaders   [][]string
+	DownstreamHeaders [][]string
+
+	/* Advance Usecase Options */
+	HostHeaderOverwrite string //Force overwrite of request "Host" header (advanced usecase)
+	NoRemoveHopByHop    bool   //Do not remove hop-by-hop headers (advanced usecase)
+
+	/* System Information Payload */
+	Version string //Version number of Zoraxy, use for X-Proxy-By
 }
 
 type requestCanceler interface {
@@ -69,8 +82,8 @@ type requestCanceler interface {
 }
 
 type DpcoreOptions struct {
-	IgnoreTLSVerification bool
-	FlushInterval         time.Duration
+	IgnoreTLSVerification bool          //Disable all TLS verification when request pass through this proxy router
+	FlushInterval         time.Duration //Duration to flush in normal requests. Stream request or keep-alive request will always flush with interval of -1 (immediately)
 }
 
 func NewDynamicProxyCore(target *url.URL, prepender string, dpcOptions *DpcoreOptions) *ReverseProxy {
@@ -96,6 +109,8 @@ func NewDynamicProxyCore(target *url.URL, prepender string, dpcOptions *DpcoreOp
 	thisTransporter.(*http.Transport).MaxConnsPerHost = optimalConcurrentConnection * 2
 	thisTransporter.(*http.Transport).DisableCompression = true
 
+	//TODO: Add user adjustable timeout option here
+
 	if dpcOptions.IgnoreTLSVerification {
 		//Ignore TLS certificate validation error
 		thisTransporter.(*http.Transport).TLSClientConfig.InsecureSkipVerify = true
@@ -177,7 +192,7 @@ var hopHeaders = []string{
 	"Te",      // canonicalized version of "TE"
 	"Trailer", // not Trailers per URL above; http://www.rfc-editor.org/errata_search.php?eid=4522
 	"Transfer-Encoding",
-	//"Upgrade",
+	//"Upgrade", // handled by websocket proxy in higher layer abstraction
 }
 
 // Copy response from src to dst with given flush interval, reference from httputil.ReverseProxy
@@ -248,79 +263,7 @@ func (p *ReverseProxy) logf(format string, args ...interface{}) {
 	}
 }
 
-func removeHeaders(header http.Header, noCache bool) {
-	// Remove hop-by-hop headers listed in the "Connection" header.
-	if c := header.Get("Connection"); c != "" {
-		for _, f := range strings.Split(c, ",") {
-			if f = strings.TrimSpace(f); f != "" {
-				header.Del(f)
-			}
-		}
-	}
-
-	// Remove hop-by-hop headers
-	for _, h := range hopHeaders {
-		if header.Get(h) != "" {
-			header.Del(h)
-		}
-	}
-
-	//Restore the Upgrade header if any
-	if header.Get("Zr-Origin-Upgrade") != "" {
-		header.Set("Upgrade", header.Get("Zr-Origin-Upgrade"))
-		header.Del("Zr-Origin-Upgrade")
-	}
-
-	//Disable cache if nocache is set
-	if noCache {
-		header.Del("Cache-Control")
-		header.Set("Cache-Control", "no-store")
-	}
-
-	//Hide Go-HTTP-Client UA if the client didnt sent us one
-	if _, ok := header["User-Agent"]; !ok {
-		// If the outbound request doesn't have a User-Agent header set,
-		// don't send the default Go HTTP client User-Agent.
-		header.Set("User-Agent", "")
-	}
-
-}
-
-func addXForwardedForHeader(req *http.Request) {
-	if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
-		// If we aren't the first proxy retain prior
-		// X-Forwarded-For information as a comma+space
-		// separated list and fold multiple headers into one.
-		if prior, ok := req.Header["X-Forwarded-For"]; ok {
-			clientIP = strings.Join(prior, ", ") + ", " + clientIP
-		}
-		req.Header.Set("X-Forwarded-For", clientIP)
-		if req.TLS != nil {
-			req.Header.Set("X-Forwarded-Proto", "https")
-		} else {
-			req.Header.Set("X-Forwarded-Proto", "http")
-		}
-
-		if req.Header.Get("X-Real-Ip") == "" {
-			//Check if CF-Connecting-IP header exists
-			CF_Connecting_IP := req.Header.Get("CF-Connecting-IP")
-			if CF_Connecting_IP != "" {
-				//Use CF Connecting IP
-				req.Header.Set("X-Real-Ip", CF_Connecting_IP)
-			} else {
-				// Not exists. Fill it in with first entry in X-Forwarded-For
-				ips := strings.Split(clientIP, ",")
-				if len(ips) > 0 {
-					req.Header.Set("X-Real-Ip", strings.TrimSpace(ips[0]))
-				}
-			}
-
-		}
-
-	}
-}
-
-func (p *ReverseProxy) ProxyHTTP(rw http.ResponseWriter, req *http.Request, rrr *ResponseRewriteRuleSet) error {
+func (p *ReverseProxy) ProxyHTTP(rw http.ResponseWriter, req *http.Request, rrr *ResponseRewriteRuleSet) (int, error) {
 	transport := p.Transport
 
 	outreq := new(http.Request)
@@ -349,7 +292,10 @@ func (p *ReverseProxy) ProxyHTTP(rw http.ResponseWriter, req *http.Request, rrr
 	outreq.Close = false
 
 	//Only skip origin rewrite iff proxy target require TLS and it is external domain name like github.com
-	if !(rrr.UseTLS && isExternalDomainName(rrr.ProxyDomain)) {
+	if rrr.HostHeaderOverwrite != "" {
+		//Use user defined overwrite header value, see issue #255
+		outreq.Host = rrr.HostHeaderOverwrite
+	} else if !(rrr.UseTLS && isExternalDomainName(rrr.ProxyDomain)) {
 		// Always use the original host, see issue #164
 		outreq.Host = rrr.OriginalHost
 	}
@@ -358,12 +304,25 @@ func (p *ReverseProxy) ProxyHTTP(rw http.ResponseWriter, req *http.Request, rrr
 	outreq.Header = make(http.Header)
 	copyHeader(outreq.Header, req.Header)
 
-	// Remove hop-by-hop headers listed in the "Connection" header, Remove hop-by-hop headers.
-	removeHeaders(outreq.Header, rrr.NoCache)
+	// Remove hop-by-hop headers.
+	if !rrr.NoRemoveHopByHop {
+		removeHeaders(outreq.Header, rrr.NoCache)
+	}
 
 	// Add X-Forwarded-For Header.
 	addXForwardedForHeader(outreq)
 
+	// Add user defined headers (to upstream)
+	injectUserDefinedHeaders(outreq.Header, rrr.UpstreamHeaders)
+
+	// Rewrite outbound UA, must be after user headers
+	rewriteUserAgent(outreq.Header, "Zoraxy/"+rrr.Version)
+
+	//Fix proxmox transfer encoding bug if detected Proxmox Cookie
+	if domainsniff.IsProxmox(req) {
+		outreq.TransferEncoding = []string{"identity"}
+	}
+
 	res, err := transport.RoundTrip(outreq)
 	if err != nil {
 		if p.Verbal {
@@ -371,11 +330,13 @@ func (p *ReverseProxy) ProxyHTTP(rw http.ResponseWriter, req *http.Request, rrr
 		}
 
 		//rw.WriteHeader(http.StatusBadGateway)
-		return err
+		return http.StatusBadGateway, err
 	}
 
 	// Remove hop-by-hop headers listed in the "Connection" header of the response, Remove hop-by-hop headers.
-	removeHeaders(res.Header, rrr.NoCache)
+	if !rrr.NoRemoveHopByHop {
+		removeHeaders(res.Header, rrr.NoCache)
+	}
 
 	//Remove the User-Agent header if exists
 	if _, ok := res.Header["User-Agent"]; ok {
@@ -390,17 +351,14 @@ func (p *ReverseProxy) ProxyHTTP(rw http.ResponseWriter, req *http.Request, rrr
 			}
 
 			//rw.WriteHeader(http.StatusBadGateway)
-			return err
+			return http.StatusBadGateway, err
 		}
 	}
 
-	//if res.StatusCode == 501 || res.StatusCode == 500 {
-	//	fmt.Println(outreq.Proto, outreq.RemoteAddr, outreq.RequestURI)
-	//	fmt.Println(">>>", outreq.Method, res.Header, res.ContentLength, res.StatusCode)
-	//	fmt.Println(outreq.Header, req.Host)
-	//}
+	//Add debug X-Proxy-By tracker
+	res.Header.Set("x-proxy-by", "zoraxy/"+rrr.Version)
 
-	//Custom header rewriter functions
+	//Custom Location header rewriter functions
 	if res.Header.Get("Location") != "" {
 		locationRewrite := res.Header.Get("Location")
 		originLocation := res.Header.Get("Location")
@@ -426,11 +384,13 @@ func (p *ReverseProxy) ProxyHTTP(rw http.ResponseWriter, req *http.Request, rrr
 		res.Header.Set("Location", locationRewrite)
 	}
 
+	// Add user defined headers (to downstream)
+	injectUserDefinedHeaders(res.Header, rrr.DownstreamHeaders)
+
 	// Copy header from response to client.
 	copyHeader(rw.Header(), res.Header)
 
 	// inject permission policy headers
-	//TODO: Load permission policy from rrr
 	permissionpolicy.InjectPermissionPolicyHeader(rw, nil)
 
 	// The "Trailer" header isn't included in the Transport's response, Build it up from Trailer.
@@ -460,14 +420,14 @@ func (p *ReverseProxy) ProxyHTTP(rw http.ResponseWriter, req *http.Request, rrr
 	res.Body.Close()
 	copyHeader(rw.Header(), res.Trailer)
 
-	return nil
+	return res.StatusCode, nil
 }
 
-func (p *ReverseProxy) ProxyHTTPS(rw http.ResponseWriter, req *http.Request) error {
+func (p *ReverseProxy) ProxyHTTPS(rw http.ResponseWriter, req *http.Request) (int, error) {
 	hij, ok := rw.(http.Hijacker)
 	if !ok {
 		p.logf("http server does not support hijacker")
-		return errors.New("http server does not support hijacker")
+		return http.StatusNotImplemented, errors.New("http server does not support hijacker")
 	}
 
 	clientConn, _, err := hij.Hijack()
@@ -475,7 +435,7 @@ func (p *ReverseProxy) ProxyHTTPS(rw http.ResponseWriter, req *http.Request) err
 		if p.Verbal {
 			p.logf("http: proxy error: %v", err)
 		}
-		return err
+		return http.StatusInternalServerError, err
 	}
 
 	proxyConn, err := net.Dial("tcp", req.URL.Host)
@@ -484,7 +444,7 @@ func (p *ReverseProxy) ProxyHTTPS(rw http.ResponseWriter, req *http.Request) err
 			p.logf("http: proxy error: %v", err)
 		}
 
-		return err
+		return http.StatusInternalServerError, err
 	}
 
 	// The returned net.Conn may have read or write deadlines
@@ -503,7 +463,7 @@ func (p *ReverseProxy) ProxyHTTPS(rw http.ResponseWriter, req *http.Request) err
 		if p.Verbal {
 			p.logf("http: proxy error: %v", err)
 		}
-		return err
+		return http.StatusGatewayTimeout, err
 	}
 
 	err = proxyConn.SetDeadline(deadline)
@@ -512,7 +472,7 @@ func (p *ReverseProxy) ProxyHTTPS(rw http.ResponseWriter, req *http.Request) err
 			p.logf("http: proxy error: %v", err)
 		}
 
-		return err
+		return http.StatusGatewayTimeout, err
 	}
 
 	_, err = clientConn.Write([]byte("HTTP/1.0 200 OK\r\n\r\n"))
@@ -521,7 +481,7 @@ func (p *ReverseProxy) ProxyHTTPS(rw http.ResponseWriter, req *http.Request) err
 			p.logf("http: proxy error: %v", err)
 		}
 
-		return err
+		return http.StatusInternalServerError, err
 	}
 
 	go func() {
@@ -534,15 +494,13 @@ func (p *ReverseProxy) ProxyHTTPS(rw http.ResponseWriter, req *http.Request) err
 	proxyConn.Close()
 	clientConn.Close()
 
-	return nil
+	return http.StatusOK, nil
 }
 
-func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request, rrr *ResponseRewriteRuleSet) error {
+func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request, rrr *ResponseRewriteRuleSet) (int, error) {
 	if req.Method == "CONNECT" {
-		err := p.ProxyHTTPS(rw, req)
-		return err
+		return p.ProxyHTTPS(rw, req)
 	} else {
-		err := p.ProxyHTTP(rw, req, rrr)
-		return err
+		return p.ProxyHTTP(rw, req, rrr)
 	}
 }

src/mod/dynamicproxy/dpcore/dpcore_test.go

@@ -1,29 +1,67 @@
 package dpcore_test
 
 import (
+	"net/url"
 	"testing"
 
 	"imuslab.com/zoraxy/mod/dynamicproxy/dpcore"
 )
 
 func TestReplaceLocationHost(t *testing.T) {
-	urlString := "http://private.com/test/newtarget/"
-	rrr := &dpcore.ResponseRewriteRuleSet{
-		OriginalHost: "test.example.com",
-		ProxyDomain:  "private.com/test",
-		UseTLS:       true,
-	}
-	useTLS := true
+	tests := []struct {
+		name           string
+		urlString      string
+		rrr            *dpcore.ResponseRewriteRuleSet
+		useTLS         bool
+		expectedResult string
+		expectError    bool
+	}{
+		{
+			name:           "Basic HTTP to HTTPS redirection",
+			urlString:      "http://example.com/resource",
+			rrr:            &dpcore.ResponseRewriteRuleSet{ProxyDomain: "example.com", OriginalHost: "proxy.example.com", UseTLS: true},
+			useTLS:         true,
+			expectedResult: "https://proxy.example.com/resource",
+			expectError:    false,
+		},
 
-	expectedResult := "https://test.example.com/newtarget/"
-
-	result, err := dpcore.ReplaceLocationHost(urlString, rrr, useTLS)
-	if err != nil {
-		t.Errorf("Error occurred: %v", err)
+		{
+			name:           "Basic HTTPS to HTTP redirection",
+			urlString:      "https://proxy.example.com/resource",
+			rrr:            &dpcore.ResponseRewriteRuleSet{ProxyDomain: "proxy.example.com", OriginalHost: "proxy.example.com", UseTLS: false},
+			useTLS:         false,
+			expectedResult: "http://proxy.example.com/resource",
+			expectError:    false,
+		},
+		{
+			name:           "No rewrite on mismatched domain",
+			urlString:      "http://anotherdomain.com/resource",
+			rrr:            &dpcore.ResponseRewriteRuleSet{ProxyDomain: "proxy.example.com", OriginalHost: "proxy.example.com", UseTLS: true},
+			useTLS:         true,
+			expectedResult: "http://anotherdomain.com/resource",
+			expectError:    false,
+		},
+		{
+			name:           "Subpath trimming with HTTPS",
+			urlString:      "https://blog.example.com/post?id=1",
+			rrr:            &dpcore.ResponseRewriteRuleSet{ProxyDomain: "blog.example.com", OriginalHost: "proxy.example.com/blog", UseTLS: true},
+			useTLS:         true,
+			expectedResult: "https://proxy.example.com/blog/post?id=1",
+			expectError:    false,
+		},
 	}
 
-	if result != expectedResult {
-		t.Errorf("Expected: %s, but got: %s", expectedResult, result)
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := dpcore.ReplaceLocationHost(tt.urlString, tt.rrr, tt.useTLS)
+			if (err != nil) != tt.expectError {
+				t.Errorf("Expected error: %v, got: %v", tt.expectError, err)
+			}
+			if result != tt.expectedResult {
+				result, _ = url.QueryUnescape(result)
+				t.Errorf("Expected result: %s, got: %s", tt.expectedResult, result)
+			}
+		})
 	}
 }
 
@@ -36,7 +74,7 @@ func TestReplaceLocationHostRelative(t *testing.T) {
 	}
 	useTLS := true
 
-	expectedResult := "https://test.example.com/api/"
+	expectedResult := "api/"
 
 	result, err := dpcore.ReplaceLocationHost(urlString, rrr, useTLS)
 	if err != nil {

src/mod/dynamicproxy/dpcore/flush.go

@@ -3,6 +3,7 @@ package dpcore
 import (
 	"mime"
 	"net/http"
+	"strings"
 	"time"
 )
 
@@ -17,6 +18,12 @@ func (p *ReverseProxy) getFlushInterval(req *http.Request, res *http.Response) t
 		return -1
 	}
 
+	// Fixed issue #235: Added auto detection for ollama / llm output stream
+	connectionHeader := req.Header["Connection"]
+	if len(connectionHeader) > 0 && strings.Contains(strings.Join(connectionHeader, ","), "keep-alive") {
+		return -1
+	}
+
 	//Cannot sniff anything. Use default value
 	return p.FlushInterval
 

src/mod/dynamicproxy/dpcore/header.go

@@ -0,0 +1,120 @@
+package dpcore
+
+import (
+	"net"
+	"net/http"
+	"strings"
+)
+
+/*
+	Header.go
+
+	This script handles headers rewrite and remove
+	in dpcore.
+
+	Added in Zoraxy v3.0.6 by tobychui
+*/
+
+// removeHeaders Remove hop-by-hop headers listed in the "Connection" header, Remove hop-by-hop headers.
+func removeHeaders(header http.Header, noCache bool) {
+	// Remove hop-by-hop headers listed in the "Connection" header.
+	if c := header.Get("Connection"); c != "" {
+		for _, f := range strings.Split(c, ",") {
+			if f = strings.TrimSpace(f); f != "" {
+				header.Del(f)
+			}
+		}
+	}
+
+	// Remove hop-by-hop headers
+	for _, h := range hopHeaders {
+		if header.Get(h) != "" {
+			header.Del(h)
+		}
+	}
+
+	//Restore the Upgrade header if any
+	if header.Get("Zr-Origin-Upgrade") != "" {
+		header.Set("Upgrade", header.Get("Zr-Origin-Upgrade"))
+		header.Del("Zr-Origin-Upgrade")
+	}
+
+	//Disable cache if nocache is set
+	if noCache {
+		header.Del("Cache-Control")
+		header.Set("Cache-Control", "no-store")
+	}
+
+}
+
+// rewriteUserAgent rewrite the user agent based on incoming request
+func rewriteUserAgent(header http.Header, UA string) {
+	//Hide Go-HTTP-Client UA if the client didnt sent us one
+	if header.Get("User-Agent") == "" {
+		// If the outbound request doesn't have a User-Agent header set,
+		// don't send the default Go HTTP client User-Agent
+		header.Del("User-Agent")
+		header.Set("User-Agent", UA)
+	}
+}
+
+// Add X-Forwarded-For Header and rewrite X-Real-Ip according to sniffing logics
+func addXForwardedForHeader(req *http.Request) {
+	if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
+		// If we aren't the first proxy retain prior
+		// X-Forwarded-For information as a comma+space
+		// separated list and fold multiple headers into one.
+		if prior, ok := req.Header["X-Forwarded-For"]; ok {
+			clientIP = strings.Join(prior, ", ") + ", " + clientIP
+		}
+		req.Header.Set("X-Forwarded-For", clientIP)
+		if req.TLS != nil {
+			req.Header.Set("X-Forwarded-Proto", "https")
+		} else {
+			req.Header.Set("X-Forwarded-Proto", "http")
+		}
+
+		if req.Header.Get("X-Real-Ip") == "" {
+			//Check if CF-Connecting-IP header exists
+			CF_Connecting_IP := req.Header.Get("CF-Connecting-IP")
+			Fastly_Client_IP := req.Header.Get("Fastly-Client-IP")
+			if CF_Connecting_IP != "" {
+				//Use CF Connecting IP
+				req.Header.Set("X-Real-Ip", CF_Connecting_IP)
+			} else if Fastly_Client_IP != "" {
+				//Use Fastly Client IP
+				req.Header.Set("X-Real-Ip", Fastly_Client_IP)
+			} else {
+				// Not exists. Fill it in with first entry in X-Forwarded-For
+				ips := strings.Split(clientIP, ",")
+				if len(ips) > 0 {
+					req.Header.Set("X-Real-Ip", strings.TrimSpace(ips[0]))
+				}
+			}
+		}
+
+	}
+}
+
+// injectUserDefinedHeaders inject the user headers from slice
+// if a value is empty string, the key will be removed from header.
+// if a key is empty string, the function will return immediately
+func injectUserDefinedHeaders(header http.Header, userHeaders [][]string) {
+	for _, userHeader := range userHeaders {
+		if len(userHeader) == 0 {
+			//End of header slice
+			return
+		}
+		headerKey := userHeader[0]
+		headerValue := userHeader[1]
+		if headerValue == "" {
+			//Remove header from head
+			header.Del(headerKey)
+			continue
+		}
+
+		//Default: Set header value
+		header.Del(headerKey) //Remove header if it already exists
+		header.Set(headerKey, headerValue)
+	}
+}

src/mod/dynamicproxy/dpcore/utils.go

@@ -1,7 +1,10 @@
 package dpcore
 
 import (
+	"bytes"
+	"io"
 	"net"
+	"net/http"
 	"net/url"
 	"strings"
 )
@@ -57,7 +60,7 @@ func replaceLocationHost(urlString string, rrr *ResponseRewriteRuleSet, useTLS b
 	return u.String(), nil
 }
 
-// Debug functions
+// Debug functions for replaceLocationHost
 func ReplaceLocationHost(urlString string, rrr *ResponseRewriteRuleSet, useTLS bool) (string, error) {
 	return replaceLocationHost(urlString, rrr, useTLS)
 }
@@ -92,3 +95,63 @@ func isExternalDomainName(hostname string) bool {
 
 	return true
 }
+
+// DeepCopyRequest returns a deep copy of the given http.Request.
+func DeepCopyRequest(req *http.Request) (*http.Request, error) {
+	// Copy the URL
+	urlCopy := *req.URL
+
+	// Copy the headers
+	headersCopy := make(http.Header, len(req.Header))
+	for k, vv := range req.Header {
+		vvCopy := make([]string, len(vv))
+		copy(vvCopy, vv)
+		headersCopy[k] = vvCopy
+	}
+
+	// Copy the cookies
+	cookiesCopy := make([]*http.Cookie, len(req.Cookies()))
+	for i, cookie := range req.Cookies() {
+		cookieCopy := *cookie
+		cookiesCopy[i] = &cookieCopy
+	}
+
+	// Copy the body, if present
+	var bodyCopy io.ReadCloser
+	if req.Body != nil {
+		var buf bytes.Buffer
+		if _, err := buf.ReadFrom(req.Body); err != nil {
+			return nil, err
+		}
+		// Reset the request body so it can be read again
+		if err := req.Body.Close(); err != nil {
+			return nil, err
+		}
+		req.Body = io.NopCloser(&buf)
+		bodyCopy = io.NopCloser(bytes.NewReader(buf.Bytes()))
+	}
+
+	// Create the new request
+	reqCopy := &http.Request{
+		Method:           req.Method,
+		URL:              &urlCopy,
+		Proto:            req.Proto,
+		ProtoMajor:       req.ProtoMajor,
+		ProtoMinor:       req.ProtoMinor,
+		Header:           headersCopy,
+		Body:             bodyCopy,
+		ContentLength:    req.ContentLength,
+		TransferEncoding: append([]string(nil), req.TransferEncoding...),
+		Close:            req.Close,
+		Host:             req.Host,
+		Form:             req.Form,
+		PostForm:         req.PostForm,
+		MultipartForm:    req.MultipartForm,
+		Trailer:          req.Trailer,
+		RemoteAddr:       req.RemoteAddr,
+		TLS:              req.TLS,
+		// Cancel and Context are not copied as it might cause issues
+	}
+
+	return reqCopy, nil
+}

src/mod/dynamicproxy/dynamicproxy.go

@@ -23,12 +23,13 @@ import (
 func NewDynamicProxy(option RouterOption) (*Router, error) {
 	proxyMap := sync.Map{}
 	thisRouter := Router{
-		Option:         &option,
-		ProxyEndpoints: &proxyMap,
-		Running:        false,
-		server:         nil,
-		routingRules:   []*RoutingRule{},
-		tldMap:         map[string]int{},
+		Option:           &option,
+		ProxyEndpoints:   &proxyMap,
+		Running:          false,
+		server:           nil,
+		routingRules:     []*RoutingRule{},
+		loadBalancer:     option.LoadBalancer,
+		rateLimitCounter: RequestCountPerIpTable{},
 	}
 
 	thisRouter.mux = &ProxyHandler{
@@ -85,6 +86,12 @@ func (router *Router) StartProxyService() error {
 		MinVersion:     uint16(minVersion),
 	}
 
+	//Start rate limitor
+	err := router.startRateLimterCounterResetTicker()
+	if err != nil {
+		return err
+	}
+
 	if router.Option.UseTls {
 		router.server = &http.Server{
 			Addr:      ":" + strconv.Itoa(router.Option.Port),
@@ -129,19 +136,35 @@ func (router *Router) StartProxyService() error {
 							}
 						}
 
+						// Rate Limit
+						if sep.RequireRateLimit {
+							if err := router.handleRateLimit(w, r, sep); err != nil {
+								return
+							}
+						}
+
 						//Validate basic auth
-						if sep.RequireBasicAuth {
+						if sep.AuthenticationProvider.AuthMethod == AuthMethodBasic {
 							err := handleBasicAuth(w, r, sep)
 							if err != nil {
 								return
 							}
 						}
 
-						sep.proxy.ServeHTTP(w, r, &dpcore.ResponseRewriteRuleSet{
-							ProxyDomain:  sep.Domain,
-							OriginalHost: originalHostHeader,
-							UseTLS:       sep.RequireTLS,
-							PathPrefix:   "",
+						selectedUpstream, err := router.loadBalancer.GetRequestUpstreamTarget(w, r, sep.ActiveOrigins, sep.UseStickySession)
+						if err != nil {
+							http.ServeFile(w, r, "./web/hosterror.html")
+							router.Option.Logger.PrintAndLog("dprouter", "failed to get upstream for hostname", err)
+							router.logRequest(r, false, 404, "vdir-http", r.Host)
+						}
+						selectedUpstream.ServeHTTP(w, r, &dpcore.ResponseRewriteRuleSet{
+							ProxyDomain:         selectedUpstream.OriginIpOrDomain,
+							OriginalHost:        originalHostHeader,
+							UseTLS:              selectedUpstream.RequireTLS,
+							HostHeaderOverwrite: sep.HeaderRewriteRules.RequestHostOverwrite,
+							NoRemoveHopByHop:    sep.HeaderRewriteRules.DisableHopByHopHeaderRemoval,
+							PathPrefix:          "",
+							Version:             sep.parent.Option.HostVersion,
 						})
 						return
 					}
@@ -173,7 +196,7 @@ func (router *Router) StartProxyService() error {
 				IdleTimeout:  120 * time.Second,
 			}
 
-			log.Println("Starting HTTP-to-HTTPS redirector (port 80)")
+			router.Option.Logger.PrintAndLog("dprouter", "Starting HTTP-to-HTTPS redirector (port 80)", nil)
 
 			//Create a redirection stop channel
 			stopChan := make(chan bool)
@@ -184,7 +207,7 @@ func (router *Router) StartProxyService() error {
 				ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 				defer cancel()
 				httpServer.Shutdown(ctx)
-				log.Println("HTTP to HTTPS redirection listener stopped")
+				router.Option.Logger.PrintAndLog("dprouter", "HTTP to HTTPS redirection listener stopped", nil)
 			}()
 
 			//Start the http server that listens to port 80 and redirect to 443
@@ -199,10 +222,10 @@ func (router *Router) StartProxyService() error {
 		}
 
 		//Start the TLS server
-		log.Println("Reverse proxy service started in the background (TLS mode)")
+		router.Option.Logger.PrintAndLog("dprouter", "Reverse proxy service started in the background (TLS mode)", nil)
 		go func() {
 			if err := router.server.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {
-				log.Fatalf("Could not start proxy server: %v\n", err)
+				router.Option.Logger.PrintAndLog("dprouter", "Could not start proxy server", err)
 			}
 		}()
 	} else {
@@ -210,10 +233,9 @@ func (router *Router) StartProxyService() error {
 		router.tlsListener = nil
 		router.server = &http.Server{Addr: ":" + strconv.Itoa(router.Option.Port), Handler: router.mux}
 		router.Running = true
-		log.Println("Reverse proxy service started in the background (Plain HTTP mode)")
+		router.Option.Logger.PrintAndLog("dprouter", "Reverse proxy service started in the background (Plain HTTP mode)", nil)
 		go func() {
 			router.server.ListenAndServe()
-			//log.Println("[DynamicProxy] " + err.Error())
 		}()
 	}
 
@@ -231,10 +253,23 @@ func (router *Router) StopProxyService() error {
 		return err
 	}
 
+	//Stop TLS listener
 	if router.tlsListener != nil {
 		router.tlsListener.Close()
 	}
 
+	//Stop rate limiter
+	if router.rateLimterStop != nil {
+		go func() {
+			// As the rate timer loop has a 1 sec ticker
+			// stop the rate limiter in go routine can prevent
+			// front end from freezing for 1 sec
+			router.rateLimterStop <- true
+		}()
+
+	}
+
+	//Stop TLS redirection (from port 80)
 	if router.tlsRedirectStop != nil {
 		router.tlsRedirectStop <- true
 	}
@@ -256,7 +291,7 @@ func (router *Router) Restart() error {
 			return err
 		}
 
-		time.Sleep(300 * time.Millisecond)
+		time.Sleep(800 * time.Millisecond)
 		// Start the server
 		err = router.StartProxyService()
 		if err != nil {

src/mod/dynamicproxy/endpoints.go

@@ -7,6 +7,8 @@ import (
 
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
+	"imuslab.com/zoraxy/mod/dynamicproxy/loadbalance"
+	"imuslab.com/zoraxy/mod/dynamicproxy/rewrite"
 )
 
 /*
@@ -25,40 +27,36 @@ import (
 
 // Check if a user define header exists in this endpoint, ignore case
 func (ep *ProxyEndpoint) UserDefinedHeaderExists(key string) bool {
-	for _, header := range ep.UserDefinedHeaders {
+	for _, header := range ep.HeaderRewriteRules.UserDefinedHeaders {
 		if strings.EqualFold(header.Key, key) {
 			return true
 		}
 	}
-
 	return false
 }
 
 // Remvoe a user defined header from the list
 func (ep *ProxyEndpoint) RemoveUserDefinedHeader(key string) error {
-	newHeaderList := []*UserDefinedHeader{}
-	for _, header := range ep.UserDefinedHeaders {
+	newHeaderList := []*rewrite.UserDefinedHeader{}
+	for _, header := range ep.HeaderRewriteRules.UserDefinedHeaders {
 		if !strings.EqualFold(header.Key, key) {
 			newHeaderList = append(newHeaderList, header)
 		}
 	}
 
-	ep.UserDefinedHeaders = newHeaderList
+	ep.HeaderRewriteRules.UserDefinedHeaders = newHeaderList
 
 	return nil
 }
 
 // Add a user defined header to the list, duplicates will be automatically removed
-func (ep *ProxyEndpoint) AddUserDefinedHeader(key string, value string) error {
-	if ep.UserDefinedHeaderExists(key) {
-		ep.RemoveUserDefinedHeader(key)
+func (ep *ProxyEndpoint) AddUserDefinedHeader(newHeaderRule *rewrite.UserDefinedHeader) error {
+	if ep.UserDefinedHeaderExists(newHeaderRule.Key) {
+		ep.RemoveUserDefinedHeader(newHeaderRule.Key)
 	}
 
-	ep.UserDefinedHeaders = append(ep.UserDefinedHeaders, &UserDefinedHeader{
-		Key:   cases.Title(language.Und, cases.NoLower).String(key), //e.g. x-proxy-by -> X-Proxy-By
-		Value: value,
-	})
-
+	newHeaderRule.Key = cases.Title(language.Und, cases.NoLower).String(newHeaderRule.Key)
+	ep.HeaderRewriteRules.UserDefinedHeaders = append(ep.HeaderRewriteRules.UserDefinedHeaders, newHeaderRule)
 	return nil
 }
 
@@ -125,9 +123,9 @@ func (ep *ProxyEndpoint) AddVirtualDirectoryRule(vdir *VirtualDirectoryEndpoint)
 		return nil, err
 	}
 
-	if ep.ProxyType == ProxyType_Root {
+	if ep.ProxyType == ProxyTypeRoot {
 		parentRouter.Root = readyRoutingRule
-	} else if ep.ProxyType == ProxyType_Host {
+	} else if ep.ProxyType == ProxyTypeHost {
 		ep.Remove()
 		parentRouter.AddProxyRouteToRuntime(readyRoutingRule)
 	} else {
@@ -137,6 +135,116 @@ func (ep *ProxyEndpoint) AddVirtualDirectoryRule(vdir *VirtualDirectoryEndpoint)
 	return readyRoutingRule, nil
 }
 
+/* Upstream related wrapper functions */
+//Check if there already exists another upstream with identical origin
+func (ep *ProxyEndpoint) UpstreamOriginExists(originURL string) bool {
+	for _, origin := range ep.ActiveOrigins {
+		if origin.OriginIpOrDomain == originURL {
+			return true
+		}
+	}
+	for _, origin := range ep.InactiveOrigins {
+		if origin.OriginIpOrDomain == originURL {
+			return true
+		}
+	}
+	return false
+}
+
+// Get a upstream origin from given origin ip or domain
+func (ep *ProxyEndpoint) GetUpstreamOriginByMatchingIP(originIpOrDomain string) (*loadbalance.Upstream, error) {
+	for _, origin := range ep.ActiveOrigins {
+		if origin.OriginIpOrDomain == originIpOrDomain {
+			return origin, nil
+		}
+	}
+
+	for _, origin := range ep.InactiveOrigins {
+		if origin.OriginIpOrDomain == originIpOrDomain {
+			return origin, nil
+		}
+	}
+	return nil, errors.New("target upstream origin not found")
+}
+
+// Add upstream to endpoint and update it to runtime
+func (ep *ProxyEndpoint) AddUpstreamOrigin(newOrigin *loadbalance.Upstream, activate bool) error {
+	//Check if the upstream already exists
+	if ep.UpstreamOriginExists(newOrigin.OriginIpOrDomain) {
+		return errors.New("upstream with same origin already exists")
+	}
+
+	if activate {
+		//Add it to the active origin list
+		err := newOrigin.StartProxy()
+		if err != nil {
+			return err
+		}
+		ep.ActiveOrigins = append(ep.ActiveOrigins, newOrigin)
+	} else {
+		//Add to inactive origin list
+		ep.InactiveOrigins = append(ep.InactiveOrigins, newOrigin)
+	}
+
+	ep.UpdateToRuntime()
+	return nil
+}
+
+// Remove upstream from endpoint and update it to runtime
+func (ep *ProxyEndpoint) RemoveUpstreamOrigin(originIpOrDomain string) error {
+	//Just to make sure there are no spaces
+	originIpOrDomain = strings.TrimSpace(originIpOrDomain)
+
+	//Check if the upstream already been removed
+	if !ep.UpstreamOriginExists(originIpOrDomain) {
+		//Not exists in the first place
+		return nil
+	}
+
+	newActiveOriginList := []*loadbalance.Upstream{}
+	for _, origin := range ep.ActiveOrigins {
+		if origin.OriginIpOrDomain != originIpOrDomain {
+			newActiveOriginList = append(newActiveOriginList, origin)
+		}
+	}
+
+	newInactiveOriginList := []*loadbalance.Upstream{}
+	for _, origin := range ep.InactiveOrigins {
+		if origin.OriginIpOrDomain != originIpOrDomain {
+			newInactiveOriginList = append(newInactiveOriginList, origin)
+		}
+	}
+	//Ok, set the origin list to the new one
+	ep.ActiveOrigins = newActiveOriginList
+	ep.InactiveOrigins = newInactiveOriginList
+	ep.UpdateToRuntime()
+	return nil
+}
+
+// Check if the proxy endpoint hostname or alias name contains subdomain wildcard
+func (ep *ProxyEndpoint) ContainsWildcardName(skipAliasCheck bool) bool {
+	hostname := ep.RootOrMatchingDomain
+	aliasHostnames := ep.MatchingDomainAlias
+
+	wildcardCheck := func(hostname string) bool {
+		return len(hostname) > 0 && hostname[0] == '*'
+	}
+
+	if wildcardCheck(hostname) {
+		return true
+	}
+
+	if !skipAliasCheck {
+		for _, aliasHostname := range aliasHostnames {
+			if wildcardCheck(aliasHostname) {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 // Create a deep clone object of the proxy endpoint
 // Note the returned object is not activated. Call to prepare function before pushing into runtime
 func (ep *ProxyEndpoint) Clone() *ProxyEndpoint {
@@ -156,5 +264,6 @@ func (ep *ProxyEndpoint) Remove() error {
 // use prepare -> remove -> add if you change anything in the endpoint
 // that effects the proxy routing src / dest
 func (ep *ProxyEndpoint) UpdateToRuntime() {
-	ep.parent.ProxyEndpoints.Store(ep.RootOrMatchingDomain, ep)
+	lookupHostname := strings.ToLower(ep.RootOrMatchingDomain)
+	ep.parent.ProxyEndpoints.Store(lookupHostname, ep)
 }

src/mod/dynamicproxy/loadbalance/loadbalance.go

@@ -0,0 +1,103 @@
+package loadbalance
+
+import (
+	"strings"
+	"sync"
+
+	"github.com/google/uuid"
+	"github.com/gorilla/sessions"
+	"imuslab.com/zoraxy/mod/dynamicproxy/dpcore"
+	"imuslab.com/zoraxy/mod/geodb"
+	"imuslab.com/zoraxy/mod/info/logger"
+)
+
+/*
+	Load Balancer
+
+	Handleing load balance request for upstream destinations
+*/
+
+type Options struct {
+	SystemUUID           string       //Use for the session store
+	UseActiveHealthCheck bool         //Use active health check, default to false
+	Geodb                *geodb.Store //GeoIP resolver for checking incoming request origin country
+	Logger               *logger.Logger
+}
+
+type RouteManager struct {
+	SessionStore           *sessions.CookieStore
+	LoadBalanceMap         sync.Map  //Sync map to store the last load balance state of a given node
+	OnlineStatusMap        sync.Map  //Sync map to store the online status of a given ip address or domain name
+	onlineStatusTickerStop chan bool //Stopping channel for the online status pinger
+	Options                Options   //Options for the load balancer
+}
+
+/* Upstream or Origin Server */
+type Upstream struct {
+	//Upstream Proxy Configs
+	OriginIpOrDomain         string //Target IP address or domain name with port
+	RequireTLS               bool   //Require TLS connection
+	SkipCertValidations      bool   //Set to true to accept self signed certs
+	SkipWebSocketOriginCheck bool   //Skip origin check on websocket upgrade connections
+
+	//Load balancing configs
+	Weight  int //Random weight for round robin, 0 for fallback only
+	MaxConn int //TODO: Maxmium connection to this server, 0 for unlimited
+
+	//currentConnectionCounts atomic.Uint64 //Counter for number of client currently connected
+	proxy *dpcore.ReverseProxy
+}
+
+// Create a new load balancer
+func NewLoadBalancer(options *Options) *RouteManager {
+	if options.SystemUUID == "" {
+		//System UUID not passed in. Use random key
+		options.SystemUUID = uuid.New().String()
+	}
+
+	//Generate a session store for stickySession
+	store := sessions.NewCookieStore([]byte(options.SystemUUID))
+	return &RouteManager{
+		SessionStore:           store,
+		LoadBalanceMap:         sync.Map{},
+		OnlineStatusMap:        sync.Map{},
+		onlineStatusTickerStop: nil,
+		Options:                *options,
+	}
+}
+
+// UpstreamsReady checks if the group of upstreams contains at least one
+// origin server that is ready
+func (m *RouteManager) UpstreamsReady(upstreams []*Upstream) bool {
+	for _, upstream := range upstreams {
+		if upstream.IsReady() {
+			return true
+		}
+	}
+	return false
+}
+
+// String format and convert a list of upstream into a string representations
+func GetUpstreamsAsString(upstreams []*Upstream) string {
+	targets := []string{}
+	for _, upstream := range upstreams {
+		targets = append(targets, upstream.String())
+	}
+	if len(targets) == 0 {
+		//No upstream
+		return "(no upstream config)"
+	}
+	return strings.Join(targets, ", ")
+}
+
+func (m *RouteManager) Close() {
+	if m.onlineStatusTickerStop != nil {
+		m.onlineStatusTickerStop <- true
+	}
+
+}
+
+// Log Println, replace all log.Println or fmt.Println with this
+func (m *RouteManager) println(message string, err error) {
+	m.Options.Logger.PrintAndLog("LoadBalancer", message, err)
+}

src/mod/dynamicproxy/loadbalance/onlineStatus.go

@@ -0,0 +1,39 @@
+package loadbalance
+
+import (
+	"net/http"
+	"time"
+)
+
+// Return the last ping status to see if the target is online
+func (m *RouteManager) IsTargetOnline(matchingDomainOrIp string) bool {
+	value, ok := m.LoadBalanceMap.Load(matchingDomainOrIp)
+	if !ok {
+		return false
+	}
+
+	isOnline, ok := value.(bool)
+	return ok && isOnline
+}
+
+// Ping a target to see if it is online
+func PingTarget(targetMatchingDomainOrIp string, requireTLS bool) bool {
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	url := targetMatchingDomainOrIp
+	if requireTLS {
+		url = "https://" + url
+	} else {
+		url = "http://" + url
+	}
+
+	resp, err := client.Get(url)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	return resp.StatusCode >= 200 && resp.StatusCode <= 600
+}

src/mod/dynamicproxy/loadbalance/originPicker.go

@@ -0,0 +1,177 @@
+package loadbalance
+
+import (
+	"errors"
+	"math/rand"
+	"net/http"
+)
+
+/*
+	Origin Picker
+
+	This script contains the code to pick the best origin
+	by this request.
+*/
+
+// GetRequestUpstreamTarget return the upstream target where this
+// request should be routed
+func (m *RouteManager) GetRequestUpstreamTarget(w http.ResponseWriter, r *http.Request, origins []*Upstream, useStickySession bool) (*Upstream, error) {
+	if len(origins) == 0 {
+		return nil, errors.New("no upstream is defined for this host")
+	}
+	var targetOrigin = origins[0]
+	if useStickySession {
+		//Use stick session, check which origins this request previously used
+		targetOriginId, err := m.getSessionHandler(r, origins)
+		if err != nil {
+			//No valid session found. Assign a new upstream
+			targetOrigin, index, err := getRandomUpstreamByWeight(origins)
+			if err != nil {
+				m.println("Unable to get random upstream", err)
+				targetOrigin = origins[0]
+				index = 0
+			}
+			m.setSessionHandler(w, r, targetOrigin.OriginIpOrDomain, index)
+			return targetOrigin, nil
+		}
+
+		//Valid session found. Resume the previous session
+		return origins[targetOriginId], nil
+	} else {
+		//Do not use stick session. Get a random one
+		var err error
+		targetOrigin, _, err = getRandomUpstreamByWeight(origins)
+		if err != nil {
+			m.println("Failed to get next origin", err)
+			targetOrigin = origins[0]
+		}
+
+	}
+
+	//fmt.Println("DEBUG: Picking origin " + targetOrigin.OriginIpOrDomain)
+	return targetOrigin, nil
+}
+
+/* Features related to session access */
+//Set a new origin for this connection by session
+func (m *RouteManager) setSessionHandler(w http.ResponseWriter, r *http.Request, originIpOrDomain string, index int) error {
+	session, err := m.SessionStore.Get(r, "STICKYSESSION")
+	if err != nil {
+		return err
+	}
+	session.Values["zr_sid_origin"] = originIpOrDomain
+	session.Values["zr_sid_index"] = index
+	session.Options.MaxAge = 86400 //1 day
+	session.Options.Path = "/"
+	err = session.Save(r, w)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Get the previous connected origin from session
+func (m *RouteManager) getSessionHandler(r *http.Request, upstreams []*Upstream) (int, error) {
+	// Get existing session
+	session, err := m.SessionStore.Get(r, "STICKYSESSION")
+	if err != nil {
+		return -1, err
+	}
+
+	// Retrieve session values for origin
+	originDomainRaw := session.Values["zr_sid_origin"]
+	originIDRaw := session.Values["zr_sid_index"]
+
+	if originDomainRaw == nil || originIDRaw == nil {
+		return -1, errors.New("no session has been set")
+	}
+	originDomain := originDomainRaw.(string)
+	originID := originIDRaw.(int)
+
+	//Check if it has been modified
+	if len(upstreams) < originID || upstreams[originID].OriginIpOrDomain != originDomain {
+		//Mismatch or upstreams has been updated
+		return -1, errors.New("upstreams has been changed")
+	}
+
+	return originID, nil
+}
+
+/* Functions related to random upstream picking */
+// Get a random upstream by the weights defined in Upstream struct, return the upstream, index value and any error
+func getRandomUpstreamByWeight(upstreams []*Upstream) (*Upstream, int, error) {
+	// If there is only one upstream, return it
+	if len(upstreams) == 1 {
+		return upstreams[0], 0, nil
+	}
+
+	// Preserve the index with upstreams
+	type upstreamWithIndex struct {
+		Upstream *Upstream
+		Index    int
+	}
+
+	// Calculate total weight for upstreams with weight > 0
+	totalWeight := 0
+	fallbackUpstreams := make([]upstreamWithIndex, 0, len(upstreams))
+
+	for index, upstream := range upstreams {
+		if upstream.Weight > 0 {
+			totalWeight += upstream.Weight
+		} else {
+			// Collect fallback upstreams
+			fallbackUpstreams = append(fallbackUpstreams, upstreamWithIndex{upstream, index})
+		}
+	}
+
+	// If there are no upstreams with weight > 0, return a fallback upstream if available
+	if totalWeight == 0 {
+		if len(fallbackUpstreams) > 0 {
+			// Randomly select one of the fallback upstreams
+			randIndex := rand.Intn(len(fallbackUpstreams))
+			return fallbackUpstreams[randIndex].Upstream, fallbackUpstreams[randIndex].Index, nil
+		}
+		// No upstreams available at all
+		return nil, -1, errors.New("no valid upstream servers available")
+	}
+
+	// Random weight between 0 and total weight
+	randomWeight := rand.Intn(totalWeight)
+
+	// Select an upstream based on the random weight
+	for index, upstream := range upstreams {
+		if upstream.Weight > 0 { // Only consider upstreams with weight > 0
+			if randomWeight < upstream.Weight {
+				// Return the selected upstream and its index
+				return upstream, index, nil
+			}
+			randomWeight -= upstream.Weight
+		}
+	}
+
+	// If we reach here, it means we should return a fallback upstream if available
+	if len(fallbackUpstreams) > 0 {
+		randIndex := rand.Intn(len(fallbackUpstreams))
+		return fallbackUpstreams[randIndex].Upstream, fallbackUpstreams[randIndex].Index, nil
+	}
+
+	return nil, -1, errors.New("failed to pick an upstream origin server")
+}
+
+// IntRange returns a random integer in the range from min to max.
+/*
+func intRange(min, max int) (int, error) {
+	var result int
+	switch {
+	case min > max:
+		// Fail with error
+		return result, errors.New("min is greater than max")
+	case max == min:
+		result = max
+	case max > min:
+		b := rand.Intn(max-min) + min
+		result = min + int(b)
+	}
+	return result, nil
+}
+*/

src/mod/dynamicproxy/loadbalance/originPicker_test.go

@@ -0,0 +1,100 @@
+package loadbalance
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"testing"
+	"time"
+)
+
+// func getRandomUpstreamByWeight(upstreams []*Upstream) (*Upstream, int, error) { ... }
+func TestRandomUpstreamSelection(t *testing.T) {
+	rand.Seed(time.Now().UnixNano()) // Seed for randomness
+
+	// Define some test upstreams
+	upstreams := []*Upstream{
+		{
+			OriginIpOrDomain:         "192.168.1.1:8080",
+			RequireTLS:               false,
+			SkipCertValidations:      false,
+			SkipWebSocketOriginCheck: false,
+			Weight:                   1,
+			MaxConn:                  0, // No connection limit for now
+		},
+		{
+			OriginIpOrDomain:         "192.168.1.2:8080",
+			RequireTLS:               false,
+			SkipCertValidations:      false,
+			SkipWebSocketOriginCheck: false,
+			Weight:                   1,
+			MaxConn:                  0,
+		},
+		{
+			OriginIpOrDomain:         "192.168.1.3:8080",
+			RequireTLS:               true,
+			SkipCertValidations:      true,
+			SkipWebSocketOriginCheck: true,
+			Weight:                   1,
+			MaxConn:                  0,
+		},
+		{
+			OriginIpOrDomain:         "192.168.1.4:8080",
+			RequireTLS:               true,
+			SkipCertValidations:      true,
+			SkipWebSocketOriginCheck: true,
+			Weight:                   1,
+			MaxConn:                  0,
+		},
+	}
+
+	// Track how many times each upstream is selected
+	selectionCount := make(map[string]int)
+	totalPicks := 10000 // Number of times to call getRandomUpstreamByWeight
+	//expectedPickCount := totalPicks / len(upstreams) // Ideal count for each upstream
+
+	// Pick upstreams and record their selection count
+	for i := 0; i < totalPicks; i++ {
+		upstream, _, err := getRandomUpstreamByWeight(upstreams)
+		if err != nil {
+			t.Fatalf("Error getting random upstream: %v", err)
+		}
+		selectionCount[upstream.OriginIpOrDomain]++
+	}
+
+	// Condition 1: Ensure every upstream has been picked at least once
+	for _, upstream := range upstreams {
+		if selectionCount[upstream.OriginIpOrDomain] == 0 {
+			t.Errorf("Upstream %s was never selected", upstream.OriginIpOrDomain)
+		}
+	}
+
+	// Condition 2: Check that the distribution is within 1-2 standard deviations
+	counts := make([]float64, len(upstreams))
+	for i, upstream := range upstreams {
+		counts[i] = float64(selectionCount[upstream.OriginIpOrDomain])
+	}
+
+	mean := float64(totalPicks) / float64(len(upstreams))
+	stddev := calculateStdDev(counts, mean)
+
+	tolerance := 2 * stddev // Allowing up to 2 standard deviations
+	for i, count := range counts {
+		if math.Abs(count-mean) > tolerance {
+			t.Errorf("Selection of upstream %s is outside acceptable range: %v picks (mean: %v, stddev: %v)", upstreams[i].OriginIpOrDomain, count, mean, stddev)
+		}
+	}
+
+	fmt.Println("Selection count:", selectionCount)
+	fmt.Printf("Mean: %.2f, StdDev: %.2f\n", mean, stddev)
+}
+
+// Helper function to calculate standard deviation
+func calculateStdDev(data []float64, mean float64) float64 {
+	var sumOfSquares float64
+	for _, value := range data {
+		sumOfSquares += (value - mean) * (value - mean)
+	}
+	variance := sumOfSquares / float64(len(data))
+	return math.Sqrt(variance)
+}

src/mod/dynamicproxy/loadbalance/upstream.go

@@ -0,0 +1,77 @@
+package loadbalance
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"imuslab.com/zoraxy/mod/dynamicproxy/dpcore"
+)
+
+// StartProxy create and start a HTTP proxy using dpcore
+// Example of webProxyEndpoint: https://example.com:443 or http://192.168.1.100:8080
+func (u *Upstream) StartProxy() error {
+	//Filter the tailing slash if any
+	domain := u.OriginIpOrDomain
+	if len(domain) == 0 {
+		return errors.New("invalid endpoint config")
+	}
+	if domain[len(domain)-1:] == "/" {
+		domain = domain[:len(domain)-1]
+	}
+
+	if !strings.HasPrefix("http://", domain) && !strings.HasPrefix("https://", domain) {
+		//TLS is not hardcoded in proxy target domain
+		if u.RequireTLS {
+			domain = "https://" + domain
+		} else {
+			domain = "http://" + domain
+		}
+	}
+
+	//Create a new proxy agent for this upstream
+	path, err := url.Parse(domain)
+	if err != nil {
+		return err
+	}
+
+	proxy := dpcore.NewDynamicProxyCore(path, "", &dpcore.DpcoreOptions{
+		IgnoreTLSVerification: u.SkipCertValidations,
+		FlushInterval:         100 * time.Millisecond,
+	})
+
+	u.proxy = proxy
+	return nil
+}
+
+// IsReady return the proxy ready state of the upstream server
+// Return false if StartProxy() is not called on this upstream before
+func (u *Upstream) IsReady() bool {
+	return u.proxy != nil
+}
+
+// Clone return a new deep copy object of the identical upstream
+func (u *Upstream) Clone() *Upstream {
+	newUpstream := Upstream{}
+	js, _ := json.Marshal(u)
+	json.Unmarshal(js, &newUpstream)
+	return &newUpstream
+}
+
+// ServeHTTP uses this upstream proxy router to route the current request, return the status code and error if any
+func (u *Upstream) ServeHTTP(w http.ResponseWriter, r *http.Request, rrr *dpcore.ResponseRewriteRuleSet) (int, error) {
+	//Auto rewrite to upstream origin if not set
+	if rrr.ProxyDomain == "" {
+		rrr.ProxyDomain = u.OriginIpOrDomain
+	}
+
+	return u.proxy.ServeHTTP(w, r, rrr)
+}
+
+// String return the string representations of endpoints in this upstream
+func (u *Upstream) String() string {
+	return u.OriginIpOrDomain
+}

src/mod/dynamicproxy/permissionpolicy/permissionpolicy.go

@@ -108,13 +108,8 @@ func GetDefaultPermissionPolicy() *PermissionsPolicy {
 	}
 }
 
-// InjectPermissionPolicyHeader inject the permission policy into headers
-func InjectPermissionPolicyHeader(w http.ResponseWriter, policy *PermissionsPolicy) {
-	//Keep the original Permission Policy if exists, or there are no policy given
-	if policy == nil || w.Header().Get("Permissions-Policy") != "" {
-		return
-	}
-
+// ToKeyValueHeader convert a permission policy struct into a key value string header
+func (policy *PermissionsPolicy) ToKeyValueHeader() []string {
 	policyHeader := []string{}
 
 	// Helper function to add policy directives
@@ -187,7 +182,16 @@ func InjectPermissionPolicyHeader(w http.ResponseWriter, policy *PermissionsPoli
 
 	// Join the directives and set the header
 	policyHeaderValue := strings.Join(policyHeader, ", ")
-
-	//Inject the new policy into the header
-	w.Header().Set("Permissions-Policy", policyHeaderValue)
+	return []string{"Permissions-Policy", policyHeaderValue}
+}
+
+// InjectPermissionPolicyHeader inject the permission policy into headers
+func InjectPermissionPolicyHeader(w http.ResponseWriter, policy *PermissionsPolicy) {
+	//Keep the original Permission Policy if exists, or there are no policy given
+	if policy == nil || w.Header().Get("Permissions-Policy") != "" {
+		return
+	}
+	headerKV := policy.ToKeyValueHeader()
+	//Inject the new policy into the header
+	w.Header().Set(headerKV[0], headerKV[1])
 }

src/mod/dynamicproxy/proxyRequestHandler.go

@@ -11,11 +11,13 @@ import (
 	"strings"
 
 	"imuslab.com/zoraxy/mod/dynamicproxy/dpcore"
+	"imuslab.com/zoraxy/mod/dynamicproxy/rewrite"
 	"imuslab.com/zoraxy/mod/netutils"
 	"imuslab.com/zoraxy/mod/statistic"
 	"imuslab.com/zoraxy/mod/websocketproxy"
 )
 
+// Check if the request URI matches any of the proxy endpoint
 func (router *Router) getTargetProxyEndpointFromRequestURI(requestURI string) *ProxyEndpoint {
 	var targetProxyEndpoint *ProxyEndpoint = nil
 	router.ProxyEndpoints.Range(func(key, value interface{}) bool {
@@ -30,8 +32,10 @@ func (router *Router) getTargetProxyEndpointFromRequestURI(requestURI string) *P
 	return targetProxyEndpoint
 }
 
+// Get the proxy endpoint from hostname, which might includes checking of wildcard certificates
 func (router *Router) getProxyEndpointFromHostname(hostname string) *ProxyEndpoint {
 	var targetSubdomainEndpoint *ProxyEndpoint = nil
+	hostname = strings.ToLower(hostname)
 	ep, ok := router.ProxyEndpoints.Load(hostname)
 	if ok {
 		//Exact hit
@@ -111,18 +115,21 @@ func (h *ProxyHandler) hostRequest(w http.ResponseWriter, r *http.Request, targe
 	r.Header.Set("X-Forwarded-Host", r.Host)
 	r.Header.Set("X-Forwarded-Server", "zoraxy-"+h.Parent.Option.HostUUID)
 
-	//Inject custom headers
-	if len(target.UserDefinedHeaders) > 0 {
-		for _, customHeader := range target.UserDefinedHeaders {
-			r.Header.Set(customHeader.Key, customHeader.Value)
-		}
+	/* Load balancing */
+	selectedUpstream, err := h.Parent.loadBalancer.GetRequestUpstreamTarget(w, r, target.ActiveOrigins, target.UseStickySession)
+	if err != nil {
+		http.ServeFile(w, r, "./web/rperror.html")
+		h.Parent.Option.Logger.PrintAndLog("proxy", "Failed to assign an upstream for this request", err)
+		h.Parent.logRequest(r, false, 521, "subdomain-http", r.URL.Hostname())
+		return
 	}
 
+	/* WebSocket automatic proxy */
 	requestURL := r.URL.String()
 	if r.Header["Upgrade"] != nil && strings.ToLower(r.Header["Upgrade"][0]) == "websocket" {
 		//Handle WebSocket request. Forward the custom Upgrade header and rewrite origin
 		r.Header.Set("Zr-Origin-Upgrade", "websocket")
-		wsRedirectionEndpoint := target.Domain
+		wsRedirectionEndpoint := selectedUpstream.OriginIpOrDomain
 		if wsRedirectionEndpoint[len(wsRedirectionEndpoint)-1:] != "/" {
 			//Append / to the end of the redirection endpoint if not exists
 			wsRedirectionEndpoint = wsRedirectionEndpoint + "/"
@@ -132,13 +139,16 @@ func (h *ProxyHandler) hostRequest(w http.ResponseWriter, r *http.Request, targe
 			requestURL = requestURL[1:]
 		}
 		u, _ := url.Parse("ws://" + wsRedirectionEndpoint + requestURL)
-		if target.RequireTLS {
+		if selectedUpstream.RequireTLS {
 			u, _ = url.Parse("wss://" + wsRedirectionEndpoint + requestURL)
 		}
-		h.logRequest(r, true, 101, "subdomain-websocket", target.Domain)
+		h.Parent.logRequest(r, true, 101, "host-websocket", selectedUpstream.OriginIpOrDomain)
 		wspHandler := websocketproxy.NewProxy(u, websocketproxy.Options{
-			SkipTLSValidation: target.SkipCertValidations,
-			SkipOriginCheck:   target.SkipWebSocketOriginCheck,
+			SkipTLSValidation:  selectedUpstream.SkipCertValidations,
+			SkipOriginCheck:    selectedUpstream.SkipWebSocketOriginCheck,
+			CopyAllHeaders:     true,
+			UserDefinedHeaders: target.HeaderRewriteRules.UserDefinedHeaders,
+			Logger:             h.Parent.Option.Logger,
 		})
 		wspHandler.ServeHTTP(w, r)
 		return
@@ -152,28 +162,45 @@ func (h *ProxyHandler) hostRequest(w http.ResponseWriter, r *http.Request, targe
 		r.URL, _ = url.Parse(originalHostHeader)
 	}
 
-	err := target.proxy.ServeHTTP(w, r, &dpcore.ResponseRewriteRuleSet{
-		ProxyDomain:  target.Domain,
-		OriginalHost: originalHostHeader,
-		UseTLS:       target.RequireTLS,
-		NoCache:      h.Parent.Option.NoCache,
-		PathPrefix:   "",
+	//Populate the user-defined headers with the values from the request
+	rewrittenUserDefinedHeaders := rewrite.PopulateRequestHeaderVariables(r, target.HeaderRewriteRules.UserDefinedHeaders)
+
+	//Build downstream and upstream header rules
+	upstreamHeaders, downstreamHeaders := rewrite.SplitUpDownStreamHeaders(&rewrite.HeaderRewriteOptions{
+		UserDefinedHeaders:           rewrittenUserDefinedHeaders,
+		HSTSMaxAge:                   target.HeaderRewriteRules.HSTSMaxAge,
+		HSTSIncludeSubdomains:        target.ContainsWildcardName(true),
+		EnablePermissionPolicyHeader: target.HeaderRewriteRules.EnablePermissionPolicyHeader,
+		PermissionPolicy:             target.HeaderRewriteRules.PermissionPolicy,
+	})
+
+	//Handle the request reverse proxy
+	statusCode, err := selectedUpstream.ServeHTTP(w, r, &dpcore.ResponseRewriteRuleSet{
+		ProxyDomain:         selectedUpstream.OriginIpOrDomain,
+		OriginalHost:        originalHostHeader,
+		UseTLS:              selectedUpstream.RequireTLS,
+		NoCache:             h.Parent.Option.NoCache,
+		PathPrefix:          "",
+		UpstreamHeaders:     upstreamHeaders,
+		DownstreamHeaders:   downstreamHeaders,
+		HostHeaderOverwrite: target.HeaderRewriteRules.RequestHostOverwrite,
+		NoRemoveHopByHop:    target.HeaderRewriteRules.DisableHopByHopHeaderRemoval,
+		Version:             target.parent.Option.HostVersion,
 	})
 
 	var dnsError *net.DNSError
 	if err != nil {
 		if errors.As(err, &dnsError) {
 			http.ServeFile(w, r, "./web/hosterror.html")
-			log.Println(err.Error())
-			h.logRequest(r, false, 404, "subdomain-http", target.Domain)
+			h.Parent.logRequest(r, false, 404, "host-http", r.URL.Hostname())
 		} else {
 			http.ServeFile(w, r, "./web/rperror.html")
-			log.Println(err.Error())
-			h.logRequest(r, false, 521, "subdomain-http", target.Domain)
+			//TODO: Take this upstream offline automatically
+			h.Parent.logRequest(r, false, 521, "host-http", r.URL.Hostname())
 		}
 	}
 
-	h.logRequest(r, true, 200, "subdomain-http", target.Domain)
+	h.Parent.logRequest(r, true, statusCode, "host-http", r.URL.Hostname())
 }
 
 // Handle vdir type request
@@ -184,13 +211,6 @@ func (h *ProxyHandler) vdirRequest(w http.ResponseWriter, r *http.Request, targe
 	r.Header.Set("X-Forwarded-Host", r.Host)
 	r.Header.Set("X-Forwarded-Server", "zoraxy-"+h.Parent.Option.HostUUID)
 
-	//Inject custom headers
-	if len(target.parent.UserDefinedHeaders) > 0 {
-		for _, customHeader := range target.parent.UserDefinedHeaders {
-			r.Header.Set(customHeader.Key, customHeader.Value)
-		}
-	}
-
 	if r.Header["Upgrade"] != nil && strings.ToLower(r.Header["Upgrade"][0]) == "websocket" {
 		//Handle WebSocket request. Forward the custom Upgrade header and rewrite origin
 		r.Header.Set("Zr-Origin-Upgrade", "websocket")
@@ -202,10 +222,13 @@ func (h *ProxyHandler) vdirRequest(w http.ResponseWriter, r *http.Request, targe
 		if target.RequireTLS {
 			u, _ = url.Parse("wss://" + wsRedirectionEndpoint + r.URL.String())
 		}
-		h.logRequest(r, true, 101, "vdir-websocket", target.Domain)
+		h.Parent.logRequest(r, true, 101, "vdir-websocket", target.Domain)
 		wspHandler := websocketproxy.NewProxy(u, websocketproxy.Options{
-			SkipTLSValidation: target.SkipCertValidations,
-			SkipOriginCheck:   target.parent.SkipWebSocketOriginCheck,
+			SkipTLSValidation:  target.SkipCertValidations,
+			SkipOriginCheck:    true, //You should not use websocket via virtual directory. But keep this to true for compatibility
+			CopyAllHeaders:     true,
+			UserDefinedHeaders: target.parent.HeaderRewriteRules.UserDefinedHeaders,
+			Logger:             h.Parent.Option.Logger,
 		})
 		wspHandler.ServeHTTP(w, r)
 		return
@@ -219,11 +242,28 @@ func (h *ProxyHandler) vdirRequest(w http.ResponseWriter, r *http.Request, targe
 		r.URL, _ = url.Parse(originalHostHeader)
 	}
 
-	err := target.proxy.ServeHTTP(w, r, &dpcore.ResponseRewriteRuleSet{
-		ProxyDomain:  target.Domain,
-		OriginalHost: originalHostHeader,
-		UseTLS:       target.RequireTLS,
-		PathPrefix:   target.MatchingPath,
+	//Populate the user-defined headers with the values from the request
+	rewrittenUserDefinedHeaders := rewrite.PopulateRequestHeaderVariables(r, target.parent.HeaderRewriteRules.UserDefinedHeaders)
+
+	//Build downstream and upstream header rules, use the parent (subdomain) endpoint's headers
+	upstreamHeaders, downstreamHeaders := rewrite.SplitUpDownStreamHeaders(&rewrite.HeaderRewriteOptions{
+		UserDefinedHeaders:           rewrittenUserDefinedHeaders,
+		HSTSMaxAge:                   target.parent.HeaderRewriteRules.HSTSMaxAge,
+		HSTSIncludeSubdomains:        target.parent.ContainsWildcardName(true),
+		EnablePermissionPolicyHeader: target.parent.HeaderRewriteRules.EnablePermissionPolicyHeader,
+		PermissionPolicy:             target.parent.HeaderRewriteRules.PermissionPolicy,
+	})
+
+	//Handle the virtual directory reverse proxy request
+	statusCode, err := target.proxy.ServeHTTP(w, r, &dpcore.ResponseRewriteRuleSet{
+		ProxyDomain:         target.Domain,
+		OriginalHost:        originalHostHeader,
+		UseTLS:              target.RequireTLS,
+		PathPrefix:          target.MatchingPath,
+		UpstreamHeaders:     upstreamHeaders,
+		DownstreamHeaders:   downstreamHeaders,
+		HostHeaderOverwrite: target.parent.HeaderRewriteRules.RequestHostOverwrite,
+		Version:             target.parent.parent.Option.HostVersion,
 	})
 
 	var dnsError *net.DNSError
@@ -231,23 +271,24 @@ func (h *ProxyHandler) vdirRequest(w http.ResponseWriter, r *http.Request, targe
 		if errors.As(err, &dnsError) {
 			http.ServeFile(w, r, "./web/hosterror.html")
 			log.Println(err.Error())
-			h.logRequest(r, false, 404, "vdir-http", target.Domain)
+			h.Parent.logRequest(r, false, 404, "vdir-http", target.Domain)
 		} else {
 			http.ServeFile(w, r, "./web/rperror.html")
 			log.Println(err.Error())
-			h.logRequest(r, false, 521, "vdir-http", target.Domain)
+			h.Parent.logRequest(r, false, 521, "vdir-http", target.Domain)
 		}
 	}
-	h.logRequest(r, true, 200, "vdir-http", target.Domain)
+	h.Parent.logRequest(r, true, statusCode, "vdir-http", target.Domain)
 
 }
 
-func (h *ProxyHandler) logRequest(r *http.Request, succ bool, statusCode int, forwardType string, target string) {
-	if h.Parent.Option.StatisticCollector != nil {
+// This logger collect data for the statistical analysis. For log to file logger, check the Logger and LogHTTPRequest handler
+func (router *Router) logRequest(r *http.Request, succ bool, statusCode int, forwardType string, target string) {
+	if router.Option.StatisticCollector != nil {
 		go func() {
 			requestInfo := statistic.RequestInfo{
 				IpAddr:                        netutils.GetRequesterIP(r),
-				RequestOriginalCountryISOCode: h.Parent.Option.GeodbStore.GetRequesterCountryISOCode(r),
+				RequestOriginalCountryISOCode: router.Option.GeodbStore.GetRequesterCountryISOCode(r),
 				Succ:                          succ,
 				StatusCode:                    statusCode,
 				ForwardType:                   forwardType,
@@ -256,7 +297,8 @@ func (h *ProxyHandler) logRequest(r *http.Request, succ bool, statusCode int, fo
 				RequestURL:                    r.Host + r.RequestURI,
 				Target:                        target,
 			}
-			h.Parent.Option.StatisticCollector.RecordRequest(requestInfo)
+			router.Option.StatisticCollector.RecordRequest(requestInfo)
 		}()
 	}
+	router.Option.Logger.LogHTTPRequest(r, forwardType, statusCode)
 }

src/mod/dynamicproxy/ratelimit.go

@@ -0,0 +1,119 @@
+package dynamicproxy
+
+import (
+	"errors"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// IpTable is a rate limiter implementation using sync.Map with atomic int64
+type RequestCountPerIpTable struct {
+	table sync.Map
+}
+
+// Increment the count of requests for a given IP
+func (t *RequestCountPerIpTable) Increment(ip string) {
+	v, _ := t.table.LoadOrStore(ip, new(int64))
+	atomic.AddInt64(v.(*int64), 1)
+}
+
+// Check if the IP is in the table and if it is, check if the count is less than the limit
+func (t *RequestCountPerIpTable) Exceeded(ip string, limit int64) bool {
+	v, ok := t.table.Load(ip)
+	if !ok {
+		return false
+	}
+	count := atomic.LoadInt64(v.(*int64))
+	return count >= limit
+}
+
+// Get the count of requests for a given IP
+func (t *RequestCountPerIpTable) GetCount(ip string) int64 {
+	v, ok := t.table.Load(ip)
+	if !ok {
+		return 0
+	}
+	return atomic.LoadInt64(v.(*int64))
+}
+
+// Clear the IP table
+func (t *RequestCountPerIpTable) Clear() {
+	t.table.Range(func(key, value interface{}) bool {
+		t.table.Delete(key)
+		return true
+	})
+}
+
+func (h *ProxyHandler) handleRateLimitRouting(w http.ResponseWriter, r *http.Request, pe *ProxyEndpoint) error {
+	err := h.Parent.handleRateLimit(w, r, pe)
+	if err != nil {
+		h.Parent.logRequest(r, false, 429, "ratelimit", r.URL.Hostname())
+	}
+	return err
+}
+
+func (router *Router) handleRateLimit(w http.ResponseWriter, r *http.Request, pe *ProxyEndpoint) error {
+	//Get the real client-ip from request header
+	clientIP := r.RemoteAddr
+	if r.Header.Get("X-Real-Ip") == "" {
+		CF_Connecting_IP := r.Header.Get("CF-Connecting-IP")
+		Fastly_Client_IP := r.Header.Get("Fastly-Client-IP")
+		if CF_Connecting_IP != "" {
+			//Use CF Connecting IP
+			clientIP = CF_Connecting_IP
+		} else if Fastly_Client_IP != "" {
+			//Use Fastly Client IP
+			clientIP = Fastly_Client_IP
+		} else {
+			ips := strings.Split(clientIP, ",")
+			if len(ips) > 0 {
+				clientIP = strings.TrimSpace(ips[0])
+			}
+		}
+	}
+
+	ip, _, err := net.SplitHostPort(clientIP)
+	if err != nil {
+		//Default allow passthrough on error
+		return nil
+	}
+
+	router.rateLimitCounter.Increment(ip)
+
+	if router.rateLimitCounter.Exceeded(ip, int64(pe.RateLimit)) {
+		w.WriteHeader(429)
+		return errors.New("rate limit exceeded")
+	}
+
+	// log.Println("Rate limit check", ip, ipTable.GetCount(ip))
+
+	return nil
+}
+
+// Start the ticker routine for reseting the rate limit counter every seconds
+func (r *Router) startRateLimterCounterResetTicker() error {
+	if r.rateLimterStop != nil {
+		return errors.New("another rate limiter ticker already running")
+	}
+	tickerStopChan := make(chan bool)
+	r.rateLimterStop = tickerStopChan
+
+	counterResetTicker := time.NewTicker(1 * time.Second)
+	go func() {
+		for {
+			select {
+			case <-tickerStopChan:
+				r.rateLimterStop = nil
+				return
+			case <-counterResetTicker.C:
+				r.rateLimitCounter.Clear()
+			}
+		}
+	}()
+
+	return nil
+}

src/mod/dynamicproxy/redirection/handler.go

@@ -1,7 +1,7 @@
 package redirection
 
 import (
-	"log"
+	"errors"
 	"net/http"
 	"strings"
 )
@@ -52,7 +52,7 @@ func (t *RuleTable) HandleRedirect(w http.ResponseWriter, r *http.Request) int {
 		//Invalid usage
 		w.WriteHeader(http.StatusInternalServerError)
 		w.Write([]byte("500 - Internal Server Error"))
-		log.Println("Target request URL do not have matching redirect rule. Check with IsRedirectable before calling HandleRedirect!")
+		t.log("Target request URL do not have matching redirect rule. Check with IsRedirectable before calling HandleRedirect!", errors.New("invalid usage"))
 		return 500
 	}
 }

src/mod/dynamicproxy/redirection/redirection.go

@@ -30,11 +30,12 @@ type RedirectRules struct {
 	StatusCode       int    //Status Code for redirection
 }
 
-func NewRuleTable(configPath string, allowRegex bool) (*RuleTable, error) {
+func NewRuleTable(configPath string, allowRegex bool, logger *logger.Logger) (*RuleTable, error) {
 	thisRuleTable := RuleTable{
 		rules:      sync.Map{},
 		configPath: configPath,
 		AllowRegex: allowRegex,
+		Logger:     logger,
 	}
 	//Load all the rules from the config path
 	if !utils.FileExists(configPath) {
@@ -67,7 +68,7 @@ func NewRuleTable(configPath string, allowRegex bool) (*RuleTable, error) {
 
 	//Map the rules into the sync map
 	for _, rule := range rules {
-		log.Println("Redirection rule added: " + rule.RedirectURL + " -> " + rule.TargetURL)
+		thisRuleTable.log("Redirection rule added: "+rule.RedirectURL+" -> "+rule.TargetURL, nil)
 		thisRuleTable.rules.Store(rule.RedirectURL, rule)
 	}
 
@@ -92,7 +93,7 @@ func (t *RuleTable) AddRedirectRule(redirectURL string, destURL string, forwardP
 	// Create a new file for writing the JSON data
 	file, err := os.Create(filepath)
 	if err != nil {
-		log.Printf("Error creating file %s: %s", filepath, err)
+		t.log("Error creating file "+filepath, err)
 		return err
 	}
 	defer file.Close()
@@ -100,7 +101,7 @@ func (t *RuleTable) AddRedirectRule(redirectURL string, destURL string, forwardP
 	// Encode the RedirectRules object to JSON and write it to the file
 	err = json.NewEncoder(file).Encode(newRule)
 	if err != nil {
-		log.Printf("Error encoding JSON to file %s: %s", filepath, err)
+		t.log("Error encoding JSON to file "+filepath, err)
 		return err
 	}
 
@@ -125,7 +126,7 @@ func (t *RuleTable) DeleteRedirectRule(redirectURL string) error {
 
 	// Delete the file
 	if err := os.Remove(filepath); err != nil {
-		log.Printf("Error deleting file %s: %s", filepath, err)
+		t.log("Error deleting file "+filepath, err)
 		return err
 	}
 
@@ -194,6 +195,6 @@ func (t *RuleTable) log(message string, err error) {
 			log.Println("[Redirect] " + message + ": " + err.Error())
 		}
 	} else {
-		t.Logger.PrintAndLog("Redirect", message, err)
+		t.Logger.PrintAndLog("redirect", message, err)
 	}
 }

src/mod/dynamicproxy/rewrite/headervars.go

@@ -0,0 +1,63 @@
+package rewrite
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+)
+
+// GetHeaderVariableValuesFromRequest returns a map of header variables and their values
+// note that variables behavior is not exactly identical to nginx variables
+func GetHeaderVariableValuesFromRequest(r *http.Request) map[string]string {
+	vars := make(map[string]string)
+
+	// Request-specific variables
+	vars["$host"] = r.Host
+	vars["$remote_addr"] = r.RemoteAddr
+	vars["$request_uri"] = r.RequestURI
+	vars["$request_method"] = r.Method
+	vars["$content_length"] = fmt.Sprintf("%d", r.ContentLength)
+	vars["$content_type"] = r.Header.Get("Content-Type")
+
+	// Parsed URI elements
+	vars["$uri"] = r.URL.Path
+	vars["$args"] = r.URL.RawQuery
+	vars["$scheme"] = r.URL.Scheme
+	vars["$query_string"] = r.URL.RawQuery
+
+	// User agent and referer
+	vars["$http_user_agent"] = r.UserAgent()
+	vars["$http_referer"] = r.Referer()
+
+	return vars
+}
+
+// CustomHeadersIncludeDynamicVariables checks if the user-defined headers contain dynamic variables
+// use for early exit when processing the headers
+func CustomHeadersIncludeDynamicVariables(userDefinedHeaders []*UserDefinedHeader) bool {
+	for _, header := range userDefinedHeaders {
+		if strings.Contains(header.Value, "$") {
+			return true
+		}
+	}
+	return false
+}
+
+// PopulateRequestHeaderVariables populates the user-defined headers with the values from the request
+func PopulateRequestHeaderVariables(r *http.Request, userDefinedHeaders []*UserDefinedHeader) []*UserDefinedHeader {
+	if !CustomHeadersIncludeDynamicVariables(userDefinedHeaders) {
+		// Early exit if there are no dynamic variables
+		return userDefinedHeaders
+	}
+	vars := GetHeaderVariableValuesFromRequest(r)
+	populatedHeaders := []*UserDefinedHeader{}
+	// Populate the user-defined headers with the values from the request
+	for _, header := range userDefinedHeaders {
+		thisHeader := header.Copy()
+		for key, value := range vars {
+			thisHeader.Value = strings.ReplaceAll(thisHeader.Value, key, value)
+		}
+		populatedHeaders = append(populatedHeaders, thisHeader)
+	}
+	return populatedHeaders
+}

src/mod/dynamicproxy/rewrite/headervars_test.go

@@ -0,0 +1,172 @@
+package rewrite
+
+import (
+	"net/http/httptest"
+	"testing"
+)
+
+func TestGetHeaderVariableValuesFromRequest(t *testing.T) {
+	// Create a sample request
+	req := httptest.NewRequest("GET", "https://example.com/test?foo=bar", nil)
+	req.Host = "example.com"
+	req.RemoteAddr = "192.168.1.1:12345"
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("User-Agent", "TestAgent")
+	req.Header.Set("Referer", "https://referer.com")
+
+	// Call the function
+	vars := GetHeaderVariableValuesFromRequest(req)
+
+	// Expected results
+	expected := map[string]string{
+		"$host":            "example.com",
+		"$remote_addr":     "192.168.1.1:12345",
+		"$request_uri":     "https://example.com/test?foo=bar",
+		"$request_method":  "GET",
+		"$content_length":  "0", // ContentLength is 0 because there's no body in the request
+		"$content_type":    "application/json",
+		"$uri":             "/test",
+		"$args":            "foo=bar",
+		"$scheme":          "https",
+		"$query_string":    "foo=bar",
+		"$http_user_agent": "TestAgent",
+		"$http_referer":    "https://referer.com",
+	}
+
+	// Check each expected variable
+	for key, expectedValue := range expected {
+		if vars[key] != expectedValue {
+			t.Errorf("Expected %s to be %s, but got %s", key, expectedValue, vars[key])
+		}
+	}
+}
+
+func TestCustomHeadersIncludeDynamicVariables(t *testing.T) {
+	tests := []struct {
+		name           string
+		headers        []*UserDefinedHeader
+		expectedHasVar bool
+	}{
+		{
+			name:           "No headers",
+			headers:        []*UserDefinedHeader{},
+			expectedHasVar: false,
+		},
+		{
+			name: "Headers without dynamic variables",
+			headers: []*UserDefinedHeader{
+				{
+					Direction: HeaderDirection_ZoraxyToUpstream,
+					Key:       "X-Custom-Header",
+					Value:     "staticValue",
+					IsRemove:  false,
+				},
+				{
+					Direction: HeaderDirection_ZoraxyToDownstream,
+					Key:       "X-Another-Header",
+					Value:     "staticValue",
+					IsRemove:  false,
+				},
+			},
+			expectedHasVar: false,
+		},
+		{
+			name: "Headers with one dynamic variable",
+			headers: []*UserDefinedHeader{
+				{
+					Direction: HeaderDirection_ZoraxyToUpstream,
+					Key:       "X-Custom-Header",
+					Value:     "$dynamicValue",
+					IsRemove:  false,
+				},
+			},
+			expectedHasVar: true,
+		},
+		{
+			name: "Headers with multiple dynamic variables",
+			headers: []*UserDefinedHeader{
+				{
+					Direction: HeaderDirection_ZoraxyToUpstream,
+					Key:       "X-Custom-Header",
+					Value:     "$dynamicValue1",
+					IsRemove:  false,
+				},
+				{
+					Direction: HeaderDirection_ZoraxyToDownstream,
+					Key:       "X-Another-Header",
+					Value:     "$dynamicValue2",
+					IsRemove:  false,
+				},
+			},
+			expectedHasVar: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			hasVar := CustomHeadersIncludeDynamicVariables(tt.headers)
+			if hasVar != tt.expectedHasVar {
+				t.Errorf("Expected %v, but got %v", tt.expectedHasVar, hasVar)
+			}
+		})
+	}
+}
+
+func TestPopulateRequestHeaderVariables(t *testing.T) {
+	// Create a sample request with specific values
+	req := httptest.NewRequest("GET", "https://example.com/test?foo=bar", nil)
+	req.Host = "example.com"
+	req.RemoteAddr = "192.168.1.1:12345"
+	req.Header.Set("User-Agent", "TestAgent")
+	req.Header.Set("Referer", "https://referer.com")
+
+	// Define user-defined headers with dynamic variables
+	userDefinedHeaders := []*UserDefinedHeader{
+		{
+			Direction: HeaderDirection_ZoraxyToUpstream,
+			Key:       "X-Forwarded-Host",
+			Value:     "$host",
+		},
+		{
+			Direction: HeaderDirection_ZoraxyToDownstream,
+			Key:       "X-Client-IP",
+			Value:     "$remote_addr",
+		},
+		{
+			Direction: HeaderDirection_ZoraxyToDownstream,
+			Key:       "X-Custom-Header",
+			Value:     "$request_uri",
+		},
+	}
+
+	// Call the function with the test data
+	resultHeaders := PopulateRequestHeaderVariables(req, userDefinedHeaders)
+
+	// Expected results after variable substitution
+	expectedHeaders := []*UserDefinedHeader{
+		{
+			Direction: HeaderDirection_ZoraxyToUpstream,
+			Key:       "X-Forwarded-Host",
+			Value:     "example.com",
+		},
+		{
+			Direction: HeaderDirection_ZoraxyToDownstream,
+			Key:       "X-Client-IP",
+			Value:     "192.168.1.1:12345",
+		},
+		{
+			Direction: HeaderDirection_ZoraxyToDownstream,
+			Key:       "X-Custom-Header",
+			Value:     "https://example.com/test?foo=bar",
+		},
+	}
+
+	// Validate results
+	for i, expected := range expectedHeaders {
+		if resultHeaders[i].Direction != expected.Direction ||
+			resultHeaders[i].Key != expected.Key ||
+			resultHeaders[i].Value != expected.Value {
+			t.Errorf("Expected header %v, but got %v", expected, resultHeaders[i])
+		}
+	}
+}

src/mod/dynamicproxy/rewrite/rewrite.go

@@ -0,0 +1,79 @@
+package rewrite
+
+/*
+ rewrite.go
+
+ This script handle the rewrite logic for custom headers
+*/
+
+import (
+	"strconv"
+
+	"imuslab.com/zoraxy/mod/dynamicproxy/permissionpolicy"
+)
+
+// SplitInboundOutboundHeaders split user defined headers into upstream and downstream headers
+// return upstream header and downstream header key-value pairs
+// if the header is expected to be deleted, the value will be set to empty string
+func SplitUpDownStreamHeaders(rewriteOptions *HeaderRewriteOptions) ([][]string, [][]string) {
+	if len(rewriteOptions.UserDefinedHeaders) == 0 && rewriteOptions.HSTSMaxAge == 0 && !rewriteOptions.EnablePermissionPolicyHeader {
+		//Early return if there are no defined headers
+		return [][]string{}, [][]string{}
+	}
+
+	//Use pre-allocation for faster performance
+	//Downstream +2 for Permission Policy and HSTS
+	upstreamHeaders := make([][]string, len(rewriteOptions.UserDefinedHeaders))
+	downstreamHeaders := make([][]string, len(rewriteOptions.UserDefinedHeaders)+2)
+	upstreamHeaderCounter := 0
+	downstreamHeaderCounter := 0
+
+	//Sort the headers into upstream or downstream
+	for _, customHeader := range rewriteOptions.UserDefinedHeaders {
+		thisHeaderSet := make([]string, 2)
+		thisHeaderSet[0] = customHeader.Key
+		thisHeaderSet[1] = customHeader.Value
+		if customHeader.IsRemove {
+			//Prevent invalid config
+			thisHeaderSet[1] = ""
+		}
+
+		//Assign to slice
+		if customHeader.Direction == HeaderDirection_ZoraxyToUpstream {
+			upstreamHeaders[upstreamHeaderCounter] = thisHeaderSet
+			upstreamHeaderCounter++
+		} else if customHeader.Direction == HeaderDirection_ZoraxyToDownstream {
+			downstreamHeaders[downstreamHeaderCounter] = thisHeaderSet
+			downstreamHeaderCounter++
+		}
+	}
+
+	//Check if the endpoint require HSTS headers
+	if rewriteOptions.HSTSMaxAge > 0 {
+		if rewriteOptions.HSTSIncludeSubdomains {
+			//Endpoint listening domain includes wildcards.
+			downstreamHeaders[downstreamHeaderCounter] = []string{"Strict-Transport-Security", "max-age=" + strconv.Itoa(int(rewriteOptions.HSTSMaxAge)) + "; includeSubdomains"}
+		} else {
+			downstreamHeaders[downstreamHeaderCounter] = []string{"Strict-Transport-Security", "max-age=" + strconv.Itoa(int(rewriteOptions.HSTSMaxAge))}
+		}
+
+		downstreamHeaderCounter++
+	}
+
+	//Check if the endpoint require Permission Policy
+	if rewriteOptions.EnablePermissionPolicyHeader {
+		var usingPermissionPolicy *permissionpolicy.PermissionsPolicy
+		if rewriteOptions.PermissionPolicy != nil {
+			//Custom permission policy
+			usingPermissionPolicy = rewriteOptions.PermissionPolicy
+		} else {
+			//Permission policy is enabled but not customized. Use default
+			usingPermissionPolicy = permissionpolicy.GetDefaultPermissionPolicy()
+		}
+
+		downstreamHeaders[downstreamHeaderCounter] = usingPermissionPolicy.ToKeyValueHeader()
+		downstreamHeaderCounter++
+	}
+
+	return upstreamHeaders, downstreamHeaders
+}

src/mod/dynamicproxy/rewrite/typedef.go

@@ -0,0 +1,51 @@
+package rewrite
+
+import (
+	"encoding/json"
+
+	"imuslab.com/zoraxy/mod/dynamicproxy/permissionpolicy"
+)
+
+/*
+	typdef.go
+
+	This script handle the type definition for custom headers
+*/
+
+/* Custom Header Related Data structure */
+// Header injection direction type
+type HeaderDirection int
+
+const (
+	HeaderDirection_ZoraxyToUpstream   HeaderDirection = 0 //Inject (or remove) header to request out-going from Zoraxy to backend server
+	HeaderDirection_ZoraxyToDownstream HeaderDirection = 1 //Inject (or remove) header to request out-going from Zoraxy to client (e.g. browser)
+)
+
+// User defined headers to add into a proxy endpoint
+type UserDefinedHeader struct {
+	Direction HeaderDirection
+	Key       string
+	Value     string
+	IsRemove  bool //Instead of set, remove this key instead
+}
+
+type HeaderRewriteOptions struct {
+	UserDefinedHeaders           []*UserDefinedHeader                //Custom headers to append when proxying requests from this endpoint
+	HSTSMaxAge                   int64                               //HSTS max age, set to 0 for disable HSTS headers
+	HSTSIncludeSubdomains        bool                                //Include subdomains in HSTS header
+	EnablePermissionPolicyHeader bool                                //Enable injection of permission policy header
+	PermissionPolicy             *permissionpolicy.PermissionsPolicy //Permission policy header
+}
+
+// Utilities for header rewrite
+func (h *UserDefinedHeader) GetDirection() HeaderDirection {
+	return h.Direction
+}
+
+// Copy eturns a deep copy of the UserDefinedHeader
+func (h *UserDefinedHeader) Copy() *UserDefinedHeader {
+	result := UserDefinedHeader{}
+	js, _ := json.Marshal(h)
+	json.Unmarshal(js, &result)
+	return &result
+}

src/mod/dynamicproxy/router.go

@@ -2,8 +2,10 @@ package dynamicproxy
 
 import (
 	"errors"
+	"log"
 	"net/url"
 	"strings"
+	"time"
 
 	"imuslab.com/zoraxy/mod/dynamicproxy/dpcore"
 )
@@ -17,41 +19,18 @@ import (
 
 // Prepare proxy route generate a proxy handler service object for your endpoint
 func (router *Router) PrepareProxyRoute(endpoint *ProxyEndpoint) (*ProxyEndpoint, error) {
-	//Filter the tailing slash if any
-	domain := endpoint.Domain
-	if len(domain) == 0 {
-		return nil, errors.New("invalid endpoint config")
-	}
-	if domain[len(domain)-1:] == "/" {
-		domain = domain[:len(domain)-1]
-	}
-	endpoint.Domain = domain
-
-	//Parse the web proxy endpoint
-	webProxyEndpoint := domain
-	if !strings.HasPrefix("http://", domain) && !strings.HasPrefix("https://", domain) {
-		//TLS is not hardcoded in proxy target domain
-		if endpoint.RequireTLS {
-			webProxyEndpoint = "https://" + webProxyEndpoint
-		} else {
-			webProxyEndpoint = "http://" + webProxyEndpoint
+	for _, thisOrigin := range endpoint.ActiveOrigins {
+		//Create the proxy routing handler
+		err := thisOrigin.StartProxy()
+		if err != nil {
+			log.Println("Unable to setup upstream " + thisOrigin.OriginIpOrDomain + ": " + err.Error())
+			continue
 		}
 	}
 
-	//Create a new proxy agent for this root
-	path, err := url.Parse(webProxyEndpoint)
-	if err != nil {
-		return nil, err
-	}
-
-	//Create the proxy routing handler
-	proxy := dpcore.NewDynamicProxyCore(path, "", &dpcore.DpcoreOptions{
-		IgnoreTLSVerification: endpoint.SkipCertValidations,
-	})
-	endpoint.proxy = proxy
 	endpoint.parent = router
 
-	//Prepare proxy routing hjandler for each of the virtual directories
+	//Prepare proxy routing handler for each of the virtual directories
 	for _, vdir := range endpoint.VirtualDirectories {
 		domain := vdir.Domain
 		if len(domain) == 0 {
@@ -63,7 +42,7 @@ func (router *Router) PrepareProxyRoute(endpoint *ProxyEndpoint) (*ProxyEndpoint
 		}
 
 		//Parse the web proxy endpoint
-		webProxyEndpoint = domain
+		webProxyEndpoint := domain
 		if !strings.HasPrefix("http://", domain) && !strings.HasPrefix("https://", domain) {
 			//TLS is not hardcoded in proxy target domain
 			if vdir.RequireTLS {
@@ -80,6 +59,7 @@ func (router *Router) PrepareProxyRoute(endpoint *ProxyEndpoint) (*ProxyEndpoint
 
 		proxy := dpcore.NewDynamicProxyCore(path, vdir.MatchingPath, &dpcore.DpcoreOptions{
 			IgnoreTLSVerification: vdir.SkipCertValidations,
+			FlushInterval:         500 * time.Millisecond,
 		})
 		vdir.proxy = proxy
 		vdir.parent = endpoint
@@ -90,18 +70,24 @@ func (router *Router) PrepareProxyRoute(endpoint *ProxyEndpoint) (*ProxyEndpoint
 
 // Add Proxy Route to current runtime. Call to PrepareProxyRoute before adding to runtime
 func (router *Router) AddProxyRouteToRuntime(endpoint *ProxyEndpoint) error {
-	if endpoint.proxy == nil {
+	lookupHostname := strings.ToLower(endpoint.RootOrMatchingDomain)
+	if len(endpoint.ActiveOrigins) == 0 {
+		//There are no active origins. No need to check for ready
+		router.ProxyEndpoints.Store(lookupHostname, endpoint)
+		return nil
+	}
+	if !router.loadBalancer.UpstreamsReady(endpoint.ActiveOrigins) {
 		//This endpoint is not prepared
 		return errors.New("proxy endpoint not ready. Use PrepareProxyRoute before adding to runtime")
 	}
 	// Push record into running subdomain endpoints
-	router.ProxyEndpoints.Store(endpoint.RootOrMatchingDomain, endpoint)
+	router.ProxyEndpoints.Store(lookupHostname, endpoint)
 	return nil
 }
 
 // Set given Proxy Route as Root. Call to PrepareProxyRoute before adding to runtime
 func (router *Router) SetProxyRouteAsRoot(endpoint *ProxyEndpoint) error {
-	if endpoint.proxy == nil {
+	if !router.loadBalancer.UpstreamsReady(endpoint.ActiveOrigins) {
 		//This endpoint is not prepared
 		return errors.New("proxy endpoint not ready. Use PrepareProxyRoute before adding to runtime")
 	}

src/mod/dynamicproxy/typedef.go

@@ -7,24 +7,33 @@ import (
 	"sync"
 
 	"imuslab.com/zoraxy/mod/access"
+	"imuslab.com/zoraxy/mod/auth/sso/authelia"
 	"imuslab.com/zoraxy/mod/dynamicproxy/dpcore"
+	"imuslab.com/zoraxy/mod/dynamicproxy/loadbalance"
+	"imuslab.com/zoraxy/mod/dynamicproxy/permissionpolicy"
 	"imuslab.com/zoraxy/mod/dynamicproxy/redirection"
+	"imuslab.com/zoraxy/mod/dynamicproxy/rewrite"
 	"imuslab.com/zoraxy/mod/geodb"
+	"imuslab.com/zoraxy/mod/info/logger"
 	"imuslab.com/zoraxy/mod/statistic"
 	"imuslab.com/zoraxy/mod/tlscert"
 )
 
+type ProxyType int
+
 const (
-	ProxyType_Root = 0
-	ProxyType_Host = 1
-	ProxyType_Vdir = 2
+	ProxyTypeRoot ProxyType = iota //Root Proxy, everything not matching will be routed here
+	ProxyTypeHost                  //Host Proxy, match by host (domain) name
+	ProxyTypeVdir                  //Virtual Directory Proxy, match by path prefix
 )
 
 type ProxyHandler struct {
 	Parent *Router
 }
 
+/* Router Object Options */
 type RouterOption struct {
+	/* Basic Settings */
 	HostUUID           string //The UUID of Zoraxy, use for heading mod
 	HostVersion        string //The version of Zoraxy, use for heading mod
 	Port               int    //Incoming port
@@ -33,28 +42,41 @@ type RouterOption struct {
 	NoCache            bool   //Force set Cache-Control: no-store
 	ListenOnPort80     bool   //Enable port 80 http listener
 	ForceHttpsRedirect bool   //Force redirection of http to https endpoint
-	TlsManager         *tlscert.Manager
-	RedirectRuleTable  *redirection.RuleTable
-	GeodbStore         *geodb.Store       //GeoIP resolver
-	AccessController   *access.Controller //Blacklist / whitelist controller
-	StatisticCollector *statistic.Collector
-	WebDirectory       string //The static web server directory containing the templates folder
+
+	/* Routing Service Managers */
+	TlsManager         *tlscert.Manager          //TLS manager for serving SAN certificates
+	RedirectRuleTable  *redirection.RuleTable    //Redirection rules handler and table
+	GeodbStore         *geodb.Store              //GeoIP resolver
+	AccessController   *access.Controller        //Blacklist / whitelist controller
+	StatisticCollector *statistic.Collector      //Statistic collector for storing stats on incoming visitors
+	WebDirectory       string                    //The static web server directory containing the templates folder
+	LoadBalancer       *loadbalance.RouteManager //Load balancer that handle load balancing of proxy target
+
+	/* Authentication Providers */
+	AutheliaRouter *authelia.AutheliaRouter //Authelia router for Authelia authentication
+
+	/* Utilities */
+	Logger *logger.Logger //Logger for reverse proxy requets
 }
 
+/* Router Object */
 type Router struct {
 	Option         *RouterOption
-	ProxyEndpoints *sync.Map
-	Running        bool
-	Root           *ProxyEndpoint
-	mux            http.Handler
-	server         *http.Server
-	tlsListener    net.Listener
-	routingRules   []*RoutingRule
+	ProxyEndpoints *sync.Map                 //Map of ProxyEndpoint objects, each ProxyEndpoint object is a routing rule that handle incoming requests
+	Running        bool                      //If the router is running
+	Root           *ProxyEndpoint            //Root proxy endpoint, default site
+	mux            http.Handler              //HTTP handler
+	server         *http.Server              //HTTP server
+	tlsListener    net.Listener              //TLS listener, handle SNI routing
+	loadBalancer   *loadbalance.RouteManager //Load balancer routing manager
+	routingRules   []*RoutingRule            //Special routing rules, handle high priority routing like ACME request handling
 
-	tlsRedirectStop chan bool      //Stop channel for tls redirection server
-	tldMap          map[string]int //Top level domain map, see tld.json
+	tlsRedirectStop  chan bool              //Stop channel for tls redirection server
+	rateLimterStop   chan bool              //Stop channel for rate limiter
+	rateLimitCounter RequestCountPerIpTable //Request counter for rate limter
 }
 
+/* Basic Auth Related Data structure*/
 // Auth credential for basic auth on certain endpoints
 type BasicAuthCredentials struct {
 	Username     string
@@ -72,11 +94,7 @@ type BasicAuthExceptionRule struct {
 	PathPrefix string
 }
 
-// User defined headers to add into a proxy endpoint
-type UserDefinedHeader struct {
-	Key   string
-	Value string
-}
+/* Routing Rule Data Structures */
 
 // A Virtual Directory endpoint, provide a subset of ProxyEndpoint for better
 // program structure than directly using ProxyEndpoint
@@ -90,42 +108,84 @@ type VirtualDirectoryEndpoint struct {
 	parent              *ProxyEndpoint       `json:"-"`
 }
 
+// Rules and settings for header rewriting
+type HeaderRewriteRules struct {
+	UserDefinedHeaders           []*rewrite.UserDefinedHeader        //Custom headers to append when proxying requests from this endpoint
+	RequestHostOverwrite         string                              //If not empty, this domain will be used to overwrite the Host field in request header
+	HSTSMaxAge                   int64                               //HSTS max age, set to 0 for disable HSTS headers
+	EnablePermissionPolicyHeader bool                                //Enable injection of permission policy header
+	PermissionPolicy             *permissionpolicy.PermissionsPolicy //Permission policy header
+	DisableHopByHopHeaderRemoval bool                                //Do not remove hop-by-hop headers
+
+}
+
+/*
+
+	Authentication Provider
+
+	TODO: Move these into a dedicated module
+*/
+
+type AuthMethod int
+
+const (
+	AuthMethodNone     AuthMethod = iota //No authentication required
+	AuthMethodBasic                      //Basic Auth
+	AuthMethodAuthelia                   //Authelia
+	AuthMethodOauth2                     //Oauth2
+)
+
+type AuthenticationProvider struct {
+	AuthMethod AuthMethod //The authentication method to use
+	/* Basic Auth Settings */
+	BasicAuthCredentials    []*BasicAuthCredentials   //Basic auth credentials
+	BasicAuthExceptionRules []*BasicAuthExceptionRule //Path to exclude in a basic auth enabled proxy target
+	BasicAuthGroupIDs       []string                  //Group IDs that are allowed to access this endpoint
+
+	/* Authelia Settings */
+	AutheliaURL string //URL of the Authelia server, leave empty to use global settings e.g. authelia.example.com
+	UseHTTPS    bool   //Whether to use HTTPS for the Authelia server
+}
+
 // A proxy endpoint record, a general interface for handling inbound routing
 type ProxyEndpoint struct {
-	ProxyType            int      //The type of this proxy, see const def
-	RootOrMatchingDomain string   //Matching domain for host, also act as key
-	MatchingDomainAlias  []string //A list of domains that alias to this rule
-	Domain               string   //Domain or IP to proxy to
+	ProxyType            ProxyType               //The type of this proxy, see const def
+	RootOrMatchingDomain string                  //Matching domain for host, also act as key
+	MatchingDomainAlias  []string                //A list of domains that alias to this rule
+	ActiveOrigins        []*loadbalance.Upstream //Activated Upstream or origin servers IP or domain to proxy to
+	InactiveOrigins      []*loadbalance.Upstream //Disabled Upstream or origin servers IP or domain to proxy to
+	UseStickySession     bool                    //Use stick session for load balancing
+	UseActiveLoadBalance bool                    //Use active loadbalancing, default passive
+	Disabled             bool                    //If the rule is disabled
 
-	//TLS/SSL Related
-	RequireTLS               bool //Target domain require TLS
-	BypassGlobalTLS          bool //Bypass global TLS setting options if TLS Listener enabled (parent.tlsListener != nil)
-	SkipCertValidations      bool //Set to true to accept self signed certs
-	SkipWebSocketOriginCheck bool //Skip origin check on websocket upgrade connections
+	//Inbound TLS/SSL Related
+	BypassGlobalTLS bool //Bypass global TLS setting options if TLS Listener enabled (parent.tlsListener != nil)
 
 	//Virtual Directories
 	VirtualDirectories []*VirtualDirectoryEndpoint
 
 	//Custom Headers
-	UserDefinedHeaders []*UserDefinedHeader //Custom headers to append when proxying requests from this endpoint
+	HeaderRewriteRules *HeaderRewriteRules
 
 	//Authentication
-	RequireBasicAuth        bool                      //Set to true to request basic auth before proxy
-	BasicAuthCredentials    []*BasicAuthCredentials   //Basic auth credentials
-	BasicAuthExceptionRules []*BasicAuthExceptionRule //Path to exclude in a basic auth enabled proxy target
+	AuthenticationProvider *AuthenticationProvider
+
+	// Rate Limiting
+	RequireRateLimit bool
+	RateLimit        int64 // Rate limit in requests per second
+
+	//Uptime Monitor
+	DisableUptimeMonitor bool //Disable uptime monitor for this endpoint
 
 	//Access Control
 	AccessFilterUUID string //Access filter ID
 
-	Disabled bool //If the rule is disabled
-
 	//Fallback routing logic (Special Rule Sets Only)
 	DefaultSiteOption int    //Fallback routing logic options
 	DefaultSiteValue  string //Fallback routing target, optional
 
 	//Internal Logic Elements
-	parent *Router              `json:"-"`
-	proxy  *dpcore.ReverseProxy `json:"-"`
+	parent *Router `json:"-"`
 }
 
 /*
@@ -141,6 +201,9 @@ const (
 	DefaultSite_ReverseProxy            = 1
 	DefaultSite_Redirect                = 2
 	DefaultSite_NotFoundPage            = 3
+	DefaultSite_NoResponse              = 4
+
+	DefaultSite_TeaPot = 418 //I'm a teapot
 )
 
 /*

src/mod/email/email.go

@@ -16,7 +16,7 @@ type Sender struct {
 	Port       int    //E.g. 587
 	Username   string //Username of the email account
 	Password   string //Password of the email account
-	SenderAddr string //e.g. admin@arozos.com
+	SenderAddr string //e.g. admin@aroz.org
 }
 
 // Create a new email sender object
@@ -42,17 +42,22 @@ SendEmail(
 )
 */
 func (s *Sender) SendEmail(to string, subject string, content string) error {
-	//Parse the email content
+	// Parse the email content
 	msg := []byte("To: " + to + "\n" +
 		"From: Zoraxy <" + s.SenderAddr + ">\n" +
 		"Subject: " + subject + "\n" +
 		"MIME-version: 1.0;\nContent-Type: text/html; charset=\"UTF-8\";\n\n" +
 		content + "\n\n")
 
-	//Login to the SMTP server
-	//Username can be username (e.g. admin) or email (e.g. admin@example.com), depending on SMTP service provider
-	auth := smtp.PlainAuth("", s.Username, s.Password, s.Hostname)
+	// Initialize the auth variable
+	var auth smtp.Auth
+	if s.Password != "" {
+		// Login to the SMTP server
+		// Username can be username (e.g. admin) or email (e.g. admin@example.com), depending on SMTP service provider
+		auth = smtp.PlainAuth("", s.Username, s.Password, s.Hostname)
+	}
 
+	// Send the email
 	err := smtp.SendMail(s.Hostname+":"+strconv.Itoa(s.Port), auth, s.SenderAddr, []string{to}, msg)
 	if err != nil {
 		return err

src/mod/forwardproxy/cproxy/default_dialer.go

@@ -18,7 +18,7 @@ func (this *defaultDialer) Dial(address string) Socket {
 	if socket, err := net.DialTimeout("tcp", address, this.timeout); err == nil {
 		return socket
 	} else {
-		this.logger.Printf("[INFO] Unable to establish connection to [%s]: %s", address, err)
+		this.logger.Printf("Unable to establish connection to [%s]: %s", address, err)
 	}
 
 	return nil

src/mod/forwardproxy/cproxy/logging_initializer.go

@@ -17,7 +17,7 @@ func (this *loggingInitializer) Initialize(client, server Socket) bool {
 	result := this.inner.Initialize(client, server)
 
 	if !result {
-		this.logger.Printf("[INFO] Connection failed [%s] -> [%s]", client.RemoteAddr(), server.RemoteAddr())
+		this.logger.Printf("Connection failed [%s] -> [%s]", client.RemoteAddr(), server.RemoteAddr())
 	}
 
 	return result

src/mod/ganserv/ganserv.go

@@ -1,6 +1,7 @@
 package ganserv
 
 import (
+	"log"
 	"net"
 
 	"imuslab.com/zoraxy/mod/database"
@@ -85,6 +86,7 @@ func NewNetworkManager(option *NetworkManagerOptions) *NetworkManager {
 	//Get controller info
 	instanceInfo, err := getControllerInfo(option.AuthToken, option.ApiPort)
 	if err != nil {
+		log.Println("ZeroTier connection failed: ", err.Error())
 		return &NetworkManager{
 			authToken:        option.AuthToken,
 			apiPort:          option.ApiPort,

src/mod/ganserv/zerotier.go

@@ -28,11 +28,17 @@ type NodeInfo struct {
 	Clock   int64  `json:"clock"`
 	Config  struct {
 		Settings struct {
-			AllowTCPFallbackRelay bool   `json:"allowTcpFallbackRelay"`
-			PortMappingEnabled    bool   `json:"portMappingEnabled"`
-			PrimaryPort           int    `json:"primaryPort"`
-			SoftwareUpdate        string `json:"softwareUpdate"`
-			SoftwareUpdateChannel string `json:"softwareUpdateChannel"`
+			AllowTCPFallbackRelay bool     `json:"allowTcpFallbackRelay,omitempty"`
+			ForceTCPRelay         bool     `json:"forceTcpRelay,omitempty"`
+			HomeDir               string   `json:"homeDir,omitempty"`
+			ListeningOn           []string `json:"listeningOn,omitempty"`
+			PortMappingEnabled    bool     `json:"portMappingEnabled,omitempty"`
+			PrimaryPort           int      `json:"primaryPort,omitempty"`
+			SecondaryPort         int      `json:"secondaryPort,omitempty"`
+			SoftwareUpdate        string   `json:"softwareUpdate,omitempty"`
+			SoftwareUpdateChannel string   `json:"softwareUpdateChannel,omitempty"`
+			SurfaceAddresses      []string `json:"surfaceAddresses,omitempty"`
+			TertiaryPort          int      `json:"tertiaryPort,omitempty"`
 		} `json:"settings"`
 	} `json:"config"`
 	Online               bool   `json:"online"`
@@ -46,7 +52,6 @@ type NodeInfo struct {
 	VersionMinor         int    `json:"versionMinor"`
 	VersionRev           int    `json:"versionRev"`
 }
-
 type ErrResp struct {
 	Message string `json:"message"`
 }

src/mod/geodb/geodb.go

@@ -3,9 +3,14 @@ package geodb
 import (
 	_ "embed"
 	"net/http"
+	"os"
+	"sync"
+	"time"
 
 	"imuslab.com/zoraxy/mod/database"
+	"imuslab.com/zoraxy/mod/info/logger"
 	"imuslab.com/zoraxy/mod/netutils"
+	"imuslab.com/zoraxy/mod/utils"
 )
 
 //go:embed geoipv4.csv
@@ -15,18 +20,23 @@ var geoipv4 []byte //Geodb dataset for ipv4
 var geoipv6 []byte //Geodb dataset for ipv6
 
 type Store struct {
-	geodb       [][]string //Parsed geodb list
-	geodbIpv6   [][]string //Parsed geodb list for ipv6
-	geotrie     *trie
-	geotrieIpv6 *trie
-	//geoipCache sync.Map
-	sysdb  *database.Database
-	option *StoreOptions
+	geodb                    [][]string //Parsed geodb list
+	geodbIpv6                [][]string //Parsed geodb list for ipv6
+	geotrie                  *trie
+	geotrieIpv6              *trie
+	sysdb                    *database.Database
+	slowLookupCacheIpv4      sync.Map     //Cache for slow lookup, ip -> cc
+	slowLookupCacheIpv6      sync.Map     //Cache for slow lookup ipv6, ip -> cc
+	cacheClearTicker         *time.Ticker //Ticker for clearing cache
+	cacheClearTickerStopChan chan bool    //Stop channel for cache clear ticker
+	option                   *StoreOptions
 }
 
 type StoreOptions struct {
-	AllowSlowIpv4LookUp bool
-	AllowSloeIpv6Lookup bool
+	AllowSlowIpv4LookUp          bool
+	AllowSlowIpv6Lookup          bool
+	Logger                       *logger.Logger
+	SlowLookupCacheClearInterval time.Duration //Clear slow lookup cache interval
 }
 
 type CountryInfo struct {
@@ -35,6 +45,23 @@ type CountryInfo struct {
 }
 
 func NewGeoDb(sysdb *database.Database, option *StoreOptions) (*Store, error) {
+	//Check if external geoDB data is available
+	if utils.FileExists("./conf/geodb/geoipv4.csv") {
+		externalV4Db, err := os.ReadFile("./conf/geodb/geoipv4.csv")
+		if err == nil {
+			option.Logger.PrintAndLog("GeoDB", "External GeoDB data found, using external IPv4 GeoIP data", nil)
+			geoipv4 = externalV4Db
+		}
+	}
+
+	if utils.FileExists("./conf/geodb/geoipv6.csv") {
+		externalV6Db, err := os.ReadFile("./conf/geodb/geoipv6.csv")
+		if err == nil {
+			option.Logger.PrintAndLog("GeoDB", "External GeoDB data found, using external IPv6 GeoIP data", nil)
+			geoipv6 = externalV6Db
+		}
+	}
+
 	parsedGeoData, err := parseCSV(geoipv4)
 	if err != nil {
 		return nil, err
@@ -51,18 +78,44 @@ func NewGeoDb(sysdb *database.Database, option *StoreOptions) (*Store, error) {
 	}
 
 	var ipv6Trie *trie
-	if !option.AllowSloeIpv6Lookup {
+	if !option.AllowSlowIpv6Lookup {
 		ipv6Trie = constrctTrieTree(parsedGeoDataIpv6)
 	}
 
-	return &Store{
-		geodb:       parsedGeoData,
-		geotrie:     ipv4Trie,
-		geodbIpv6:   parsedGeoDataIpv6,
-		geotrieIpv6: ipv6Trie,
-		sysdb:       sysdb,
-		option:      option,
-	}, nil
+	if option.SlowLookupCacheClearInterval == 0 {
+		option.SlowLookupCacheClearInterval = 30 * time.Minute
+	}
+
+	//Create a new store
+	thisGeoDBStore := &Store{
+		geodb:                    parsedGeoData,
+		geotrie:                  ipv4Trie,
+		geodbIpv6:                parsedGeoDataIpv6,
+		geotrieIpv6:              ipv6Trie,
+		sysdb:                    sysdb,
+		slowLookupCacheIpv4:      sync.Map{},
+		slowLookupCacheIpv6:      sync.Map{},
+		cacheClearTicker:         time.NewTicker(option.SlowLookupCacheClearInterval),
+		cacheClearTickerStopChan: make(chan bool),
+		option:                   option,
+	}
+
+	//Start cache clear ticker
+	if option.AllowSlowIpv4LookUp || option.AllowSlowIpv6Lookup {
+		go func(store *Store) {
+			for {
+				select {
+				case <-store.cacheClearTickerStopChan:
+					return
+				case <-thisGeoDBStore.cacheClearTicker.C:
+					thisGeoDBStore.slowLookupCacheIpv4 = sync.Map{}
+					thisGeoDBStore.slowLookupCacheIpv6 = sync.Map{}
+				}
+			}
+		}(thisGeoDBStore)
+	}
+
+	return thisGeoDBStore, nil
 }
 
 func (s *Store) ResolveCountryCodeFromIP(ipstring string) (*CountryInfo, error) {
@@ -74,8 +127,12 @@ func (s *Store) ResolveCountryCodeFromIP(ipstring string) (*CountryInfo, error)
 
 }
 
+// Close the store
 func (s *Store) Close() {
-
+	if s.option.AllowSlowIpv4LookUp || s.option.AllowSlowIpv6Lookup {
+		//Stop cache clear ticker
+		s.cacheClearTickerStopChan <- true
+	}
 }
 
 func (s *Store) GetRequesterCountryISOCode(r *http.Request) string {

src/mod/geodb/geodb_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"imuslab.com/zoraxy/mod/geodb"
+	"imuslab.com/zoraxy/mod/info/logger"
 )
 
 /*
@@ -42,8 +43,10 @@ func TestTrieConstruct(t *testing.T) {
 func TestResolveCountryCodeFromIP(t *testing.T) {
 	// Create a new store
 	store, err := geodb.NewGeoDb(nil, &geodb.StoreOptions{
-		false,
-		false,
+		true,
+		true,
+		&logger.Logger{},
+		0,
 	})
 	if err != nil {
 		t.Errorf("error creating store: %v", err)
@@ -56,6 +59,7 @@ func TestResolveCountryCodeFromIP(t *testing.T) {
 		{"176.113.115.113", "RU"},
 		{"65.21.233.213", "FI"},
 		{"94.23.207.193", "FR"},
+		{"77.131.21.232", "FR"},
 	}
 
 	for _, testcase := range knownIpCountryMap {
@@ -82,4 +86,24 @@ func TestResolveCountryCodeFromIP(t *testing.T) {
 	if info.CountryIsoCode != expected {
 		t.Errorf("expected country code %s, but got %s for IP %s", expected, info.CountryIsoCode, ip)
 	}
+
+	// Test for issue #401
+	// Create 100 concurrent goroutines to resolve country code for random IP addresses in the test cases above
+	for i := 0; i < 100; i++ {
+		go func() {
+			for _, testcase := range knownIpCountryMap {
+				ip := testcase[0]
+				expected := testcase[1]
+				info, err := store.ResolveCountryCodeFromIP(ip)
+				if err != nil {
+					t.Errorf("error resolving country code for IP %s: %v", ip, err)
+					return
+				}
+				if info.CountryIsoCode != expected {
+					t.Errorf("expected country code %s, but got %s for IP %s", expected, info.CountryIsoCode, ip)
+				}
+			}
+		}()
+	}
+
 }

src/mod/geodb/geoloader.go

@@ -16,13 +16,6 @@ func (s *Store) search(ip string) string {
 		ip = strings.Split(ip, ",")[0]
 		ip = strings.TrimSpace(ip)
 	}
-	//See if there are cached country code for this ip
-	/*
-		ccc, ok := s.geoipCache.Load(ip)
-		if ok {
-			return ccc.(string)
-		}
-	*/
 
 	//Search in geotrie tree
 	cc := ""

src/mod/geodb/locale.go

@@ -0,0 +1,67 @@
+package geodb
+
+import "net/http"
+
+// GetRequesterCountryISOCode get the locale of the requester
+func (s *Store) GetLocaleFromRequest(r *http.Request) (string, error) {
+	cc := s.GetRequesterCountryISOCode(r)
+	return GetLocaleFromCountryCode(cc), nil
+}
+
+// GetLocaleFromCountryCode get the locale given the country code
+func GetLocaleFromCountryCode(cc string) string {
+	//If you find your country is not in the list, please add it here
+	mapCountryToLocale := map[string]string{
+		"aa": "ar_AA",
+		"by": "be_BY",
+		"bg": "bg_BG",
+		"es": "ca_ES",
+		"cz": "cs_CZ",
+		"dk": "da_DK",
+		"ch": "de_CH",
+		"de": "de_DE",
+		"gr": "el_GR",
+		"au": "en_AU",
+		"be": "en_BE",
+		"gb": "en_GB",
+		"jp": "en_JP",
+		"us": "en_US",
+		"za": "en_ZA",
+		"fi": "fi_FI",
+		"ca": "fr_CA",
+		"fr": "fr_FR",
+		"hr": "hr_HR",
+		"hu": "hu_HU",
+		"is": "is_IS",
+		"it": "it_IT",
+		"il": "iw_IL",
+		"kr": "ko_KR",
+		"lt": "lt_LT",
+		"lv": "lv_LV",
+		"mk": "mk_MK",
+		"nl": "nl_NL",
+		"no": "no_NO",
+		"pl": "pl_PL",
+		"br": "pt_BR",
+		"pt": "pt_PT",
+		"ro": "ro_RO",
+		"ru": "ru_RU",
+		"sp": "sh_SP",
+		"sk": "sk_SK",
+		"sl": "sl_SL",
+		"al": "sq_AL",
+		"se": "sv_SE",
+		"th": "th_TH",
+		"tr": "tr_TR",
+		"ua": "uk_UA",
+		"cn": "zh_CN",
+		"tw": "zh_TW",
+		"hk": "zh_HK",
+	}
+	locale, ok := mapCountryToLocale[cc]
+	if !ok {
+		return "en-US"
+	}
+
+	return locale
+}

src/mod/geodb/slowSearch.go

@@ -53,6 +53,16 @@ func isIPv6InRange(startIP, endIP, testIP string) (bool, error) {
 
 // Slow country code lookup for
 func (s *Store) slowSearchIpv4(ipAddr string) string {
+	if isReservedIP(ipAddr) {
+		return ""
+	}
+
+	//Check if already in cache
+	cc := s.GetSlowSearchCachedIpv4(ipAddr)
+	if cc != "" {
+		return cc
+	}
+
 	for _, ipRange := range s.geodb {
 		startIp := ipRange[0]
 		endIp := ipRange[1]
@@ -60,6 +70,8 @@ func (s *Store) slowSearchIpv4(ipAddr string) string {
 
 		inRange, _ := isIPv4InRange(startIp, endIp, ipAddr)
 		if inRange {
+			//Add to cache
+			s.slowLookupCacheIpv4.Store(ipAddr, cc)
 			return cc
 		}
 	}
@@ -67,6 +79,16 @@ func (s *Store) slowSearchIpv4(ipAddr string) string {
 }
 
 func (s *Store) slowSearchIpv6(ipAddr string) string {
+	if isReservedIP(ipAddr) {
+		return ""
+	}
+
+	//Check if already in cache
+	cc := s.GetSlowSearchCachedIpv6(ipAddr)
+	if cc != "" {
+		return cc
+	}
+
 	for _, ipRange := range s.geodbIpv6 {
 		startIp := ipRange[0]
 		endIp := ipRange[1]
@@ -74,8 +96,28 @@ func (s *Store) slowSearchIpv6(ipAddr string) string {
 
 		inRange, _ := isIPv6InRange(startIp, endIp, ipAddr)
 		if inRange {
+			//Add to cache
+			s.slowLookupCacheIpv6.Store(ipAddr, cc)
 			return cc
 		}
 	}
 	return ""
 }
+
+// GetSlowSearchCachedIpv4 return the country code for the given ipv4 address, return empty string if not found
+func (s *Store) GetSlowSearchCachedIpv4(ipAddr string) string {
+	cc, ok := s.slowLookupCacheIpv4.Load(ipAddr)
+	if ok {
+		return cc.(string)
+	}
+	return ""
+}
+
+// GetSlowSearchCachedIpv6 return the country code for the given ipv6 address, return empty string if not found
+func (s *Store) GetSlowSearchCachedIpv6(ipAddr string) string {
+	cc, ok := s.slowLookupCacheIpv6.Load(ipAddr)
+	if ok {
+		return cc.(string)
+	}
+	return ""
+}

src/mod/geodb/trie.go

@@ -1,7 +1,6 @@
 package geodb
 
 import (
-	"math"
 	"net"
 )
 
@@ -41,14 +40,10 @@ func (t *trie) insert(ipAddr string, cc string) {
 	ipBytes := ipToBytes(ipAddr)
 	current := t.root
 	for _, b := range ipBytes {
-		//For each byte in the ip address
+		//For each byte in the ip address (4 / 16 bytes)
 		//each byte is 8 bit
-		for j := 0; j < 8; j++ {
-			bitwise := (b&uint8(math.Pow(float64(2), float64(j))) > 0)
-			bit := 0b0000
-			if bitwise {
-				bit = 0b0001
-			}
+		for j := 7; j >= 0; j-- {
+			bit := int(b >> j & 1)
 			if current.childrens[bit] == nil {
 				current.childrens[bit] = &trie_Node{
 					childrens: [2]*trie_Node{},
@@ -58,21 +53,9 @@ func (t *trie) insert(ipAddr string, cc string) {
 			current = current.childrens[bit]
 		}
 	}
-
-	/*
-		for i := 63; i >= 0; i-- {
-			bit := (ipInt64 >> uint(i)) & 1
-			if current.childrens[bit] == nil {
-				current.childrens[bit] = &trie_Node{
-					childrens: [2]*trie_Node{},
-					cc:        cc,
-				}
-			}
-			current = current.childrens[bit]
-		}
-	*/
 }
 
+// isReservedIP check if the given ip address is NOT a public ip address
 func isReservedIP(ip string) bool {
 	parsedIP := net.ParseIP(ip)
 	if parsedIP == nil {
@@ -86,12 +69,10 @@ func isReservedIP(ip string) bool {
 	if parsedIP.IsLinkLocalUnicast() || parsedIP.IsLinkLocalMulticast() {
 		return true
 	}
-
+	//Check if the IP is in the reserved private range
 	if parsedIP.IsPrivate() {
 		return true
 	}
-
-	// If the IP address is not a reserved address, return false
 	return false
 }
 
@@ -106,27 +87,15 @@ func (t *trie) search(ipAddr string) string {
 	for _, b := range ipBytes {
 		//For each byte in the ip address
 		//each byte is 8 bit
-		for j := 0; j < 8; j++ {
-			bitwise := (b&uint8(math.Pow(float64(2), float64(j))) > 0)
-			bit := 0b0000
-			if bitwise {
-				bit = 0b0001
-			}
+		for j := 7; j >= 0; j-- {
+			bit := int(b >> j & 1)
 			if current.childrens[bit] == nil {
 				return current.cc
 			}
 			current = current.childrens[bit]
 		}
 	}
-	/*
-		for i := 63; i >= 0; i-- {
-			bit := (ipInt64 >> uint(i)) & 1
-			if current.childrens[bit] == nil {
-				return current.cc
-			}
-			current = current.childrens[bit]
-		}
-	*/
+
 	if len(current.childrens) == 0 {
 		return current.cc
 	}

src/mod/geodb/updater.go

@@ -0,0 +1,56 @@
+package geodb
+
+import (
+	"io"
+	"log"
+	"net/http"
+	"os"
+
+	"imuslab.com/zoraxy/mod/utils"
+)
+
+const (
+	ipv4UpdateSource = "https://cdn.jsdelivr.net/npm/@ip-location-db/geo-whois-asn-country/geo-whois-asn-country-ipv4.csv"
+	ipv6UpdateSource = "https://cdn.jsdelivr.net/npm/@ip-location-db/geo-whois-asn-country/geo-whois-asn-country-ipv6.csv"
+)
+
+// DownloadGeoDBUpdate download the latest geodb update
+func DownloadGeoDBUpdate(externalGeoDBStoragePath string) {
+	//Create the storage path if not exist
+	if !utils.FileExists(externalGeoDBStoragePath) {
+		os.MkdirAll(externalGeoDBStoragePath, 0755)
+	}
+
+	//Download the update
+	log.Println("Downloading IPv4 database update...")
+	err := downloadFile(ipv4UpdateSource, externalGeoDBStoragePath+"/geoipv4.csv")
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	log.Println("Downloading IPv6 database update...")
+	err = downloadFile(ipv6UpdateSource, externalGeoDBStoragePath+"/geoipv6.csv")
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	log.Println("GeoDB update stored at: " + externalGeoDBStoragePath)
+	log.Println("Exiting...")
+}
+
+// Utility functions
+func downloadFile(url string, savepath string) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	fileContent, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(savepath, fileContent, 0644)
+}

src/mod/info/logger/logger.go

@@ -13,29 +13,31 @@ import (
 	Zoraxy Logger
 
 	This script is designed to make a managed log for the Zoraxy
-	and replace the ton of log.Println in the system core
+	and replace the ton of log.Println in the system core.
+	The core logger is based in golang's build-in log package
 */
 
 type Logger struct {
-	LogToFile      bool   //Set enable write to file
 	Prefix         string //Prefix for log files
 	LogFolder      string //Folder to store the log  file
 	CurrentLogFile string //Current writing filename
+	logger         *log.Logger
 	file           *os.File
 }
 
-func NewLogger(logFilePrefix string, logFolder string, logToFile bool) (*Logger, error) {
+// Create a new logger that log to files
+func NewLogger(logFilePrefix string, logFolder string) (*Logger, error) {
 	err := os.MkdirAll(logFolder, 0775)
 	if err != nil {
 		return nil, err
 	}
 
 	thisLogger := Logger{
-		LogToFile: logToFile,
 		Prefix:    logFilePrefix,
 		LogFolder: logFolder,
 	}
 
+	//Create the log file if not exists
 	logFilePath := thisLogger.getLogFilepath()
 	f, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0755)
 	if err != nil {
@@ -43,9 +45,26 @@ func NewLogger(logFilePrefix string, logFolder string, logToFile bool) (*Logger,
 	}
 	thisLogger.CurrentLogFile = logFilePath
 	thisLogger.file = f
+
+	//Start the logger
+	logger := log.New(f, "", log.Flags()&^(log.Ldate|log.Ltime))
+	logger.SetFlags(0)
+	logger.SetOutput(f)
+	thisLogger.logger = logger
 	return &thisLogger, nil
 }
 
+// Create a fmt logger that only log to STDOUT
+func NewFmtLogger() (*Logger, error) {
+	return &Logger{
+		Prefix:         "",
+		LogFolder:      "",
+		CurrentLogFile: "",
+		logger:         nil,
+		file:           nil,
+	}, nil
+}
+
 func (l *Logger) getLogFilepath() string {
 	year, month, _ := time.Now().Date()
 	return filepath.Join(l.LogFolder, l.Prefix+"_"+strconv.Itoa(year)+"-"+strconv.Itoa(int(month))+".log")
@@ -54,9 +73,8 @@ func (l *Logger) getLogFilepath() string {
 // PrintAndLog will log the message to file and print the log to STDOUT
 func (l *Logger) PrintAndLog(title string, message string, originalError error) {
 	go func() {
-		l.Log(title, message, originalError)
+		l.Log(title, message, originalError, true)
 	}()
-	log.Println("[" + title + "] " + message)
 }
 
 // Println is a fast snap-in replacement for log.Println
@@ -64,18 +82,26 @@ func (l *Logger) Println(v ...interface{}) {
 	//Convert the array of interfaces into string
 	message := fmt.Sprint(v...)
 	go func() {
-		l.Log("info", string(message), nil)
+		l.Log("internal", string(message), nil, true)
 	}()
-	log.Println("[INFO] " + string(message))
 }
 
-func (l *Logger) Log(title string, errorMessage string, originalError error) {
+func (l *Logger) Log(title string, errorMessage string, originalError error, copyToSTDOUT bool) {
 	l.ValidateAndUpdateLogFilepath()
-	if l.LogToFile {
+	if l.logger == nil || copyToSTDOUT {
+		//Use STDOUT instead of logger
 		if originalError == nil {
-			l.file.WriteString(time.Now().Format("2006-01-02 15:04:05.000000") + "|" + fmt.Sprintf("%-16s", title) + " [INFO]" + errorMessage + "\n")
+			fmt.Println("[" + time.Now().Format("2006-01-02 15:04:05.000000") + "] [" + title + "] [system:info] " + errorMessage)
 		} else {
-			l.file.WriteString(time.Now().Format("2006-01-02 15:04:05.000000") + "|" + fmt.Sprintf("%-16s", title) + " [ERROR]" + errorMessage + " " + originalError.Error() + "\n")
+			fmt.Println("[" + time.Now().Format("2006-01-02 15:04:05.000000") + "] [" + title + "] [system:error] " + errorMessage + ": " + originalError.Error())
+		}
+	}
+
+	if l.logger != nil {
+		if originalError == nil {
+			l.logger.Println("[" + time.Now().Format("2006-01-02 15:04:05.000000") + "] [" + title + "] [system:info] " + errorMessage)
+		} else {
+			l.logger.Println("[" + time.Now().Format("2006-01-02 15:04:05.000000") + "] [" + title + "] [system:error] " + errorMessage + ": " + originalError.Error())
 		}
 	}
 
@@ -83,18 +109,28 @@ func (l *Logger) Log(title string, errorMessage string, originalError error) {
 
 // Validate if the logging target is still valid (detect any months change)
 func (l *Logger) ValidateAndUpdateLogFilepath() {
+	if l.file == nil {
+		return
+	}
 	expectedCurrentLogFilepath := l.getLogFilepath()
 	if l.CurrentLogFile != expectedCurrentLogFilepath {
 		//Change of month. Update to a new log file
 		l.file.Close()
+		l.file = nil
+
+		//Create a new log file
 		f, err := os.OpenFile(expectedCurrentLogFilepath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0755)
 		if err != nil {
-			log.Println("[Logger] Unable to create new log. Logging to file disabled.")
-			l.LogToFile = false
+			log.Println("Unable to create new log. Logging is disabled: ", err.Error())
+			l.logger = nil
 			return
 		}
 		l.CurrentLogFile = expectedCurrentLogFilepath
 		l.file = f
+
+		//Start a new logger
+		logger := log.New(f, "", log.Default().Flags())
+		l.logger = logger
 	}
 }
 

src/mod/info/logger/trafficlog.go

@@ -0,0 +1,32 @@
+package logger
+
+/*
+	Traffic Log
+
+	This script log the traffic of HTTP requests
+
+*/
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"imuslab.com/zoraxy/mod/netutils"
+)
+
+// Log HTTP request. Note that this must run in go routine to prevent any blocking
+// in reverse proxy router
+func (l *Logger) LogHTTPRequest(r *http.Request, reqclass string, statusCode int) {
+	go func() {
+		l.ValidateAndUpdateLogFilepath()
+		if l.logger == nil || l.file == nil {
+			//logger is not initiated. Do not log http request
+			return
+		}
+		clientIP := netutils.GetRequesterIP(r)
+		requestURI := r.RequestURI
+		statusCodeString := strconv.Itoa(statusCode)
+		//fmt.Println("[" + time.Now().Format("2006-01-02 15:04:05.000000") + "] [router:" + reqclass + "] [client " + clientIP + "] " + r.Method + " " + requestURI + " " + statusCodeString)
+		l.logger.Println("[" + time.Now().Format("2006-01-02 15:04:05.000000") + "] [router:" + reqclass + "] [origin:" + r.URL.Hostname() + "] [client " + clientIP + "] " + r.Method + " " + requestURI + " " + statusCodeString)
+	}()
+}

src/mod/info/logviewer/logviewer.go

@@ -51,13 +51,7 @@ func (v *Viewer) HandleReadLog(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	catergory, err := utils.GetPara(r, "catergory")
-	if err != nil {
-		utils.SendErrorResponse(w, "invalid catergory given")
-		return
-	}
-
-	content, err := v.LoadLogFile(strings.TrimSpace(filepath.Base(catergory)), strings.TrimSpace(filepath.Base(filename)))
+	content, err := v.LoadLogFile(strings.TrimSpace(filepath.Base(filename)))
 	if err != nil {
 		utils.SendErrorResponse(w, err.Error())
 		return
@@ -106,8 +100,10 @@ func (v *Viewer) ListLogFiles(showFullpath bool) map[string][]*LogFile {
 	return result
 }
 
-func (v *Viewer) LoadLogFile(catergory string, filename string) (string, error) {
-	logFilepath := filepath.Join(v.option.RootFolder, catergory, filename)
+func (v *Viewer) LoadLogFile(filename string) (string, error) {
+	filename = filepath.ToSlash(filename)
+	filename = strings.ReplaceAll(filename, "../", "")
+	logFilepath := filepath.Join(v.option.RootFolder, filename)
 	if utils.FileExists(logFilepath) {
 		//Load it
 		content, err := os.ReadFile(logFilepath)

src/mod/ipscan/handlers.go

@@ -0,0 +1,79 @@
+package ipscan
+
+/*
+	ipscan http handlers
+
+	This script provide http handlers for ipscan module
+*/
+
+import (
+	"encoding/json"
+	"net"
+	"net/http"
+
+	"imuslab.com/zoraxy/mod/utils"
+)
+
+// HandleScanPort is the HTTP handler for scanning opened ports on a given IP address
+func HandleScanPort(w http.ResponseWriter, r *http.Request) {
+	targetIp, err := utils.GetPara(r, "ip")
+	if err != nil {
+		utils.SendErrorResponse(w, "target IP address not given")
+		return
+	}
+
+	// Check if the IP is a valid IP address
+	ip := net.ParseIP(targetIp)
+	if ip == nil {
+		utils.SendErrorResponse(w, "invalid IP address")
+		return
+	}
+
+	// Scan the ports
+	openPorts := ScanPorts(targetIp)
+	jsonData, err := json.Marshal(openPorts)
+	if err != nil {
+		utils.SendErrorResponse(w, "failed to marshal JSON")
+		return
+	}
+
+	utils.SendJSONResponse(w, string(jsonData))
+}
+
+// HandleIpScan is the HTTP handler for scanning IP addresses in a given range or CIDR
+func HandleIpScan(w http.ResponseWriter, r *http.Request) {
+	cidr, err := utils.PostPara(r, "cidr")
+	if err != nil {
+		//Ip range mode
+		start, err := utils.PostPara(r, "start")
+		if err != nil {
+			utils.SendErrorResponse(w, "missing start ip")
+			return
+		}
+
+		end, err := utils.PostPara(r, "end")
+		if err != nil {
+			utils.SendErrorResponse(w, "missing end ip")
+			return
+		}
+
+		discoveredHosts, err := ScanIpRange(start, end)
+		if err != nil {
+			utils.SendErrorResponse(w, err.Error())
+			return
+		}
+
+		js, _ := json.Marshal(discoveredHosts)
+		utils.SendJSONResponse(w, string(js))
+	} else {
+		//CIDR mode
+		discoveredHosts, err := ScanCIDRRange(cidr)
+		if err != nil {
+			utils.SendErrorResponse(w, err.Error())
+			return
+		}
+
+		js, _ := json.Marshal(discoveredHosts)
+		utils.SendJSONResponse(w, string(js))
+	}
+}

src/mod/ipscan/ipscan.go

@@ -27,7 +27,7 @@ type DiscoveredHost struct {
 	HttpsPortDetected bool
 }
 
-//Scan an IP range given the start and ending ip address
+// Scan an IP range given the start and ending ip address
 func ScanIpRange(start, end string) ([]*DiscoveredHost, error) {
 	ipStart := net.ParseIP(start)
 	ipEnd := net.ParseIP(end)
@@ -57,7 +57,6 @@ func ScanIpRange(start, end string) ([]*DiscoveredHost, error) {
 			host.CheckHostname()
 			host.CheckPort("http", 80, &host.HttpPortDetected)
 			host.CheckPort("https", 443, &host.HttpsPortDetected)
-			fmt.Println("OK", host)
 			hosts = append(hosts, host)
 
 		}(thisIp)
@@ -118,7 +117,7 @@ func (host *DiscoveredHost) CheckPing() error {
 func (host *DiscoveredHost) CheckHostname() {
 	// lookup the hostname for the IP address
 	names, err := net.LookupAddr(host.IP)
-	fmt.Println(names, err)
+	//fmt.Println(names, err)
 	if err == nil && len(names) > 0 {
 		host.Hostname = names[0]
 	}

src/mod/ipscan/portscan.go

@@ -0,0 +1,48 @@
+package ipscan
+
+/*
+	Port Scanner
+
+	This module scan the given IP address and scan all the opened port
+
+*/
+
+import (
+	"fmt"
+	"net"
+	"sync"
+	"time"
+)
+
+// OpenedPort holds information about an open port and its service type
+type OpenedPort struct {
+	Port  int
+	IsTCP bool
+}
+
+// ScanPorts scans all the opened ports on a given host IP (both IPv4 and IPv6)
+func ScanPorts(host string) []*OpenedPort {
+	var openPorts []*OpenedPort
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+
+	for port := 1; port <= 65535; port++ {
+		wg.Add(1)
+		go func(port int) {
+			defer wg.Done()
+			address := fmt.Sprintf("%s:%d", host, port)
+
+			// Check TCP
+			conn, err := net.DialTimeout("tcp", address, 5*time.Second)
+			if err == nil {
+				mu.Lock()
+				openPorts = append(openPorts, &OpenedPort{Port: port, IsTCP: true})
+				mu.Unlock()
+				conn.Close()
+			}
+		}(port)
+	}
+
+	wg.Wait()
+	return openPorts
+}