Added UI for plugin system and upnp example

- Added wip UI for plugin tag system
- Added upnp port forwarder plugin
- Added error and fatal printout for plugins
- Optimized UI flow for plugin context window
- Added dev web server for plugin development purpose

src/mod/plugins/groups.go

@@ -0,0 +1,101 @@
+package plugins
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+
+	"imuslab.com/zoraxy/mod/plugins/zoraxy_plugin"
+)
+
+// ListPluginGroups returns a map of plugin groups
+func (m *Manager) ListPluginGroups() map[string][]string {
+	pluginGroup := map[string][]string{}
+	m.Options.pluginGroupsMutex.RLock()
+	for k, v := range m.Options.PluginGroups {
+		pluginGroup[k] = append([]string{}, v...)
+	}
+	m.Options.pluginGroupsMutex.RUnlock()
+	return pluginGroup
+}
+
+// AddPluginToGroup adds a plugin to a group
+func (m *Manager) AddPluginToGroup(tag, pluginID string) error {
+	//Check if the plugin exists
+	plugin, ok := m.LoadedPlugins[pluginID]
+	if !ok {
+		return errors.New("plugin not found")
+	}
+
+	//Check if the plugin is a router type plugin
+	if plugin.Spec.Type != zoraxy_plugin.PluginType_Router {
+		return errors.New("plugin is not a router type plugin")
+	}
+
+	m.Options.pluginGroupsMutex.Lock()
+	//Check if the tag exists
+	_, ok = m.Options.PluginGroups[tag]
+	if !ok {
+		m.Options.PluginGroups[tag] = []string{pluginID}
+		m.Options.pluginGroupsMutex.Unlock()
+		return nil
+	}
+
+	//Add the plugin to the group
+	m.Options.PluginGroups[tag] = append(m.Options.PluginGroups[tag], pluginID)
+
+	m.Options.pluginGroupsMutex.Unlock()
+	return nil
+}
+
+// RemovePluginFromGroup removes a plugin from a group
+func (m *Manager) RemovePluginFromGroup(tag, pluginID string) error {
+	m.Options.pluginGroupsMutex.Lock()
+	defer m.Options.pluginGroupsMutex.Unlock()
+	//Check if the tag exists
+	_, ok := m.Options.PluginGroups[tag]
+	if !ok {
+		return errors.New("tag not found")
+	}
+
+	//Remove the plugin from the group
+	pluginList := m.Options.PluginGroups[tag]
+	for i, id := range pluginList {
+		if id == pluginID {
+			pluginList = append(pluginList[:i], pluginList[i+1:]...)
+			m.Options.PluginGroups[tag] = pluginList
+			return nil
+		}
+	}
+	return errors.New("plugin not found")
+}
+
+// RemovePluginGroup removes a plugin group
+func (m *Manager) RemovePluginGroup(tag string) error {
+	m.Options.pluginGroupsMutex.Lock()
+	defer m.Options.pluginGroupsMutex.Unlock()
+	_, ok := m.Options.PluginGroups[tag]
+	if !ok {
+		return errors.New("tag not found")
+	}
+	delete(m.Options.PluginGroups, tag)
+	return nil
+}
+
+// SavePluginGroupsFromFile loads plugin groups from a file
+func (m *Manager) SavePluginGroupsToFile() error {
+	m.Options.pluginGroupsMutex.RLock()
+	pluginGroupsCopy := make(map[string][]string)
+	for k, v := range m.Options.PluginGroups {
+		pluginGroupsCopy[k] = append([]string{}, v...)
+	}
+	m.Options.pluginGroupsMutex.RUnlock()
+
+	//Write to file
+	js, _ := json.Marshal(pluginGroupsCopy)
+	err := os.WriteFile(m.Options.PluginGroupsConfig, js, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}

src/mod/plugins/handler.go

@@ -11,6 +11,146 @@ import (
 	"imuslab.com/zoraxy/mod/utils"
 )
 
+/* Plugin Groups */
+// HandleListPluginGroups handles the request to list all plugin groups
+func (m *Manager) HandleListPluginGroups(w http.ResponseWriter, r *http.Request) {
+	targetTag, err := utils.GetPara(r, "tag")
+	if err != nil {
+		//List all tags
+		pluginGroups := m.ListPluginGroups()
+		js, _ := json.Marshal(pluginGroups)
+		utils.SendJSONResponse(w, string(js))
+	} else {
+		//List the plugins under the tag
+		m.tagPluginListMutex.RLock()
+		plugins, ok := m.tagPluginList[targetTag]
+		m.tagPluginListMutex.RUnlock()
+		if !ok {
+			//Return empty array
+			js, _ := json.Marshal([]string{})
+			utils.SendJSONResponse(w, string(js))
+			return
+		}
+
+		//Sort the plugin by its name
+		sort.Slice(plugins, func(i, j int) bool {
+			return plugins[i].Spec.Name < plugins[j].Spec.Name
+		})
+
+		js, err := json.Marshal(plugins)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		utils.SendJSONResponse(w, string(js))
+	}
+}
+
+// HandleAddPluginToGroup handles the request to add a plugin to a group
+func (m *Manager) HandleAddPluginToGroup(w http.ResponseWriter, r *http.Request) {
+	tag, err := utils.PostPara(r, "tag")
+	if err != nil {
+		utils.SendErrorResponse(w, "tag not found")
+		return
+	}
+
+	pluginID, err := utils.PostPara(r, "plugin_id")
+	if err != nil {
+		utils.SendErrorResponse(w, "plugin_id not found")
+		return
+	}
+
+	//Check if plugin exists
+	_, err = m.GetPluginByID(pluginID)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	//Add the plugin to the group
+	err = m.AddPluginToGroup(tag, pluginID)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	//Save the plugin groups to file
+	err = m.SavePluginGroupsToFile()
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	//Update the radix tree mapping
+	m.UpdateTagsToPluginMaps()
+
+	utils.SendOK(w)
+}
+
+// HandleRemovePluginFromGroup handles the request to remove a plugin from a group
+func (m *Manager) HandleRemovePluginFromGroup(w http.ResponseWriter, r *http.Request) {
+	tag, err := utils.PostPara(r, "tag")
+	if err != nil {
+		utils.SendErrorResponse(w, "tag not found")
+		return
+	}
+
+	pluginID, err := utils.PostPara(r, "plugin_id")
+	if err != nil {
+		utils.SendErrorResponse(w, "plugin_id not found")
+		return
+	}
+
+	//Remove the plugin from the group
+	err = m.RemovePluginFromGroup(tag, pluginID)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	//Save the plugin groups to file
+	err = m.SavePluginGroupsToFile()
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	//Update the radix tree mapping
+	m.UpdateTagsToPluginMaps()
+
+	utils.SendOK(w)
+}
+
+// HandleRemovePluginGroup handles the request to remove a plugin group
+func (m *Manager) HandleRemovePluginGroup(w http.ResponseWriter, r *http.Request) {
+	tag, err := utils.PostPara(r, "tag")
+	if err != nil {
+		utils.SendErrorResponse(w, "tag not found")
+		return
+	}
+
+	//Remove the plugin group
+	err = m.RemovePluginGroup(tag)
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	//Save the plugin groups to file
+	err = m.SavePluginGroupsToFile()
+	if err != nil {
+		utils.SendErrorResponse(w, err.Error())
+		return
+	}
+
+	//Update the radix tree mapping
+	m.UpdateTagsToPluginMaps()
+
+	utils.SendOK(w)
+}
+
+/* Plugin APIs */
 // HandleListPlugins handles the request to list all loaded plugins
 func (m *Manager) HandleListPlugins(w http.ResponseWriter, r *http.Request) {
 	plugins, err := m.ListLoadedPlugins()

src/mod/plugins/lifecycle.go

@@ -2,7 +2,6 @@ package plugins
 
 import (
 	"encoding/json"
-	"fmt"
 	"io"
 	"net/http"
 	"net/url"
@@ -53,10 +52,16 @@ func (m *Manager) StartPlugin(pluginID string) error {
 		return err
 	}
 
+	stdErrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return err
+	}
+
 	if err := cmd.Start(); err != nil {
 		return err
 	}
 
+	//Create a goroutine to handle the STDOUT of the plugin
 	go func() {
 		buf := make([]byte, 1)
 		lineBuf := ""
@@ -82,6 +87,48 @@ func (m *Manager) StartPlugin(pluginID string) error {
 		}
 	}()
 
+	//Create a goroutine to handle the STDERR of the plugin
+	go func() {
+		buf := make([]byte, 1)
+		lineBuf := ""
+		for {
+			n, err := stdErrPipe.Read(buf)
+			if n > 0 {
+				lineBuf += string(buf[:n])
+				for {
+					if idx := strings.IndexByte(lineBuf, '\n'); idx != -1 {
+						m.handlePluginSTDERR(pluginID, lineBuf[:idx])
+						lineBuf = lineBuf[idx+1:]
+					} else {
+						break
+					}
+				}
+			}
+			if err != nil {
+				if err != io.EOF {
+					m.handlePluginSTDERR(pluginID, lineBuf) // handle any remaining data
+				}
+				break
+			}
+		}
+	}()
+
+	//Create a goroutine to wait for the plugin to exit
+	go func() {
+		err := cmd.Wait()
+		if err != nil {
+			//In theory this should not happen except for a crash
+			m.Log("plugin "+thisPlugin.Spec.ID+" encounted a fatal error. Disabling plugin...", err)
+
+			//Set the plugin state to disabled
+			thisPlugin.Enabled = false
+
+			//Generate a new static forwarder radix tree
+			m.UpdateTagsToPluginMaps()
+			return
+		}
+	}()
+
 	//Create a UI forwarder if the plugin has UI
 	err = m.StartUIHandlerForPlugin(thisPlugin, pluginConfiguration.Port)
 	if err != nil {
@@ -119,8 +166,6 @@ func (m *Manager) StartUIHandlerForPlugin(targetPlugin *Plugin, pluginListeningP
 		return err
 	}
 
-	fmt.Println("DEBUG: Requesting Plugin UI URL: ", pluginUIURL)
-
 	// Generate the plugin subpath to be trimmed
 	pluginMatchingPath := filepath.ToSlash(filepath.Join("/plugin.ui/"+targetPlugin.Spec.ID+"/")) + "/"
 	if targetPlugin.Spec.UIPath != "" {
@@ -148,6 +193,19 @@ func (m *Manager) handlePluginSTDOUT(pluginID string, line string) {
 	m.Log("["+thisPlugin.Spec.Name+":"+strconv.Itoa(processID)+"] "+line, nil)
 }
 
+func (m *Manager) handlePluginSTDERR(pluginID string, line string) {
+	thisPlugin, err := m.GetPluginByID(pluginID)
+	if err != nil {
+		return
+	}
+	processID := -1
+	if thisPlugin.process != nil && thisPlugin.process.Process != nil {
+		// Get the process ID of the plugin
+		processID = thisPlugin.process.Process.Pid
+	}
+	m.Log("["+thisPlugin.Spec.Name+":"+strconv.Itoa(processID)+"] "+line, nil)
+}
+
 // StopPlugin stops a plugin, it is garanteed that the plugin is stopped after this function
 func (m *Manager) StopPlugin(pluginID string) error {
 	thisPlugin, err := m.GetPluginByID(pluginID)

src/mod/plugins/plugins.go

@@ -82,6 +82,7 @@ func (m *Manager) LoadPluginsFromDisk() error {
 			m.Log("Loaded plugin: "+thisPlugin.Spec.Name, nil)
 
 			// If the plugin was enabled, start it now
+			fmt.Println("Plugin enabled state", m.GetPluginPreviousEnableState(thisPlugin.Spec.ID))
 			if m.GetPluginPreviousEnableState(thisPlugin.Spec.ID) {
 				err = m.StartPlugin(thisPlugin.Spec.ID)
 				if err != nil {
@@ -118,11 +119,11 @@ func (m *Manager) GetPluginByID(pluginID string) (*Plugin, error) {
 
 // EnablePlugin enables a plugin
 func (m *Manager) EnablePlugin(pluginID string) error {
+	m.Options.Database.Write("plugins", pluginID, true)
 	err := m.StartPlugin(pluginID)
 	if err != nil {
 		return err
 	}
-	m.Options.Database.Write("plugins", pluginID, true)
 	//Generate the static forwarder radix tree
 	m.UpdateTagsToPluginMaps()
 	return nil

src/mod/plugins/traffic_router.go

@@ -13,8 +13,6 @@ func (m *Manager) HandleRoute(w http.ResponseWriter, r *http.Request, tags []str
 		return false
 	}
 
-	return false
-
 	//For each tag, check if the request path matches the static capture path                    //Wait group for the goroutines
 	var staticRoutehandlers []*Plugin          //The handler for the request, can be multiple plugins
 	var longestPrefixAcrossAlltags string = "" //The longest prefix across all tags

src/mod/plugins/zoraxy_plugin/dev_webserver.go

@@ -0,0 +1,145 @@
+package zoraxy_plugin
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+type PluginUiDebugRouter struct {
+	PluginID         string //The ID of the plugin
+	TargetDir        string //The directory where the UI files are stored
+	HandlerPrefix    string //The prefix of the handler used to route this router, e.g. /ui
+	EnableDebug      bool   //Enable debug mode
+	terminateHandler func() //The handler to be called when the plugin is terminated
+}
+
+// NewPluginFileSystemUIRouter creates a new PluginUiRouter with file system
+// The targetDir is the directory where the UI files are stored (e.g. ./www)
+// The handlerPrefix is the prefix of the handler used to route this router
+// The handlerPrefix should start with a slash (e.g. /ui) that matches the http.Handle path
+// All prefix should not end with a slash
+func NewPluginFileSystemUIRouter(pluginID string, targetDir string, handlerPrefix string) *PluginUiDebugRouter {
+	//Make sure all prefix are in /prefix format
+	if !strings.HasPrefix(handlerPrefix, "/") {
+		handlerPrefix = "/" + handlerPrefix
+	}
+	handlerPrefix = strings.TrimSuffix(handlerPrefix, "/")
+
+	//Return the PluginUiRouter
+	return &PluginUiDebugRouter{
+		PluginID:      pluginID,
+		TargetDir:     targetDir,
+		HandlerPrefix: handlerPrefix,
+	}
+}
+
+func (p *PluginUiDebugRouter) populateCSRFToken(r *http.Request, fsHandler http.Handler) http.Handler {
+	//Get the CSRF token from header
+	csrfToken := r.Header.Get("X-Zoraxy-Csrf")
+	if csrfToken == "" {
+		csrfToken = "missing-csrf-token"
+	}
+
+	//Return the middleware
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Check if the request is for an HTML file
+		if strings.HasSuffix(r.URL.Path, ".html") {
+			//Read the target file from file system
+			targetFilePath := strings.TrimPrefix(r.URL.Path, "/")
+			targetFilePath = p.TargetDir + "/" + targetFilePath
+			targetFilePath = strings.TrimPrefix(targetFilePath, "/")
+			targetFileContent, err := os.ReadFile(targetFilePath)
+			if err != nil {
+				http.Error(w, "File not found", http.StatusNotFound)
+				return
+			}
+			body := string(targetFileContent)
+			body = strings.ReplaceAll(body, "{{.csrfToken}}", csrfToken)
+			w.Header().Set("Content-Type", "text/html")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(body))
+			return
+		} else if strings.HasSuffix(r.URL.Path, "/") {
+			//Check if the request is for a directory
+			//Check if the directory has an index.html file
+			targetFilePath := strings.TrimPrefix(r.URL.Path, "/")
+			targetFilePath = p.TargetDir + "/" + targetFilePath + "index.html"
+			targetFilePath = strings.TrimPrefix(targetFilePath, "/")
+			if _, err := os.Stat(targetFilePath); err == nil {
+				//Serve the index.html file
+				targetFileContent, err := os.ReadFile(targetFilePath)
+				if err != nil {
+					http.Error(w, "File not found", http.StatusNotFound)
+					return
+				}
+				body := string(targetFileContent)
+				body = strings.ReplaceAll(body, "{{.csrfToken}}", csrfToken)
+				w.Header().Set("Content-Type", "text/html")
+				w.WriteHeader(http.StatusOK)
+				w.Write([]byte(body))
+				return
+			}
+		}
+
+		//Call the next handler
+		fsHandler.ServeHTTP(w, r)
+	})
+
+}
+
+// GetHttpHandler returns the http.Handler for the PluginUiRouter
+func (p *PluginUiDebugRouter) Handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		//Remove the plugin UI handler path prefix
+		if p.EnableDebug {
+			fmt.Print("Request URL:", r.URL.Path, " rewriting to ")
+		}
+
+		rewrittenURL := r.RequestURI
+		rewrittenURL = strings.TrimPrefix(rewrittenURL, p.HandlerPrefix)
+		rewrittenURL = strings.ReplaceAll(rewrittenURL, "//", "/")
+		r.URL.Path = rewrittenURL
+		r.RequestURI = rewrittenURL
+		if p.EnableDebug {
+			fmt.Println(r.URL.Path)
+		}
+
+		//Serve the file from the file system
+		fsHandler := http.FileServer(http.Dir(p.TargetDir))
+
+		// Replace {{csrf_token}} with the actual CSRF token and serve the file
+		p.populateCSRFToken(r, fsHandler).ServeHTTP(w, r)
+	})
+}
+
+// RegisterTerminateHandler registers the terminate handler for the PluginUiRouter
+// The terminate handler will be called when the plugin is terminated from Zoraxy plugin manager
+// if mux is nil, the handler will be registered to http.DefaultServeMux
+func (p *PluginUiDebugRouter) RegisterTerminateHandler(termFunc func(), mux *http.ServeMux) {
+	p.terminateHandler = termFunc
+	if mux == nil {
+		mux = http.DefaultServeMux
+	}
+	mux.HandleFunc(p.HandlerPrefix+"/term", func(w http.ResponseWriter, r *http.Request) {
+		p.terminateHandler()
+		w.WriteHeader(http.StatusOK)
+		go func() {
+			//Make sure the response is sent before the plugin is terminated
+			time.Sleep(100 * time.Millisecond)
+			os.Exit(0)
+		}()
+	})
+}
+
+// Attach the file system UI handler to the target http.ServeMux
+func (p *PluginUiDebugRouter) AttachHandlerToMux(mux *http.ServeMux) {
+	if mux == nil {
+		mux = http.DefaultServeMux
+	}
+
+	p.HandlerPrefix = strings.TrimSuffix(p.HandlerPrefix, "/")
+	mux.Handle(p.HandlerPrefix+"/", p.Handler())
+}

src/mod/plugins/zoraxy_plugin/embed_webserver.go

@@ -74,6 +74,20 @@ func (p *PluginUiRouter) populateCSRFToken(r *http.Request, fsHandler http.Handl
 			w.WriteHeader(http.StatusOK)
 			w.Write([]byte(body))
 			return
+		} else if strings.HasSuffix(r.URL.Path, "/") {
+			// Check if the directory has an index.html file
+			indexFilePath := strings.TrimPrefix(r.URL.Path, "/") + "index.html"
+			indexFilePath = p.TargetFsPrefix + "/" + indexFilePath
+			indexFilePath = strings.TrimPrefix(indexFilePath, "/")
+			indexFileContent, err := fs.ReadFile(*p.TargetFs, indexFilePath)
+			if err == nil {
+				body := string(indexFileContent)
+				body = strings.ReplaceAll(body, "{{.csrfToken}}", csrfToken)
+				w.Header().Set("Content-Type", "text/html")
+				w.WriteHeader(http.StatusOK)
+				w.Write([]byte(body))
+				return
+			}
 		}
 
 		//Call the next handler
@@ -130,3 +144,13 @@ func (p *PluginUiRouter) RegisterTerminateHandler(termFunc func(), mux *http.Ser
 		}()
 	})
 }
+
+// Attach the embed UI handler to the target http.ServeMux
+func (p *PluginUiRouter) AttachHandlerToMux(mux *http.ServeMux) {
+	if mux == nil {
+		mux = http.DefaultServeMux
+	}
+
+	p.HandlerPrefix = strings.TrimSuffix(p.HandlerPrefix, "/")
+	mux.Handle(p.HandlerPrefix+"/", p.Handler())
+}