Merge pull request #746 from AnthonyMichaelTDM/plugin-improvements-api-keys

feat(plugins): Implement plugin API key management and authentication middleware

src/def.go

@@ -149,6 +149,9 @@ var (
 	loadBalancer       *loadbalance.RouteManager //Global scope loadbalancer, store the state of the lb routing
 	pluginManager      *plugins.Manager          //Plugin manager for managing plugins
 
+	//Plugin auth related
+	pluginApiKeyManager *auth.APIKeyManager //API key manager for plugin authentication
+
 	//Authentication Provider
 	forwardAuthRouter *forward.AuthRouter  // Forward Auth router for Authelia/Authentik/etc authentication
 	oauth2Router      *oauth2.OAuth2Router //OAuth2Router router for OAuth2Router authentication

src/main.go

@@ -115,6 +115,7 @@ func main() {
 	//Initiate management interface APIs
 	requireAuth = !(*noauth)
 	initAPIs(webminPanelMux)
+	initRestAPI(webminPanelMux)
 
 	//Start the reverse proxy server in go routine
 	go func() {

src/mod/auth/plugin_apikey_manager.go

@@ -0,0 +1,112 @@
+package auth
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"sync"
+	"time"
+
+	"imuslab.com/zoraxy/mod/plugins/zoraxy_plugin"
+)
+
+// PluginAPIKey represents an API key for a plugin
+type PluginAPIKey struct {
+	PluginID           string
+	APIKey             string
+	PermittedEndpoints []zoraxy_plugin.PermittedAPIEndpoint // List of permitted API endpoints
+	CreatedAt          time.Time
+}
+
+// APIKeyManager manages API keys for plugins
+type APIKeyManager struct {
+	keys  map[string]*PluginAPIKey // key: API key, value: plugin info
+	mutex sync.RWMutex
+}
+
+// NewAPIKeyManager creates a new API key manager
+func NewAPIKeyManager() *APIKeyManager {
+	return &APIKeyManager{
+		keys:  make(map[string]*PluginAPIKey),
+		mutex: sync.RWMutex{},
+	}
+}
+
+// GenerateAPIKey generates a new API key for a plugin
+func (m *APIKeyManager) GenerateAPIKey(pluginID string, permittedEndpoints []zoraxy_plugin.PermittedAPIEndpoint) (*PluginAPIKey, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	// Generate a cryptographically secure random key
+	bytes := make([]byte, 32)
+	if _, err := rand.Read(bytes); err != nil {
+		return nil, fmt.Errorf("failed to generate random bytes: %w", err)
+	}
+
+	// Hash the random bytes to create the API key
+	hash := sha256.Sum256(bytes)
+	apiKey := hex.EncodeToString(hash[:])
+
+	// Create the plugin API key
+	pluginAPIKey := &PluginAPIKey{
+		PluginID:           pluginID,
+		APIKey:             apiKey,
+		PermittedEndpoints: permittedEndpoints,
+		CreatedAt:          time.Now(),
+	}
+
+	// Store the API key
+	m.keys[apiKey] = pluginAPIKey
+
+	return pluginAPIKey, nil
+}
+
+// ValidateAPIKey validates an API key and returns the associated plugin information
+func (m *APIKeyManager) ValidateAPIKey(apiKey string) (*PluginAPIKey, error) {
+	m.mutex.RLock()
+	defer m.mutex.RUnlock()
+
+	pluginAPIKey, exists := m.keys[apiKey]
+	if !exists {
+		return nil, fmt.Errorf("invalid API key")
+	}
+
+	return pluginAPIKey, nil
+}
+
+// ValidateAPIKeyForEndpoint validates an API key for a specific endpoint
+func (m *APIKeyManager) ValidateAPIKeyForEndpoint(endpoint string, method string, apiKey string) (*PluginAPIKey, error) {
+	pluginAPIKey, err := m.ValidateAPIKey(apiKey)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if the endpoint is permitted
+	for _, permittedEndpoint := range pluginAPIKey.PermittedEndpoints {
+		if permittedEndpoint.Endpoint == endpoint && permittedEndpoint.Method == method {
+			return pluginAPIKey, nil
+		}
+	}
+
+	return nil, fmt.Errorf("endpoint not permitted for this API key")
+}
+
+// RevokeAPIKeysForPlugin revokes all API keys for a specific plugin
+func (m *APIKeyManager) RevokeAPIKeysForPlugin(pluginID string) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	keysToRemove := []string{}
+	for apiKey, pluginAPIKey := range m.keys {
+		if pluginAPIKey.PluginID == pluginID {
+			keysToRemove = append(keysToRemove, apiKey)
+		}
+	}
+
+	for _, apiKey := range keysToRemove {
+		delete(m.keys, apiKey)
+	}
+
+	return nil
+}

src/mod/auth/plugin_middleware.go

@@ -0,0 +1,94 @@
+// Handles the API-Key based authentication for plugins
+
+package auth
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+)
+
+const (
+	PLUGIN_API_PREFIX = "/plugin"
+)
+
+type PluginMiddlewareOptions struct {
+	DeniedHandler http.HandlerFunc //Thing(s) to do when request is rejected
+	ApiKeyManager *APIKeyManager
+	TargetMux     *http.ServeMux
+}
+
+// PluginAuthMiddleware provides authentication middleware for plugin API requests
+type PluginAuthMiddleware struct {
+	option    PluginMiddlewareOptions
+	endpoints map[string]http.HandlerFunc
+}
+
+// NewPluginAuthMiddleware creates a new plugin authentication middleware
+func NewPluginAuthMiddleware(option PluginMiddlewareOptions) *PluginAuthMiddleware {
+	return &PluginAuthMiddleware{
+		option:    option,
+		endpoints: make(map[string]http.HandlerFunc),
+	}
+}
+
+func (m *PluginAuthMiddleware) HandleAuthCheck(w http.ResponseWriter, r *http.Request, handler http.HandlerFunc) {
+	// Check for API key in the Authorization header
+	authHeader := r.Header.Get("Authorization")
+	if authHeader == "" {
+		// No authorization header
+		m.option.DeniedHandler(w, r)
+		return
+	}
+
+	// Check if it's a plugin API key (Bearer token)
+	if !strings.HasPrefix(authHeader, "Bearer ") {
+		// Not a Bearer token
+		m.option.DeniedHandler(w, r)
+		return
+	}
+
+	// Extract the API key
+	apiKey := strings.TrimPrefix(authHeader, "Bearer ")
+
+	// Validate the API key for this endpoint
+	_, err := m.option.ApiKeyManager.ValidateAPIKeyForEndpoint(r.URL.Path, r.Method, apiKey)
+	if err != nil {
+		// Invalid API key or endpoint not permitted
+		m.option.DeniedHandler(w, r)
+		return
+	}
+
+	// Call the original handler
+	handler(w, r)
+}
+
+// wraps an HTTP handler with plugin authentication middleware
+func (m *PluginAuthMiddleware) HandleFunc(endpoint string, handler http.HandlerFunc) error {
+	// ensure the endpoint is prefixed with PLUGIN_API_PREFIX
+	if !strings.HasPrefix(endpoint, PLUGIN_API_PREFIX) {
+		endpoint = PLUGIN_API_PREFIX + endpoint
+	}
+
+	// Check if the endpoint already registered
+	if _, exist := m.endpoints[endpoint]; exist {
+		fmt.Println("WARNING! Duplicated registering of plugin api endpoint: " + endpoint)
+		return errors.New("endpoint register duplicated")
+	}
+
+	m.endpoints[endpoint] = handler
+
+	wrappedHandler := func(w http.ResponseWriter, r *http.Request) {
+		m.HandleAuthCheck(w, r, handler)
+	}
+
+	// Ok. Register handler
+	if m.option.TargetMux == nil {
+		http.HandleFunc(endpoint, wrappedHandler)
+	} else {
+		m.option.TargetMux.HandleFunc(endpoint, wrappedHandler)
+	}
+
+	return nil
+}

src/mod/plugins/lifecycle.go

@@ -41,6 +41,18 @@ func (m *Manager) StartPlugin(pluginID string) error {
 		Port:         getRandomPortNumber(),
 		RuntimeConst: *m.Options.SystemConst,
 	}
+
+	// Generate API key if the plugin has permitted endpoints
+	if len(thisPlugin.Spec.PermittedAPIEndpoints) > 0 {
+		apiKey, err := m.Options.APIKeyManager.GenerateAPIKey(thisPlugin.Spec.ID, thisPlugin.Spec.PermittedAPIEndpoints)
+		if err != nil {
+			return err
+		}
+		pluginConfiguration.APIKey = apiKey.APIKey
+		pluginConfiguration.ZoraxyPort = m.Options.ZoraxyPort
+		m.Log("Generated API key for plugin "+thisPlugin.Spec.Name, nil)
+	}
+
 	js, _ := json.Marshal(pluginConfiguration)
 
 	//Start the plugin with given configuration
@@ -270,6 +282,13 @@ func (m *Manager) StopPlugin(pluginID string) error {
 	thisPlugin.Enabled = false
 	thisPlugin.StopAllStaticPathRouters()
 	thisPlugin.StopDynamicForwardRouter()
+
+	//Clean up API key
+	err = m.Options.APIKeyManager.RevokeAPIKeysForPlugin(thisPlugin.Spec.ID)
+	if err != nil {
+		m.Log("Failed to revoke API keys for plugin "+thisPlugin.Spec.Name, err)
+	}
+
 	return nil
 }
 

src/mod/plugins/typdef.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 	"time"
 
+	"imuslab.com/zoraxy/mod/auth"
 	"imuslab.com/zoraxy/mod/database"
 	"imuslab.com/zoraxy/mod/dynamicproxy/dpcore"
 	"imuslab.com/zoraxy/mod/info/logger"
@@ -42,10 +43,14 @@ type ManagerOptions struct {
 
 	/* Runtime */
 	SystemConst  *zoraxyPlugin.RuntimeConstantValue //The system constant value
+	ZoraxyPort   int                                //The port of the Zoraxy instance, used for API calls
 	CSRFTokenGen func(*http.Request) string         `json:"-"` //The CSRF token generator function
 	Database     *database.Database                 `json:"-"`
 	Logger       *logger.Logger                     `json:"-"`
 
+	/* API Key Management */
+	APIKeyManager *auth.APIKeyManager `json:"-"` //The API key manager for the plugins
+
 	/* Development */
 	EnableHotReload   bool //Check if the plugin file is changed and reload the plugin automatically
 	HotReloadInterval int  //The interval for checking the plugin file change, in seconds

src/mod/plugins/zoraxy_plugin/zoraxy_plugin.go

@@ -47,6 +47,12 @@ type RuntimeConstantValue struct {
 	DevelopmentBuild bool   `json:"development_build"` //Whether the Zoraxy is a development build or not
 }
 
+type PermittedAPIEndpoint struct {
+	Method   string `json:"method"`   //HTTP method for the API endpoint (e.g., GET, POST)
+	Endpoint string `json:"endpoint"` //The API endpoint that the plugin can access
+	Reason   string `json:"reason"`   //The reason why the plugin needs to access this endpoint
+}
+
 /*
 IntroSpect Payload
 
@@ -97,7 +103,10 @@ type IntroSpect struct {
 
 	/* Subscriptions Settings */
 	SubscriptionPath    string            `json:"subscription_path"`    //Subscription event path of your plugin (e.g. /notifyme), a POST request with SubscriptionEvent as body will be sent to this path when the event is triggered
-	SubscriptionsEvents map[string]string `json:"subscriptions_events"` //Subscriptions events of your plugin, see Zoraxy documentation for more details
+	SubscriptionsEvents map[string]string `json:"subscriptions_events"` //Subscriptions events of your plugin, paired with comments describing how the event is used, see Zoraxy documentation for more details
+
+	/* API Access Control */
+	PermittedAPIEndpoints []PermittedAPIEndpoint `json:"permitted_api_endpoints"` //List of API endpoints this plugin can access, and a description of why the plugin needs to access this endpoint
 }
 
 /*
@@ -126,8 +135,10 @@ by the supplied values like starting a web server at given port
 that listens to 127.0.0.1:port
 */
 type ConfigureSpec struct {
-	Port         int                  `json:"port"`          //Port to listen
-	RuntimeConst RuntimeConstantValue `json:"runtime_const"` //Runtime constant values
+	Port         int                  `json:"port"`                  //Port to listen
+	RuntimeConst RuntimeConstantValue `json:"runtime_const"`         //Runtime constant values
+	APIKey       string               `json:"api_key,omitempty"`     //API key for accessing Zoraxy APIs, if the plugin has permitted endpoints
+	ZoraxyPort   int                  `json:"zoraxy_port,omitempty"` //The port that Zoraxy is running on, used for making API calls to Zoraxy
 	//To be expanded
 }
 

src/plugin_api.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+	"net/http"
+
+	"imuslab.com/zoraxy/mod/auth"
+	"imuslab.com/zoraxy/mod/netstat"
+)
+
+// Register the APIs for HTTP proxy management functions
+func RegisterHTTPProxyRestAPI(authMiddleware *auth.PluginAuthMiddleware) {
+	/* Reverse Proxy Settings & Status */
+	authMiddleware.HandleFunc("/api/proxy/status", ReverseProxyStatus)
+	authMiddleware.HandleFunc("/api/proxy/list", ReverseProxyList)
+	authMiddleware.HandleFunc("/api/proxy/listTags", ReverseProxyListTags)
+	authMiddleware.HandleFunc("/api/proxy/detail", ReverseProxyListDetail)
+	/* Reverse proxy upstream (load balance) */
+	authMiddleware.HandleFunc("/api/proxy/upstream/list", ReverseProxyUpstreamList)
+	/* Reverse proxy virtual directory */
+	authMiddleware.HandleFunc("/api/proxy/vdir/list", ReverseProxyListVdir)
+	/* Reverse proxy user-defined header */
+	authMiddleware.HandleFunc("/api/proxy/header/list", HandleCustomHeaderList)
+	/* Reverse proxy auth related */
+	authMiddleware.HandleFunc("/api/proxy/auth/exceptions/list", ListProxyBasicAuthExceptionPaths)
+}
+
+// Register the APIs for redirection rules management functions
+func RegisterRedirectionRestAPI(authRouter *auth.PluginAuthMiddleware) {
+	authRouter.HandleFunc("/api/redirect/list", handleListRedirectionRules)
+}
+
+// Register the APIs for access rules management functions
+func RegisterAccessRuleRestAPI(authRouter *auth.PluginAuthMiddleware) {
+	/* Access Rules Settings & Status */
+	authRouter.HandleFunc("/api/access/list", handleListAccessRules)
+	// authRouter.HandleFunc("/api/access/attach", handleAttachRuleToHost)
+	// authRouter.HandleFunc("/api/access/create", handleCreateAccessRule)
+	// authRouter.HandleFunc("/api/access/remove", handleRemoveAccessRule)
+	// authRouter.HandleFunc("/api/access/update", handleUpadateAccessRule)
+	/* Blacklist */
+	authRouter.HandleFunc("/api/blacklist/list", handleListBlacklisted)
+	authRouter.HandleFunc("/api/blacklist/country/add", handleCountryBlacklistAdd)
+	authRouter.HandleFunc("/api/blacklist/country/remove", handleCountryBlacklistRemove)
+	authRouter.HandleFunc("/api/blacklist/ip/add", handleIpBlacklistAdd)
+	authRouter.HandleFunc("/api/blacklist/ip/remove", handleIpBlacklistRemove)
+	authRouter.HandleFunc("/api/blacklist/enable", handleBlacklistEnable)
+	/* Whitelist */
+	authRouter.HandleFunc("/api/whitelist/list", handleListWhitelisted)
+	authRouter.HandleFunc("/api/whitelist/country/add", handleCountryWhitelistAdd)
+	authRouter.HandleFunc("/api/whitelist/country/remove", handleCountryWhitelistRemove)
+	authRouter.HandleFunc("/api/whitelist/ip/add", handleIpWhitelistAdd)
+	authRouter.HandleFunc("/api/whitelist/ip/remove", handleIpWhitelistRemove)
+	authRouter.HandleFunc("/api/whitelist/enable", handleWhitelistEnable)
+	authRouter.HandleFunc("/api/whitelist/allowLocal", handleWhitelistAllowLoopback)
+	/* Quick Ban List */
+	authRouter.HandleFunc("/api/quickban/list", handleListQuickBan)
+}
+
+// Register the APIs for path blocking rules management functions, WIP
+func RegisterPathRuleRestAPI(authRouter *auth.PluginAuthMiddleware) {
+	authRouter.HandleFunc("/api/pathrule/list", pathRuleHandler.HandleListBlockingPath)
+}
+
+// Register the APIs statistic anlysis and uptime monitoring functions
+func RegisterStatisticalRestAPI(authRouter *auth.PluginAuthMiddleware) {
+	/* Traffic Summary */
+	authRouter.HandleFunc("/api/stats/summary", statisticCollector.HandleTodayStatLoad)
+	authRouter.HandleFunc("/api/stats/countries", HandleCountryDistrSummary)
+	authRouter.HandleFunc("/api/stats/netstat", netstatBuffers.HandleGetNetworkInterfaceStats)
+	authRouter.HandleFunc("/api/stats/netstatgraph", netstatBuffers.HandleGetBufferedNetworkInterfaceStats)
+	authRouter.HandleFunc("/api/stats/listnic", netstat.HandleListNetworkInterfaces)
+	/* Zoraxy Analytic */
+	authRouter.HandleFunc("/api/analytic/list", AnalyticLoader.HandleSummaryList)
+	authRouter.HandleFunc("/api/analytic/load", AnalyticLoader.HandleLoadTargetDaySummary)
+	authRouter.HandleFunc("/api/analytic/loadRange", AnalyticLoader.HandleLoadTargetRangeSummary)
+	authRouter.HandleFunc("/api/analytic/exportRange", AnalyticLoader.HandleRangeExport)
+	authRouter.HandleFunc("/api/analytic/resetRange", AnalyticLoader.HandleRangeReset)
+	/* UpTime Monitor */
+	authRouter.HandleFunc("/api/utm/list", HandleUptimeMonitorListing)
+}
+
+// Register the APIs for Stream (TCP / UDP) Proxy management functions
+func RegisterStreamProxyRestAPI(authRouter *auth.PluginAuthMiddleware) {
+	authRouter.HandleFunc("/api/streamprox/config/list", streamProxyManager.HandleListConfigs)
+	authRouter.HandleFunc("/api/streamprox/config/status", streamProxyManager.HandleGetProxyStatus)
+}
+
+// Register the APIs for mDNS service management functions
+func RegisterMDNSRestAPI(authRouter *auth.PluginAuthMiddleware) {
+	authRouter.HandleFunc("/api/mdns/list", HandleMdnsListing)
+}
+
+// Register the APIs for Static Web Server management functions
+func RegisterStaticWebServerRestAPI(authRouter *auth.PluginAuthMiddleware) {
+	/* Static Web Server Controls */
+	authRouter.HandleFunc("/api/webserv/status", staticWebServer.HandleGetStatus)
+
+	/* File Manager */
+	if *allowWebFileManager {
+		authRouter.HandleFunc("/api/fs/list", staticWebServer.FileManager.HandleList)
+	}
+}
+
+func RegisterPluginRestAPI(authRouter *auth.PluginAuthMiddleware) {
+	authRouter.HandleFunc("/api/plugins/list", pluginManager.HandleListPlugins)
+	authRouter.HandleFunc("/api/plugins/info", pluginManager.HandlePluginInfo)
+
+	authRouter.HandleFunc("/api/plugins/groups/list", pluginManager.HandleListPluginGroups)
+
+	authRouter.HandleFunc("/api/plugins/store/list", pluginManager.HandleListDownloadablePlugins)
+}
+
+/* Register all the APIs */
+func initRestAPI(targetMux *http.ServeMux) {
+	authMiddleware := auth.NewPluginAuthMiddleware(
+		auth.PluginMiddlewareOptions{
+			TargetMux:     targetMux,
+			ApiKeyManager: pluginApiKeyManager,
+			DeniedHandler: func(w http.ResponseWriter, r *http.Request) {
+				http.Error(w, "401 - Unauthorized", http.StatusUnauthorized)
+			},
+		},
+	)
+
+	//Register the APIs
+	RegisterHTTPProxyRestAPI(authMiddleware)
+	RegisterRedirectionRestAPI(authMiddleware)
+	RegisterAccessRuleRestAPI(authMiddleware)
+	RegisterPathRuleRestAPI(authMiddleware)
+	RegisterStatisticalRestAPI(authMiddleware)
+	RegisterStreamProxyRestAPI(authMiddleware)
+	RegisterMDNSRestAPI(authMiddleware)
+	RegisterStaticWebServerRestAPI(authMiddleware)
+	RegisterPluginRestAPI(authMiddleware)
+}

src/start.go

@@ -3,6 +3,7 @@ package main
 import (
 	"log"
 	"net/http"
+	"net/netip"
 	"os"
 	"runtime"
 	"strconv"
@@ -99,6 +100,9 @@ func startupSequence() {
 		http.Redirect(w, r, "/login.html", http.StatusTemporaryRedirect)
 	})
 
+	// Create an API key manager for plugin authentication
+	pluginApiKeyManager = auth.NewAPIKeyManager()
+
 	//Create a TLS certificate manager
 	tlsCertManager, err = tlscert.NewManager(CONF_CERT_STORE, SystemWideLogger)
 	if err != nil {
@@ -313,11 +317,18 @@ func startupSequence() {
 	*/
 	pluginFolder := *path_plugin
 	pluginFolder = strings.TrimSuffix(pluginFolder, "/")
+	ZoraxyAddrPort, err := netip.ParseAddrPort(*webUIPort)
+	ZoraxyPort := 8000
+	if err == nil && ZoraxyAddrPort.IsValid() && ZoraxyAddrPort.Port() > 0 {
+		ZoraxyPort = int(ZoraxyAddrPort.Port())
+	}
 	pluginManager = plugins.NewPluginManager(&plugins.ManagerOptions{
 		PluginDir:          pluginFolder,
 		Database:           sysdb,
 		Logger:             SystemWideLogger,
 		PluginGroupsConfig: CONF_PLUGIN_GROUPS,
+		APIKeyManager:      pluginApiKeyManager,
+		ZoraxyPort:         ZoraxyPort,
 		CSRFTokenGen: func(r *http.Request) string {
 			return csrf.Token(r)
 		},

src/web/snippet/pluginInfo.html

@@ -139,6 +139,26 @@
                         </tbody>
                     </table>
                     <div class="ui divider"></div>
+                    <h4>Plugin IntroSpect Permitted API Endpoints</h4>
+                    <p>The following API endpoints are registered by this plugin and will be accessible by the plugin's API key:</p>
+                    <table class="ui basic celled unstackable table">
+                        <thead>
+                            <tr>
+                                <th>Endpoint</th>
+                                <th>Method</th>
+                                <th>Reason</th>
+                            </tr>
+                        </thead>
+                        <!-- This tbody will be filled by JavaScript -->
+                        <tbody id="plugin_permitted_api_endpoints">
+                        </tbody>
+                    </table>
+                    <p>
+                        Note that the API endpoints are only accessible by the plugin's API key.
+                        If the plugin does not have an API key, it will not be able to access these endpoints.
+                        API keys are generated automatically by Zoraxy when a plugin with permitted API endpoints is enabled.
+                    </p>
+                    <div class="ui divider"></div>
                 </div>
             </div>
         </div>
@@ -219,6 +239,22 @@
                 $("#dynamic_capture_sniffing_path").text(dynamicCaptureSniffingPath);
                 $("#dynamic_capture_ingress").text(dynamicCaptureIngress);
                 $("#registered_ui_proxy_path").text(registeredUIProxyPath);
+
+                //Update permitted API endpoints
+                let apiEndpoints = data.Spec.permitted_api_endpoints;
+                if (apiEndpoints == null || apiEndpoints.length == 0) {
+                    $("#plugin_permitted_api_endpoints").html('<tr><td colspan="3">No API endpoints registered</td></tr>');
+                } else {
+                    let endpointRows = '';
+                    apiEndpoints.forEach(function(endpoint) {
+                        endpointRows += `<tr>
+                            <td>${endpoint.endpoint}</td>
+                            <td>${endpoint.method}</td>
+                            <td>${endpoint.reason}</td>
+                        </tr>`;
+                    });
+                    $("#plugin_permitted_api_endpoints").html(endpointRows);
+                }
                 });
             }