Added embed server for plugin library

- Added embeded resources server for plugin library
- Added ztnc plugin for global area network
- Added wide mode for side wrapper
This commit is contained in:
Toby Chui
2025-02-28 15:46:57 +08:00
parent bddff0cf2f
commit 53657e8716
54 changed files with 4870 additions and 42 deletions

View File

@@ -0,0 +1,146 @@
package database
/*
ArOZ Online Database Access Module
author: tobychui
This is an improved Object oriented base solution to the original
aroz online database script.
*/
import (
"log"
"runtime"
"aroz.org/zoraxy/ztnc/mod/database/dbinc"
)
type Database struct {
Db interface{} //This will be nil on openwrt, leveldb.DB on x64 platforms or bolt.DB on other platforms
BackendType dbinc.BackendType
Backend dbinc.Backend
}
func NewDatabase(dbfile string, backendType dbinc.BackendType) (*Database, error) {
if runtime.GOARCH == "riscv64" {
log.Println("RISCV hardware detected, ignoring the backend type and using FS emulated database")
}
return newDatabase(dbfile, backendType)
}
// Get the recommended backend type for the current system
func GetRecommendedBackendType() dbinc.BackendType {
//Check if the system is running on RISCV hardware
if runtime.GOARCH == "riscv64" {
//RISCV hardware, currently only support FS emulated database
return dbinc.BackendFSOnly
} else if runtime.GOOS == "windows" || (runtime.GOOS == "linux" && runtime.GOARCH == "amd64") {
//Powerful hardware
return dbinc.BackendBoltDB
//return dbinc.BackendLevelDB
}
//Default to BoltDB, the safest option
return dbinc.BackendBoltDB
}
/*
Create / Drop a table
Usage:
err := sysdb.NewTable("MyTable")
err := sysdb.DropTable("MyTable")
*/
// Create a new table
func (d *Database) NewTable(tableName string) error {
return d.newTable(tableName)
}
// Check is table exists
func (d *Database) TableExists(tableName string) bool {
return d.tableExists(tableName)
}
// Drop the given table
func (d *Database) DropTable(tableName string) error {
return d.dropTable(tableName)
}
/*
Write to database with given tablename and key. Example Usage:
type demo struct{
content string
}
thisDemo := demo{
content: "Hello World",
}
err := sysdb.Write("MyTable", "username/message",thisDemo);
*/
func (d *Database) Write(tableName string, key string, value interface{}) error {
return d.write(tableName, key, value)
}
/*
Read from database and assign the content to a given datatype. Example Usage:
type demo struct{
content string
}
thisDemo := new(demo)
err := sysdb.Read("MyTable", "username/message",&thisDemo);
*/
func (d *Database) Read(tableName string, key string, assignee interface{}) error {
return d.read(tableName, key, assignee)
}
/*
Check if a key exists in the database table given tablename and key
if sysdb.KeyExists("MyTable", "username/message"){
log.Println("Key exists")
}
*/
func (d *Database) KeyExists(tableName string, key string) bool {
return d.keyExists(tableName, key)
}
/*
Delete a value from the database table given tablename and key
err := sysdb.Delete("MyTable", "username/message");
*/
func (d *Database) Delete(tableName string, key string) error {
return d.delete(tableName, key)
}
/*
//List table example usage
//Assume the value is stored as a struct named "groupstruct"
entries, err := sysdb.ListTable("test")
if err != nil {
panic(err)
}
for _, keypairs := range entries{
log.Println(string(keypairs[0]))
group := new(groupstruct)
json.Unmarshal(keypairs[1], &group)
log.Println(group);
}
*/
func (d *Database) ListTable(tableName string) ([][][]byte, error) {
return d.listTable(tableName)
}
/*
Close the database connection
*/
func (d *Database) Close() {
d.close()
}

View File

@@ -0,0 +1,70 @@
//go:build !mipsle && !riscv64
// +build !mipsle,!riscv64
package database
import (
"errors"
"aroz.org/zoraxy/ztnc/mod/database/dbbolt"
"aroz.org/zoraxy/ztnc/mod/database/dbinc"
"aroz.org/zoraxy/ztnc/mod/database/dbleveldb"
)
func newDatabase(dbfile string, backendType dbinc.BackendType) (*Database, error) {
if backendType == dbinc.BackendFSOnly {
return nil, errors.New("Unsupported backend type for this platform")
}
if backendType == dbinc.BackendLevelDB {
db, err := dbleveldb.NewDB(dbfile)
return &Database{
Db: nil,
BackendType: backendType,
Backend: db,
}, err
}
db, err := dbbolt.NewBoltDatabase(dbfile)
return &Database{
Db: nil,
BackendType: backendType,
Backend: db,
}, err
}
func (d *Database) newTable(tableName string) error {
return d.Backend.NewTable(tableName)
}
func (d *Database) tableExists(tableName string) bool {
return d.Backend.TableExists(tableName)
}
func (d *Database) dropTable(tableName string) error {
return d.Backend.DropTable(tableName)
}
func (d *Database) write(tableName string, key string, value interface{}) error {
return d.Backend.Write(tableName, key, value)
}
func (d *Database) read(tableName string, key string, assignee interface{}) error {
return d.Backend.Read(tableName, key, assignee)
}
func (d *Database) keyExists(tableName string, key string) bool {
return d.Backend.KeyExists(tableName, key)
}
func (d *Database) delete(tableName string, key string) error {
return d.Backend.Delete(tableName, key)
}
func (d *Database) listTable(tableName string) ([][][]byte, error) {
return d.Backend.ListTable(tableName)
}
func (d *Database) close() {
d.Backend.Close()
}

View File

@@ -0,0 +1,196 @@
//go:build mipsle || riscv64
// +build mipsle riscv64
package database
import (
"encoding/json"
"errors"
"log"
"os"
"path/filepath"
"strings"
"aroz.org/zoraxy/zerotiernc/mod/database/dbinc"
)
/*
OpenWRT or RISCV backend
For OpenWRT or RISCV platform, we will use the filesystem as the database backend
as boltdb or leveldb is not supported on these platforms, including boltDB and LevelDB
in conditional compilation will create a build error on these platforms
*/
func newDatabase(dbfile string, backendType dbinc.BackendType) (*Database, error) {
dbRootPath := filepath.ToSlash(filepath.Clean(dbfile))
dbRootPath = "fsdb/" + dbRootPath
err := os.MkdirAll(dbRootPath, 0755)
if err != nil {
return nil, err
}
log.Println("Filesystem Emulated Key-value Database Service Started: " + dbRootPath)
return &Database{
Db: dbRootPath,
BackendType: dbinc.BackendFSOnly,
Backend: nil,
}, nil
}
func (d *Database) dump(filename string) ([]string, error) {
//Get all file objects from root
rootfiles, err := filepath.Glob(filepath.Join(d.Db.(string), "/*"))
if err != nil {
return []string{}, err
}
//Filter out the folders
rootFolders := []string{}
for _, file := range rootfiles {
if !isDirectory(file) {
rootFolders = append(rootFolders, filepath.Base(file))
}
}
return rootFolders, nil
}
func (d *Database) newTable(tableName string) error {
tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
if !fileExists(tablePath) {
return os.MkdirAll(tablePath, 0755)
}
return nil
}
func (d *Database) tableExists(tableName string) bool {
tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
if _, err := os.Stat(tablePath); errors.Is(err, os.ErrNotExist) {
return false
}
if !isDirectory(tablePath) {
return false
}
return true
}
func (d *Database) dropTable(tableName string) error {
tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
if d.tableExists(tableName) {
return os.RemoveAll(tablePath)
} else {
return errors.New("table not exists")
}
}
func (d *Database) write(tableName string, key string, value interface{}) error {
tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
js, err := json.Marshal(value)
if err != nil {
return err
}
key = strings.ReplaceAll(key, "/", "-SLASH_SIGN-")
return os.WriteFile(filepath.Join(tablePath, key+".entry"), js, 0755)
}
func (d *Database) read(tableName string, key string, assignee interface{}) error {
if !d.keyExists(tableName, key) {
return errors.New("key not exists")
}
key = strings.ReplaceAll(key, "/", "-SLASH_SIGN-")
tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
entryPath := filepath.Join(tablePath, key+".entry")
content, err := os.ReadFile(entryPath)
if err != nil {
return err
}
err = json.Unmarshal(content, &assignee)
return err
}
func (d *Database) keyExists(tableName string, key string) bool {
key = strings.ReplaceAll(key, "/", "-SLASH_SIGN-")
tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
entryPath := filepath.Join(tablePath, key+".entry")
return fileExists(entryPath)
}
func (d *Database) delete(tableName string, key string) error {
if !d.keyExists(tableName, key) {
return errors.New("key not exists")
}
key = strings.ReplaceAll(key, "/", "-SLASH_SIGN-")
tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
entryPath := filepath.Join(tablePath, key+".entry")
return os.Remove(entryPath)
}
func (d *Database) listTable(tableName string) ([][][]byte, error) {
if !d.tableExists(tableName) {
return [][][]byte{}, errors.New("table not exists")
}
tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
entries, err := filepath.Glob(filepath.Join(tablePath, "/*.entry"))
if err != nil {
return [][][]byte{}, err
}
var results [][][]byte = [][][]byte{}
for _, entry := range entries {
if !isDirectory(entry) {
//Read it
key := filepath.Base(entry)
key = strings.TrimSuffix(key, filepath.Ext(key))
key = strings.ReplaceAll(key, "-SLASH_SIGN-", "/")
bkey := []byte(key)
bval := []byte("")
c, err := os.ReadFile(entry)
if err != nil {
break
}
bval = c
results = append(results, [][]byte{bkey, bval})
}
}
return results, nil
}
func (d *Database) close() {
//Nothing to close as it is file system
}
func isDirectory(path string) bool {
fileInfo, err := os.Stat(path)
if err != nil {
return false
}
return fileInfo.IsDir()
}
func fileExists(name string) bool {
_, err := os.Stat(name)
if err == nil {
return true
}
if errors.Is(err, os.ErrNotExist) {
return false
}
return false
}

View File

@@ -0,0 +1,141 @@
package dbbolt
import (
"encoding/json"
"errors"
"github.com/boltdb/bolt"
)
type Database struct {
Db interface{} //This is the bolt database object
}
func NewBoltDatabase(dbfile string) (*Database, error) {
db, err := bolt.Open(dbfile, 0600, nil)
if err != nil {
return nil, err
}
return &Database{
Db: db,
}, err
}
// Create a new table
func (d *Database) NewTable(tableName string) error {
err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(tableName))
if err != nil {
return err
}
return nil
})
return err
}
// Check is table exists
func (d *Database) TableExists(tableName string) bool {
return d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(tableName))
if b == nil {
return errors.New("table not exists")
}
return nil
}) == nil
}
// Drop the given table
func (d *Database) DropTable(tableName string) error {
err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
err := tx.DeleteBucket([]byte(tableName))
if err != nil {
return err
}
return nil
})
return err
}
// Write to table
func (d *Database) Write(tableName string, key string, value interface{}) error {
jsonString, err := json.Marshal(value)
if err != nil {
return err
}
err = d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(tableName))
if err != nil {
return err
}
b := tx.Bucket([]byte(tableName))
err = b.Put([]byte(key), jsonString)
return err
})
return err
}
func (d *Database) Read(tableName string, key string, assignee interface{}) error {
err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(tableName))
v := b.Get([]byte(key))
json.Unmarshal(v, &assignee)
return nil
})
return err
}
func (d *Database) KeyExists(tableName string, key string) bool {
resultIsNil := false
if !d.TableExists(tableName) {
//Table not exists. Do not proceed accessing key
//log.Println("[DB] ERROR: Requesting key from table that didn't exist!!!")
return false
}
err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(tableName))
v := b.Get([]byte(key))
if v == nil {
resultIsNil = true
}
return nil
})
if err != nil {
return false
} else {
if resultIsNil {
return false
} else {
return true
}
}
}
func (d *Database) Delete(tableName string, key string) error {
err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
tx.Bucket([]byte(tableName)).Delete([]byte(key))
return nil
})
return err
}
func (d *Database) ListTable(tableName string) ([][][]byte, error) {
var results [][][]byte
err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(tableName))
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
results = append(results, [][]byte{k, v})
}
return nil
})
return results, err
}
func (d *Database) Close() {
d.Db.(*bolt.DB).Close()
}

View File

@@ -0,0 +1,67 @@
package dbbolt_test
import (
"os"
"testing"
"aroz.org/zoraxy/ztnc/mod/database/dbbolt"
)
func TestNewBoltDatabase(t *testing.T) {
dbfile := "test.db"
defer os.Remove(dbfile)
db, err := dbbolt.NewBoltDatabase(dbfile)
if err != nil {
t.Fatalf("Failed to create new Bolt database: %v", err)
}
defer db.Close()
if db.Db == nil {
t.Fatalf("Expected non-nil database object")
}
}
func TestNewTable(t *testing.T) {
dbfile := "test.db"
defer os.Remove(dbfile)
db, err := dbbolt.NewBoltDatabase(dbfile)
if err != nil {
t.Fatalf("Failed to create new Bolt database: %v", err)
}
defer db.Close()
err = db.NewTable("testTable")
if err != nil {
t.Fatalf("Failed to create new table: %v", err)
}
}
func TestTableExists(t *testing.T) {
dbfile := "test.db"
defer os.Remove(dbfile)
db, err := dbbolt.NewBoltDatabase(dbfile)
if err != nil {
t.Fatalf("Failed to create new Bolt database: %v", err)
}
defer db.Close()
tableName := "testTable"
err = db.NewTable(tableName)
if err != nil {
t.Fatalf("Failed to create new table: %v", err)
}
exists := db.TableExists(tableName)
if !exists {
t.Fatalf("Expected table %s to exist", tableName)
}
nonExistentTable := "nonExistentTable"
exists = db.TableExists(nonExistentTable)
if exists {
t.Fatalf("Expected table %s to not exist", nonExistentTable)
}
}

View File

@@ -0,0 +1,39 @@
package dbinc
/*
dbinc is the interface for all database backend
*/
type BackendType int
const (
BackendBoltDB BackendType = iota //Default backend
BackendFSOnly //OpenWRT or RISCV backend
BackendLevelDB //LevelDB backend
BackEndAuto = BackendBoltDB
)
type Backend interface {
NewTable(tableName string) error
TableExists(tableName string) bool
DropTable(tableName string) error
Write(tableName string, key string, value interface{}) error
Read(tableName string, key string, assignee interface{}) error
KeyExists(tableName string, key string) bool
Delete(tableName string, key string) error
ListTable(tableName string) ([][][]byte, error)
Close()
}
func (b BackendType) String() string {
switch b {
case BackendBoltDB:
return "BoltDB"
case BackendFSOnly:
return "File System Emulated Key-Value Store"
case BackendLevelDB:
return "LevelDB"
default:
return "Unknown"
}
}

View File

@@ -0,0 +1,152 @@
package dbleveldb
import (
"encoding/json"
"log"
"path/filepath"
"strings"
"sync"
"time"
"aroz.org/zoraxy/ztnc/mod/database/dbinc"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/util"
)
// Ensure the DB struct implements the Backend interface
var _ dbinc.Backend = (*DB)(nil)
type DB struct {
db *leveldb.DB
Table sync.Map //For emulating table creation
batch leveldb.Batch //Batch write
writeFlushTicker *time.Ticker //Ticker for flushing data into disk
writeFlushStop chan bool //Stop channel for write flush ticker
}
func NewDB(path string) (*DB, error) {
//If the path is not a directory (e.g. /tmp/dbfile.db), convert the filename to directory
if filepath.Ext(path) != "" {
path = strings.ReplaceAll(path, ".", "_")
}
db, err := leveldb.OpenFile(path, nil)
if err != nil {
return nil, err
}
thisDB := &DB{
db: db,
Table: sync.Map{},
batch: leveldb.Batch{},
}
//Create a ticker to flush data into disk every 1 seconds
writeFlushTicker := time.NewTicker(1 * time.Second)
writeFlushStop := make(chan bool)
go func() {
for {
select {
case <-writeFlushTicker.C:
if thisDB.batch.Len() == 0 {
//No flushing needed
continue
}
err = db.Write(&thisDB.batch, nil)
if err != nil {
log.Println("[LevelDB] Failed to flush data into disk: ", err)
}
thisDB.batch.Reset()
case <-writeFlushStop:
return
}
}
}()
thisDB.writeFlushTicker = writeFlushTicker
thisDB.writeFlushStop = writeFlushStop
return thisDB, nil
}
func (d *DB) NewTable(tableName string) error {
//Create a table entry in the sync.Map
d.Table.Store(tableName, true)
return nil
}
func (d *DB) TableExists(tableName string) bool {
_, ok := d.Table.Load(tableName)
return ok
}
func (d *DB) DropTable(tableName string) error {
d.Table.Delete(tableName)
iter := d.db.NewIterator(nil, nil)
defer iter.Release()
for iter.Next() {
key := iter.Key()
if filepath.Dir(string(key)) == tableName {
err := d.db.Delete(key, nil)
if err != nil {
return err
}
}
}
return nil
}
func (d *DB) Write(tableName string, key string, value interface{}) error {
data, err := json.Marshal(value)
if err != nil {
return err
}
d.batch.Put([]byte(filepath.ToSlash(filepath.Join(tableName, key))), data)
return nil
}
func (d *DB) Read(tableName string, key string, assignee interface{}) error {
data, err := d.db.Get([]byte(filepath.ToSlash(filepath.Join(tableName, key))), nil)
if err != nil {
return err
}
return json.Unmarshal(data, assignee)
}
func (d *DB) KeyExists(tableName string, key string) bool {
_, err := d.db.Get([]byte(filepath.ToSlash(filepath.Join(tableName, key))), nil)
return err == nil
}
func (d *DB) Delete(tableName string, key string) error {
return d.db.Delete([]byte(filepath.ToSlash(filepath.Join(tableName, key))), nil)
}
func (d *DB) ListTable(tableName string) ([][][]byte, error) {
iter := d.db.NewIterator(util.BytesPrefix([]byte(tableName+"/")), nil)
defer iter.Release()
var result [][][]byte
for iter.Next() {
key := iter.Key()
//The key contains the table name as prefix. Trim it before returning
value := iter.Value()
result = append(result, [][]byte{[]byte(strings.TrimPrefix(string(key), tableName+"/")), value})
}
err := iter.Error()
if err != nil {
return nil, err
}
return result, nil
}
func (d *DB) Close() {
//Write the remaining data in batch back into disk
d.writeFlushStop <- true
d.writeFlushTicker.Stop()
d.db.Write(&d.batch, nil)
d.db.Close()
}

View File

@@ -0,0 +1,141 @@
package dbleveldb_test
import (
"os"
"testing"
"aroz.org/zoraxy/ztnc/mod/database/dbleveldb"
)
func TestNewDB(t *testing.T) {
path := "/tmp/testdb"
defer os.RemoveAll(path)
db, err := dbleveldb.NewDB(path)
if err != nil {
t.Fatalf("Failed to create new DB: %v", err)
}
defer db.Close()
}
func TestNewTable(t *testing.T) {
path := "/tmp/testdb"
defer os.RemoveAll(path)
db, err := dbleveldb.NewDB(path)
if err != nil {
t.Fatalf("Failed to create new DB: %v", err)
}
defer db.Close()
err = db.NewTable("testTable")
if err != nil {
t.Fatalf("Failed to create new table: %v", err)
}
}
func TestTableExists(t *testing.T) {
path := "/tmp/testdb"
defer os.RemoveAll(path)
db, err := dbleveldb.NewDB(path)
if err != nil {
t.Fatalf("Failed to create new DB: %v", err)
}
defer db.Close()
db.NewTable("testTable")
if !db.TableExists("testTable") {
t.Fatalf("Table should exist")
}
}
func TestDropTable(t *testing.T) {
path := "/tmp/testdb"
defer os.RemoveAll(path)
db, err := dbleveldb.NewDB(path)
if err != nil {
t.Fatalf("Failed to create new DB: %v", err)
}
defer db.Close()
db.NewTable("testTable")
err = db.DropTable("testTable")
if err != nil {
t.Fatalf("Failed to drop table: %v", err)
}
if db.TableExists("testTable") {
t.Fatalf("Table should not exist")
}
}
func TestWriteAndRead(t *testing.T) {
path := "/tmp/testdb"
defer os.RemoveAll(path)
db, err := dbleveldb.NewDB(path)
if err != nil {
t.Fatalf("Failed to create new DB: %v", err)
}
defer db.Close()
db.NewTable("testTable")
err = db.Write("testTable", "testKey", "testValue")
if err != nil {
t.Fatalf("Failed to write to table: %v", err)
}
var value string
err = db.Read("testTable", "testKey", &value)
if err != nil {
t.Fatalf("Failed to read from table: %v", err)
}
if value != "testValue" {
t.Fatalf("Expected 'testValue', got '%v'", value)
}
}
func TestListTable(t *testing.T) {
path := "/tmp/testdb"
defer os.RemoveAll(path)
db, err := dbleveldb.NewDB(path)
if err != nil {
t.Fatalf("Failed to create new DB: %v", err)
}
defer db.Close()
db.NewTable("testTable")
err = db.Write("testTable", "testKey1", "testValue1")
if err != nil {
t.Fatalf("Failed to write to table: %v", err)
}
err = db.Write("testTable", "testKey2", "testValue2")
if err != nil {
t.Fatalf("Failed to write to table: %v", err)
}
result, err := db.ListTable("testTable")
if err != nil {
t.Fatalf("Failed to list table: %v", err)
}
if len(result) != 2 {
t.Fatalf("Expected 2 entries, got %v", len(result))
}
expected := map[string]string{
"testTable/testKey1": "\"testValue1\"",
"testTable/testKey2": "\"testValue2\"",
}
for _, entry := range result {
key := string(entry[0])
value := string(entry[1])
if expected[key] != value {
t.Fatalf("Expected value '%v' for key '%v', got '%v'", expected[key], key, value)
}
}
}