marcel/zoraxy
1
0
Fork 0
mirror of https://github.com/tobychui/zoraxy.git synced 2025-06-01 13:17:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

Database update

Browse Source 
- Removed read-only mode
- Added LevelDB for big data storage

TODO: Update backup utilities to support new db structure
This commit is contained in:
Toby Chui 2024-12-06 23:34:21 +08:00
parent 2f1a6b5ba4
commit cc08c704de
14 changed files with 612 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/config.go
View File

@ -167,7 +167,6 @@ func ExportConfigAsZip(w http.ResponseWriter, r *http.Request) {
if includeSysDBRaw == "true" {
//Include the system database in backup snapshot
//Temporary set it to read only
sysdb.ReadOnly = true
includeSysDB = true
}
@ -241,8 +240,6 @@ func ExportConfigAsZip(w http.ResponseWriter, r *http.Request) {
return
}
//Restore sysdb state
sysdb.ReadOnly = false
}
if err != nil {

1
src/def.go
View File

@ -74,6 +74,7 @@ const (
/* System Startup Flags */
var (
webUIPort = flag.String("port", ":8000", "Management web interface listening port")
databaseBackend = flag.String("db", "auto", "Database backend to use (leveldb, boltdb, auto) Note that fsdb will be used on unsupported platforms like RISCV")
noauth = flag.Bool("noauth", false, "Disable authentication for management interface")
showver = flag.Bool("version", false, "Show version of this server")
allowSshLoopback = flag.Bool("sshlb", false, "Allow loopback web ssh connection (DANGER)")

2
src/go.mod
View File

@ -28,9 +28,11 @@ require (
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang/snappy v0.0.1 // indirect
github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.114 // indirect
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 // indirect
github.com/tidwall/buntdb v1.1.2 // indirect
github.com/tidwall/gjson v1.12.1 // indirect

6
src/go.sum
View File

@ -277,6 +277,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@ -528,6 +530,7 @@ github.com/nzdjb/go-metaname v1.0.0 h1:sNASlZC1RM3nSudtBTE1a3ZVTDyTpjqI5WXRPrdZ9
github.com/nzdjb/go-metaname v1.0.0/go.mod h1:0GR0LshZax1Lz4VrOrfNSE4dGvTp7HGjiemdczXT2H4=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
@ -536,6 +539,7 @@ github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3
github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY=
github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
@ -660,6 +664,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.1002 h1:RE84sHFFx6t24DJvSnF9fS1DzBNv9OpctzHK3t7AY+I=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.1002/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dnspod v1.0.1002 h1:QwE0dRkAAbdf+eACnkNULgDn9ZKUJpPWRyXdqJolP5E=

68
src/mod/database/database.go
View File

@ -9,17 +9,37 @@ package database
*/
import (
"sync"
"log"
"runtime"
"imuslab.com/zoraxy/mod/database/dbinc"
)
type Database struct {
Db interface{} //This will be nil on openwrt and *bolt.DB in the rest of the systems
Tables sync.Map
ReadOnly bool
Db interface{} //This will be nil on openwrt, leveldb.DB on x64 platforms or bolt.DB on other platforms
BackendType dbinc.BackendType
Backend dbinc.Backend
}
func NewDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
return newDatabase(dbfile, readOnlyMode)
func NewDatabase(dbfile string, backendType dbinc.BackendType) (*Database, error) {
if runtime.GOARCH == "riscv64" {
log.Println("RISCV hardware detected, ignoring the backend type and using FS emulated database")
}
return newDatabase(dbfile, backendType)
}
func GetRecommendedBackendType() dbinc.BackendType {
//Check if the system is running on RISCV hardware
if runtime.GOARCH == "riscv64" {
//RISCV hardware, currently only support FS emulated database
return dbinc.BackendFSOnly
} else if runtime.GOOS == "windows" || (runtime.GOOS == "linux" && runtime.GOARCH == "amd64") {
//Powerful hardware, use LevelDB
return dbinc.BackendLevelDB
}
//Default to BoltDB, the safest option
return dbinc.BackendBoltDB
}
/*
@ -29,39 +49,33 @@ func NewDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
err := sysdb.DropTable("MyTable")
*/
func (d *Database) UpdateReadWriteMode(readOnly bool) {
d.ReadOnly = readOnly
}
//Dump the whole db into a log file
func (d *Database) Dump(filename string) ([]string, error) {
return d.dump(filename)
}
//Create a new table
// Create a new table
func (d *Database) NewTable(tableName string) error {
return d.newTable(tableName)
}
//Check is table exists
// Check is table exists
func (d *Database) TableExists(tableName string) bool {
return d.tableExists(tableName)
}
//Drop the given table
// Drop the given table
func (d *Database) DropTable(tableName string) error {
return d.dropTable(tableName)
}
/*
Write to database with given tablename and key. Example Usage:
Write to database with given tablename and key. Example Usage:
type demo struct{
content string
}
thisDemo := demo{
content: "Hello World",
}
err := sysdb.Write("MyTable", "username/message",thisDemo);
err := sysdb.Write("MyTable", "username/message",thisDemo);
*/
func (d *Database) Write(tableName string, key string, value interface{}) error {
return d.write(tableName, key, value)
@ -81,14 +95,21 @@ func (d *Database) Read(tableName string, key string, assignee interface{}) erro
return d.read(tableName, key, assignee)
}
/*
Check if a key exists in the database table given tablename and key
if sysdb.KeyExists("MyTable", "username/message"){
log.Println("Key exists")
}
*/
func (d *Database) KeyExists(tableName string, key string) bool {
return d.keyExists(tableName, key)
}
/*
Delete a value from the database table given tablename and key
Delete a value from the database table given tablename and key
err := sysdb.Delete("MyTable", "username/message");
err := sysdb.Delete("MyTable", "username/message");
*/
func (d *Database) Delete(tableName string, key string) error {
return d.delete(tableName, key)
@ -115,6 +136,9 @@ func (d *Database) ListTable(tableName string) ([][][]byte, error) {
return d.listTable(tableName)
}
/*
Close the database connection
*/
func (d *Database) Close() {
d.close()
}

170
src/mod/database/database_core.go
View File

@ -4,183 +4,67 @@
package database
import (
"encoding/json"
"errors"
"log"
"sync"
"github.com/boltdb/bolt"
"imuslab.com/zoraxy/mod/database/dbbolt"
"imuslab.com/zoraxy/mod/database/dbinc"
"imuslab.com/zoraxy/mod/database/dbleveldb"
)
func newDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
db, err := bolt.Open(dbfile, 0600, nil)
if err != nil {
return nil, err
func newDatabase(dbfile string, backendType dbinc.BackendType) (*Database, error) {
if backendType == dbinc.BackendFSOnly {
return nil, errors.New("Unsupported backend type for this platform")
}
tableMap := sync.Map{}
//Build the table list from database
err = db.View(func(tx *bolt.Tx) error {
return tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
tableMap.Store(string(name), "")
return nil
})
})
if backendType == dbinc.BackendLevelDB {
db, err := dbleveldb.NewDB(dbfile)
return &Database{
Db: nil,
BackendType: backendType,
Backend: db,
}, err
}
db, err := dbbolt.NewBoltDatabase(dbfile)
return &Database{
Db: db,
Tables: tableMap,
ReadOnly: readOnlyMode,
Db: nil,
BackendType: backendType,
Backend: db,
}, err
}
//Dump the whole db into a log file
func (d *Database) dump(filename string) ([]string, error) {
results := []string{}
d.Tables.Range(func(tableName, v interface{}) bool {
entries, err := d.ListTable(tableName.(string))
if err != nil {
log.Println("Reading table " + tableName.(string) + " failed: " + err.Error())
return false
}
for _, keypairs := range entries {
results = append(results, string(keypairs[0])+":"+string(keypairs[1])+"\n")
}
return true
})
return results, nil
}
//Create a new table
func (d *Database) newTable(tableName string) error {
if d.ReadOnly == true {
return errors.New("Operation rejected in ReadOnly mode")
}
err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(tableName))
if err != nil {
return err
}
return nil
})
d.Tables.Store(tableName, "")
return err
return d.Backend.NewTable(tableName)
}
//Check is table exists
func (d *Database) tableExists(tableName string) bool {
if _, ok := d.Tables.Load(tableName); ok {
return true
}
return false
return d.Backend.TableExists(tableName)
}
//Drop the given table
func (d *Database) dropTable(tableName string) error {
if d.ReadOnly == true {
return errors.New("Operation rejected in ReadOnly mode")
}
err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
err := tx.DeleteBucket([]byte(tableName))
if err != nil {
return err
}
return nil
})
return err
return d.Backend.DropTable(tableName)
}
//Write to table
func (d *Database) write(tableName string, key string, value interface{}) error {
if d.ReadOnly {
return errors.New("Operation rejected in ReadOnly mode")
}
jsonString, err := json.Marshal(value)
if err != nil {
return err
}
err = d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(tableName))
if err != nil {
return err
}
b := tx.Bucket([]byte(tableName))
err = b.Put([]byte(key), jsonString)
return err
})
return err
return d.Backend.Write(tableName, key, value)
}
func (d *Database) read(tableName string, key string, assignee interface{}) error {
err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(tableName))
v := b.Get([]byte(key))
json.Unmarshal(v, &assignee)
return nil
})
return err
return d.Backend.Read(tableName, key, assignee)
}
func (d *Database) keyExists(tableName string, key string) bool {
resultIsNil := false
if !d.TableExists(tableName) {
//Table not exists. Do not proceed accessing key
log.Println("[DB] ERROR: Requesting key from table that didn't exist!!!")
return false
}
err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(tableName))
v := b.Get([]byte(key))
if v == nil {
resultIsNil = true
}
return nil
})
if err != nil {
return false
} else {
if resultIsNil {
return false
} else {
return true
}
}
return d.Backend.KeyExists(tableName, key)
}
func (d *Database) delete(tableName string, key string) error {
if d.ReadOnly {
return errors.New("Operation rejected in ReadOnly mode")
}
err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
tx.Bucket([]byte(tableName)).Delete([]byte(key))
return nil
})
return err
return d.Backend.Delete(tableName, key)
}
func (d *Database) listTable(tableName string) ([][][]byte, error) {
var results [][][]byte
err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(tableName))
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
results = append(results, [][]byte{k, v})
}
return nil
})
return results, err
return d.Backend.ListTable(tableName)
}
func (d *Database) close() {
d.Db.(*bolt.DB).Close()
d.Backend.Close()
}

48
src/mod/database/database_openwrt.go
View File

@ -10,10 +10,19 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"imuslab.com/zoraxy/mod/database/dbinc"
)
func newDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
/*
OpenWRT or RISCV backend
For OpenWRT or RISCV platform, we will use the filesystem as the database backend
as boltdb or leveldb is not supported on these platforms, including boltDB and LevelDB
in conditional compilation will create a build error on these platforms
*/
func newDatabase(dbfile string, backendType dbinc.BackendType) (*Database, error) {
dbRootPath := filepath.ToSlash(filepath.Clean(dbfile))
dbRootPath = "fsdb/" + dbRootPath
err := os.MkdirAll(dbRootPath, 0755)
@ -21,24 +30,11 @@ func newDatabase(dbfile string, readOnlyMode bool) (*Database, error) {
return nil, err
}
tableMap := sync.Map{}
//build the table list from file system
files, err := filepath.Glob(filepath.Join(dbRootPath, "/*"))
if err != nil {
return nil, err
}
for _, file := range files {
if isDirectory(file) {
tableMap.Store(filepath.Base(file), "")
}
}
log.Println("Filesystem Emulated Key-value Database Service Started: " + dbRootPath)
return &Database{
Db: dbRootPath,
Tables: tableMap,
ReadOnly: readOnlyMode,
Db: dbRootPath,
BackendType: dbinc.BackendFSOnly,
Backend: nil,
}, nil
}
@ -61,9 +57,7 @@ func (d *Database) dump(filename string) ([]string, error) {
}
func (d *Database) newTable(tableName string) error {
if d.ReadOnly {
return errors.New("Operation rejected in ReadOnly mode")
}
tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
if !fileExists(tablePath) {
return os.MkdirAll(tablePath, 0755)
@ -85,9 +79,7 @@ func (d *Database) tableExists(tableName string) bool {
}
func (d *Database) dropTable(tableName string) error {
if d.ReadOnly {
return errors.New("Operation rejected in ReadOnly mode")
}
tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
if d.tableExists(tableName) {
return os.RemoveAll(tablePath)
@ -98,9 +90,7 @@ func (d *Database) dropTable(tableName string) error {
}
func (d *Database) write(tableName string, key string, value interface{}) error {
if d.ReadOnly {
return errors.New("Operation rejected in ReadOnly mode")
}
tablePath := filepath.Join(d.Db.(string), filepath.Base(tableName))
js, err := json.Marshal(value)
if err != nil {
@ -138,9 +128,7 @@ func (d *Database) keyExists(tableName string, key string) bool {
}
func (d *Database) delete(tableName string, key string) error {
if d.ReadOnly {
return errors.New("Operation rejected in ReadOnly mode")
}
if !d.keyExists(tableName, key) {
return errors.New("key not exists")
}

141
src/mod/database/dbbolt/dbbolt.go Normal file
View File

@ -0,0 +1,141 @@
package dbbolt
import (
"encoding/json"
"errors"
"github.com/boltdb/bolt"
)
type Database struct {
Db interface{} //This is the bolt database object
}
func NewBoltDatabase(dbfile string) (*Database, error) {
db, err := bolt.Open(dbfile, 0600, nil)
if err != nil {
return nil, err
}
return &Database{
Db: db,
}, err
}
// Create a new table
func (d *Database) NewTable(tableName string) error {
err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(tableName))
if err != nil {
return err
}
return nil
})
return err
}
// Check is table exists
func (d *Database) TableExists(tableName string) bool {
return d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(tableName))
if b == nil {
return errors.New("table not exists")
}
return nil
}) == nil
}
// Drop the given table
func (d *Database) DropTable(tableName string) error {
err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
err := tx.DeleteBucket([]byte(tableName))
if err != nil {
return err
}
return nil
})
return err
}
// Write to table
func (d *Database) Write(tableName string, key string, value interface{}) error {
jsonString, err := json.Marshal(value)
if err != nil {
return err
}
err = d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(tableName))
if err != nil {
return err
}
b := tx.Bucket([]byte(tableName))
err = b.Put([]byte(key), jsonString)
return err
})
return err
}
func (d *Database) Read(tableName string, key string, assignee interface{}) error {
err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(tableName))
v := b.Get([]byte(key))
json.Unmarshal(v, &assignee)
return nil
})
return err
}
func (d *Database) KeyExists(tableName string, key string) bool {
resultIsNil := false
if !d.TableExists(tableName) {
//Table not exists. Do not proceed accessing key
//log.Println("[DB] ERROR: Requesting key from table that didn't exist!!!")
return false
}
err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(tableName))
v := b.Get([]byte(key))
if v == nil {
resultIsNil = true
}
return nil
})
if err != nil {
return false
} else {
if resultIsNil {
return false
} else {
return true
}
}
}
func (d *Database) Delete(tableName string, key string) error {
err := d.Db.(*bolt.DB).Update(func(tx *bolt.Tx) error {
tx.Bucket([]byte(tableName)).Delete([]byte(key))
return nil
})
return err
}
func (d *Database) ListTable(tableName string) ([][][]byte, error) {
var results [][][]byte
err := d.Db.(*bolt.DB).View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(tableName))
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
results = append(results, [][]byte{k, v})
}
return nil
})
return results, err
}
func (d *Database) Close() {
d.Db.(*bolt.DB).Close()
}

67
src/mod/database/dbbolt/dbbolt_test.go Normal file
View File

@ -0,0 +1,67 @@
package dbbolt_test
import (
"os"
"testing"
"imuslab.com/zoraxy/mod/database/dbbolt"
)
func TestNewBoltDatabase(t *testing.T) {
dbfile := "test.db"
defer os.Remove(dbfile)
db, err := dbbolt.NewBoltDatabase(dbfile)
if err != nil {
t.Fatalf("Failed to create new Bolt database: %v", err)
}
defer db.Close()
if db.Db == nil {
t.Fatalf("Expected non-nil database object")
}
}
func TestNewTable(t *testing.T) {
dbfile := "test.db"
defer os.Remove(dbfile)
db, err := dbbolt.NewBoltDatabase(dbfile)
if err != nil {
t.Fatalf("Failed to create new Bolt database: %v", err)
}
defer db.Close()
err = db.NewTable("testTable")
if err != nil {
t.Fatalf("Failed to create new table: %v", err)
}
}
func TestTableExists(t *testing.T) {
dbfile := "test.db"
defer os.Remove(dbfile)
db, err := dbbolt.NewBoltDatabase(dbfile)
if err != nil {
t.Fatalf("Failed to create new Bolt database: %v", err)
}
defer db.Close()
tableName := "testTable"
err = db.NewTable(tableName)
if err != nil {
t.Fatalf("Failed to create new table: %v", err)
}
exists := db.TableExists(tableName)
if !exists {
t.Fatalf("Expected table %s to exist", tableName)
}
nonExistentTable := "nonExistentTable"
exists = db.TableExists(nonExistentTable)
if exists {
t.Fatalf("Expected table %s to not exist", nonExistentTable)
}
}

39
src/mod/database/dbinc/dbinc.go Normal file
View File

@ -0,0 +1,39 @@
package dbinc
/*
dbinc is the interface for all database backend
*/
type BackendType int
const (
BackendBoltDB BackendType = iota //Default backend
BackendFSOnly //OpenWRT or RISCV backend
BackendLevelDB //LevelDB backend
BackEndAuto = BackendBoltDB
)
type Backend interface {
NewTable(tableName string) error
TableExists(tableName string) bool
DropTable(tableName string) error
Write(tableName string, key string, value interface{}) error
Read(tableName string, key string, assignee interface{}) error
KeyExists(tableName string, key string) bool
Delete(tableName string, key string) error
ListTable(tableName string) ([][][]byte, error)
Close()
}
func (b BackendType) String() string {
switch b {
case BackendBoltDB:
return "BoltDB"
case BackendFSOnly:
return "File System Emulated Key-Value Store"
case BackendLevelDB:
return "LevelDB"
default:
return "Unknown"
}
}

110
src/mod/database/dbleveldb/dbleveldb.go Normal file
View File

@ -0,0 +1,110 @@
package dbleveldb
import (
"encoding/json"
"path/filepath"
"strings"
"sync"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/util"
"imuslab.com/zoraxy/mod/database/dbinc"
)
// Ensure the DB struct implements the Backend interface
var _ dbinc.Backend = (*DB)(nil)
type DB struct {
db *leveldb.DB
Table sync.Map //For emulating table creation
}
func NewDB(path string) (*DB, error) {
//If the path is not a directory (e.g. /tmp/dbfile.db), convert the filename to directory
if filepath.Ext(path) != "" {
path = strings.ReplaceAll(path, ".", "_")
}
db, err := leveldb.OpenFile(path, nil)
if err != nil {
return nil, err
}
return &DB{db: db, Table: sync.Map{}}, nil
}
func (d *DB) NewTable(tableName string) error {
//Create a table entry in the sync.Map
d.Table.Store(tableName, true)
return nil
}
func (d *DB) TableExists(tableName string) bool {
_, ok := d.Table.Load(tableName)
return ok
}
func (d *DB) DropTable(tableName string) error {
d.Table.Delete(tableName)
iter := d.db.NewIterator(nil, nil)
defer iter.Release()
for iter.Next() {
key := iter.Key()
if filepath.Dir(string(key)) == tableName {
err := d.db.Delete(key, nil)
if err != nil {
return err
}
}
}
return nil
}
func (d *DB) Write(tableName string, key string, value interface{}) error {
data, err := json.Marshal(value)
if err != nil {
return err
}
return d.db.Put([]byte(filepath.ToSlash(filepath.Join(tableName, key))), data, nil)
}
func (d *DB) Read(tableName string, key string, assignee interface{}) error {
data, err := d.db.Get([]byte(filepath.ToSlash(filepath.Join(tableName, key))), nil)
if err != nil {
return err
}
return json.Unmarshal(data, assignee)
}
func (d *DB) KeyExists(tableName string, key string) bool {
_, err := d.db.Get([]byte(filepath.ToSlash(filepath.Join(tableName, key))), nil)
return err == nil
}
func (d *DB) Delete(tableName string, key string) error {
return d.db.Delete([]byte(filepath.ToSlash(filepath.Join(tableName, key))), nil)
}
func (d *DB) ListTable(tableName string) ([][][]byte, error) {
iter := d.db.NewIterator(util.BytesPrefix([]byte(tableName+"/")), nil)
defer iter.Release()
var result [][][]byte
for iter.Next() {
key := iter.Key()
//The key contains the table name as prefix. Trim it before returning
value := iter.Value()
result = append(result, [][]byte{[]byte(strings.TrimPrefix(string(key), tableName+"/")), value})
}
err := iter.Error()
if err != nil {
return nil, err
}
return result, nil
}
func (d *DB) Close() {
d.db.Close()
}

141
src/mod/database/dbleveldb/dbleveldb_test.go Normal file
View File

@ -0,0 +1,141 @@
package dbleveldb_test
import (
"os"
"testing"
"imuslab.com/zoraxy/mod/database/dbleveldb"
)
func TestNewDB(t *testing.T) {
path := "/tmp/testdb"
defer os.RemoveAll(path)
db, err := dbleveldb.NewDB(path)
if err != nil {
t.Fatalf("Failed to create new DB: %v", err)
}
defer db.Close()
}
func TestNewTable(t *testing.T) {
path := "/tmp/testdb"
defer os.RemoveAll(path)
db, err := dbleveldb.NewDB(path)
if err != nil {
t.Fatalf("Failed to create new DB: %v", err)
}
defer db.Close()
err = db.NewTable("testTable")
if err != nil {
t.Fatalf("Failed to create new table: %v", err)
}
}
func TestTableExists(t *testing.T) {
path := "/tmp/testdb"
defer os.RemoveAll(path)
db, err := dbleveldb.NewDB(path)
if err != nil {
t.Fatalf("Failed to create new DB: %v", err)
}
defer db.Close()
db.NewTable("testTable")
if !db.TableExists("testTable") {
t.Fatalf("Table should exist")
}
}
func TestDropTable(t *testing.T) {
path := "/tmp/testdb"
defer os.RemoveAll(path)
db, err := dbleveldb.NewDB(path)
if err != nil {
t.Fatalf("Failed to create new DB: %v", err)
}
defer db.Close()
db.NewTable("testTable")
err = db.DropTable("testTable")
if err != nil {
t.Fatalf("Failed to drop table: %v", err)
}
if db.TableExists("testTable") {
t.Fatalf("Table should not exist")
}
}
func TestWriteAndRead(t *testing.T) {
path := "/tmp/testdb"
defer os.RemoveAll(path)
db, err := dbleveldb.NewDB(path)
if err != nil {
t.Fatalf("Failed to create new DB: %v", err)
}
defer db.Close()
db.NewTable("testTable")
err = db.Write("testTable", "testKey", "testValue")
if err != nil {
t.Fatalf("Failed to write to table: %v", err)
}
var value string
err = db.Read("testTable", "testKey", &value)
if err != nil {
t.Fatalf("Failed to read from table: %v", err)
}
if value != "testValue" {
t.Fatalf("Expected 'testValue', got '%v'", value)
}
}
func TestListTable(t *testing.T) {
path := "/tmp/testdb"
defer os.RemoveAll(path)
db, err := dbleveldb.NewDB(path)
if err != nil {
t.Fatalf("Failed to create new DB: %v", err)
}
defer db.Close()
db.NewTable("testTable")
err = db.Write("testTable", "testKey1", "testValue1")
if err != nil {
t.Fatalf("Failed to write to table: %v", err)
}
err = db.Write("testTable", "testKey2", "testValue2")
if err != nil {
t.Fatalf("Failed to write to table: %v", err)
}
result, err := db.ListTable("testTable")
if err != nil {
t.Fatalf("Failed to list table: %v", err)
}
if len(result) != 2 {
t.Fatalf("Expected 2 entries, got %v", len(result))
}
expected := map[string]string{
"testTable/testKey1": "\"testValue1\"",
"testTable/testKey2": "\"testValue2\"",
}
for _, entry := range result {
key := string(entry[0])
value := string(entry[1])
if expected[key] != value {
t.Fatalf("Expected value '%v' for key '%v', got '%v'", expected[key], key, value)
}
}
}

6
src/mod/webserv/handler.go
View File

@ -83,7 +83,11 @@ func (ws *WebServer) SetEnableDirectoryListing(w http.ResponseWriter, r *http.Re
utils.SendErrorResponse(w, "invalid setting given")
return
}
err = ws.option.Sysdb.Write("webserv", "dirlist", enableList)
if err != nil {
utils.SendErrorResponse(w, "unable to save setting")
return
}
ws.option.EnableDirectoryListing = enableList
utils.SendOK(w)
}

10
src/start.go
View File

@ -13,6 +13,7 @@ import (
"imuslab.com/zoraxy/mod/acme"
"imuslab.com/zoraxy/mod/auth"
"imuslab.com/zoraxy/mod/database"
"imuslab.com/zoraxy/mod/database/dbinc"
"imuslab.com/zoraxy/mod/dockerux"
"imuslab.com/zoraxy/mod/dynamicproxy/loadbalance"
"imuslab.com/zoraxy/mod/dynamicproxy/redirection"
@ -64,7 +65,14 @@ func startupSequence() {
})
//Create database
db, err := database.NewDatabase(DATABASE_PATH, false)
backendType := database.GetRecommendedBackendType()
if *databaseBackend == "leveldb" {
backendType = dbinc.BackendLevelDB
} else if *databaseBackend == "boltdb" {
backendType = dbinc.BackendBoltDB
}
l.PrintAndLog("database", "Using "+backendType.String()+" as the database backend", nil)
db, err := database.NewDatabase(DATABASE_PATH, backendType)
if err != nil {
log.Fatal(err)
}