From dff79ed87f5c7ce5a0c6c321df4e884e45e1af37 Mon Sep 17 00:00:00 2001 From: hexxa Date: Thu, 21 Jul 2022 22:25:38 +0800 Subject: [PATCH] feat(dep/fileindex): enable fileindex in file managing --- src/depidx/deps.go | 10 +++++ src/handlers/fileshdr/handlers.go | 36 +++++++++++++++ src/search/fileindex/fsearch.go | 14 +++++- src/server/config.go | 30 +++++++------ src/server/config_load_test.go | 60 +++++++++++++------------ src/server/server.go | 5 +++ src/server/testdata/test_quickshare.db | Bin 131072 -> 131072 bytes 7 files changed, 111 insertions(+), 44 deletions(-) diff --git a/src/depidx/deps.go b/src/depidx/deps.go index 788cedf..b2a14ff 100644 --- a/src/depidx/deps.go +++ b/src/depidx/deps.go @@ -14,6 +14,7 @@ import ( "github.com/ihexxa/quickshare/src/idgen" "github.com/ihexxa/quickshare/src/iolimiter" "github.com/ihexxa/quickshare/src/kvstore" + "github.com/ihexxa/quickshare/src/search/fileindex" "github.com/ihexxa/quickshare/src/worker" ) @@ -38,6 +39,7 @@ type Deps struct { workers worker.IWorkerPool boltStore *boltstore.BoltStore cron cron.ICron + fileIndex fileindex.IFileIndex } func NewDeps(cfg gocfg.ICfg) *Deps { @@ -139,3 +141,11 @@ func (deps *Deps) Cron() cron.ICron { func (deps *Deps) SetCron(cronImp cron.ICron) { deps.cron = cronImp } + +func (deps *Deps) FileIndex() fileindex.IFileIndex { + return deps.fileIndex +} + +func (deps *Deps) SetIFileIndex(index fileindex.IFileIndex) { + deps.fileIndex = index +} diff --git a/src/handlers/fileshdr/handlers.go b/src/handlers/fileshdr/handlers.go index 6e2de70..b17590e 100644 --- a/src/handlers/fileshdr/handlers.go +++ b/src/handlers/fileshdr/handlers.go @@ -17,6 +17,7 @@ import ( "github.com/ihexxa/gocfg" "github.com/ihexxa/multipart" + "github.com/ihexxa/fsearch" "github.com/ihexxa/quickshare/src/db" "github.com/ihexxa/quickshare/src/db/userstore" "github.com/ihexxa/quickshare/src/depidx" @@ -210,6 +211,12 @@ func (h *FileHandlers) Create(c *gin.Context) { return } + err = h.deps.FileIndex().AddPath(fsFilePath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + c.JSON(q.Resp(200)) return } @@ -290,6 +297,12 @@ func (h *FileHandlers) Delete(c *gin.Context) { txErr = err return } + + err = h.deps.FileIndex().DelPath(filePath) + if err != nil && !errors.Is(err, fsearch.ErrNotFound) { + txErr = err + return + } }) if txErr != nil { @@ -364,6 +377,11 @@ func (h *FileHandlers) Mkdir(c *gin.Context) { return } + err = h.deps.FileIndex().AddPath(dirPath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } c.JSON(q.Resp(200)) } @@ -421,6 +439,18 @@ func (h *FileHandlers) Move(c *gin.Context) { return } + newPathBase := filepath.Base(newPath) + err = h.deps.FileIndex().AddPath(newPathBase) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + err = h.deps.FileIndex().MovePath(oldPath, newPathBase) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + c.JSON(q.Resp(200)) } @@ -527,6 +557,12 @@ func (h *FileHandlers) UploadChunk(c *gin.Context) { txErr, statusCode = err, 500 return } + + err = h.deps.FileIndex().AddPath(fsFilePath) + if err != nil { + txErr, statusCode = err, 500 + return + } } if txErr != nil { diff --git a/src/search/fileindex/fsearch.go b/src/search/fileindex/fsearch.go index 26dc7cf..de13328 100644 --- a/src/search/fileindex/fsearch.go +++ b/src/search/fileindex/fsearch.go @@ -12,16 +12,26 @@ import ( "github.com/ihexxa/quickshare/src/fs" ) +type IFileIndex interface { + Search(keyword string) ([]string, error) + AddPath(pathname string) error + DelPath(pathname string) error + RenamePath(pathname, newName string) error + MovePath(pathname, dstParentPath string) error + WriteTo(pathname string) error + ReadFrom(pathname string) error +} + type FileTreeIndex struct { fs fs.ISimpleFS index *fsearch.FSearch } -func NewFileTreeIndex(fs fs.ISimpleFS) *FileTreeIndex { +func NewFileTreeIndex(fs fs.ISimpleFS, pathSeparator string, maxResultSize int) *FileTreeIndex { return &FileTreeIndex{ fs: fs, // TODO: support max result size config - index: fsearch.New("/", 1024), + index: fsearch.New(pathSeparator, maxResultSize), } } diff --git a/src/server/config.go b/src/server/config.go index c5dc3f5..ad54f8a 100644 --- a/src/server/config.go +++ b/src/server/config.go @@ -41,13 +41,14 @@ type Secrets struct { } type ServerCfg struct { - Debug bool `json:"debug" yaml:"debug"` - Host string `json:"host" yaml:"host"` - Port int `json:"port" yaml:"port" cfg:"env"` - ReadTimeout int `json:"readTimeout" yaml:"readTimeout"` - WriteTimeout int `json:"writeTimeout" yaml:"writeTimeout"` - MaxHeaderBytes int `json:"maxHeaderBytes" yaml:"maxHeaderBytes"` - PublicPath string `json:"publicPath" yaml:"publicPath"` + Debug bool `json:"debug" yaml:"debug"` + Host string `json:"host" yaml:"host"` + Port int `json:"port" yaml:"port" cfg:"env"` + ReadTimeout int `json:"readTimeout" yaml:"readTimeout"` + WriteTimeout int `json:"writeTimeout" yaml:"writeTimeout"` + MaxHeaderBytes int `json:"maxHeaderBytes" yaml:"maxHeaderBytes"` + PublicPath string `json:"publicPath" yaml:"publicPath"` + SearchResultLimit int `json:"searchResultLimit" yaml:"searchResultLimit"` } type WorkerPoolCfg struct { @@ -105,13 +106,14 @@ func DefaultConfigStruct() *Config { TokenSecret: "", // it will auto generated if it is left as empty }, Server: &ServerCfg{ - Debug: false, - Host: "0.0.0.0", - Port: 8686, - ReadTimeout: 2000, - WriteTimeout: 1000 * 3600 * 24, // 1 day - MaxHeaderBytes: 512, - PublicPath: "static/public", + Debug: false, + Host: "0.0.0.0", + Port: 8686, + ReadTimeout: 2000, + WriteTimeout: 1000 * 3600 * 24, // 1 day + MaxHeaderBytes: 512, + PublicPath: "static/public", + SearchResultLimit: 16, }, Workers: &WorkerPoolCfg{ QueueSize: 1024, diff --git a/src/server/config_load_test.go b/src/server/config_load_test.go index b6c25e4..78d5c78 100644 --- a/src/server/config_load_test.go +++ b/src/server/config_load_test.go @@ -126,13 +126,14 @@ func TestLoadCfg(t *testing.T) { TokenSecret: "1", }, Server: &ServerCfg{ - Debug: true, - Host: "1", - Port: 1, - ReadTimeout: 1, - WriteTimeout: 1, - MaxHeaderBytes: 1, - PublicPath: "1", + Debug: true, + Host: "1", + Port: 1, + ReadTimeout: 1, + WriteTimeout: 1, + MaxHeaderBytes: 1, + PublicPath: "1", + SearchResultLimit: 16, }, Workers: &WorkerPoolCfg{ QueueSize: 1, @@ -192,13 +193,14 @@ func TestLoadCfg(t *testing.T) { TokenSecret: "4", }, Server: &ServerCfg{ - Debug: false, - Host: "4", - Port: 4, - ReadTimeout: 4, - WriteTimeout: 4, - MaxHeaderBytes: 4, - PublicPath: "4", + Debug: false, + Host: "4", + Port: 4, + ReadTimeout: 4, + WriteTimeout: 4, + MaxHeaderBytes: 4, + PublicPath: "4", + SearchResultLimit: 16, }, Workers: &WorkerPoolCfg{ QueueSize: 4, @@ -260,13 +262,14 @@ func TestLoadCfg(t *testing.T) { TokenSecret: "4", }, Server: &ServerCfg{ - Debug: false, - Host: "4", - Port: 4, - ReadTimeout: 4, - WriteTimeout: 4, - MaxHeaderBytes: 4, - PublicPath: "4", + Debug: false, + Host: "4", + Port: 4, + ReadTimeout: 4, + WriteTimeout: 4, + MaxHeaderBytes: 4, + PublicPath: "4", + SearchResultLimit: 16, }, Workers: &WorkerPoolCfg{ QueueSize: 4, @@ -328,13 +331,14 @@ func TestLoadCfg(t *testing.T) { TokenSecret: "4", }, Server: &ServerCfg{ - Debug: false, - Host: "4", - Port: 4, - ReadTimeout: 4, - WriteTimeout: 4, - MaxHeaderBytes: 4, - PublicPath: "4", + Debug: false, + Host: "4", + Port: 4, + ReadTimeout: 4, + WriteTimeout: 4, + MaxHeaderBytes: 4, + PublicPath: "4", + SearchResultLimit: 16, }, Workers: &WorkerPoolCfg{ QueueSize: 4, diff --git a/src/server/server.go b/src/server/server.go index 674a2ca..1647442 100644 --- a/src/server/server.go +++ b/src/server/server.go @@ -38,6 +38,7 @@ import ( "github.com/ihexxa/quickshare/src/iolimiter" "github.com/ihexxa/quickshare/src/kvstore" "github.com/ihexxa/quickshare/src/kvstore/boltdbpvd" + "github.com/ihexxa/quickshare/src/search/fileindex" "github.com/ihexxa/quickshare/src/worker/localworker" qsstatic "github.com/ihexxa/quickshare/static" ) @@ -140,6 +141,9 @@ func initDeps(cfg gocfg.ICfg) *depidx.Deps { filesystem := local.NewLocalFS(rootPath, 0660, opensLimit, openTTL, readerTTL, ider) jwtEncDec := jwt.NewJWTEncDec(secret) + searchResultLimit := cfg.GrabInt("Server.SearchResultLimit") + fileIndex := fileindex.NewFileTreeIndex(filesystem, "/", searchResultLimit) + dbPath := cfg.GrabString("Db.DbPath") dbDir := filepath.Dir(dbPath) if err := filesystem.MkdirAll(dbDir); err != nil { @@ -196,6 +200,7 @@ func initDeps(cfg gocfg.ICfg) *depidx.Deps { deps.SetID(ider) deps.SetLog(logger) deps.SetLimiter(limiter) + deps.SetIFileIndex(fileIndex) queueSize := cfg.GrabInt("Workers.QueueSize") sleepCyc := cfg.GrabInt("Workers.SleepCyc") diff --git a/src/server/testdata/test_quickshare.db b/src/server/testdata/test_quickshare.db index 876d3c20efea23fcda33108bee95322aeef8d8cc..7a66137c49c9380fed38a26b1032aac5ade8219b 100644 GIT binary patch delta 64 zcmZo@;Am*znBc%4B*Op(raPoE!=F`7KKNX6W5NvnNdgW6fl#S}ZRgc~YW~aHp16Q< M1AmjkG6lv50DtZkSpWb4 delta 63 zcmZo@;Am*znBX8_C(Qr`Dpt-X<7-*ewkIxN+`vCcz(K$kD)UizX5RlBPn{t$O$y5t H7#{!t9i