From 17b454448759440945100c9dd885120bb29a0374 Mon Sep 17 00:00:00 2001 From: hexxa Date: Sat, 5 Mar 2022 00:16:04 +0800 Subject: [PATCH] fix(files): add boltdb store and refactor files handlers --- src/client/users.go | 6 +- src/db/boltstore/bolt_store.go | 377 ++++++++++++++++++++++ src/db/common.go | 64 +++- src/db/fileinfostore/file_info_store.go | 79 +++-- src/db/fileinfostore/upload_info_store.go | 40 +-- src/db/userstore/user_store.go | 117 +++---- src/depidx/deps.go | 10 + src/handlers/fileshdr/handlers.go | 248 ++++++++------ src/handlers/fileshdr/upload_mgr.go | 126 -------- src/handlers/multiusers/handlers.go | 35 +- src/iolimiter/iolimiter.go | 5 +- src/kvstore/boltdbpvd/interface.go | 7 + src/kvstore/boltdbpvd/provider.go | 4 +- src/server/server.go | 9 +- src/server/server_files_test.go | 4 +- src/server/server_permission_test.go | 3 +- src/server/server_space_limit_test.go | 4 +- src/server/server_users_test.go | 15 +- src/server/testdata/test_quickshare.db | Bin 524288 -> 524288 bytes 19 files changed, 751 insertions(+), 402 deletions(-) create mode 100644 src/db/boltstore/bolt_store.go delete mode 100644 src/handlers/fileshdr/upload_mgr.go create mode 100644 src/kvstore/boltdbpvd/interface.go diff --git a/src/client/users.go b/src/client/users.go index 76c2d09..b41fc73 100644 --- a/src/client/users.go +++ b/src/client/users.go @@ -5,7 +5,7 @@ import ( "fmt" "net/http" - "github.com/ihexxa/quickshare/src/db/userstore" + "github.com/ihexxa/quickshare/src/db" "github.com/ihexxa/quickshare/src/handlers" "github.com/ihexxa/quickshare/src/handlers/multiusers" "github.com/parnurzeal/gorequest" @@ -63,7 +63,7 @@ func (cl *SingleUserClient) ForceSetPwd(userID, newPwd string, token *http.Cooki End() } -func (cl *SingleUserClient) SetUser(ID uint64, role string, quota *userstore.Quota, token *http.Cookie) (*http.Response, string, []error) { +func (cl *SingleUserClient) SetUser(ID uint64, role string, quota *db.Quota, token *http.Cookie) (*http.Response, string, []error) { return cl.r.Patch(cl.url("/v1/users/")). Send(multiusers.SetUserReq{ ID: ID, @@ -173,7 +173,7 @@ func (cl *SingleUserClient) Self(token *http.Cookie) (*http.Response, *multiuser return resp, selfResp, errs } -func (cl *SingleUserClient) SetPreferences(prefers *userstore.Preferences, token *http.Cookie) (*http.Response, string, []error) { +func (cl *SingleUserClient) SetPreferences(prefers *db.Preferences, token *http.Cookie) (*http.Response, string, []error) { return cl.r.Patch(cl.url("/v1/users/preferences")). Send(multiusers.SetPreferencesReq{ Preferences: prefers, diff --git a/src/db/boltstore/bolt_store.go b/src/db/boltstore/bolt_store.go new file mode 100644 index 0000000..c472f93 --- /dev/null +++ b/src/db/boltstore/bolt_store.go @@ -0,0 +1,377 @@ +package boltstore + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/boltdb/bolt" + "github.com/ihexxa/quickshare/src/db" +) + +type BoltStore struct { + boltdb *bolt.DB +} + +func NewBoltStore(boltdb *bolt.DB) (*BoltStore, error) { + bs := &BoltStore{ + boltdb: boltdb, + } + return bs, nil +} + +func (bs *BoltStore) getUserInfo(tx *bolt.Tx, userID uint64) (*db.User, error) { + var err error + + usersBucket := tx.Bucket([]byte(db.UsersNs)) + if usersBucket == nil { + return nil, db.ErrBucketNotFound + } + + uidStr := fmt.Sprint(userID) + infoBytes := usersBucket.Get([]byte(uidStr)) + if infoBytes == nil { + return nil, db.ErrKeyNotFound + } + + userInfo := &db.User{} + err = json.Unmarshal(infoBytes, userInfo) + if err != nil { + return nil, err + } else if userInfo.ID != userID { + return nil, fmt.Errorf("user id key(%d) info(%d) does match", userID, userInfo.ID) + } + + return userInfo, nil +} + +func (bs *BoltStore) setUserInfo(tx *bolt.Tx, userID uint64, userInfo *db.User) error { + var err error + + usersBucket := tx.Bucket([]byte(db.UsersNs)) + if usersBucket == nil { + return db.ErrBucketNotFound + } + + userInfoBytes, err := json.Marshal(userInfo) + if err != nil { + return err + } + + uidStr := fmt.Sprint(userID) + return usersBucket.Put([]byte(uidStr), userInfoBytes) +} + +func (bs *BoltStore) getUploadInfo(tx *bolt.Tx, userID uint64, itemPath string) (*db.UploadInfo, error) { + var err error + + uidStr := fmt.Sprint(userID) + userUploadNS := db.UploadNS(uidStr) + + uploadInfoBucket := tx.Bucket([]byte(userUploadNS)) + if uploadInfoBucket == nil { + return nil, bolt.ErrBucketNotFound + + } + + uploadInfoBytes := uploadInfoBucket.Get([]byte(itemPath)) + if uploadInfoBytes == nil { + return nil, db.ErrKeyNotFound + } + + uploadInfo := &db.UploadInfo{} + err = json.Unmarshal(uploadInfoBytes, uploadInfo) + if err != nil { + return nil, err + } + return uploadInfo, nil +} + +func (bs *BoltStore) setUploadInfo(tx *bolt.Tx, userID uint64, uploadPath string, uploadInfo *db.UploadInfo, overWrite bool) error { + var err error + + uidStr := fmt.Sprint(userID) + userUploadNS := db.UploadNS(uidStr) + uploadInfoBucket := tx.Bucket([]byte(userUploadNS)) + if uploadInfoBucket == nil { + uploadInfoBucket, err = tx.CreateBucket([]byte(userUploadNS)) + if err != nil { + return err + } + } + + existingInfo := uploadInfoBucket.Get([]byte(uploadPath)) + if existingInfo != nil && !overWrite { + return db.ErrCreateExisting + } + + uploadInfoBytes, err := json.Marshal(uploadInfo) + if err != nil { + return err + } + + return uploadInfoBucket.Put([]byte(uploadPath), uploadInfoBytes) +} + +func (bs *BoltStore) getFileInfo(tx *bolt.Tx, userID uint64, itemPath string) (*db.FileInfo, error) { + var err error + + fileInfoBucket := tx.Bucket([]byte(db.InfoNs)) + if fileInfoBucket == nil { + return nil, db.ErrBucketNotFound + } + + fileInfoBytes := fileInfoBucket.Get([]byte(itemPath)) + if fileInfoBytes == nil { + return nil, db.ErrKeyNotFound + } + + fileInfo := &db.FileInfo{} + err = json.Unmarshal(fileInfoBytes, fileInfo) + if err != nil { + return nil, err + } + return fileInfo, nil +} + +func (bs *BoltStore) setFileInfo(tx *bolt.Tx, userID uint64, itemPath string, fileInfo *db.FileInfo) error { + var err error + + fileInfoBucket := tx.Bucket([]byte(db.InfoNs)) + if fileInfoBucket == nil { + return db.ErrBucketNotFound + } + + fileInfoBytes, err := json.Marshal(fileInfo) + if err != nil { + return err + } + + return fileInfoBucket.Put([]byte(itemPath), fileInfoBytes) +} + +func (bs *BoltStore) AddUploadInfos(userID uint64, tmpPath, filePath string, info *db.FileInfo) error { + return bs.boltdb.Update(func(tx *bolt.Tx) error { + var err error + // check space quota + userInfo, err := bs.getUserInfo(tx, userID) + if err != nil { + return err + } + + if userInfo.UsedSpace+info.Size > int64(userInfo.Quota.SpaceLimit) { + return errors.New("space limit is reached") + } + + // update used space + userInfo.UsedSpace += info.Size + err = bs.setUserInfo(tx, userID, userInfo) + if err != nil { + return err + } + + // add upload info + uploadInfo := &db.UploadInfo{ + RealFilePath: filePath, + Size: info.Size, + Uploaded: 0, + } + return bs.setUploadInfo(tx, userID, tmpPath, uploadInfo, false) + }) +} + +func (bs *BoltStore) DelUploadingInfos(userID uint64, uploadPath string) error { + return bs.boltdb.Update(func(tx *bolt.Tx) error { + var err error + + // delete upload info + uidStr := fmt.Sprint(userID) + userUploadNS := db.UploadNS(uidStr) + + uploadInfoBucket := tx.Bucket([]byte(userUploadNS)) + if uploadInfoBucket == nil { + return db.ErrBucketNotFound + } + + uploadInfoBytes := uploadInfoBucket.Get([]byte(uploadPath)) + if uploadInfoBytes == nil { + return db.ErrKeyNotFound + } + + uploadInfo := &db.UploadInfo{} + err = json.Unmarshal(uploadInfoBytes, uploadInfo) + if err != nil { + return err + } + + err = uploadInfoBucket.Delete([]byte(uploadPath)) + if err != nil { + return err + } + + // decr used space + userInfo, err := bs.getUserInfo(tx, userID) + if err != nil { + return err + } + + userInfo.UsedSpace -= uploadInfo.Size + return bs.setUserInfo(tx, userID, userInfo) + }) +} + +func (bs *BoltStore) MoveUploadingInfos(userID uint64, uploadPath, itemPath string) error { + return bs.boltdb.Update(func(tx *bolt.Tx) error { + var err error + + // delete upload info + uidStr := fmt.Sprint(userID) + userUploadNS := db.UploadNS(uidStr) + + uploadInfoBucket := tx.Bucket([]byte(userUploadNS)) + if uploadInfoBucket == nil { + return db.ErrBucketNotFound + } + + uploadInfoBytes := uploadInfoBucket.Get([]byte(uploadPath)) + if uploadInfoBytes == nil { + return db.ErrKeyNotFound + } + + uploadInfo := &db.UploadInfo{} + err = json.Unmarshal(uploadInfoBytes, uploadInfo) + if err != nil { + return err + } + + err = uploadInfoBucket.Delete([]byte(uploadPath)) + if err != nil { + return err + } + + // create file info + fileInfo := &db.FileInfo{ + IsDir: false, + Size: uploadInfo.Size, + } + return bs.setFileInfo(tx, userID, itemPath, fileInfo) + }) +} + +func (bs *BoltStore) delShareID(tx *bolt.Tx, itemPath string) error { + var err error + + shareIDBucket := tx.Bucket([]byte(db.ShareIDNs)) + if shareIDBucket == nil { + return db.ErrBucketNotFound + } + + shareIDtoDir := map[string]string{} + shareIDBucket.ForEach(func(k, v []byte) error { + shareIDtoDir[string(k)] = string(v) + return nil + }) + + // because before this version, shareIDs are not removed correctly + // so it iterates all shareIDs and cleans remaining entries + for shareID, shareDir := range shareIDtoDir { + if shareDir == itemPath { + err = shareIDBucket.Delete([]byte(shareID)) + if err != nil { + return err + } + } + } + + return nil +} + +func (bs *BoltStore) DelInfos(userID uint64, itemPath string) error { + return bs.boltdb.Update(func(tx *bolt.Tx) error { + var err error + + // delete file info + fileInfoBucket := tx.Bucket([]byte(db.InfoNs)) + if fileInfoBucket == nil { + return db.ErrBucketNotFound + } + + fileInfoBytes := fileInfoBucket.Get([]byte(itemPath)) + if fileInfoBucket == nil { + return db.ErrKeyNotFound + } + + fileInfo := &db.FileInfo{} + err = json.Unmarshal(fileInfoBytes, fileInfo) + if err != nil { + return err + } + + err = fileInfoBucket.Delete([]byte(itemPath)) + if err != nil { + return err + } + + // decr used space + userInfo, err := bs.getUserInfo(tx, userID) + if err != nil { + return err + } + + userInfo.UsedSpace -= fileInfo.Size + err = bs.setUserInfo(tx, userID, userInfo) + if err != nil { + return err + } + + // delete share id + if fileInfo.IsDir { + err = bs.delShareID(tx, itemPath) + if err != nil { + return err + } + } + + return nil + }) +} + +func (bs *BoltStore) MoveInfos(userID uint64, oldPath, newPath string, isDir bool) error { + return bs.boltdb.Update(func(tx *bolt.Tx) error { + var err error + + fileInfoBucket := tx.Bucket([]byte(db.InfoNs)) + if fileInfoBucket == nil { + return db.ErrBucketNotFound + } + + fileInfo, err := bs.getFileInfo(tx, userID, oldPath) + if err != nil { + if errors.Is(err, db.ErrKeyNotFound) && isDir { + // the file info for the dir does not exist + return nil + } + return err + } + + // delete old shareID + if isDir { + fileInfo.Shared = false + fileInfo.ShareID = "" + err = bs.delShareID(tx, oldPath) + if err != nil { + return err + } + } + + // delete old info + err = fileInfoBucket.Delete([]byte(oldPath)) + if err != nil { + return err + } + + fmt.Println("\n\n\n4", err) + // add new info + return bs.setFileInfo(tx, userID, newPath, fileInfo) + }) +} diff --git a/src/db/common.go b/src/db/common.go index f9b6006..eec5aa0 100644 --- a/src/db/common.go +++ b/src/db/common.go @@ -1,3 +1,65 @@ package db -const SchemaV2 = "v2" // add size to file info +import ( + "errors" + "fmt" + + "github.com/ihexxa/quickshare/src/db/sitestore" +) + +const ( + SchemaV2 = "v2" // add size to file info + + UsersNs = "users" + InfoNs = "sharing" + ShareIDNs = "sharingKey" + + uploadsPrefix = "uploads" +) + +var ( + ErrBucketNotFound = errors.New("bucket not found") + ErrKeyNotFound = errors.New("key not found") + ErrCreateExisting = errors.New("create upload info which already exists") +) + +type FileInfo struct { + IsDir bool `json:"isDir"` + Shared bool `json:"shared"` + ShareID string `json:"shareID"` // for short url + Sha1 string `json:"sha1"` + Size int64 `json:"size"` +} + +type Quota struct { + SpaceLimit int64 `json:"spaceLimit,string"` + UploadSpeedLimit int `json:"uploadSpeedLimit"` + DownloadSpeedLimit int `json:"downloadSpeedLimit"` +} + +type Preferences struct { + Bg *sitestore.BgConfig `json:"bg"` + CSSURL string `json:"cssURL"` + LanPackURL string `json:"lanPackURL"` + Lan string `json:"lan"` +} + +type User struct { + ID uint64 `json:"id,string"` + Name string `json:"name"` + Pwd string `json:"pwd"` + Role string `json:"role"` + UsedSpace int64 `json:"usedSpace,string"` + Quota *Quota `json:"quota"` + Preferences *Preferences `json:"preferences"` +} + +type UploadInfo struct { + RealFilePath string `json:"realFilePath"` + Size int64 `json:"size"` + Uploaded int64 `json:"uploaded"` +} + +func UploadNS(user string) string { + return fmt.Sprintf("%s/%s", uploadsPrefix, user) +} diff --git a/src/db/fileinfostore/file_info_store.go b/src/db/fileinfostore/file_info_store.go index 6812ede..463dbaf 100644 --- a/src/db/fileinfostore/file_info_store.go +++ b/src/db/fileinfostore/file_info_store.go @@ -11,12 +11,11 @@ import ( "github.com/ihexxa/quickshare/src/db" "github.com/ihexxa/quickshare/src/kvstore" + "github.com/ihexxa/quickshare/src/kvstore/boltdbpvd" ) const ( InitNs = "Init" - InfoNs = "sharing" - ShareIDNs = "sharingKey" InitTimeKey = "initTime" SchemaVerKey = "SchemaVersion" SchemaV1 = "v1" @@ -35,36 +34,29 @@ func IsNotFound(err error) bool { return err == ErrNotFound } -type FileInfo struct { - IsDir bool `json:"isDir"` - Shared bool `json:"shared"` - ShareID string `json:"shareID"` // for short url - Sha1 string `json:"sha1"` - Size int64 `json:"size"` -} - type IFileInfoStore interface { AddSharing(dirPath string) error DelSharing(dirPath string) error GetSharing(dirPath string) (bool, bool) ListSharings(prefix string) (map[string]string, error) - GetInfo(itemPath string) (*FileInfo, error) - SetInfo(itemPath string, info *FileInfo) error + GetInfo(itemPath string) (*db.FileInfo, error) + SetInfo(itemPath string, info *db.FileInfo) error DelInfo(itemPath string) error SetSha1(itemPath, sign string) error - GetInfos(itemPaths []string) (map[string]*FileInfo, error) + GetInfos(itemPaths []string) (map[string]*db.FileInfo, error) GetSharingDir(hashID string) (string, error) // upload info AddUploadInfo(user, filePath, tmpPath string, fileSize int64) error SetUploadInfo(user, filePath string, newUploaded int64) error GetUploadInfo(user, filePath string) (string, int64, int64, error) DelUploadInfo(user, filePath string) error - ListUploadInfo(user string) ([]*UploadInfo, error) + ListUploadInfo(user string) ([]*db.UploadInfo, error) } type FileInfoStore struct { - mtx *sync.RWMutex - store kvstore.IKVStore + mtx *sync.RWMutex + store kvstore.IKVStore + boltdb boltdbpvd.BoltProvider } func migrate(fi *FileInfoStore) error { @@ -77,7 +69,7 @@ func migrate(fi *FileInfoStore) error { switch ver { case "v0": // add ShareID to FileInfos - infoStrs, err := fi.store.ListStringsIn(InfoNs) + infoStrs, err := fi.store.ListStringsIn(db.InfoNs) if err != nil { return err } @@ -103,13 +95,13 @@ func migrate(fi *FileInfoStore) error { } shareID = dirShareID - err = fi.store.SetStringIn(ShareIDNs, shareID, itemPath) + err = fi.store.SetStringIn(db.ShareIDNs, shareID, itemPath) if err != nil { return err } } - newInfo := &FileInfo{ + newInfo := &db.FileInfo{ IsDir: infoV0.IsDir, Shared: infoV0.Shared, ShareID: shareID, @@ -126,7 +118,7 @@ func migrate(fi *FileInfoStore) error { } case "v1": // add size to file info - infoStrs, err := fi.store.ListStringsIn(InfoNs) + infoStrs, err := fi.store.ListStringsIn(db.InfoNs) if err != nil { return err } @@ -145,7 +137,7 @@ func migrate(fi *FileInfoStore) error { return fmt.Errorf("list sharing error: %w", err) } - newInfo := &FileInfo{ + newInfo := &db.FileInfo{ IsDir: infoV1.IsDir, Shared: infoV1.Shared, ShareID: infoV1.ShareID, @@ -174,8 +166,8 @@ func NewFileInfoStore(store kvstore.IKVStore) (*FileInfoStore, error) { var err error for _, nsName := range []string{ InitNs, - InfoNs, - ShareIDNs, + db.InfoNs, + db.ShareIDNs, } { if !store.HasNamespace(nsName) { if err = store.AddNamespace(nsName); err != nil { @@ -184,9 +176,12 @@ func NewFileInfoStore(store kvstore.IKVStore) (*FileInfoStore, error) { } } + boltdb := store.(boltdbpvd.BoltProvider) + fi := &FileInfoStore{ - store: store, - mtx: &sync.RWMutex{}, + store: store, + boltdb: boltdb, + mtx: &sync.RWMutex{}, } if err = migrate(fi); err != nil { return nil, err @@ -203,7 +198,7 @@ func (fi *FileInfoStore) AddSharing(dirPath string) error { if !IsNotFound(err) { return err } - info = &FileInfo{ + info = &db.FileInfo{ IsDir: true, } } @@ -213,7 +208,7 @@ func (fi *FileInfoStore) AddSharing(dirPath string) error { if err != nil { return err } - err = fi.store.SetStringIn(ShareIDNs, shareID, dirPath) + err = fi.store.SetStringIn(db.ShareIDNs, shareID, dirPath) if err != nil { return err } @@ -238,14 +233,14 @@ func (fi *FileInfoStore) DelSharing(dirPath string) error { // because before this version, shareIDs are not removed correctly // so it iterates all shareIDs and cleans remaining entries - shareIDtoDir, err := fi.store.ListStringsIn(ShareIDNs) + shareIDtoDir, err := fi.store.ListStringsIn(db.ShareIDNs) if err != nil { return err } for shareID, shareDir := range shareIDtoDir { if shareDir == dirPath { - err = fi.store.DelStringIn(ShareIDNs, shareID) + err = fi.store.DelStringIn(db.ShareIDNs, shareID) if err != nil { return err } @@ -270,12 +265,12 @@ func (fi *FileInfoStore) GetSharing(dirPath string) (bool, bool) { } func (fi *FileInfoStore) ListSharings(prefix string) (map[string]string, error) { - infoStrs, err := fi.store.ListStringsByPrefixIn(prefix, InfoNs) + infoStrs, err := fi.store.ListStringsByPrefixIn(prefix, db.InfoNs) if err != nil { return nil, err } - info := &FileInfo{} + info := &db.FileInfo{} sharings := map[string]string{} for itemPath, infoStr := range infoStrs { err = json.Unmarshal([]byte(infoStr), info) @@ -291,13 +286,13 @@ func (fi *FileInfoStore) ListSharings(prefix string) (map[string]string, error) return sharings, nil } -func (fi *FileInfoStore) GetInfo(itemPath string) (*FileInfo, error) { - infoStr, ok := fi.store.GetStringIn(InfoNs, itemPath) +func (fi *FileInfoStore) GetInfo(itemPath string) (*db.FileInfo, error) { + infoStr, ok := fi.store.GetStringIn(db.InfoNs, itemPath) if !ok { return nil, ErrNotFound } - info := &FileInfo{} + info := &db.FileInfo{} err := json.Unmarshal([]byte(infoStr), info) if err != nil { return nil, fmt.Errorf("get file info: %w", err) @@ -305,8 +300,8 @@ func (fi *FileInfoStore) GetInfo(itemPath string) (*FileInfo, error) { return info, nil } -func (fi *FileInfoStore) GetInfos(itemPaths []string) (map[string]*FileInfo, error) { - infos := map[string]*FileInfo{} +func (fi *FileInfoStore) GetInfos(itemPaths []string) (map[string]*db.FileInfo, error) { + infos := map[string]*db.FileInfo{} for _, itemPath := range itemPaths { info, err := fi.GetInfo(itemPath) if err != nil { @@ -321,13 +316,13 @@ func (fi *FileInfoStore) GetInfos(itemPaths []string) (map[string]*FileInfo, err return infos, nil } -func (fi *FileInfoStore) SetInfo(itemPath string, info *FileInfo) error { +func (fi *FileInfoStore) SetInfo(itemPath string, info *db.FileInfo) error { infoStr, err := json.Marshal(info) if err != nil { return fmt.Errorf("set file info: %w", err) } - err = fi.store.SetStringIn(InfoNs, itemPath, string(infoStr)) + err = fi.store.SetStringIn(db.InfoNs, itemPath, string(infoStr)) if err != nil { return fmt.Errorf("set file info: %w", err) } @@ -335,7 +330,7 @@ func (fi *FileInfoStore) SetInfo(itemPath string, info *FileInfo) error { } func (fi *FileInfoStore) DelInfo(itemPath string) error { - return fi.store.DelStringIn(InfoNs, itemPath) + return fi.store.DelStringIn(db.InfoNs, itemPath) } func (fi *FileInfoStore) SetSha1(itemPath, sign string) error { @@ -347,7 +342,7 @@ func (fi *FileInfoStore) SetSha1(itemPath, sign string) error { if !IsNotFound(err) { return err } - info = &FileInfo{ + info = &db.FileInfo{ IsDir: false, Shared: false, } @@ -370,7 +365,7 @@ func (fi *FileInfoStore) getShareID(payload string) (string, error) { } shareID := fmt.Sprintf("%x", h.Sum(nil))[:7] - shareDir, ok := fi.store.GetStringIn(ShareIDNs, shareID) + shareDir, ok := fi.store.GetStringIn(db.ShareIDNs, shareID) if !ok { return shareID, nil } else if ok && shareDir == payload { @@ -382,7 +377,7 @@ func (fi *FileInfoStore) getShareID(payload string) (string, error) { } func (fi *FileInfoStore) GetSharingDir(hashID string) (string, error) { - dirPath, ok := fi.store.GetStringIn(ShareIDNs, hashID) + dirPath, ok := fi.store.GetStringIn(db.ShareIDNs, hashID) if !ok { return "", ErrSharingNotFound } diff --git a/src/db/fileinfostore/upload_info_store.go b/src/db/fileinfostore/upload_info_store.go index d352fab..234bd38 100644 --- a/src/db/fileinfostore/upload_info_store.go +++ b/src/db/fileinfostore/upload_info_store.go @@ -3,29 +3,17 @@ package fileinfostore import ( "encoding/json" "errors" - "fmt" + + "github.com/ihexxa/quickshare/src/db" ) var ( - ErrCreateExisting = errors.New("create upload info which already exists") ErrGreaterThanSize = errors.New("uploaded is greater than file size") ErrUploadNotFound = errors.New("upload info not found") - - uploadsPrefix = "uploads" ) -type UploadInfo struct { - RealFilePath string `json:"realFilePath"` - Size int64 `json:"size"` - Uploaded int64 `json:"uploaded"` -} - -func UploadNS(user string) string { - return fmt.Sprintf("%s/%s", uploadsPrefix, user) -} - func (fi *FileInfoStore) AddUploadInfo(user, filePath, tmpPath string, fileSize int64) error { - ns := UploadNS(user) + ns := db.UploadNS(user) err := fi.store.AddNamespace(ns) if err != nil { return err @@ -33,10 +21,10 @@ func (fi *FileInfoStore) AddUploadInfo(user, filePath, tmpPath string, fileSize _, ok := fi.store.GetStringIn(ns, tmpPath) if ok { - return ErrCreateExisting + return db.ErrCreateExisting } - info := &UploadInfo{ + info := &db.UploadInfo{ RealFilePath: filePath, Size: fileSize, Uploaded: 0, @@ -57,7 +45,7 @@ func (fi *FileInfoStore) SetUploadInfo(user, filePath string, newUploaded int64) return ErrGreaterThanSize } - newInfo := &UploadInfo{ + newInfo := &db.UploadInfo{ RealFilePath: realFilePath, Size: fileSize, Uploaded: newUploaded, @@ -66,17 +54,17 @@ func (fi *FileInfoStore) SetUploadInfo(user, filePath string, newUploaded int64) if err != nil { return err } - return fi.store.SetStringIn(UploadNS(user), filePath, string(newInfoBytes)) + return fi.store.SetStringIn(db.UploadNS(user), filePath, string(newInfoBytes)) } func (fi *FileInfoStore) GetUploadInfo(user, filePath string) (string, int64, int64, error) { - ns := UploadNS(user) + ns := db.UploadNS(user) infoBytes, ok := fi.store.GetStringIn(ns, filePath) if !ok { return "", 0, 0, ErrUploadNotFound } - info := &UploadInfo{} + info := &db.UploadInfo{} err := json.Unmarshal([]byte(infoBytes), info) if err != nil { return "", 0, 0, err @@ -86,11 +74,11 @@ func (fi *FileInfoStore) GetUploadInfo(user, filePath string) (string, int64, in } func (fi *FileInfoStore) DelUploadInfo(user, filePath string) error { - return fi.store.DelInt64In(UploadNS(user), filePath) + return fi.store.DelInt64In(db.UploadNS(user), filePath) } -func (fi *FileInfoStore) ListUploadInfo(user string) ([]*UploadInfo, error) { - ns := UploadNS(user) +func (fi *FileInfoStore) ListUploadInfo(user string) ([]*db.UploadInfo, error) { + ns := db.UploadNS(user) if !fi.store.HasNamespace(ns) { return nil, nil } @@ -100,9 +88,9 @@ func (fi *FileInfoStore) ListUploadInfo(user string) ([]*UploadInfo, error) { return nil, err } - infos := []*UploadInfo{} + infos := []*db.UploadInfo{} for _, infoStr := range infoMap { - info := &UploadInfo{} + info := &db.UploadInfo{} err = json.Unmarshal([]byte(infoStr), info) if err != nil { return nil, err diff --git a/src/db/userstore/user_store.go b/src/db/userstore/user_store.go index 3a9cc51..d2292bc 100644 --- a/src/db/userstore/user_store.go +++ b/src/db/userstore/user_store.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/ihexxa/quickshare/src/db" "github.com/ihexxa/quickshare/src/db/sitestore" "github.com/ihexxa/quickshare/src/kvstore" ) @@ -17,7 +18,6 @@ const ( VisitorRole = "visitor" InitNs = "usersInit" IDsNs = "ids" - UsersNs = "users" RoleListNs = "roleList" InitTimeKey = "initTime" VisitorID = uint64(1) @@ -34,7 +34,7 @@ var ( ErrReachedLimit = errors.New("reached space limit") ErrNotFound = errors.New("not found") - DefaultPreferences = Preferences{ + DefaultPreferences = db.Preferences{ Bg: &sitestore.BgConfig{ Url: "", Repeat: "no-repeat", @@ -51,48 +51,25 @@ func IsReachedLimitErr(err error) bool { return err == ErrReachedLimit } -type Quota struct { - SpaceLimit int64 `json:"spaceLimit,string"` - UploadSpeedLimit int `json:"uploadSpeedLimit"` - DownloadSpeedLimit int `json:"downloadSpeedLimit"` -} - -type Preferences struct { - Bg *sitestore.BgConfig `json:"bg"` - CSSURL string `json:"cssURL"` - LanPackURL string `json:"lanPackURL"` - Lan string `json:"lan"` -} - type UserCfg struct { Name string `json:"name"` Role string `json:"role"` Pwd string `json:"pwd"` } -type User struct { - ID uint64 `json:"id,string"` - Name string `json:"name"` - Pwd string `json:"pwd"` - Role string `json:"role"` - UsedSpace int64 `json:"usedSpace,string"` - Quota *Quota `json:"quota"` - Preferences *Preferences `json:"preferences"` -} - type IUserStore interface { Init(rootName, rootPwd string) error IsInited() bool - AddUser(user *User) error + AddUser(user *db.User) error DelUser(id uint64) error - GetUser(id uint64) (*User, error) - GetUserByName(name string) (*User, error) - SetInfo(id uint64, user *User) error + GetUser(id uint64) (*db.User, error) + GetUserByName(name string) (*db.User, error) + SetInfo(id uint64, user *db.User) error CanIncrUsed(id uint64, capacity int64) (bool, error) SetUsed(id uint64, incr bool, capacity int64) error SetPwd(id uint64, pwd string) error - SetPreferences(id uint64, settings *Preferences) error - ListUsers() ([]*User, error) + SetPreferences(id uint64, settings *db.Preferences) error + ListUsers() ([]*db.User, error) ListUserIDs() (map[string]string, error) AddRole(role string) error DelRole(role string) error @@ -110,7 +87,7 @@ func NewKVUserStore(store kvstore.IKVStore) (*KVUserStore, error) { var err error for _, nsName := range []string{ IDsNs, - UsersNs, + db.UsersNs, InitNs, RoleListNs, } { @@ -129,12 +106,12 @@ func NewKVUserStore(store kvstore.IKVStore) (*KVUserStore, error) { func (us *KVUserStore) Init(rootName, rootPwd string) error { var err error adminPreferences := DefaultPreferences - admin := &User{ + admin := &db.User{ ID: 0, Name: rootName, Pwd: rootPwd, Role: AdminRole, - Quota: &Quota{ + Quota: &db.Quota{ SpaceLimit: defaultSpaceLimit, UploadSpeedLimit: defaultUploadSpeedLimit, DownloadSpeedLimit: defaultDownloadSpeedLimit, @@ -143,12 +120,12 @@ func (us *KVUserStore) Init(rootName, rootPwd string) error { } visitorPreferences := DefaultPreferences - visitor := &User{ + visitor := &db.User{ ID: VisitorID, Name: VisitorName, Pwd: rootPwd, Role: VisitorRole, - Quota: &Quota{ + Quota: &db.Quota{ SpaceLimit: 0, UploadSpeedLimit: visitorUploadSpeedLimit, DownloadSpeedLimit: visitorDownloadSpeedLimit, @@ -156,7 +133,7 @@ func (us *KVUserStore) Init(rootName, rootPwd string) error { Preferences: &visitorPreferences, } - for _, user := range []*User{admin, visitor} { + for _, user := range []*db.User{admin, visitor} { err = us.AddUser(user) if err != nil { return err @@ -178,12 +155,12 @@ func (us *KVUserStore) IsInited() bool { return ok } -func (us *KVUserStore) AddUser(user *User) error { +func (us *KVUserStore) AddUser(user *db.User) error { us.mtx.Lock() defer us.mtx.Unlock() userID := fmt.Sprint(user.ID) - _, ok := us.store.GetStringIn(UsersNs, userID) + _, ok := us.store.GetStringIn(db.UsersNs, userID) if ok { return fmt.Errorf("userID (%d) exists", user.ID) } @@ -203,7 +180,7 @@ func (us *KVUserStore) AddUser(user *User) error { if err != nil { return err } - return us.store.SetStringIn(UsersNs, userID, string(userBytes)) + return us.store.SetStringIn(db.UsersNs, userID, string(userBytes)) } func (us *KVUserStore) DelUser(id uint64) error { @@ -211,11 +188,11 @@ func (us *KVUserStore) DelUser(id uint64) error { defer us.mtx.Unlock() userID := fmt.Sprint(id) - infoStr, ok := us.store.GetStringIn(UsersNs, userID) + infoStr, ok := us.store.GetStringIn(db.UsersNs, userID) if !ok { return fmt.Errorf("userID (%s) does not exist", userID) } - user := &User{} + user := &db.User{} err := json.Unmarshal([]byte(infoStr), user) if err != nil { return err @@ -223,24 +200,24 @@ func (us *KVUserStore) DelUser(id uint64) error { // TODO: add complement operations if part of the actions fails err1 := us.store.DelStringIn(IDsNs, user.Name) - err2 := us.store.DelStringIn(UsersNs, userID) + err2 := us.store.DelStringIn(db.UsersNs, userID) if err1 != nil || err2 != nil { return fmt.Errorf("DelUser: err1(%s) err2(%s)", err1, err2) } return nil } -func (us *KVUserStore) GetUser(id uint64) (*User, error) { +func (us *KVUserStore) GetUser(id uint64) (*db.User, error) { us.mtx.RLock() defer us.mtx.RUnlock() userID := fmt.Sprint(id) - infoStr, ok := us.store.GetStringIn(UsersNs, userID) + infoStr, ok := us.store.GetStringIn(db.UsersNs, userID) if !ok { return nil, fmt.Errorf("user (%s) not found", userID) } - user := &User{} + user := &db.User{} err := json.Unmarshal([]byte(infoStr), user) if err != nil { return nil, err @@ -258,7 +235,7 @@ func (us *KVUserStore) GetUser(id uint64) (*User, error) { } -func (us *KVUserStore) GetUserByName(name string) (*User, error) { +func (us *KVUserStore) GetUserByName(name string) (*db.User, error) { us.mtx.RLock() defer us.mtx.RUnlock() @@ -266,12 +243,12 @@ func (us *KVUserStore) GetUserByName(name string) (*User, error) { if !ok { return nil, ErrNotFound } - infoStr, ok := us.store.GetStringIn(UsersNs, userID) + infoStr, ok := us.store.GetStringIn(db.UsersNs, userID) if !ok { return nil, ErrNotFound } - user := &User{} + user := &db.User{} err := json.Unmarshal([]byte(infoStr), user) if err != nil { return nil, err @@ -290,11 +267,11 @@ func (us *KVUserStore) SetPwd(id uint64, pwd string) error { defer us.mtx.Unlock() userID := fmt.Sprint(id) - infoStr, ok := us.store.GetStringIn(UsersNs, userID) + infoStr, ok := us.store.GetStringIn(db.UsersNs, userID) if !ok { return fmt.Errorf("user (%d) does not exist", id) } - gotUser := &User{} + gotUser := &db.User{} err := json.Unmarshal([]byte(infoStr), gotUser) if err != nil { return err @@ -307,19 +284,19 @@ func (us *KVUserStore) SetPwd(id uint64, pwd string) error { if err != nil { return err } - return us.store.SetStringIn(UsersNs, userID, string(infoBytes)) + return us.store.SetStringIn(db.UsersNs, userID, string(infoBytes)) } -func (us *KVUserStore) SetPreferences(id uint64, prefers *Preferences) error { +func (us *KVUserStore) SetPreferences(id uint64, prefers *db.Preferences) error { us.mtx.Lock() defer us.mtx.Unlock() userID := fmt.Sprint(id) - infoStr, ok := us.store.GetStringIn(UsersNs, userID) + infoStr, ok := us.store.GetStringIn(db.UsersNs, userID) if !ok { return fmt.Errorf("user (%d) does not exist", id) } - gotUser := &User{} + gotUser := &db.User{} err := json.Unmarshal([]byte(infoStr), gotUser) if err != nil { return err @@ -332,19 +309,19 @@ func (us *KVUserStore) SetPreferences(id uint64, prefers *Preferences) error { if err != nil { return err } - return us.store.SetStringIn(UsersNs, userID, string(infoBytes)) + return us.store.SetStringIn(db.UsersNs, userID, string(infoBytes)) } func (us *KVUserStore) CanIncrUsed(id uint64, capacity int64) (bool, error) { us.mtx.Lock() defer us.mtx.Unlock() userID := fmt.Sprint(id) - infoStr, ok := us.store.GetStringIn(UsersNs, userID) + infoStr, ok := us.store.GetStringIn(db.UsersNs, userID) if !ok { return false, fmt.Errorf("user (%d) does not exist", id) } - gotUser := &User{} + gotUser := &db.User{} err := json.Unmarshal([]byte(infoStr), gotUser) if err != nil { return false, err @@ -360,12 +337,12 @@ func (us *KVUserStore) SetUsed(id uint64, incr bool, capacity int64) error { defer us.mtx.Unlock() userID := fmt.Sprint(id) - infoStr, ok := us.store.GetStringIn(UsersNs, userID) + infoStr, ok := us.store.GetStringIn(db.UsersNs, userID) if !ok { return fmt.Errorf("user (%d) does not exist", id) } - gotUser := &User{} + gotUser := &db.User{} err := json.Unmarshal([]byte(infoStr), gotUser) if err != nil { return err @@ -389,19 +366,19 @@ func (us *KVUserStore) SetUsed(id uint64, incr bool, capacity int64) error { if err != nil { return err } - return us.store.SetStringIn(UsersNs, userID, string(infoBytes)) + return us.store.SetStringIn(db.UsersNs, userID, string(infoBytes)) } -func (us *KVUserStore) SetInfo(id uint64, user *User) error { +func (us *KVUserStore) SetInfo(id uint64, user *db.User) error { us.mtx.Lock() defer us.mtx.Unlock() userID := fmt.Sprint(id) - infoStr, ok := us.store.GetStringIn(UsersNs, userID) + infoStr, ok := us.store.GetStringIn(db.UsersNs, userID) if !ok { return fmt.Errorf("user (%d) does not exist", id) } - gotUser := &User{} + gotUser := &db.User{} err := json.Unmarshal([]byte(infoStr), gotUser) if err != nil { return err @@ -425,14 +402,14 @@ func (us *KVUserStore) SetInfo(id uint64, user *User) error { return err } - return us.store.SetStringIn(UsersNs, userID, string(infoBytes)) + return us.store.SetStringIn(db.UsersNs, userID, string(infoBytes)) } -func (us *KVUserStore) ListUsers() ([]*User, error) { +func (us *KVUserStore) ListUsers() ([]*db.User, error) { us.mtx.RLock() defer us.mtx.RUnlock() - idToInfo, err := us.store.ListStringsIn(UsersNs) + idToInfo, err := us.store.ListStringsIn(db.UsersNs) if err != nil { return nil, err } @@ -441,9 +418,9 @@ func (us *KVUserStore) ListUsers() ([]*User, error) { return nil, err } - users := []*User{} + users := []*db.User{} for _, infoStr := range idToInfo { - user := &User{} + user := &db.User{} err = json.Unmarshal([]byte(infoStr), user) if err != nil { return nil, err @@ -459,7 +436,7 @@ func (us *KVUserStore) ListUsers() ([]*User, error) { for _, user := range users { _, ok := nameToID[user.Name] if !ok { - err = us.store.DelStringIn(UsersNs, fmt.Sprint(user.ID)) + err = us.store.DelStringIn(db.UsersNs, fmt.Sprint(user.ID)) if err != nil { return nil, err } diff --git a/src/depidx/deps.go b/src/depidx/deps.go index 09dfd4c..29f8265 100644 --- a/src/depidx/deps.go +++ b/src/depidx/deps.go @@ -5,6 +5,7 @@ import ( "go.uber.org/zap" "github.com/ihexxa/quickshare/src/cryptoutil" + "github.com/ihexxa/quickshare/src/db/boltstore" "github.com/ihexxa/quickshare/src/db/fileinfostore" "github.com/ihexxa/quickshare/src/db/sitestore" "github.com/ihexxa/quickshare/src/db/userstore" @@ -34,6 +35,7 @@ type Deps struct { logger *zap.SugaredLogger limiter iolimiter.ILimiter workers worker.IWorkerPool + boltStore *boltstore.BoltStore } func NewDeps(cfg gocfg.ICfg) *Deps { @@ -119,3 +121,11 @@ func (deps *Deps) Workers() worker.IWorkerPool { func (deps *Deps) SetWorkers(workers worker.IWorkerPool) { deps.workers = workers } + +func (deps *Deps) BoltStore() *boltstore.BoltStore { + return deps.boltStore +} + +func (deps *Deps) SetBoltStore(boltStore *boltstore.BoltStore) { + deps.boltStore = boltStore +} diff --git a/src/handlers/fileshdr/handlers.go b/src/handlers/fileshdr/handlers.go index b215a7d..4b5f59a 100644 --- a/src/handlers/fileshdr/handlers.go +++ b/src/handlers/fileshdr/handlers.go @@ -18,7 +18,7 @@ import ( "github.com/ihexxa/gocfg" "github.com/ihexxa/multipart" - "github.com/ihexxa/quickshare/src/db/fileinfostore" + "github.com/ihexxa/quickshare/src/db" "github.com/ihexxa/quickshare/src/db/userstore" "github.com/ihexxa/quickshare/src/depidx" q "github.com/ihexxa/quickshare/src/handlers" @@ -40,16 +40,14 @@ const ( ) type FileHandlers struct { - cfg gocfg.ICfg - deps *depidx.Deps - uploadMgr *UploadMgr + cfg gocfg.ICfg + deps *depidx.Deps } func NewFileHandlers(cfg gocfg.ICfg, deps *depidx.Deps) (*FileHandlers, error) { handlers := &FileHandlers{ - cfg: cfg, - deps: deps, - uploadMgr: NewUploadMgr(deps.KV()), + cfg: cfg, + deps: deps, } deps.Workers().AddHandler(MsgTypeSha1, handlers.genSha1) @@ -149,15 +147,28 @@ func (h *FileHandlers) Create(c *gin.Context) { c.JSON(q.ErrResp(c, 500, err)) return } + tmpFilePath := q.UploadPath(userName, req.Path) if req.FileSize == 0 { - err = h.deps.FS().MkdirAll(filepath.Dir(req.Path)) + // TODO: limit the number of files with 0 byte + err = h.deps.BoltStore().AddUploadInfos(userIDInt, tmpFilePath, fsFilePath, &db.FileInfo{ + Size: req.FileSize, + }) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + err = h.deps.BoltStore().MoveUploadingInfos(userIDInt, tmpFilePath, fsFilePath) if err != nil { c.JSON(q.ErrResp(c, 500, err)) return } - // TODO: limit the number of files with 0 byte + err = h.deps.FS().MkdirAll(filepath.Dir(req.Path)) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } err = h.deps.FS().Create(fsFilePath) if err != nil { @@ -193,18 +204,36 @@ func (h *FileHandlers) Create(c *gin.Context) { return } - tmpFilePath := q.UploadPath(userName, req.Path) + err = h.deps.BoltStore().AddUploadInfos(userIDInt, tmpFilePath, fsFilePath, &db.FileInfo{ + Size: req.FileSize, + }) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + locker := h.NewAutoLocker(c, lockName(tmpFilePath)) locker.Exec(func() { - // TODO: - ok, err := h.deps.Users().CanIncrUsed(userIDInt, req.FileSize) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } else if !ok { - c.JSON(q.ErrResp(c, 429, userstore.ErrReachedLimit)) - return - } + // ok, err := h.deps.Users().CanIncrUsed(userIDInt, req.FileSize) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } else if !ok { + // c.JSON(q.ErrResp(c, 429, userstore.ErrReachedLimit)) + // return + // } + + // err = h.deps.FileInfos().AddUploadInfo(userID, req.Path, tmpFilePath, req.FileSize) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } + + // err = h.deps.Users().SetUsed(userIDInt, true, req.FileSize) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } err = h.deps.FS().Create(tmpFilePath) if err != nil { @@ -215,11 +244,6 @@ func (h *FileHandlers) Create(c *gin.Context) { } return } - err = h.uploadMgr.AddInfo(userID, req.Path, tmpFilePath, req.FileSize) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } err = h.deps.FS().MkdirAll(filepath.Dir(req.Path)) if err != nil { @@ -227,12 +251,6 @@ func (h *FileHandlers) Create(c *gin.Context) { return } - err = h.deps.Users().SetUsed(userIDInt, true, req.FileSize) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } - c.JSON(q.Resp(200)) }) } @@ -259,32 +277,37 @@ func (h *FileHandlers) Delete(c *gin.Context) { locker := h.NewAutoLocker(c, lockName(filePath)) locker.Exec(func() { - info, err := h.deps.FileInfos().GetInfo(filePath) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } - err = h.deps.FS().Remove(filePath) if err != nil { c.JSON(q.ErrResp(c, 500, err)) return } - - err = h.deps.Users().SetUsed(userIDInt, false, info.Size) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } - - err = h.deps.FileInfos().DelInfo(filePath) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } - - c.JSON(q.Resp(200)) }) + + // info, err := h.deps.FileInfos().GetInfo(filePath) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } + + // err = h.deps.Users().SetUsed(userIDInt, false, info.Size) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } + + // err = h.deps.FileInfos().DelInfo(filePath) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } + + err = h.deps.BoltStore().DelInfos(userIDInt, filePath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + c.JSON(q.Resp(200)) } type MetadataResp struct { @@ -364,13 +387,19 @@ func (h *FileHandlers) Move(c *gin.Context) { return } role := c.MustGet(q.RoleParam).(string) + userID := c.MustGet(q.UserIDParam).(string) userName := c.MustGet(q.UserParam).(string) if !h.canAccess(userName, role, "move", req.OldPath) || !h.canAccess(userName, role, "move", req.NewPath) { c.JSON(q.ErrResp(c, 403, q.ErrAccessDenied)) return } + userIDInt, err := strconv.ParseUint(userID, 10, 64) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } - _, err := h.deps.FS().Stat(req.OldPath) + itemInfo, err := h.deps.FS().Stat(req.OldPath) if err != nil { c.JSON(q.ErrResp(c, 500, err)) return @@ -385,6 +414,12 @@ func (h *FileHandlers) Move(c *gin.Context) { return } + err = h.deps.BoltStore().MoveInfos(userIDInt, req.OldPath, req.NewPath, itemInfo.IsDir()) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + err = h.deps.FS().Rename(req.OldPath, req.NewPath) if err != nil { c.JSON(q.ErrResp(c, 500, err)) @@ -469,45 +504,52 @@ func (h *FileHandlers) UploadChunk(c *gin.Context) { return } + err = h.deps.BoltStore().MoveUploadingInfos(userIDInt, tmpFilePath, fsFilePath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + err = h.deps.FS().Rename(tmpFilePath, fsFilePath) if err != nil { c.JSON(q.ErrResp(c, 500, fmt.Errorf("%s error: %w", req.Path, err))) return } - err = h.deps.FileInfos().DelUploadInfo(userID, tmpFilePath) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } - fsInfo, err := h.deps.FS().Stat(fsFilePath) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } + // err = h.deps.FileInfos().DelUploadInfo(userID, tmpFilePath) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } - err = h.deps.FileInfos().SetInfo(fsFilePath, &fileinfostore.FileInfo{ - IsDir: false, - Shared: false, - Size: fsInfo.Size(), - }) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } + // fsInfo, err := h.deps.FS().Stat(fsFilePath) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } + + // err = h.deps.FileInfos().SetInfo(fsFilePath, &db.FileInfo{ + // IsDir: false, + // Shared: false, + // Size: fsInfo.Size(), + // }) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } // TODO: check space quota? - if fsInfo.Size()-fileSize != 0 { - sizeDiff := fsInfo.Size() - fileSize - if sizeDiff < 0 { - sizeDiff = -sizeDiff - } - err = h.deps.Users().SetUsed(userIDInt, fsInfo.Size()-fileSize > 0, sizeDiff) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } - } + // if fsInfo.Size()-fileSize != 0 { + // sizeDiff := fsInfo.Size() - fileSize + // if sizeDiff < 0 { + // sizeDiff = -sizeDiff + // } + // err = h.deps.Users().SetUsed(userIDInt, fsInfo.Size()-fileSize > 0, sizeDiff) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } + // } msg, err := json.Marshal(Sha1Params{ FilePath: fsFilePath, @@ -842,7 +884,7 @@ func lockName(filePath string) string { } type ListUploadingsResp struct { - UploadInfos []*fileinfostore.UploadInfo `json:"uploadInfos"` + UploadInfos []*db.UploadInfo `json:"uploadInfos"` } func (h *FileHandlers) ListUploadings(c *gin.Context) { @@ -854,7 +896,7 @@ func (h *FileHandlers) ListUploadings(c *gin.Context) { return } if infos == nil { - infos = []*fileinfostore.UploadInfo{} + infos = []*db.UploadInfo{} } c.JSON(200, &ListUploadingsResp{UploadInfos: infos}) } @@ -884,11 +926,11 @@ func (h *FileHandlers) DelUploading(c *gin.Context) { tmpFilePath := q.UploadPath(userName, filePath) locker := h.NewAutoLocker(c, lockName(tmpFilePath)) locker.Exec(func() { - _, size, _, err := h.deps.FileInfos().GetUploadInfo(userID, tmpFilePath) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } + // _, size, _, err := h.deps.FileInfos().GetUploadInfo(userID, tmpFilePath) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } _, err = h.deps.FS().Stat(tmpFilePath) if err != nil { @@ -906,20 +948,26 @@ func (h *FileHandlers) DelUploading(c *gin.Context) { } } - err = h.deps.FileInfos().DelUploadInfo(userID, tmpFilePath) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } + // err = h.deps.FileInfos().DelUploadInfo(userID, tmpFilePath) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } - err = h.deps.Users().SetUsed(userIDInt, false, size) - if err != nil { - c.JSON(q.ErrResp(c, 500, err)) - return - } - - c.JSON(q.Resp(200)) + // err = h.deps.Users().SetUsed(userIDInt, false, size) + // if err != nil { + // c.JSON(q.ErrResp(c, 500, err)) + // return + // } }) + + err = h.deps.BoltStore().DelUploadingInfos(userIDInt, tmpFilePath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + + c.JSON(q.Resp(200)) } type SharingReq struct { diff --git a/src/handlers/fileshdr/upload_mgr.go b/src/handlers/fileshdr/upload_mgr.go deleted file mode 100644 index 9326c2a..0000000 --- a/src/handlers/fileshdr/upload_mgr.go +++ /dev/null @@ -1,126 +0,0 @@ -package fileshdr - -import ( - "encoding/json" - "errors" - "fmt" - - "github.com/ihexxa/quickshare/src/kvstore" -) - -var ( - ErrCreateExisting = errors.New("create upload info which already exists") - ErrGreaterThanSize = errors.New("uploaded is greater than file size") - ErrNotFound = errors.New("upload info not found") - - uploadsPrefix = "uploads" -) - -type UploadInfo struct { - RealFilePath string `json:"realFilePath"` - Size int64 `json:"size"` - Uploaded int64 `json:"uploaded"` -} - -type UploadMgr struct { - kv kvstore.IKVStore -} - -func UploadNS(user string) string { - return fmt.Sprintf("%s/%s", uploadsPrefix, user) -} - -func NewUploadMgr(kv kvstore.IKVStore) *UploadMgr { - return &UploadMgr{ - kv: kv, - } -} - -func (um *UploadMgr) AddInfo(user, filePath, tmpPath string, fileSize int64) error { - ns := UploadNS(user) - err := um.kv.AddNamespace(ns) - if err != nil { - return err - } - - _, ok := um.kv.GetStringIn(ns, tmpPath) - if ok { - return ErrCreateExisting - } - - info := &UploadInfo{ - RealFilePath: filePath, - Size: fileSize, - Uploaded: 0, - } - infoBytes, err := json.Marshal(info) - if err != nil { - return err - } - - return um.kv.SetStringIn(ns, tmpPath, string(infoBytes)) -} - -func (um *UploadMgr) SetInfo(user, filePath string, newUploaded int64) error { - realFilePath, fileSize, _, err := um.GetInfo(user, filePath) - if err != nil { - return err - } else if newUploaded > fileSize { - return ErrGreaterThanSize - } - - newInfo := &UploadInfo{ - RealFilePath: realFilePath, - Size: fileSize, - Uploaded: newUploaded, - } - newInfoBytes, err := json.Marshal(newInfo) - if err != nil { - return err - } - return um.kv.SetStringIn(UploadNS(user), filePath, string(newInfoBytes)) -} - -func (um *UploadMgr) GetInfo(user, filePath string) (string, int64, int64, error) { - ns := UploadNS(user) - infoBytes, ok := um.kv.GetStringIn(ns, filePath) - if !ok { - return "", 0, 0, ErrNotFound - } - - info := &UploadInfo{} - err := json.Unmarshal([]byte(infoBytes), info) - if err != nil { - return "", 0, 0, err - } - - return info.RealFilePath, info.Size, info.Uploaded, nil -} - -func (um *UploadMgr) DelInfo(user, filePath string) error { - return um.kv.DelInt64In(UploadNS(user), filePath) -} - -func (um *UploadMgr) ListInfo(user string) ([]*UploadInfo, error) { - ns := UploadNS(user) - if !um.kv.HasNamespace(ns) { - return nil, nil - } - - infoMap, err := um.kv.ListStringsIn(ns) - if err != nil { - return nil, err - } - - infos := []*UploadInfo{} - for _, infoStr := range infoMap { - info := &UploadInfo{} - err = json.Unmarshal([]byte(infoStr), info) - if err != nil { - return nil, err - } - infos = append(infos, info) - } - - return infos, nil -} diff --git a/src/handlers/multiusers/handlers.go b/src/handlers/multiusers/handlers.go index 9bab2f9..d699d48 100644 --- a/src/handlers/multiusers/handlers.go +++ b/src/handlers/multiusers/handlers.go @@ -12,6 +12,7 @@ import ( "github.com/ihexxa/gocfg" "golang.org/x/crypto/bcrypt" + "github.com/ihexxa/quickshare/src/db" "github.com/ihexxa/quickshare/src/db/userstore" "github.com/ihexxa/quickshare/src/depidx" q "github.com/ihexxa/quickshare/src/handlers" @@ -179,12 +180,12 @@ func (h *MultiUsersSvc) Init(adminName, adminPwd string) (string, error) { } preferences := userstore.DefaultPreferences - user := &userstore.User{ + user := &db.User{ ID: h.deps.ID().Gen(), Name: userCfg.Name, Pwd: string(pwdHash), Role: userCfg.Role, - Quota: &userstore.Quota{ + Quota: &db.Quota{ SpaceLimit: spaceLimit, UploadSpeedLimit: uploadSpeedLimit, DownloadSpeedLimit: downloadSpeedLimit, @@ -424,12 +425,12 @@ func (h *MultiUsersSvc) AddUser(c *gin.Context) { } newPreferences := userstore.DefaultPreferences - err = h.deps.Users().AddUser(&userstore.User{ + err = h.deps.Users().AddUser(&db.User{ ID: uid, Name: req.Name, Pwd: string(pwdHash), Role: req.Role, - Quota: &userstore.Quota{ + Quota: &db.Quota{ SpaceLimit: int64(h.cfg.IntOr("Users.SpaceLimit", 100*1024*1024)), // TODO: support int64 UploadSpeedLimit: h.cfg.IntOr("Users.UploadSpeedLimit", 100*1024), DownloadSpeedLimit: h.cfg.IntOr("Users.DownloadSpeedLimit", 100*1024), @@ -486,7 +487,7 @@ func (h *MultiUsersSvc) DelUser(c *gin.Context) { } type ListUsersResp struct { - Users []*userstore.User `json:"users"` + Users []*db.User `json:"users"` } func (h *MultiUsersSvc) ListUsers(c *gin.Context) { @@ -623,12 +624,12 @@ func (h *MultiUsersSvc) isValidRole(role string) error { } type SelfResp struct { - ID string `json:"id"` - Name string `json:"name"` - Role string `json:"role"` - Quota *userstore.Quota `json:"quota"` - UsedSpace int64 `json:"usedSpace,string"` - Preferences *userstore.Preferences `json:"preferences"` + ID string `json:"id"` + Name string `json:"name"` + Role string `json:"role"` + Quota *db.Quota `json:"quota"` + UsedSpace int64 `json:"usedSpace,string"` + Preferences *db.Preferences `json:"preferences"` } func (h *MultiUsersSvc) Self(c *gin.Context) { @@ -655,10 +656,10 @@ func (h *MultiUsersSvc) Self(c *gin.Context) { } type SetUserReq struct { - ID uint64 `json:"id,string"` - Role string `json:"role"` - UsedSpace int64 `json:"usedSpace,string"` - Quota *userstore.Quota `json:"quota"` + ID uint64 `json:"id,string"` + Role string `json:"role"` + UsedSpace int64 `json:"usedSpace,string"` + Quota *db.Quota `json:"quota"` } func (h *MultiUsersSvc) SetUser(c *gin.Context) { @@ -668,7 +669,7 @@ func (h *MultiUsersSvc) SetUser(c *gin.Context) { return } - err := h.deps.Users().SetInfo(req.ID, &userstore.User{ + err := h.deps.Users().SetInfo(req.ID, &db.User{ Role: req.Role, Quota: req.Quota, }) @@ -681,7 +682,7 @@ func (h *MultiUsersSvc) SetUser(c *gin.Context) { } type SetPreferencesReq struct { - Preferences *userstore.Preferences `json:"preferences"` + Preferences *db.Preferences `json:"preferences"` } func (h *MultiUsersSvc) SetPreferences(c *gin.Context) { diff --git a/src/iolimiter/iolimiter.go b/src/iolimiter/iolimiter.go index fea2350..6a80354 100644 --- a/src/iolimiter/iolimiter.go +++ b/src/iolimiter/iolimiter.go @@ -4,6 +4,7 @@ import ( "fmt" "sync" + "github.com/ihexxa/quickshare/src/db" "github.com/ihexxa/quickshare/src/db/userstore" "github.com/ihexxa/quickshare/src/golimiter" ) @@ -20,7 +21,7 @@ type IOLimiter struct { UploadLimiter *golimiter.Limiter DownloadLimiter *golimiter.Limiter users userstore.IUserStore - quotaCache map[uint64]*userstore.Quota + quotaCache map[uint64]*db.Quota } func NewIOLimiter(cap, cyc int, users userstore.IUserStore) *IOLimiter { @@ -29,7 +30,7 @@ func NewIOLimiter(cap, cyc int, users userstore.IUserStore) *IOLimiter { UploadLimiter: golimiter.New(cap, cyc), DownloadLimiter: golimiter.New(cap, cyc), users: users, - quotaCache: map[uint64]*userstore.Quota{}, + quotaCache: map[uint64]*db.Quota{}, } } diff --git a/src/kvstore/boltdbpvd/interface.go b/src/kvstore/boltdbpvd/interface.go new file mode 100644 index 0000000..40ab7a0 --- /dev/null +++ b/src/kvstore/boltdbpvd/interface.go @@ -0,0 +1,7 @@ +package boltdbpvd + +import "github.com/boltdb/bolt" + +type BoltProvider interface { + Bolt() *bolt.DB +} diff --git a/src/kvstore/boltdbpvd/provider.go b/src/kvstore/boltdbpvd/provider.go index f274aa4..34c23e0 100644 --- a/src/kvstore/boltdbpvd/provider.go +++ b/src/kvstore/boltdbpvd/provider.go @@ -433,6 +433,6 @@ func (bp *BoltPvd) ListStringsByPrefixIn(prefix, ns string) (map[string]string, return results, err } -func (bp *BoltPvd) StartUpdateTxBolt(op func(*bolt.Tx) error) error { - return bp.db.Update(op) +func (bp *BoltPvd) Bolt() *bolt.DB { + return bp.db } diff --git a/src/server/server.go b/src/server/server.go index 3f1e029..0e52849 100644 --- a/src/server/server.go +++ b/src/server/server.go @@ -23,6 +23,7 @@ import ( "golang.org/x/crypto/bcrypt" "github.com/ihexxa/quickshare/src/cryptoutil/jwt" + "github.com/ihexxa/quickshare/src/db/boltstore" "github.com/ihexxa/quickshare/src/db/fileinfostore" "github.com/ihexxa/quickshare/src/db/sitestore" "github.com/ihexxa/quickshare/src/db/userstore" @@ -141,6 +142,7 @@ func initDeps(cfg gocfg.ICfg) *depidx.Deps { if err := filesystem.MkdirAll(dbPath); err != nil { panic(fmt.Sprintf("fail to create path for db: %s", err)) } + kv := boltdbpvd.New(dbPath, 1024) users, err := userstore.NewKVUserStore(kv) if err != nil { @@ -152,7 +154,11 @@ func initDeps(cfg gocfg.ICfg) *depidx.Deps { } siteStore, err := sitestore.NewSiteStore(kv) if err != nil { - panic(fmt.Sprintf("fail to new site config store: %s", err)) + panic(fmt.Sprintf("fail to init site config store: %s", err)) + } + boltDB, err := boltstore.NewBoltStore(kv.Bolt()) + if err != nil { + panic(fmt.Sprintf("fail to init bolt store: %s", err)) } err = siteStore.Init(&sitestore.SiteConfig{ @@ -182,6 +188,7 @@ func initDeps(cfg gocfg.ICfg) *depidx.Deps { deps.SetUsers(users) deps.SetFileInfos(fileInfos) deps.SetSiteStore(siteStore) + deps.SetBoltStore(boltDB) deps.SetID(ider) deps.SetLog(logger) deps.SetLimiter(limiter) diff --git a/src/server/server_files_test.go b/src/server/server_files_test.go index fdaceeb..85af468 100644 --- a/src/server/server_files_test.go +++ b/src/server/server_files_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/ihexxa/quickshare/src/client" - "github.com/ihexxa/quickshare/src/db/fileinfostore" + "github.com/ihexxa/quickshare/src/db" q "github.com/ihexxa/quickshare/src/handlers" "github.com/ihexxa/quickshare/src/handlers/fileshdr" ) @@ -631,7 +631,7 @@ func TestFileHandlers(t *testing.T) { t.Fatal(res.StatusCode) } - gotInfos := map[string]*fileinfostore.UploadInfo{} + gotInfos := map[string]*db.UploadInfo{} for _, info := range lResp.UploadInfos { gotInfos[info.RealFilePath] = info } diff --git a/src/server/server_permission_test.go b/src/server/server_permission_test.go index 65669f9..2fcec96 100644 --- a/src/server/server_permission_test.go +++ b/src/server/server_permission_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/ihexxa/quickshare/src/client" + "github.com/ihexxa/quickshare/src/db" "github.com/ihexxa/quickshare/src/db/userstore" q "github.com/ihexxa/quickshare/src/handlers" "github.com/ihexxa/quickshare/src/handlers/settings" @@ -105,7 +106,7 @@ func TestPermissions(t *testing.T) { testUsersAPIs := func(desc, user string, pwd string, requireAuth bool, expectedCodes map[string]int) { newPwd := "12345" newRole := userstore.AdminRole - newQuota := &userstore.Quota{ + newQuota := &db.Quota{ SpaceLimit: int64(2046), UploadSpeedLimit: int(8 * 1024 * 1024), DownloadSpeedLimit: int(8 * 1024 * 1024), diff --git a/src/server/server_space_limit_test.go b/src/server/server_space_limit_test.go index d385b54..e01a804 100644 --- a/src/server/server_space_limit_test.go +++ b/src/server/server_space_limit_test.go @@ -125,8 +125,8 @@ func TestSpaceLimit(t *testing.T) { res, _, errs := cl.Create(filePath, 1) if len(errs) > 0 { t.Fatal(errs) - } else if res.StatusCode != 429 { - t.Fatal("(space limit): this request should be rejected") + } else if res.StatusCode != 500 { + t.Fatalf("(space limit): this request should be rejected: %d", res.StatusCode) } for i := 0; i < spaceLimit/fileSize; i++ { diff --git a/src/server/server_users_test.go b/src/server/server_users_test.go index 627fe90..635e0b3 100644 --- a/src/server/server_users_test.go +++ b/src/server/server_users_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/ihexxa/quickshare/src/client" + "github.com/ihexxa/quickshare/src/db" "github.com/ihexxa/quickshare/src/db/sitestore" "github.com/ihexxa/quickshare/src/db/userstore" q "github.com/ihexxa/quickshare/src/handlers" @@ -120,14 +121,14 @@ func TestUsersHandlers(t *testing.T) { }) t.Run("test users APIs: Login-Self-SetPwd-Logout-Login", func(t *testing.T) { - users := []*userstore.User{ + users := []*db.User{ { ID: 0, Name: adminName, Pwd: adminPwd, Role: userstore.AdminRole, UsedSpace: 0, - Quota: &userstore.Quota{ + Quota: &db.Quota{ UploadSpeedLimit: 50 * 1024 * 1024, DownloadSpeedLimit: 50 * 1024 * 1024, SpaceLimit: 1024 * 1024 * 1024, @@ -139,7 +140,7 @@ func TestUsersHandlers(t *testing.T) { Pwd: "Quicksh@re", Role: userstore.UserRole, UsedSpace: 0, - Quota: &userstore.Quota{ + Quota: &db.Quota{ UploadSpeedLimit: 409600, DownloadSpeedLimit: 409600, SpaceLimit: 1024, @@ -309,7 +310,7 @@ func TestUsersHandlers(t *testing.T) { } } - newRole, newQuota := userstore.AdminRole, &userstore.Quota{ + newRole, newQuota := userstore.AdminRole, &db.Quota{ SpaceLimit: 3, UploadSpeedLimit: 3, DownloadSpeedLimit: 3, @@ -445,8 +446,8 @@ func TestUsersHandlers(t *testing.T) { token := client.GetCookie(resp.Cookies(), su.TokenCookie) - prefers := []*userstore.Preferences{ - &userstore.Preferences{ + prefers := []*db.Preferences{ + &db.Preferences{ Bg: &sitestore.BgConfig{ Url: "/bgurl", Repeat: "no-repeat", @@ -456,7 +457,7 @@ func TestUsersHandlers(t *testing.T) { CSSURL: "/cssurl", LanPackURL: "/lanpack", }, - &userstore.Preferences{ + &db.Preferences{ Bg: &sitestore.BgConfig{ Url: "/bgurl2", Repeat: "no-repeat2", diff --git a/src/server/testdata/test_quickshare.db b/src/server/testdata/test_quickshare.db index b512ca2612902f52271dd2b59072a3441a3ecb8f..f5ed8a88d7735fa2a2891e33e41d4790a5bfe2ca 100644 GIT binary patch delta 81 zcmZo@P-tjSnBX8Vhk=m+1Pm|qcTKPL)!A+sz_@{bl7NH2T&Rraigl0X7^E$M$TTUm aDKNGvFtsT#w<)l+DX_LFuq{(ye*gf>lNW>l delta 81 zcmZo@P-tjSnBX8Vfq{_$1lBIU6jz*>v|+np0OJP!NdgW66QMF!6`!SYwyH}&WSSJ( a6d2nSnA#MW+Z0&Z6j<97*p?}