doesnm/quickshare
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(files): add boltdb store and refactor files handlers

Browse source
This commit is contained in:
hexxa 2022-03-05 00:16:04 +08:00 • committed by Hexxa
parent 044cdea1d4
commit 17b4544487
19 changed files with 751 additions and 402 deletions
Download patch file Download diff file Expand all files Collapse all files

248
src/handlers/fileshdr/handlers.go
View file

@ -18,7 +18,7 @@ import (
"github.com/ihexxa/gocfg"
"github.com/ihexxa/multipart"
"github.com/ihexxa/quickshare/src/db/fileinfostore"
"github.com/ihexxa/quickshare/src/db"
"github.com/ihexxa/quickshare/src/db/userstore"
"github.com/ihexxa/quickshare/src/depidx"
q "github.com/ihexxa/quickshare/src/handlers"
@ -40,16 +40,14 @@ const (
)
type FileHandlers struct {
cfg gocfg.ICfg
deps *depidx.Deps
uploadMgr *UploadMgr
cfg gocfg.ICfg
deps *depidx.Deps
}
func NewFileHandlers(cfg gocfg.ICfg, deps *depidx.Deps) (*FileHandlers, error) {
handlers := &FileHandlers{
cfg: cfg,
deps: deps,
uploadMgr: NewUploadMgr(deps.KV()),
cfg: cfg,
deps: deps,
}
deps.Workers().AddHandler(MsgTypeSha1, handlers.genSha1)
@ -149,15 +147,28 @@ func (h *FileHandlers) Create(c *gin.Context) {
c.JSON(q.ErrResp(c, 500, err))
return
}
tmpFilePath := q.UploadPath(userName, req.Path)
if req.FileSize == 0 {
err = h.deps.FS().MkdirAll(filepath.Dir(req.Path))
// TODO: limit the number of files with 0 byte
err = h.deps.BoltStore().AddUploadInfos(userIDInt, tmpFilePath, fsFilePath, &db.FileInfo{
Size: req.FileSize,
})
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
err = h.deps.BoltStore().MoveUploadingInfos(userIDInt, tmpFilePath, fsFilePath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
// TODO: limit the number of files with 0 byte
err = h.deps.FS().MkdirAll(filepath.Dir(req.Path))
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
err = h.deps.FS().Create(fsFilePath)
if err != nil {
@ -193,18 +204,36 @@ func (h *FileHandlers) Create(c *gin.Context) {
return
}
tmpFilePath := q.UploadPath(userName, req.Path)
err = h.deps.BoltStore().AddUploadInfos(userIDInt, tmpFilePath, fsFilePath, &db.FileInfo{
Size: req.FileSize,
})
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
locker := h.NewAutoLocker(c, lockName(tmpFilePath))
locker.Exec(func() {
// TODO:
ok, err := h.deps.Users().CanIncrUsed(userIDInt, req.FileSize)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
} else if !ok {
c.JSON(q.ErrResp(c, 429, userstore.ErrReachedLimit))
return
}
// ok, err := h.deps.Users().CanIncrUsed(userIDInt, req.FileSize)
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// } else if !ok {
// c.JSON(q.ErrResp(c, 429, userstore.ErrReachedLimit))
// return
// }
// err = h.deps.FileInfos().AddUploadInfo(userID, req.Path, tmpFilePath, req.FileSize)
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// }
// err = h.deps.Users().SetUsed(userIDInt, true, req.FileSize)
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// }
err = h.deps.FS().Create(tmpFilePath)
if err != nil {
@ -215,11 +244,6 @@ func (h *FileHandlers) Create(c *gin.Context) {
}
return
}
err = h.uploadMgr.AddInfo(userID, req.Path, tmpFilePath, req.FileSize)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
err = h.deps.FS().MkdirAll(filepath.Dir(req.Path))
if err != nil {
@ -227,12 +251,6 @@ func (h *FileHandlers) Create(c *gin.Context) {
return
}
err = h.deps.Users().SetUsed(userIDInt, true, req.FileSize)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
c.JSON(q.Resp(200))
})
}
@ -259,32 +277,37 @@ func (h *FileHandlers) Delete(c *gin.Context) {
locker := h.NewAutoLocker(c, lockName(filePath))
locker.Exec(func() {
info, err := h.deps.FileInfos().GetInfo(filePath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
err = h.deps.FS().Remove(filePath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
err = h.deps.Users().SetUsed(userIDInt, false, info.Size)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
err = h.deps.FileInfos().DelInfo(filePath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
c.JSON(q.Resp(200))
})
// info, err := h.deps.FileInfos().GetInfo(filePath)
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// }
// err = h.deps.Users().SetUsed(userIDInt, false, info.Size)
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// }
// err = h.deps.FileInfos().DelInfo(filePath)
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// }
err = h.deps.BoltStore().DelInfos(userIDInt, filePath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
c.JSON(q.Resp(200))
}
type MetadataResp struct {
@ -364,13 +387,19 @@ func (h *FileHandlers) Move(c *gin.Context) {
return
}
role := c.MustGet(q.RoleParam).(string)
userID := c.MustGet(q.UserIDParam).(string)
userName := c.MustGet(q.UserParam).(string)
if !h.canAccess(userName, role, "move", req.OldPath) || !h.canAccess(userName, role, "move", req.NewPath) {
c.JSON(q.ErrResp(c, 403, q.ErrAccessDenied))
return
}
userIDInt, err := strconv.ParseUint(userID, 10, 64)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
_, err := h.deps.FS().Stat(req.OldPath)
itemInfo, err := h.deps.FS().Stat(req.OldPath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
@ -385,6 +414,12 @@ func (h *FileHandlers) Move(c *gin.Context) {
return
}
err = h.deps.BoltStore().MoveInfos(userIDInt, req.OldPath, req.NewPath, itemInfo.IsDir())
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
err = h.deps.FS().Rename(req.OldPath, req.NewPath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
@ -469,45 +504,52 @@ func (h *FileHandlers) UploadChunk(c *gin.Context) {
return
}
err = h.deps.BoltStore().MoveUploadingInfos(userIDInt, tmpFilePath, fsFilePath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
err = h.deps.FS().Rename(tmpFilePath, fsFilePath)
if err != nil {
c.JSON(q.ErrResp(c, 500, fmt.Errorf("%s error: %w", req.Path, err)))
return
}
err = h.deps.FileInfos().DelUploadInfo(userID, tmpFilePath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
fsInfo, err := h.deps.FS().Stat(fsFilePath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
// err = h.deps.FileInfos().DelUploadInfo(userID, tmpFilePath)
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// }
err = h.deps.FileInfos().SetInfo(fsFilePath, &fileinfostore.FileInfo{
IsDir: false,
Shared: false,
Size: fsInfo.Size(),
})
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
// fsInfo, err := h.deps.FS().Stat(fsFilePath)
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// }
// err = h.deps.FileInfos().SetInfo(fsFilePath, &db.FileInfo{
// IsDir: false,
// Shared: false,
// Size: fsInfo.Size(),
// })
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// }
// TODO: check space quota?
if fsInfo.Size()-fileSize != 0 {
sizeDiff := fsInfo.Size() - fileSize
if sizeDiff < 0 {
sizeDiff = -sizeDiff
}
err = h.deps.Users().SetUsed(userIDInt, fsInfo.Size()-fileSize > 0, sizeDiff)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
}
// if fsInfo.Size()-fileSize != 0 {
// sizeDiff := fsInfo.Size() - fileSize
// if sizeDiff < 0 {
// sizeDiff = -sizeDiff
// }
// err = h.deps.Users().SetUsed(userIDInt, fsInfo.Size()-fileSize > 0, sizeDiff)
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// }
// }
msg, err := json.Marshal(Sha1Params{
FilePath: fsFilePath,
@ -842,7 +884,7 @@ func lockName(filePath string) string {
}
type ListUploadingsResp struct {
UploadInfos []*fileinfostore.UploadInfo `json:"uploadInfos"`
UploadInfos []*db.UploadInfo `json:"uploadInfos"`
}
func (h *FileHandlers) ListUploadings(c *gin.Context) {
@ -854,7 +896,7 @@ func (h *FileHandlers) ListUploadings(c *gin.Context) {
return
}
if infos == nil {
infos = []*fileinfostore.UploadInfo{}
infos = []*db.UploadInfo{}
}
c.JSON(200, &ListUploadingsResp{UploadInfos: infos})
}
@ -884,11 +926,11 @@ func (h *FileHandlers) DelUploading(c *gin.Context) {
tmpFilePath := q.UploadPath(userName, filePath)
locker := h.NewAutoLocker(c, lockName(tmpFilePath))
locker.Exec(func() {
_, size, _, err := h.deps.FileInfos().GetUploadInfo(userID, tmpFilePath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
// _, size, _, err := h.deps.FileInfos().GetUploadInfo(userID, tmpFilePath)
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// }
_, err = h.deps.FS().Stat(tmpFilePath)
if err != nil {
@ -906,20 +948,26 @@ func (h *FileHandlers) DelUploading(c *gin.Context) {
}
}
err = h.deps.FileInfos().DelUploadInfo(userID, tmpFilePath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
// err = h.deps.FileInfos().DelUploadInfo(userID, tmpFilePath)
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// }
err = h.deps.Users().SetUsed(userIDInt, false, size)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
c.JSON(q.Resp(200))
// err = h.deps.Users().SetUsed(userIDInt, false, size)
// if err != nil {
// c.JSON(q.ErrResp(c, 500, err))
// return
// }
})
err = h.deps.BoltStore().DelUploadingInfos(userIDInt, tmpFilePath)
if err != nil {
c.JSON(q.ErrResp(c, 500, err))
return
}
c.JSON(q.Resp(200))
}
type SharingReq struct {

126
src/handlers/fileshdr/upload_mgr.go
View file

@ -1,126 +0,0 @@
package fileshdr
import (
"encoding/json"
"errors"
"fmt"
"github.com/ihexxa/quickshare/src/kvstore"
)
var (
ErrCreateExisting = errors.New("create upload info which already exists")
ErrGreaterThanSize = errors.New("uploaded is greater than file size")
ErrNotFound = errors.New("upload info not found")
uploadsPrefix = "uploads"
)
type UploadInfo struct {
RealFilePath string `json:"realFilePath"`
Size int64 `json:"size"`
Uploaded int64 `json:"uploaded"`
}
type UploadMgr struct {
kv kvstore.IKVStore
}
func UploadNS(user string) string {
return fmt.Sprintf("%s/%s", uploadsPrefix, user)
}
func NewUploadMgr(kv kvstore.IKVStore) *UploadMgr {
return &UploadMgr{
kv: kv,
}
}
func (um *UploadMgr) AddInfo(user, filePath, tmpPath string, fileSize int64) error {
ns := UploadNS(user)
err := um.kv.AddNamespace(ns)
if err != nil {
return err
}
_, ok := um.kv.GetStringIn(ns, tmpPath)
if ok {
return ErrCreateExisting
}
info := &UploadInfo{
RealFilePath: filePath,
Size: fileSize,
Uploaded: 0,
}
infoBytes, err := json.Marshal(info)
if err != nil {
return err
}
return um.kv.SetStringIn(ns, tmpPath, string(infoBytes))
}
func (um *UploadMgr) SetInfo(user, filePath string, newUploaded int64) error {
realFilePath, fileSize, _, err := um.GetInfo(user, filePath)
if err != nil {
return err
} else if newUploaded > fileSize {
return ErrGreaterThanSize
}
newInfo := &UploadInfo{
RealFilePath: realFilePath,
Size: fileSize,
Uploaded: newUploaded,
}
newInfoBytes, err := json.Marshal(newInfo)
if err != nil {
return err
}
return um.kv.SetStringIn(UploadNS(user), filePath, string(newInfoBytes))
}
func (um *UploadMgr) GetInfo(user, filePath string) (string, int64, int64, error) {
ns := UploadNS(user)
infoBytes, ok := um.kv.GetStringIn(ns, filePath)
if !ok {
return "", 0, 0, ErrNotFound
}
info := &UploadInfo{}
err := json.Unmarshal([]byte(infoBytes), info)
if err != nil {
return "", 0, 0, err
}
return info.RealFilePath, info.Size, info.Uploaded, nil
}
func (um *UploadMgr) DelInfo(user, filePath string) error {
return um.kv.DelInt64In(UploadNS(user), filePath)
}
func (um *UploadMgr) ListInfo(user string) ([]*UploadInfo, error) {
ns := UploadNS(user)
if !um.kv.HasNamespace(ns) {
return nil, nil
}
infoMap, err := um.kv.ListStringsIn(ns)
if err != nil {
return nil, err
}
infos := []*UploadInfo{}
for _, infoStr := range infoMap {
info := &UploadInfo{}
err = json.Unmarshal([]byte(infoStr), info)
if err != nil {
return nil, err
}
infos = append(infos, info)
}
return infos, nil
}

35
src/handlers/multiusers/handlers.go
View file

@ -12,6 +12,7 @@ import (
"github.com/ihexxa/gocfg"
"golang.org/x/crypto/bcrypt"
"github.com/ihexxa/quickshare/src/db"
"github.com/ihexxa/quickshare/src/db/userstore"
"github.com/ihexxa/quickshare/src/depidx"
q "github.com/ihexxa/quickshare/src/handlers"
@ -179,12 +180,12 @@ func (h *MultiUsersSvc) Init(adminName, adminPwd string) (string, error) {
}
preferences := userstore.DefaultPreferences
user := &userstore.User{
user := &db.User{
ID: h.deps.ID().Gen(),
Name: userCfg.Name,
Pwd: string(pwdHash),
Role: userCfg.Role,
Quota: &userstore.Quota{
Quota: &db.Quota{
SpaceLimit: spaceLimit,
UploadSpeedLimit: uploadSpeedLimit,
DownloadSpeedLimit: downloadSpeedLimit,
@ -424,12 +425,12 @@ func (h *MultiUsersSvc) AddUser(c *gin.Context) {
}
newPreferences := userstore.DefaultPreferences
err = h.deps.Users().AddUser(&userstore.User{
err = h.deps.Users().AddUser(&db.User{
ID: uid,
Name: req.Name,
Pwd: string(pwdHash),
Role: req.Role,
Quota: &userstore.Quota{
Quota: &db.Quota{
SpaceLimit: int64(h.cfg.IntOr("Users.SpaceLimit", 100*1024*1024)), // TODO: support int64
UploadSpeedLimit: h.cfg.IntOr("Users.UploadSpeedLimit", 100*1024),
DownloadSpeedLimit: h.cfg.IntOr("Users.DownloadSpeedLimit", 100*1024),
@ -486,7 +487,7 @@ func (h *MultiUsersSvc) DelUser(c *gin.Context) {
}
type ListUsersResp struct {
Users []*userstore.User `json:"users"`
Users []*db.User `json:"users"`
}
func (h *MultiUsersSvc) ListUsers(c *gin.Context) {
@ -623,12 +624,12 @@ func (h *MultiUsersSvc) isValidRole(role string) error {
}
type SelfResp struct {
ID string `json:"id"`
Name string `json:"name"`
Role string `json:"role"`
Quota *userstore.Quota `json:"quota"`
UsedSpace int64 `json:"usedSpace,string"`
Preferences *userstore.Preferences `json:"preferences"`
ID string `json:"id"`
Name string `json:"name"`
Role string `json:"role"`
Quota *db.Quota `json:"quota"`
UsedSpace int64 `json:"usedSpace,string"`
Preferences *db.Preferences `json:"preferences"`
}
func (h *MultiUsersSvc) Self(c *gin.Context) {
@ -655,10 +656,10 @@ func (h *MultiUsersSvc) Self(c *gin.Context) {
}
type SetUserReq struct {
ID uint64 `json:"id,string"`
Role string `json:"role"`
UsedSpace int64 `json:"usedSpace,string"`
Quota *userstore.Quota `json:"quota"`
ID uint64 `json:"id,string"`
Role string `json:"role"`
UsedSpace int64 `json:"usedSpace,string"`
Quota *db.Quota `json:"quota"`
}
func (h *MultiUsersSvc) SetUser(c *gin.Context) {
@ -668,7 +669,7 @@ func (h *MultiUsersSvc) SetUser(c *gin.Context) {
return
}
err := h.deps.Users().SetInfo(req.ID, &userstore.User{
err := h.deps.Users().SetInfo(req.ID, &db.User{
Role: req.Role,
Quota: req.Quota,
})
@ -681,7 +682,7 @@ func (h *MultiUsersSvc) SetUser(c *gin.Context) {
}
type SetPreferencesReq struct {
Preferences *userstore.Preferences `json:"preferences"`
Preferences *db.Preferences `json:"preferences"`
}
func (h *MultiUsersSvc) SetPreferences(c *gin.Context) {