doesnm/quickshare
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(qs2) add qs2 framework

Browse source
This commit is contained in:
hexxa 2020-12-05 10:30:03 +08:00
parent 6ae65fe09b
commit 83100007e3
33 changed files with 2934 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

49
src/server/config.go Normal file
View file

@ -0,0 +1,49 @@
package server
type FSConfig struct {
Root string `json:"root"`
OpensLimit int `json:"opensLimit"`
OpenTTL int `json:"openTTL"`
}
type Secrets struct {
TokenSecret string `json:"tokenSecret" cfg:"env"`
}
type ServerCfg struct {
ProdMode bool `json:"prodMode"`
Addr string `json:"addr"`
ReadTimeout int `json:"readTimeout"`
WriteTimeout int `json:"writeTimeout"`
MaxHeaderBytes int `json:"maxHeaderBytes"`
}
type Config struct {
Fs *FSConfig `json:"fs"`
Secrets *Secrets `json:"secrets"`
Server *ServerCfg `json:"server"`
}
func NewEmptyConfig() *Config {
return &Config{}
}
func NewDefaultConfig() *Config {
return &Config{
Fs: &FSConfig{
Root: ".",
OpensLimit: 128,
OpenTTL: 60, // 1 min
},
Secrets: &Secrets{
TokenSecret: "",
},
Server: &ServerCfg{
ProdMode: true,
Addr: "127.0.0.1:8888",
ReadTimeout: 2000,
WriteTimeout: 2000,
MaxHeaderBytes: 512,
},
}
}

BIN
src/server/quickshare.db Normal file
View file

Binary file not shown.

134
src/server/server.go
View file

@ -1,67 +1,149 @@
package server
import (
"context"
"crypto/rand"
"fmt"
"net/http"
"time"
"github.com/gin-gonic/gin"
"github.com/ihexxa/gocfg"
"github.com/ihexxa/quickshare/src/handlers"
"github.com/ihexxa/quickshare/src/cryptoutil/jwt"
"github.com/ihexxa/quickshare/src/depidx"
"github.com/ihexxa/quickshare/src/fs"
"github.com/ihexxa/quickshare/src/fs/local"
"github.com/ihexxa/quickshare/src/handlers/fileshdr"
"github.com/ihexxa/quickshare/src/handlers/singleuserhdr"
"github.com/ihexxa/quickshare/src/idgen/simpleidgen"
"github.com/ihexxa/quickshare/src/kvstore"
"github.com/ihexxa/quickshare/src/kvstore/boltdbpvd"
"github.com/ihexxa/quickshare/src/logging/simplelog"
"github.com/ihexxa/quickshare/src/uploadmgr"
)
type Server struct {
server *http.Server
}
type ServerCfg struct {
Addr string `json:"addr"`
ReadTimeout int `json:"readTimeout"`
WriteTimeout int `json:"writeTimeout"`
MaxHeaderBytes int `json:"maxHeaderBytes"`
deps *depidx.Deps
}
func NewServer(cfg gocfg.ICfg) (*Server, error) {
// gin.SetMode(gin.ReleaseMode)
deps := initDeps(cfg)
if cfg.BoolOr("Server.ProdMode", true) {
gin.SetMode(gin.ReleaseMode)
}
router := gin.Default()
router, err := addHandlers(router)
router, err := addHandlers(router, cfg, deps)
if err != nil {
return nil, err
}
srv := &http.Server{
// TODO: set more options
Addr: cfg.StringOr("ServerCfg.Addr", ":8080"),
Addr: cfg.GrabString("Server.Addr"),
Handler: router,
ReadTimeout: time.Duration(cfg.IntOr("ServerCfg.ReadTimeout", 1)) * time.Second,
WriteTimeout: time.Duration(cfg.IntOr("ServerCfg.ReadTimeout", 1)) * time.Second,
MaxHeaderBytes: cfg.IntOr("ServerCfg.MaxHeaderBytes", 512),
ReadTimeout: time.Duration(cfg.GrabInt("Server.ReadTimeout")) * time.Millisecond,
WriteTimeout: time.Duration(cfg.GrabInt("Server.WriteTimeout")) * time.Millisecond,
MaxHeaderBytes: cfg.GrabInt("Server.MaxHeaderBytes"),
}
return &Server{
server: srv,
deps: deps,
}, nil
}
func addHandlers(router *gin.Engine) (*gin.Engine, error) {
func (s *Server) depsFS() fs.ISimpleFS {
return s.deps.FS()
}
func (s *Server) depsKVStore() kvstore.IKVStore {
return s.deps.KV()
}
func makeRandToken() string {
b := make([]byte, 32)
_, err := rand.Read(b)
if err != nil {
panic(err)
}
return string(b)
}
func initDeps(cfg gocfg.ICfg) *depidx.Deps {
secret, ok := cfg.String("ENV.TOKENSECRET")
if !ok {
secret = makeRandToken()
fmt.Println("warning: TOKENSECRET is not given, using generated token")
}
rootPath := cfg.GrabString("Fs.Root")
opensLimit := cfg.GrabInt("Fs.OpensLimit")
openTTL := cfg.GrabInt("Fs.OpenTTL")
filesystem := local.NewLocalFS(rootPath, 0660, opensLimit, openTTL)
jwtEncDec := jwt.NewJWTEncDec(secret)
logger := simplelog.NewSimpleLogger()
kv := boltdbpvd.New(".", 1024)
ider := simpleidgen.New()
deps := depidx.NewDeps(cfg)
deps.SetFS(filesystem)
deps.SetToken(jwtEncDec)
deps.SetLog(logger)
deps.SetKV(kv)
deps.SetID(ider)
uploadMgr, err := uploadmgr.NewUploadMgr(deps)
if err != nil {
panic(err)
}
deps.SetUploader(uploadMgr)
return deps
}
func addHandlers(router *gin.Engine, cfg gocfg.ICfg, deps *depidx.Deps) (*gin.Engine, error) {
v1 := router.Group("/v1")
users := v1.Group("/users")
users.POST("/login", handlers.Login)
users.POST("/logout", handlers.Logout)
userHdrs := singleuserhdr.NewSimpleUserHandlers(cfg, deps)
users.POST("/login", userHdrs.Login)
users.POST("/logout", userHdrs.Logout)
files := v1.Group("files")
files.POST("/upload/", handlers.Upload)
files.GET("/list/", handlers.List)
files.DELETE("/delete/", handlers.Delete)
files.GET("/metadata/", handlers.Metadata)
files.POST("/copy/", handlers.Copy)
files.POST("/move/", handlers.Move)
filesSvc := v1.Group("/fs")
fileHdrs, err := fileshdr.NewFileHandlers(cfg, deps)
if err != nil {
panic(err)
}
filesSvc.POST("/files", fileHdrs.Create)
filesSvc.DELETE("/files", fileHdrs.Delete)
filesSvc.GET("/files", fileHdrs.Download)
filesSvc.PATCH("/files/chunks", fileHdrs.UploadChunk)
filesSvc.GET("/files/chunks", fileHdrs.UploadStatus)
filesSvc.PATCH("/files/copy", fileHdrs.Copy)
filesSvc.PATCH("/files/move", fileHdrs.Move)
filesSvc.GET("/dirs", fileHdrs.List)
filesSvc.POST("/dirs", fileHdrs.Mkdir)
// files.POST("/dirs/copy", fileHdrs.CopyDir)
filesSvc.GET("/metadata", fileHdrs.Metadata)
return router, nil
}
func (s *Server) Start() error {
return s.server.ListenAndServe()
err := s.server.ListenAndServe()
if err != http.ErrServerClosed {
return err
}
return nil
}
func (s *Server) Shutdown() error {
// TODO: add timeout
return s.server.Shutdown(context.Background())
}

250
src/server/server_files_test.go Normal file
View file

@ -0,0 +1,250 @@
package server
import (
"crypto/sha1"
"fmt"
"os"
"path"
"path/filepath"
"testing"
"time"
"github.com/ihexxa/gocfg"
"github.com/ihexxa/quickshare/src/client"
"github.com/ihexxa/quickshare/src/handlers/fileshdr"
)
func startTestServer(config string) *Server {
cfg, err := gocfg.New(NewDefaultConfig()).
Load(gocfg.JSONStr(config))
if err != nil {
panic(err)
}
srv, err := NewServer(cfg)
if err != nil {
panic(err)
}
go srv.Start()
return srv
}
func TestFileHandlers(t *testing.T) {
addr := "http://127.0.0.1:8888"
root := "./testData"
chunkSize := 2
config := `{
"Server": {
"ProdMode": false
},
"FS": {
"Root": "./testData"
}
}`
srv := startTestServer(config)
defer srv.Shutdown()
// kv := srv.depsKVStore()
fs := srv.depsFS()
defer os.RemoveAll(root)
cl := client.NewQSClient(addr)
// TODO: remove this
time.Sleep(500)
t.Run("test file APIs: Create-UploadChunk-UploadStatus-Metadata-Delete", func(t *testing.T) {
for filePath, content := range map[string]string{
"path1/f1.md": "11111",
"path1/path2/f2.md": "101010",
} {
fileSize := int64(len([]byte(content)))
// create a file
res, _, errs := cl.Create(filePath, fileSize)
if len(errs) > 0 {
t.Fatal(errs)
} else if res.StatusCode != 200 {
t.Fatal(res.StatusCode)
}
// check uploading file
uploadFilePath := path.Join(fileshdr.UploadDir, fmt.Sprintf("%x", sha1.Sum([]byte(filePath))))
info, err := fs.Stat(uploadFilePath)
if err != nil {
t.Fatal(err)
} else if info.Name() != filepath.Base(uploadFilePath) {
t.Fatal(info.Name(), filepath.Base(uploadFilePath))
}
// upload a chunk
i := 0
contentBytes := []byte(content)
for i < len(contentBytes) {
right := i + chunkSize
if right > len(contentBytes) {
right = len(contentBytes)
}
res, _, errs = cl.UploadChunk(filePath, string(contentBytes[i:right]), int64(i))
i = right
if len(errs) > 0 {
t.Fatal(errs)
} else if res.StatusCode != 200 {
t.Fatal(res.StatusCode)
}
if int64(i) != fileSize {
_, statusResp, errs := cl.UploadStatus(filePath)
if len(errs) > 0 {
t.Fatal(errs)
} else if statusResp.Path != filePath ||
statusResp.IsDir ||
statusResp.FileSize != fileSize ||
statusResp.Uploaded != int64(i) {
t.Fatal("incorrect uploadinfo info", statusResp)
}
}
}
// check uploaded file
fsFilePath := filepath.Join(fileshdr.FsDir, filePath)
info, err = fs.Stat(fsFilePath)
if err != nil {
t.Fatal(err)
} else if info.Name() != filepath.Base(fsFilePath) {
t.Fatal(info.Name(), filepath.Base(fsFilePath))
}
// metadata
_, mRes, errs := cl.Metadata(filePath)
if len(errs) > 0 {
t.Fatal(errs)
} else if mRes.Name != info.Name() ||
mRes.IsDir != info.IsDir() ||
mRes.Size != info.Size() {
// TODO: modTime is not checked
t.Fatal("incorrect uploaded info", mRes)
}
// delete file
res, _, errs = cl.Delete(filePath)
if len(errs) > 0 {
t.Fatal(errs)
} else if res.StatusCode != 200 {
t.Fatal(res.StatusCode)
}
}
})
t.Run("test file APIs: Mkdir-Create-UploadChunk-List", func(t *testing.T) {
for dirPath, files := range map[string]map[string]string{
"dir/path1/": map[string]string{
"f1.md": "11111",
"f2.md": "22222222222",
},
"dir/path1/path2": map[string]string{
"f3.md": "3333333",
},
} {
res, _, errs := cl.Mkdir(dirPath)
if len(errs) > 0 {
t.Fatal(errs)
} else if res.StatusCode != 200 {
t.Fatal(res.StatusCode)
}
for fileName, content := range files {
filePath := filepath.Join(dirPath, fileName)
fileSize := int64(len([]byte(content)))
// create a file
res, _, errs := cl.Create(filePath, fileSize)
if len(errs) > 0 {
t.Fatal(errs)
} else if res.StatusCode != 200 {
t.Fatal(res.StatusCode)
}
res, _, errs = cl.UploadChunk(filePath, content, 0)
if len(errs) > 0 {
t.Fatal(errs)
} else if res.StatusCode != 200 {
t.Fatal(res.StatusCode)
}
}
_, lResp, errs := cl.List(dirPath)
if len(errs) > 0 {
t.Fatal(errs)
}
for _, metadata := range lResp.Metadatas {
content, ok := files[metadata.Name]
if !ok {
t.Fatalf("%s not found", metadata.Name)
} else if int64(len(content)) != metadata.Size {
t.Fatalf("size not match %d %d \n", len(content), metadata.Size)
}
}
}
})
t.Run("test file APIs: Mkdir-Create-UploadChunk-Move-List", func(t *testing.T) {
srcDir := "move/src"
dstDir := "move/dst"
for _, dirPath := range []string{srcDir, dstDir} {
res, _, errs := cl.Mkdir(dirPath)
if len(errs) > 0 {
t.Fatal(errs)
} else if res.StatusCode != 200 {
t.Fatal(res.StatusCode)
}
}
files := map[string]string{
"f1.md": "111",
"f2.md": "22222",
}
for fileName, content := range files {
oldPath := filepath.Join(srcDir, fileName)
newPath := filepath.Join(dstDir, fileName)
fileSize := int64(len([]byte(content)))
// create a file
res, _, errs := cl.Create(oldPath, fileSize)
if len(errs) > 0 {
t.Fatal(errs)
} else if res.StatusCode != 200 {
t.Fatal(res.StatusCode)
}
res, _, errs = cl.UploadChunk(oldPath, content, 0)
if len(errs) > 0 {
t.Fatal(errs)
} else if res.StatusCode != 200 {
t.Fatal(res.StatusCode)
}
res, _, errs = cl.Move(oldPath, newPath)
if len(errs) > 0 {
t.Fatal(errs)
} else if res.StatusCode != 200 {
t.Fatal(res.StatusCode)
}
}
_, lResp, errs := cl.List(dstDir)
if len(errs) > 0 {
t.Fatal(errs)
}
for _, metadata := range lResp.Metadatas {
content, ok := files[metadata.Name]
if !ok {
t.Fatalf("%s not found", metadata.Name)
} else if int64(len(content)) != metadata.Size {
t.Fatalf("size not match %d %d \n", len(content), metadata.Size)
}
}
})
}