fix(test, travis): fix build and test error
This commit is contained in:
hexxa
parent
d726ac5aa4
commit
472fe8dadb
6 changed files with 16 additions and 300 deletions
|
|
@ -6,7 +6,6 @@ import (
|
|||
"github.com/ihexxa/quickshare/src/fs"
|
||||
"github.com/ihexxa/quickshare/src/idgen"
|
||||
"github.com/ihexxa/quickshare/src/kvstore"
|
||||
"github.com/ihexxa/quickshare/src/logging"
|
||||
)
|
||||
|
||||
type IUploader interface {
|
||||
|
|
@ -20,7 +19,6 @@ type IUploader interface {
|
|||
type Deps struct {
|
||||
fs fs.ISimpleFS
|
||||
token cryptoutil.ITokenEncDec
|
||||
log logging.ILogger
|
||||
kv kvstore.IKVStore
|
||||
uploader IUploader
|
||||
id idgen.IIDGen
|
||||
|
|
@ -46,14 +44,6 @@ func (deps *Deps) SetToken(tokenMaker cryptoutil.ITokenEncDec) {
|
|||
deps.token = tokenMaker
|
||||
}
|
||||
|
||||
func (deps *Deps) Log() logging.ILogger {
|
||||
return deps.log
|
||||
}
|
||||
|
||||
func (deps *Deps) SetLog(logger logging.ILogger) {
|
||||
deps.log = logger
|
||||
}
|
||||
|
||||
func (deps *Deps) KV() kvstore.IKVStore {
|
||||
return deps.kv
|
||||
}
|
||||
|
|
@ -62,14 +52,6 @@ func (deps *Deps) SetKV(kvstore kvstore.IKVStore) {
|
|||
deps.kv = kvstore
|
||||
}
|
||||
|
||||
func (deps *Deps) Uploader() IUploader {
|
||||
return deps.uploader
|
||||
}
|
||||
|
||||
func (deps *Deps) SetUploader(uploader IUploader) {
|
||||
deps.uploader = uploader
|
||||
}
|
||||
|
||||
func (deps *Deps) ID() idgen.IIDGen {
|
||||
return deps.id
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package memstore
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
|
||||
"github.com/ihexxa/quickshare/src/kvstore"
|
||||
|
|
@ -164,3 +165,16 @@ func (st *MemStore) Unlock(key string) error {
|
|||
delete(st.locks, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (st *MemStore) AddNamespace(nsName string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (st *MemStore) GetStringIn(namespace, key string) (string, bool) {
|
||||
panic("not implemented")
|
||||
return "", false
|
||||
}
|
||||
|
||||
func (st *MemStore) SetStringIn(namespace, key, val string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,8 +20,6 @@ import (
|
|||
"github.com/ihexxa/quickshare/src/idgen/simpleidgen"
|
||||
"github.com/ihexxa/quickshare/src/kvstore"
|
||||
"github.com/ihexxa/quickshare/src/kvstore/boltdbpvd"
|
||||
"github.com/ihexxa/quickshare/src/logging/simplelog"
|
||||
"github.com/ihexxa/quickshare/src/uploadmgr"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
|
|
@ -70,22 +68,14 @@ func initDeps(cfg gocfg.ICfg) *depidx.Deps {
|
|||
ider := simpleidgen.New()
|
||||
filesystem := local.NewLocalFS(rootPath, 0660, opensLimit, openTTL)
|
||||
jwtEncDec := jwt.NewJWTEncDec(secret)
|
||||
logger := simplelog.NewSimpleLogger()
|
||||
kv := boltdbpvd.New(rootPath, 1024)
|
||||
|
||||
deps := depidx.NewDeps(cfg)
|
||||
deps.SetFS(filesystem)
|
||||
deps.SetToken(jwtEncDec)
|
||||
deps.SetLog(logger)
|
||||
deps.SetKV(kv)
|
||||
deps.SetID(ider)
|
||||
|
||||
uploadMgr, err := uploadmgr.NewUploadMgr(deps)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
deps.SetUploader(uploadMgr)
|
||||
|
||||
return deps
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,110 +0,0 @@
|
|||
package uploadmgr
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
|
||||
"github.com/ihexxa/quickshare/src/depidx"
|
||||
)
|
||||
|
||||
// TODO:
|
||||
// uploading resumption test
|
||||
// rename file after uploaded
|
||||
// differetiate file and dir
|
||||
|
||||
var ErrBadData = errors.New("file size or uploaded not found for a file")
|
||||
var ErrUploaded = errors.New("file already uploaded")
|
||||
var ErrWriteUploaded = errors.New("try to write acknowledge part")
|
||||
|
||||
type UploadMgr struct {
|
||||
deps *depidx.Deps
|
||||
}
|
||||
|
||||
func NewUploadMgr(deps *depidx.Deps) (*UploadMgr, error) {
|
||||
if deps.KV() == nil {
|
||||
return nil, errors.New("kvstore is not found in deps")
|
||||
}
|
||||
if deps.FS() == nil {
|
||||
return nil, errors.New("fs is not found in deps")
|
||||
}
|
||||
|
||||
return &UploadMgr{
|
||||
deps: deps,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func fileSizeKey(filePath string) string { return fmt.Sprintf("%s:size", filePath) }
|
||||
func fileUploadedKey(filePath string) string { return fmt.Sprintf("%s:uploaded", filePath) }
|
||||
|
||||
func (mgr *UploadMgr) Create(filePath string, size int64) error {
|
||||
// _, found := mgr.deps.KV().GetBool(filePath)
|
||||
// if found {
|
||||
// return os.ErrExist
|
||||
// }
|
||||
|
||||
dirPath := path.Dir(filePath)
|
||||
if dirPath != "" {
|
||||
err := mgr.deps.FS().MkdirAll(dirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err := mgr.deps.FS().Create(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// mgr.deps.KV().SetBool(filePath, true)
|
||||
// mgr.deps.KV().SetInt64(fileSizeKey(filePath), size)
|
||||
// mgr.deps.KV().SetInt64(fileUploadedKey(filePath), 0)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mgr *UploadMgr) WriteChunk(filePath string, chunk []byte, off int64) (int, error) {
|
||||
// _, found := mgr.deps.KV().GetBool(filePath)
|
||||
// if !found {
|
||||
// return 0, os.ErrNotExist
|
||||
// }
|
||||
|
||||
// fileSize, ok1 := mgr.deps.KV().GetInt64(fileSizeKey(filePath))
|
||||
// uploaded, ok2 := mgr.deps.KV().GetInt64(fileUploadedKey(filePath))
|
||||
// if !ok1 || !ok2 {
|
||||
// return 0, ErrBadData
|
||||
// } else if uploaded == fileSize {
|
||||
// return 0, ErrUploaded
|
||||
// } else if off != uploaded {
|
||||
// return 0, ErrWriteUploaded
|
||||
// }
|
||||
|
||||
wrote, err := mgr.deps.FS().WriteAt(filePath, chunk, off)
|
||||
if err != nil {
|
||||
return wrote, err
|
||||
}
|
||||
|
||||
// mgr.deps.KV().SetInt64(fileUploadedKey(filePath), off+int64(wrote))
|
||||
return wrote, nil
|
||||
}
|
||||
|
||||
func (mgr *UploadMgr) Status(filePath string) (int64, bool, error) {
|
||||
// _, found := mgr.deps.KV().GetBool(filePath)
|
||||
// if !found {
|
||||
// return 0, false, os.ErrNotExist
|
||||
// }
|
||||
|
||||
fileSize, ok1 := mgr.deps.KV().GetInt64(fileSizeKey(filePath))
|
||||
fileUploaded, ok2 := mgr.deps.KV().GetInt64(fileUploadedKey(filePath))
|
||||
if !ok1 || !ok2 {
|
||||
return 0, false, ErrBadData
|
||||
}
|
||||
return fileUploaded, fileSize == fileUploaded, nil
|
||||
}
|
||||
|
||||
func (mgr *UploadMgr) Close() error {
|
||||
return mgr.deps.FS().Close()
|
||||
}
|
||||
|
||||
func (mgr *UploadMgr) Sync() error {
|
||||
return mgr.deps.FS().Sync()
|
||||
}
|
||||
|
|
@ -1,155 +0,0 @@
|
|||
package uploadmgr
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/ihexxa/gocfg"
|
||||
"github.com/ihexxa/quickshare/src/depidx"
|
||||
"github.com/ihexxa/quickshare/src/fs"
|
||||
"github.com/ihexxa/quickshare/src/fs/local"
|
||||
"github.com/ihexxa/quickshare/src/kvstore/memstore"
|
||||
"github.com/ihexxa/quickshare/src/server"
|
||||
)
|
||||
|
||||
var debug = flag.Bool("d", false, "debug mode")
|
||||
|
||||
// TODO: teardown after each test case
|
||||
|
||||
func TestUploadMgr(t *testing.T) {
|
||||
rootPath, err := ioutil.TempDir("./", "quickshare_test_")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
|
||||
newTestUploadMgr := func() (*UploadMgr, fs.ISimpleFS) {
|
||||
cfg := gocfg.New()
|
||||
err := cfg.Load(gocfg.JSONStr("{}"), server.NewEmptyConfig())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
filesystem := local.NewLocalFS(rootPath, 0660, 32, 10)
|
||||
kvstore := memstore.New()
|
||||
|
||||
deps := depidx.NewDeps(cfg)
|
||||
deps.SetFS(filesystem)
|
||||
deps.SetKV(kvstore)
|
||||
|
||||
mgr, err := NewUploadMgr(deps)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return mgr, filesystem
|
||||
}
|
||||
|
||||
t.Run("normal upload", func(t *testing.T) {
|
||||
mgr, _ := newTestUploadMgr()
|
||||
defer mgr.Close()
|
||||
|
||||
testCases := map[string]string{
|
||||
"foo.md": "",
|
||||
"bar.md": "1",
|
||||
"path1/foobar.md": "1110011",
|
||||
}
|
||||
|
||||
for filePath, content := range testCases {
|
||||
err = mgr.Create(filePath, int64(len([]byte(content))))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
bytes := []byte(content)
|
||||
for i := 0; i < len(bytes); i++ {
|
||||
wrote, err := mgr.WriteChunk(filePath, bytes[i:i+1], int64(i))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if wrote != 1 {
|
||||
t.Fatalf("wrote(%d) != 1", wrote)
|
||||
}
|
||||
}
|
||||
|
||||
if err = mgr.Sync(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
gotBytes, err := ioutil.ReadFile(path.Join(rootPath, filePath))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(gotBytes) != content {
|
||||
t.Errorf("content not same expected(%s) got(%s)", content, string(gotBytes))
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("concurrently upload", func(t *testing.T) {
|
||||
mgr, _ := newTestUploadMgr()
|
||||
defer mgr.Close()
|
||||
|
||||
testCases := []map[string]string{
|
||||
map[string]string{
|
||||
"file20.md": "111",
|
||||
"file21.md": "2222000",
|
||||
"path1/file22.md": "1010011",
|
||||
"path2/file22.md": "1010011",
|
||||
},
|
||||
}
|
||||
|
||||
uploadWorker := func(id int, filePath, content string, wg *sync.WaitGroup) {
|
||||
err = mgr.Create(filePath, int64(len([]byte(content))))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
bytes := []byte(content)
|
||||
for i := 0; i < len(bytes); i++ {
|
||||
wrote, err := mgr.WriteChunk(filePath, bytes[i:i+1], int64(i))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if wrote != 1 {
|
||||
t.Fatalf("wrote(%d) != 1", wrote)
|
||||
}
|
||||
if *debug {
|
||||
fmt.Printf("worker-%d wrote %s\n", id, string(bytes[i:i+1]))
|
||||
}
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
}
|
||||
|
||||
for _, files := range testCases {
|
||||
wg := &sync.WaitGroup{}
|
||||
workerID := 0
|
||||
for filePath, content := range files {
|
||||
wg.Add(1)
|
||||
go uploadWorker(workerID, filePath, content, wg)
|
||||
workerID++
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
if err = mgr.Sync(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for filePath, content := range files {
|
||||
gotBytes, err := ioutil.ReadFile(path.Join(rootPath, filePath))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(gotBytes) != content {
|
||||
t.Errorf("content not same expected(%s) got(%s)", content, string(gotBytes))
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue