From 83100007e33ba5437a5c6a69b89e36f8ef51ce17 Mon Sep 17 00:00:00 2001 From: hexxa Date: Sat, 5 Dec 2020 10:30:03 +0800 Subject: [PATCH] feat(qs2) add qs2 framework --- cmd/main.go | 33 +- cmd/quickshare.db | Bin 0 -> 32768 bytes go.mod | 13 +- go.sum | 85 ++++- src/cfg/cfg.go | 0 src/client/http.go | 119 +++++++ src/cryptoutil/cryptoutil_interface.go | 6 + src/cryptoutil/jwt/jwt.go | 51 +++ src/depidx/deps.go | 79 +++++ src/downloadmgr/mgr.go | 7 + src/fs/fs_interface.go | 28 ++ src/fs/local/fs.go | 314 +++++++++++++++++++ src/fs/mem/fs.go | 187 +++++++++++ src/handlers/file_mgmt.go | 12 - src/handlers/fileshdr/handlers.go | 414 +++++++++++++++++++++++++ src/handlers/fileshdr/upload_mgr.go | 85 +++++ src/handlers/simple_user.go | 9 - src/handlers/singleuserhdr/handlers.go | 75 +++++ src/handlers/util.go | 104 +++++++ src/idgen/idgen_interface.go | 5 + src/idgen/simpleidgen/simple_id_gen.go | 27 ++ src/kvstore/boltdbpvd/provider.go | 225 ++++++++++++++ src/kvstore/kvstore_interface.go | 26 ++ src/kvstore/memstore/provider.go | 166 ++++++++++ src/kvstore/test/provider_test.go | 185 +++++++++++ src/logging/log_interface.go | 9 + src/logging/simplelog/simple_log.go | 32 ++ src/server/config.go | 49 +++ src/server/quickshare.db | Bin 0 -> 32768 bytes src/server/server.go | 134 ++++++-- src/server/server_files_test.go | 250 +++++++++++++++ src/uploadmgr/mgr.go | 110 +++++++ src/uploadmgr/mgr_test.go | 155 +++++++++ 33 files changed, 2934 insertions(+), 60 deletions(-) create mode 100644 cmd/quickshare.db delete mode 100644 src/cfg/cfg.go create mode 100644 src/client/http.go create mode 100644 src/cryptoutil/cryptoutil_interface.go create mode 100644 src/cryptoutil/jwt/jwt.go create mode 100644 src/depidx/deps.go create mode 100644 src/downloadmgr/mgr.go create mode 100644 src/fs/fs_interface.go create mode 100644 src/fs/local/fs.go create mode 100644 src/fs/mem/fs.go delete mode 100644 src/handlers/file_mgmt.go create mode 100644 src/handlers/fileshdr/handlers.go create mode 100644 src/handlers/fileshdr/upload_mgr.go delete mode 100644 src/handlers/simple_user.go create mode 100644 src/handlers/singleuserhdr/handlers.go create mode 100644 src/handlers/util.go create mode 100644 src/idgen/idgen_interface.go create mode 100644 src/idgen/simpleidgen/simple_id_gen.go create mode 100644 src/kvstore/boltdbpvd/provider.go create mode 100644 src/kvstore/kvstore_interface.go create mode 100644 src/kvstore/memstore/provider.go create mode 100644 src/kvstore/test/provider_test.go create mode 100644 src/logging/log_interface.go create mode 100644 src/logging/simplelog/simple_log.go create mode 100644 src/server/config.go create mode 100644 src/server/quickshare.db create mode 100644 src/server/server_files_test.go create mode 100644 src/uploadmgr/mgr.go create mode 100644 src/uploadmgr/mgr_test.go diff --git a/cmd/main.go b/cmd/main.go index d60ef24..2161900 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,16 +1,41 @@ package main import ( - "github.com/ihexxa/quickshare/src/server" - "github.com/ihexxa/gocfg" + goflags "github.com/jessevdk/go-flags" + + "github.com/ihexxa/quickshare/src/server" ) +var opts struct { + host string `short:"h" long:"host" description:"server hostname"` + port string `short:"f" long:"file" description:"A file"` + configs []string `short:"c" description:"config path"` +} + func main() { - cfg := gocfg.New() + _, err := goflags.Parse(&opts) + if err != nil { + panic(err) + } + + cfg := gocfg.New(server.NewDefaultConfig()) + if len(opts.configs) > 0 { + for _, configPath := range opts.configs { + cfg, err = cfg.Load(gocfg.JSON(configPath)) + if err != nil { + panic(err) + } + } + } + srv, err := server.NewServer(cfg) if err != nil { panic(err) } - srv.Start() + + err = srv.Start() + if err != nil { + panic(err) + } } diff --git a/cmd/quickshare.db b/cmd/quickshare.db new file mode 100644 index 0000000000000000000000000000000000000000..b55996e478e25f6787be73468e1ca5942e641839 GIT binary patch literal 32768 zcmeI(O-jQ+6ae5@|3E}~gxo>#7@`}O;twjt(t@okkI_{L@+ z7D{v%=offoUh@i*udA6iEsFWIQQc=ByR%LjcYl9gJuWt@!Pdh4Yv-&uJ=q)OI(GsD z2oNAZfB*pk1PBlyK%hMWYt>Yit^a@a0&3G;uf}=(zg^{%{rBnC=h0iO)Q|uH0t5&U zAV7cs0RjXF5NL@&Uf*|$`HDX62b@QwK7SdJ@`s4@w2VmkB%+=_N1pZ%%82y*HKP9g zY2<02pp3X24*N?F3FOJ#^oJMY#;V?6e0cDes>YiB@anFyXf(d>4Q?CD;vHQC2oNAZ zfB*pk1PBlyK%i{`xu4%H=8Nxq4j`ZNZ`*v$hX4Tr1PBlyK!5-N0t5&U_`g72?{}(y z_NV^+aYVg;e;s+f-~YIfKSf^e|Gz}u>VCgge*8i%0t5&UAV7cs0RjXF5LhLF+~?2z z|9k{Lp9@$e3-3yR009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 Q2oNAZfB*pk1llU_4Hq6Q-~a#s literal 0 HcmV?d00001 diff --git a/go.mod b/go.mod index 67d76d7..d3fa310 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,22 @@ -module github.com/ihexxa/quickshare/v2 +module github.com/ihexxa/quickshare go 1.13 require ( + github.com/boltdb/bolt v1.3.1 github.com/gin-gonic/gin v1.6.3 github.com/ihexxa/gocfg v0.0.0-00010101000000-000000000000 - github.com/ihexxa/quickshare v0.0.0-00010101000000-000000000000 + github.com/ihexxa/multipart v0.0.0-00010101000000-000000000000 + github.com/jessevdk/go-flags v1.4.0 + github.com/parnurzeal/gorequest v0.2.16 + github.com/pkg/errors v0.9.1 // indirect github.com/robbert229/jwt v2.0.0+incompatible github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c + golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb // indirect + moul.io/http2curl v1.0.0 // indirect ) replace github.com/ihexxa/gocfg => /home/hexxa/ws/github.com/ihexxa/gocfg -replace github.com/ihexxa/quickshare => /home/hexxa/ws/github.com/ihexxa/quickshare +replace github.com/ihexxa/multipart => /home/hexxa/ws/github.com/ihexxa/multipart + diff --git a/go.sum b/go.sum index 7dae471..2cb18f3 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,17 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4 h1:EBTWhcAX7rNQ80RLwLCpHZBBrJuzallFHnF+yMXo928= +github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elazarl/goproxy v0.0.0-20201021153353-00ad82a08272 h1:Am81SElhR3XCQBunTisljzNkNese2T1FiV8jP79+dqg= +github.com/elazarl/goproxy v0.0.0-20201021153353-00ad82a08272/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= @@ -16,12 +27,25 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/ihexxa/gocfg v0.0.0-20201125104145-475ceb31d336 h1:MNVCDezNiNP/LB55VUpLdeB8zx7Io3fuRb5+VE94Gpw= -github.com/ihexxa/gocfg v0.0.0-20201125104145-475ceb31d336/go.mod h1:oqDTq1ywx4Qi9DdhFwwMHoPCYv6Txrfj2SY5WWcgiJs= -github.com/ihexxa/quickshare v0.0.0-20180618025333-c25851175220 h1:ASMUDsbG5xb7GJj8eMlnr0JUYBM+SWZhL4jOTn0ro6M= -github.com/ihexxa/quickshare v0.0.0-20180618025333-c25851175220/go.mod h1:IE+2BSx6Ti6RGouYQKO4m2FgLALOpPzndeNb2ulmodE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= @@ -30,14 +54,24 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OH github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ= +github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robbert229/jwt v2.0.0+incompatible h1:5Pc2FCpA2ahofO4QrWzXXQc0RZYfrZu0TSWHLcTOLz0= github.com/robbert229/jwt v2.0.0+incompatible/go.mod h1:I0pqJYBbhfQce4mJL2X6pYnk3T1oaAuF2ou8rSWpMBo= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c h1:fyKiXKO1/I/B6Y2U8T7WdQGWzwehOuGIrljPtt7YTTI= github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -46,14 +80,53 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= +moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= diff --git a/src/cfg/cfg.go b/src/cfg/cfg.go deleted file mode 100644 index e69de29..0000000 diff --git a/src/client/http.go b/src/client/http.go new file mode 100644 index 0000000..83f6b4b --- /dev/null +++ b/src/client/http.go @@ -0,0 +1,119 @@ +package client + +import ( + "encoding/json" + "fmt" + + "github.com/ihexxa/quickshare/src/handlers/fileshdr" + "github.com/parnurzeal/gorequest" +) + +type QSClient struct { + addr string + r *gorequest.SuperAgent +} + +func NewQSClient(addr string) *QSClient { + gr := gorequest.New() + return &QSClient{ + addr: addr, + r: gr, + } +} + +func (cl *QSClient) url(urlpath string) string { + return fmt.Sprintf("%s%s", cl.addr, urlpath) +} + +func (cl *QSClient) Create(filepath string, size int64) (gorequest.Response, string, []error) { + return cl.r.Post(cl.url("/v1/fs/files")). + Send(fileshdr.CreateReq{ + Path: filepath, + FileSize: size, + }). + End() +} + +func (cl *QSClient) Delete(filepath string) (gorequest.Response, string, []error) { + return cl.r.Delete(cl.url("/v1/fs/files")). + Param(fileshdr.FilePathQuery, filepath). + End() +} + +func (cl *QSClient) Metadata(filepath string) (gorequest.Response, *fileshdr.MetadataResp, []error) { + resp, body, errs := cl.r.Get(cl.url("/v1/fs/metadata")). + Param(fileshdr.FilePathQuery, filepath). + End() + + mResp := &fileshdr.MetadataResp{} + err := json.Unmarshal([]byte(body), mResp) + if err != nil { + errs = append(errs, err) + return nil, nil, errs + } + return resp, mResp, nil +} + +func (cl *QSClient) Mkdir(dirpath string) (gorequest.Response, string, []error) { + return cl.r.Post(cl.url("/v1/fs/dirs")). + Send(fileshdr.MkdirReq{Path: dirpath}). + End() +} + +func (cl *QSClient) Move(oldpath, newpath string) (gorequest.Response, string, []error) { + return cl.r.Patch(cl.url("/v1/fs/files/move")). + Send(fileshdr.MoveReq{ + OldPath: oldpath, + NewPath: newpath, + }). + End() +} + +func (cl *QSClient) UploadChunk(filepath string, content string, offset int64) (gorequest.Response, string, []error) { + return cl.r.Patch(cl.url("/v1/fs/files/chunks")). + Send(fileshdr.UploadChunkReq{ + Path: filepath, + Content: content, + Offset: offset, + }). + End() +} + +func (cl *QSClient) UploadStatus(filepath string) (gorequest.Response, *fileshdr.UploadStatusResp, []error) { + resp, body, errs := cl.r.Get(cl.url("/v1/fs/files/chunks")). + Param(fileshdr.FilePathQuery, filepath). + End() + + uResp := &fileshdr.UploadStatusResp{} + err := json.Unmarshal([]byte(body), uResp) + if err != nil { + errs = append(errs, err) + return nil, nil, errs + } + return resp, uResp, nil +} + +func (cl *QSClient) Download(filepath string, headers map[string]string) (gorequest.Response, string, []error) { + r := cl.r.Get(cl.url("/v1/fs/files/chunks")). + Param(fileshdr.FilePathQuery, filepath) + for key, val := range headers { + r = r.Set(key, val) + } + return r.End() +} + +func (cl *QSClient) List(dirPath string) (gorequest.Response, *fileshdr.ListResp, []error) { + resp, body, errs := cl.r.Get(cl.url("/v1/fs/dirs")). + Param(fileshdr.ListDirQuery, dirPath). + End() + if len(errs) > 0 { + return nil, nil, errs + } + + lResp := &fileshdr.ListResp{} + err := json.Unmarshal([]byte(body), lResp) + if err != nil { + return nil, nil, append(errs, err) + } + return resp, lResp, nil +} diff --git a/src/cryptoutil/cryptoutil_interface.go b/src/cryptoutil/cryptoutil_interface.go new file mode 100644 index 0000000..16aa9d5 --- /dev/null +++ b/src/cryptoutil/cryptoutil_interface.go @@ -0,0 +1,6 @@ +package cryptoutil + +type ITokenEncDec interface { + FromToken(token string, kvs map[string]string) (map[string]string, error) + ToToken(kvs map[string]string) (string, error) +} diff --git a/src/cryptoutil/jwt/jwt.go b/src/cryptoutil/jwt/jwt.go new file mode 100644 index 0000000..3d8f02c --- /dev/null +++ b/src/cryptoutil/jwt/jwt.go @@ -0,0 +1,51 @@ +package jwt + +import ( + "errors" + + jwtpkg "github.com/robbert229/jwt" +) + +type JWTEncDec struct { + alg jwtpkg.Algorithm +} + +func NewJWTEncDec(secret string) *JWTEncDec { + return &JWTEncDec{ + alg: jwtpkg.HmacSha256(secret), + } +} + +func (ed *JWTEncDec) FromToken(token string, kvs map[string]string) (map[string]string, error) { + claims, err := ed.alg.Decode(token) + if err != nil { + return nil, err + } + + for key := range kvs { + iVal, err := claims.Get(key) + if err != nil { + return nil, err + } + strVal, ok := iVal.(string) + if !ok { + return nil, errors.New("incorrect JWT claim") + } + + kvs[key] = strVal + } + return kvs, nil +} + +func (ed *JWTEncDec) ToToken(kvs map[string]string) (string, error) { + claims := jwtpkg.NewClaim() + for key, val := range kvs { + claims.Set(key, val) + } + + token, err := ed.alg.Encode(claims) + if err != nil { + return "", err + } + return token, nil +} diff --git a/src/depidx/deps.go b/src/depidx/deps.go new file mode 100644 index 0000000..ea63f7c --- /dev/null +++ b/src/depidx/deps.go @@ -0,0 +1,79 @@ +package depidx + +import ( + "github.com/ihexxa/gocfg" + "github.com/ihexxa/quickshare/src/cryptoutil" + "github.com/ihexxa/quickshare/src/fs" + "github.com/ihexxa/quickshare/src/idgen" + "github.com/ihexxa/quickshare/src/kvstore" + "github.com/ihexxa/quickshare/src/logging" +) + +type IUploader interface { + Create(filePath string, size int64) error + WriteChunk(filePath string, chunk []byte, off int64) (int, error) + Status(filePath string) (int64, bool, error) + Close() error + Sync() error +} + +type Deps struct { + fs fs.ISimpleFS + token cryptoutil.ITokenEncDec + log logging.ILogger + kv kvstore.IKVStore + uploader IUploader + id idgen.IIDGen +} + +func NewDeps(cfg gocfg.ICfg) *Deps { + return &Deps{} +} + +func (deps *Deps) FS() fs.ISimpleFS { + return deps.fs +} + +func (deps *Deps) SetFS(filesystem fs.ISimpleFS) { + deps.fs = filesystem +} + +func (deps *Deps) Token() cryptoutil.ITokenEncDec { + return deps.token +} + +func (deps *Deps) SetToken(tokenMaker cryptoutil.ITokenEncDec) { + deps.token = tokenMaker +} + +func (deps *Deps) Log() logging.ILogger { + return deps.log +} + +func (deps *Deps) SetLog(logger logging.ILogger) { + deps.log = logger +} + +func (deps *Deps) KV() kvstore.IKVStore { + return deps.kv +} + +func (deps *Deps) SetKV(kvstore kvstore.IKVStore) { + deps.kv = kvstore +} + +func (deps *Deps) Uploader() IUploader { + return deps.uploader +} + +func (deps *Deps) SetUploader(uploader IUploader) { + deps.uploader = uploader +} + +func (deps *Deps) ID() idgen.IIDGen { + return deps.id +} + +func (deps *Deps) SetID(ider idgen.IIDGen) { + deps.id = ider +} diff --git a/src/downloadmgr/mgr.go b/src/downloadmgr/mgr.go new file mode 100644 index 0000000..14f6d1c --- /dev/null +++ b/src/downloadmgr/mgr.go @@ -0,0 +1,7 @@ +package downloadmgr + +type DownloadMgr struct{} + +func NewDownloadMgr() *DownloadMgr { + return &DownloadMgr{} +} diff --git a/src/fs/fs_interface.go b/src/fs/fs_interface.go new file mode 100644 index 0000000..41754d9 --- /dev/null +++ b/src/fs/fs_interface.go @@ -0,0 +1,28 @@ +package fs + +import ( + "io" + "os" +) + +type ReadCloseSeeker interface { + io.Reader + io.ReaderFrom + io.Closer + io.Seeker +} + +type ISimpleFS interface { + Create(path string) error + MkdirAll(path string) error + Remove(path string) error + Rename(oldpath, newpath string) error + ReadAt(path string, b []byte, off int64) (n int, err error) + WriteAt(path string, b []byte, off int64) (n int, err error) + Stat(path string) (os.FileInfo, error) + Close() error + Sync() error + GetFileReader(path string) (ReadCloseSeeker, error) + Root() string + ListDir(path string) ([]os.FileInfo, error) +} diff --git a/src/fs/local/fs.go b/src/fs/local/fs.go new file mode 100644 index 0000000..e9225be --- /dev/null +++ b/src/fs/local/fs.go @@ -0,0 +1,314 @@ +package local + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/ihexxa/quickshare/src/fs" +) + +var ErrTooManyOpens = errors.New("too many opened files") + +type LocalFS struct { + root string + defaultPerm os.FileMode + defaultDirPerm os.FileMode + opens map[string]*fileInfo + opensLimit int + opensMtx *sync.RWMutex + opensCleanSize int + openTTL time.Duration + readers map[string]*fileInfo +} + +type fileInfo struct { + lastAccess time.Time + fd *os.File +} + +func NewLocalFS(root string, defaultPerm uint32, opensLimit, openTTL int) *LocalFS { + if root == "" { + root = "." + } + return &LocalFS{ + root: root, + defaultPerm: os.FileMode(defaultPerm), + defaultDirPerm: os.FileMode(0775), + opens: map[string]*fileInfo{}, + opensLimit: opensLimit, + openTTL: time.Duration(openTTL) * time.Second, + opensMtx: &sync.RWMutex{}, + opensCleanSize: 10, + readers: map[string]*fileInfo{}, // TODO: track readers and close idles + } +} + +func (fs *LocalFS) Root() string { + return fs.root +} + +// closeOpens assumes that it is called after opensMtx.Lock() +func (fs *LocalFS) closeOpens(closeAll bool) error { + batch := fs.opensCleanSize + + var err error + for key, info := range fs.opens { + if batch <= 0 && !closeAll { + break + } + batch-- + + if info.lastAccess.Add(fs.openTTL).Before(time.Now()) { + delete(fs.opens, key) + if err = info.fd.Sync(); err != nil { + return err + } + if err := info.fd.Close(); err != nil { + return err + } + } + } + + return nil +} + +func (fs *LocalFS) Sync() error { + fs.opensMtx.Lock() + defer fs.opensMtx.Unlock() + return fs.closeOpens(true) +} + +// check refers implementation of Dir.Open() in http package +func (fs *LocalFS) translate(name string) (string, error) { + if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) { + return "", errors.New("invalid character in file path") + } + return filepath.Join(fs.root, filepath.FromSlash(path.Clean("/"+name))), nil +} + +func (fs *LocalFS) Create(path string) error { + fullpath, err := fs.translate(path) + if err != nil { + return err + } + + fd, err := os.OpenFile(fullpath, os.O_CREATE|os.O_RDWR|os.O_EXCL, fs.defaultPerm) + if err != nil { + return err + } + + fs.opensMtx.Lock() + defer fs.opensMtx.Unlock() + if len(fs.opens) > fs.opensLimit { + return ErrTooManyOpens + } + + fs.opens[fullpath] = &fileInfo{ + lastAccess: time.Now(), + fd: fd, + } + return nil +} + +func (fs *LocalFS) MkdirAll(path string) error { + fullpath, err := fs.translate(path) + if err != nil { + return err + } + return os.MkdirAll(fullpath, fs.defaultDirPerm) +} + +func (fs *LocalFS) Remove(entryPath string) error { + fullpath, err := fs.translate(entryPath) + if err != nil { + return err + } + return os.Remove(fullpath) +} + +func (fs *LocalFS) Rename(oldpath, newpath string) error { + fullOldPath, err := fs.translate(oldpath) + if err != nil { + return err + } + _, err = os.Stat(fullOldPath) + if err != nil { + return err + } + + fullNewPath, err := fs.translate(newpath) + if err != nil { + return err + } + + // avoid replacing existing file/folder + _, err = os.Stat(fullNewPath) + if err != nil { + if os.IsNotExist(err) { + return os.Rename(fullOldPath, fullNewPath) + } + return err + } + return os.ErrExist +} + +func (fs *LocalFS) ReadAt(path string, b []byte, off int64) (int, error) { + fullpath, err := fs.translate(path) + if err != nil { + return 0, err + } + + info, err := func() (*fileInfo, error) { + fs.opensMtx.Lock() + defer fs.opensMtx.Unlock() + + info, ok := fs.opens[fullpath] + if !ok { + if len(fs.opens) > fs.opensLimit { + return nil, ErrTooManyOpens + } + + fd, err := os.OpenFile(fullpath, os.O_RDWR|os.O_APPEND, fs.defaultPerm) + if err != nil { + return nil, err + } + info = &fileInfo{ + fd: fd, + lastAccess: time.Now(), + } + fs.opens[fullpath] = info + fs.closeOpens(false) + } + + return info, nil + }() + if err != nil { + return 0, err + } + + newOffset, err := info.fd.Seek(off, os.SEEK_SET) + if err != nil { + return 0, err + } else if newOffset != off { + // TODO: will this happen? + return 0, fmt.Errorf("seek offset (%d) != required(%d)", newOffset, off) + } + + return info.fd.ReadAt(b, off) +} + +func (fs *LocalFS) WriteAt(path string, b []byte, off int64) (int, error) { + fullpath, err := fs.translate(path) + if err != nil { + return 0, err + } + + info, err := func() (*fileInfo, error) { + fs.opensMtx.Lock() + defer fs.opensMtx.Unlock() + + info, ok := fs.opens[fullpath] + if !ok { + if len(fs.opens) > fs.opensLimit { + return nil, ErrTooManyOpens + } + + // it does NOT create file for writing + fd, err := os.OpenFile(fullpath, os.O_RDWR|os.O_APPEND, fs.defaultPerm) + if err != nil { + return nil, err + } + info = &fileInfo{ + fd: fd, + lastAccess: time.Now(), + } + fs.opens[fullpath] = info + fs.closeOpens(false) + } + + return info, nil + }() + if err != nil { + return 0, err + } + + newOffset, err := info.fd.Seek(off, os.SEEK_SET) + if err != nil { + return 0, err + } else if newOffset != off { + // TODO: will this happen? + return 0, fmt.Errorf("seek offset (%d) != required(%d)", newOffset, off) + } + + return info.fd.WriteAt(b, off) +} + +func (fs *LocalFS) Stat(path string) (os.FileInfo, error) { + fullpath, err := fs.translate(path) + if err != nil { + return nil, err + } + + fs.opensMtx.RLock() + info, ok := fs.opens[fullpath] + fs.opensMtx.RUnlock() + if ok { + return info.fd.Stat() + } + return os.Stat(fullpath) +} + +func (fs *LocalFS) Close() error { + fs.opensMtx.Lock() + defer fs.opensMtx.Unlock() + + var err error + for filePath, info := range fs.opens { + err = info.fd.Sync() + if err != nil { + return err + } + err = info.fd.Close() + if err != nil { + return err + } + delete(fs.opens, filePath) + } + + return nil +} + +// readers are not tracked by opens +func (fs *LocalFS) GetFileReader(path string) (fs.ReadCloseSeeker, error) { + fullpath, err := fs.translate(path) + if err != nil { + return nil, err + } + + fd, err := os.OpenFile(fullpath, os.O_RDONLY, fs.defaultPerm) + if err != nil { + return nil, err + } + + fs.readers[fullpath] = &fileInfo{ + fd: fd, + lastAccess: time.Now(), + } + return fd, nil +} + +func (fs *LocalFS) ListDir(path string) ([]os.FileInfo, error) { + fullpath, err := fs.translate(path) + if err != nil { + return nil, err + } + + return ioutil.ReadDir(fullpath) +} diff --git a/src/fs/mem/fs.go b/src/fs/mem/fs.go new file mode 100644 index 0000000..81fdc76 --- /dev/null +++ b/src/fs/mem/fs.go @@ -0,0 +1,187 @@ +package mem + +// type MemFS struct { +// files map[string][]byte +// dirs map[string][]string +// } + +// type MemFileInfo struct { +// name string +// size int64 +// isDir bool +// } + +// func (fi *MemFileInfo) Name() string { +// return fi.name +// } +// func (fi *MemFileInfo) Size() int64 { +// return fi.size +// } +// func (fi *MemFileInfo) Mode() os.FileMode { +// return 0666 +// } +// func (fi *MemFileInfo) ModTime() time.Time { +// return time.Now() +// } +// func (fi *MemFileInfo) IsDir() bool { +// return fi.isDir +// } +// func (fi *MemFileInfo) Sys() interface{} { +// return "" +// } + +// func NewMemFS() *MemFS { +// return &MemFS{ +// files: map[string][]byte{}, +// dirs: map[string][]string{}, +// } +// } + +// // Create(filePath string) error +// // MkdirAll(filePath string) error +// // Remove(filePath string) error + +// func (fs *MemFS) Create(filePath string) error { +// dirPath := path.Dir(filePath) +// files, ok := fs.dirs[dirPath] +// if !ok { +// fs.dirs[dirPath] = []string{} +// } + +// fs.dirs[dirPath] = append(fs.dirs[dirPath], filePath) +// fs.files[filePath] = []byte("") +// return nil +// } + +// func (fs *MemFS) MkdirAll(dirPath string) error { +// _, ok := fs.dirs[dirPath] +// if ok { +// return os.ErrExist +// } +// fs.dirs[dirPath] = []string{} +// return nil +// } + +// func (fs *MemFS) Remove(filePath string) error { +// files, ok := fs.dirs[filePath] +// if ok { +// for _, fileName := range files { +// d +// } +// } + +// delete(fs.dirs, filePath) +// delete(fs.files, filePath) +// return nil +// } + +// func (fs *MemFS) Rename(oldpath, newpath string) error { +// content, ok := fs.files[oldpath] +// if !ok { +// return os.ErrNotExist +// } +// delete(fs.files, oldpath) + +// newDir := path.Dir(newpath) +// _, ok = fs.dirs[newDir] +// if !ok { +// fs.dirs[newDir] = []string{} +// } +// fs.dirs[newDir] = append(fs.dirs[newDir], newpath) +// fs.files[newpath] = content +// return nil +// } + +// func (fs *MemFS) ReadAt(filePath string, b []byte, off int64) (n int, err error) { +// content, ok := fs.files[filePath] +// if !ok { +// return 0, os.ErrNotExist +// } + +// if off >= int64(len(content)) { +// return 0, errors.New("offset > fileSize") +// } +// right := off + int64(len(b)) +// if right > int64(len(content)) { +// right = int64(len(content)) +// } +// return copy(b, content[off:right]), nil +// } + +// func (fs *MemFS) WriteAt(filePath string, b []byte, off int64) (n int, err error) { +// content, ok := fs.files[filePath] +// if !ok { +// return 0, os.ErrNotExist +// } + +// if off >= int64(len(content)) { +// return 0, errors.New("offset > fileSize") +// } else if off+int64(len(b)) > int64(len(content)) { +// fs.files[filePath] = append( +// fs.files[filePath], +// make([]byte, off+int64(len(b))-int64(len(content)))..., +// ) +// } + +// copy(fs.files[filePath][off:], b) +// return len(b), nil +// } + +// func (fs *MemFS) Stat(filePath string) (os.FileInfo, error) { +// _, ok := fs.dirs[filePath] +// if ok { +// return &MemFileInfo{ +// name: filePath, +// size: 0, +// isDir: true, +// }, nil +// } + +// content, ok := fs.files[filePath] +// if ok { +// return &MemFileInfo{ +// name: filePath, +// size: int64(len(content)), +// isDir: false, +// }, nil +// } +// return nil, os.ErrNotExist +// } + +// func (fs *MemFS) Close() error { +// return nil +// } + +// func (fs *MemFS) Sync() error { +// return nil +// } + +// func (fs *MemFS) GetFileReader(filePath string) (ReadCloseSeeker, error) { +// content, ok := fs.files[filePath] +// if !ok { +// return nil, os.ErrNotExist +// } +// return bytes.NewReader(content) +// } + +// func (fs *MemFS) Root() string { +// return "" +// } + +// func (fs *MemFS) ListDir(filePath string) ([]os.FileInfo, error) { +// files, ok := fs.dirs[filePath] +// if !ok { +// return nil, os.ErrNotExist +// } + +// infos := []*MemFileInfo{} +// for _, fileName := range files { +// infos = append(infos, &MemFileInfo{ +// name: fileName, +// size: int64(len(fs.files[fileName])), +// isDir: false, +// }) +// } + +// return infos +// } diff --git a/src/handlers/file_mgmt.go b/src/handlers/file_mgmt.go deleted file mode 100644 index 124a773..0000000 --- a/src/handlers/file_mgmt.go +++ /dev/null @@ -1,12 +0,0 @@ -package handlers - -import ( - "github.com/gin-gonic/gin" -) - -func Upload(ctx *gin.Context) {} -func List(ctx *gin.Context) {} -func Delete(ctx *gin.Context) {} -func Metadata(ctx *gin.Context) {} -func Copy(ctx *gin.Context) {} -func Move(ctx *gin.Context) {} diff --git a/src/handlers/fileshdr/handlers.go b/src/handlers/fileshdr/handlers.go new file mode 100644 index 0000000..675da84 --- /dev/null +++ b/src/handlers/fileshdr/handlers.go @@ -0,0 +1,414 @@ +package fileshdr + +import ( + "crypto/sha1" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "time" + + "github.com/gin-gonic/gin" + "github.com/ihexxa/gocfg" + "github.com/ihexxa/multipart" + + "github.com/ihexxa/quickshare/src/depidx" + q "github.com/ihexxa/quickshare/src/handlers" +) + +var ( + // dirs + UploadDir = "uploadings" + FsDir = "files" + + // queries + FilePathQuery = "fp" + ListDirQuery = "dp" + + // headers + rangeHeader = "Range" + acceptRangeHeader = "Accept-Range" + ifRangeHeader = "If-Range" +) + +type FileHandlers struct { + cfg gocfg.ICfg + deps *depidx.Deps + uploadMgr *UploadMgr +} + +func NewFileHandlers(cfg gocfg.ICfg, deps *depidx.Deps) (*FileHandlers, error) { + var err error + if err = deps.FS().MkdirAll(UploadDir); err != nil { + return nil, err + } + if err = deps.FS().MkdirAll(FsDir); err != nil { + return nil, err + } + + return &FileHandlers{ + cfg: cfg, + deps: deps, + uploadMgr: NewUploadMgr(deps.KV()), + }, err +} + +type AutoLocker struct { + h *FileHandlers + c *gin.Context + key string +} + +func (h *FileHandlers) NewAutoLocker(c *gin.Context, key string) *AutoLocker { + return &AutoLocker{ + h: h, + c: c, + key: key, + } +} + +func (lk *AutoLocker) Exec(handler func()) { + var err error + kv := lk.h.deps.KV() + if err = kv.TryLock(lk.key); err != nil { + lk.c.JSON(q.Resp(500)) + return + } + + handler() + + if err = kv.Unlock(lk.key); err != nil { + // TODO: use logger + fmt.Println(err) + } +} + +type CreateReq struct { + Path string `json:"path"` + FileSize int64 `json:"fileSize"` +} + +func (h *FileHandlers) Create(c *gin.Context) { + req := &CreateReq{} + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + + tmpFilePath := h.GetTmpPath(req.Path) + locker := h.NewAutoLocker(c, tmpFilePath) + locker.Exec(func() { + err := h.deps.FS().Create(tmpFilePath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + err = h.uploadMgr.AddInfo(req.Path, tmpFilePath, req.FileSize, false) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + + fileDir := h.FsPath(filepath.Dir(req.Path)) + err = h.deps.FS().MkdirAll(fileDir) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + }) + + c.JSON(q.Resp(200)) +} + +func (h *FileHandlers) Delete(c *gin.Context) { + filePath := c.Query(FilePathQuery) + if filePath == "" { + c.JSON(q.ErrResp(c, 400, errors.New("invalid file path"))) + return + } + + filePath = h.FsPath(filePath) + err := h.deps.FS().Remove(filePath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + + c.JSON(q.Resp(200)) +} + +type MetadataResp struct { + Name string `json:"name"` + Size int64 `json:"size"` + ModTime time.Time `json:"modTime"` + IsDir bool `json:"isDir"` +} + +func (h *FileHandlers) Metadata(c *gin.Context) { + filePath := c.Query(FilePathQuery) + if filePath == "" { + c.JSON(q.ErrResp(c, 400, errors.New("invalid file path"))) + return + } + + filePath = h.FsPath(filePath) + info, err := h.deps.FS().Stat(filePath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + + c.JSON(200, MetadataResp{ + Name: info.Name(), + Size: info.Size(), + ModTime: info.ModTime(), + IsDir: info.IsDir(), + }) +} + +type MkdirReq struct { + Path string `json:"path"` +} + +func (h *FileHandlers) Mkdir(c *gin.Context) { + req := &MkdirReq{} + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(q.ErrResp(c, 400, err)) + return + } + + dirPath := h.FsPath(req.Path) + err := h.deps.FS().MkdirAll(dirPath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + + c.JSON(q.Resp(200)) +} + +type MoveReq struct { + OldPath string `json:"oldPath"` + NewPath string `json:"newPath"` +} + +func (h *FileHandlers) Move(c *gin.Context) { + req := &MoveReq{} + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(q.ErrResp(c, 400, err)) + return + } + + oldPath := h.FsPath(req.OldPath) + newPath := h.FsPath(req.NewPath) + _, err := h.deps.FS().Stat(oldPath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + _, err = h.deps.FS().Stat(newPath) + if err != nil && !os.IsNotExist(err) { + c.JSON(q.ErrResp(c, 500, err)) + return + } else if err == nil { + // err is nil because file exists + c.JSON(q.ErrResp(c, 400, os.ErrExist)) + return + } + + err = h.deps.FS().Rename(oldPath, newPath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + + c.JSON(q.Resp(200)) +} + +type UploadChunkReq struct { + Path string `json:"path"` + Content string `json:"content"` + Offset int64 `json:"offset"` +} + +func (h *FileHandlers) UploadChunk(c *gin.Context) { + req := &UploadChunkReq{} + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + + tmpFilePath := h.GetTmpPath(req.Path) + locker := h.NewAutoLocker(c, tmpFilePath) + locker.Exec(func() { + var err error + + _, fileSize, uploaded, err := h.uploadMgr.GetInfo(tmpFilePath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } else if uploaded != req.Offset { + c.JSON(q.ErrResp(c, 500, errors.New("offset != uploaded"))) + return + } + + wrote, err := h.deps.FS().WriteAt(tmpFilePath, []byte(req.Content), req.Offset) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + err = h.uploadMgr.IncreUploaded(tmpFilePath, int64(wrote)) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + + // move the file from uploading dir to uploaded dir + if uploaded+int64(wrote) == fileSize { + fsFilePath := h.FsPath(req.Path) + err = h.deps.FS().Rename(tmpFilePath, fsFilePath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + err = h.uploadMgr.DelInfo(tmpFilePath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + } + + c.JSON(200, &UploadStatusResp{ + Path: req.Path, + IsDir: false, + FileSize: fileSize, + Uploaded: uploaded + int64(wrote), + }) + }) +} + +type UploadStatusResp struct { + Path string `json:"path"` + IsDir bool `json:"isDir"` + FileSize int64 `json:"fileSize"` + Uploaded int64 `json:"uploaded"` +} + +func (h *FileHandlers) UploadStatus(c *gin.Context) { + filePath := c.Query(FilePathQuery) + if filePath == "" { + c.JSON(q.ErrResp(c, 400, errors.New("invalid file name"))) + } + + tmpFilePath := h.GetTmpPath(filePath) + locker := h.NewAutoLocker(c, tmpFilePath) + locker.Exec(func() { + _, fileSize, uploaded, err := h.uploadMgr.GetInfo(tmpFilePath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + + c.JSON(200, &UploadStatusResp{ + Path: filePath, + IsDir: false, + FileSize: fileSize, + Uploaded: uploaded, + }) + }) +} + +// TODO: support ETag +// TODO: use correct content type +func (h *FileHandlers) Download(c *gin.Context) { + rangeVal := c.GetHeader(rangeHeader) + ifRangeVal := c.GetHeader(ifRangeHeader) + filePath := c.Query(FilePathQuery) + if filePath == "" { + c.JSON(q.ErrResp(c, 400, errors.New("invalid file name"))) + } + + // concurrency relies on os's mechanism + filePath = h.FsPath(filePath) + info, err := h.deps.FS().Stat(filePath) + if err != nil { + c.JSON(q.ErrResp(c, 400, err)) + return + } else if info.IsDir() { + c.JSON(q.ErrResp(c, 501, errors.New("downloading a folder is not supported"))) + } + + r, err := h.deps.FS().GetFileReader(filePath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + } + + // respond to normal requests + if ifRangeVal != "" || rangeVal == "" { + c.DataFromReader(200, info.Size(), "application/octet-stream", r, map[string]string{}) + return + } + + // respond to range requests + parts, err := multipart.RangeToParts(rangeVal, "application/octet-stream", fmt.Sprintf("%d", info.Size())) + if err != nil { + c.JSON(q.ErrResp(c, 400, err)) + } + pr, pw := io.Pipe() + err = multipart.WriteResponse(r, pw, filePath, parts) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + } + extraHeaders := map[string]string{ + "Content-Disposition": fmt.Sprintf(`attachment; filename="%s"`, filePath), + } + c.DataFromReader(206, info.Size(), "application/octet-stream", pr, extraHeaders) +} + +type ListResp struct { + Metadatas []*MetadataResp `json:"metadatas"` +} + +func (h *FileHandlers) List(c *gin.Context) { + dirPath := c.Query(ListDirQuery) + if dirPath == "" { + c.JSON(q.ErrResp(c, 400, errors.New("incorrect path name"))) + return + } + + dirPath = h.FsPath(dirPath) + infos, err := h.deps.FS().ListDir(dirPath) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + metadatas := []*MetadataResp{} + for _, info := range infos { + metadatas = append(metadatas, &MetadataResp{ + Name: info.Name(), + Size: info.Size(), + ModTime: info.ModTime(), + IsDir: info.IsDir(), + }) + } + + c.JSON(200, &ListResp{Metadatas: metadatas}) +} + +func (h *FileHandlers) Copy(c *gin.Context) { + c.JSON(q.NewMsgResp(501, "Not Implemented")) +} + +func (h *FileHandlers) CopyDir(c *gin.Context) { + c.JSON(q.NewMsgResp(501, "Not Implemented")) +} + +func (h *FileHandlers) GetTmpPath(filePath string) string { + return path.Join(UploadDir, fmt.Sprintf("%x", sha1.Sum([]byte(filePath)))) +} + +func (h *FileHandlers) FsPath(filePath string) string { + return path.Join(FsDir, filePath) +} diff --git a/src/handlers/fileshdr/upload_mgr.go b/src/handlers/fileshdr/upload_mgr.go new file mode 100644 index 0000000..73008db --- /dev/null +++ b/src/handlers/fileshdr/upload_mgr.go @@ -0,0 +1,85 @@ +package fileshdr + +import ( + "errors" + "fmt" + "os" + + "github.com/ihexxa/quickshare/src/kvstore" +) + +var ( + isDirKey = "isDir" + fileSizeKey = "fileSize" + uploadedKey = "uploaded" + filePathKey = "fileName" +) + +type UploadMgr struct { + kv kvstore.IKVStore +} + +func NewUploadMgr(kv kvstore.IKVStore) *UploadMgr { + return &UploadMgr{ + kv: kv, + } +} + +func (um *UploadMgr) AddInfo(fileName, tmpName string, fileSize int64, isDir bool) error { + err := um.kv.SetInt64(infoKey(tmpName, fileSizeKey), fileSize) + if err != nil { + return err + } + err = um.kv.SetInt64(infoKey(tmpName, uploadedKey), 0) + if err != nil { + return err + } + return um.kv.SetString(infoKey(tmpName, filePathKey), fileName) +} + +func (um *UploadMgr) IncreUploaded(fileName string, newUploaded int64) error { + fileSize, ok := um.kv.GetInt64(infoKey(fileName, fileSizeKey)) + if !ok { + return fmt.Errorf("file size %s not found", fileName) + } + preUploaded, ok := um.kv.GetInt64(infoKey(fileName, uploadedKey)) + if !ok { + return fmt.Errorf("file uploaded %s not found", fileName) + } + if newUploaded+preUploaded <= fileSize { + um.kv.SetInt64(infoKey(fileName, uploadedKey), newUploaded+preUploaded) + return nil + } + return errors.New("uploaded is greater than file size") +} + +func (um *UploadMgr) GetInfo(fileName string) (string, int64, int64, error) { + realFilePath, ok := um.kv.GetString(infoKey(fileName, filePathKey)) + if !ok { + return "", 0, 0, os.ErrNotExist + } + fileSize, ok := um.kv.GetInt64(infoKey(fileName, fileSizeKey)) + if !ok { + return "", 0, 0, os.ErrNotExist + } + uploaded, ok := um.kv.GetInt64(infoKey(fileName, uploadedKey)) + if !ok { + return "", 0, 0, os.ErrNotExist + } + + return realFilePath, fileSize, uploaded, nil +} + +func (um *UploadMgr) DelInfo(fileName string) error { + if err := um.kv.DelInt64(infoKey(fileName, fileSizeKey)); err != nil { + return err + } + if err := um.kv.DelInt64(infoKey(fileName, uploadedKey)); err != nil { + return err + } + return um.kv.DelString(infoKey(fileName, filePathKey)) +} + +func infoKey(fileName, key string) string { + return fmt.Sprintf("%s:%s", fileName, key) +} diff --git a/src/handlers/simple_user.go b/src/handlers/simple_user.go deleted file mode 100644 index c1c66e8..0000000 --- a/src/handlers/simple_user.go +++ /dev/null @@ -1,9 +0,0 @@ -package handlers - -import ( - "github.com/gin-gonic/gin" -) - -func Login(ctx *gin.Context) {} - -func Logout(ctx *gin.Context) {} diff --git a/src/handlers/singleuserhdr/handlers.go b/src/handlers/singleuserhdr/handlers.go new file mode 100644 index 0000000..efb54fa --- /dev/null +++ b/src/handlers/singleuserhdr/handlers.go @@ -0,0 +1,75 @@ +package singleuserhdr + +import ( + "errors" + + "github.com/gin-gonic/gin" + "github.com/ihexxa/gocfg" + + "github.com/ihexxa/quickshare/src/depidx" + q "github.com/ihexxa/quickshare/src/handlers" +) + +var ErrInvalidUser = errors.New("invalid user name or password") + +type SimpleUserHandlers struct { + cfg gocfg.ICfg + deps *depidx.Deps +} + +func NewSimpleUserHandlers(cfg gocfg.ICfg, deps *depidx.Deps) *SimpleUserHandlers { + return &SimpleUserHandlers{ + cfg: cfg, + deps: deps, + } +} + +func (hdr *SimpleUserHandlers) Login(c *gin.Context) { + userName := c.Query("username") + pwd := c.Query("pwd") + if userName == "" || pwd == "" { + c.JSON(q.ErrResp(c, 400, ErrInvalidUser)) + return + } + + expectedName, ok1 := hdr.deps.KV().GetString("username") + expectedPwd, ok2 := hdr.deps.KV().GetString("pwd") + if !ok1 || !ok2 { + c.JSON(q.ErrResp(c, 400, ErrInvalidUser)) + return + } + + if userName != expectedName || pwd != expectedPwd { + c.JSON(q.ErrResp(c, 400, ErrInvalidUser)) + return + } + token, err := hdr.deps.Token().ToToken(map[string]string{ + "username": expectedName, + }) + if err != nil { + c.JSON(q.ErrResp(c, 500, err)) + return + } + + // TODO: use config + c.SetCookie("token", token, 3600, "/", "localhost", false, true) + c.JSON(q.Resp(200)) +} + +func (hdr *SimpleUserHandlers) Logout(c *gin.Context) { + token, err := c.Cookie("token") + if err != nil { + c.JSON(q.ErrResp(c, 400, err)) + return + } + + // TODO: // check if token expired + _, err = hdr.deps.Token().FromToken(token, map[string]string{"token": ""}) + if err != nil { + c.JSON(q.ErrResp(c, 400, err)) + return + } + + c.SetCookie("token", "", 0, "/", "localhost", false, true) + c.JSON(q.Resp(200)) +} diff --git a/src/handlers/util.go b/src/handlers/util.go new file mode 100644 index 0000000..e03b801 --- /dev/null +++ b/src/handlers/util.go @@ -0,0 +1,104 @@ +package handlers + +import ( + "fmt" + + "github.com/gin-gonic/gin" +) + +var statusCodes = map[int]string{ + 100: "Continue", // RFC 7231, 6.2.1 + 101: "SwitchingProtocols", // RFC 7231, 6.2.2 + 102: "Processing", // RFC 2518, 10.1 + 103: "EarlyHints", // RFC 8297 + 200: "OK", // RFC 7231, 6.3.1 + 201: "Created", // RFC 7231, 6.3.2 + 202: "Accepted", // RFC 7231, 6.3.3 + 203: "NonAuthoritativeInfo", // RFC 7231, 6.3.4 + 204: "NoContent", // RFC 7231, 6.3.5 + 205: "ResetContent", // RFC 7231, 6.3.6 + 206: "PartialContent", // RFC 7233, 4.1 + 207: "MultiStatus", // RFC 4918, 11.1 + 208: "AlreadyReported", // RFC 5842, 7.1 + 226: "IMUsed", // RFC 3229, 10.4.1 + 300: "MultipleChoices", // RFC 7231, 6.4.1 + 301: "MovedPermanently", // RFC 7231, 6.4.2 + 302: "Found", // RFC 7231, 6.4.3 + 303: "SeeOther", // RFC 7231, 6.4.4 + 304: "NotModified", // RFC 7232, 4.1 + 305: "UseProxy", // RFC 7231, 6.4.5 + 307: "TemporaryRedirect", // RFC 7231, 6.4.7 + 308: "PermanentRedirect", // RFC 7538, 3 + 400: "BadRequest", // RFC 7231, 6.5.1 + 401: "Unauthorized", // RFC 7235, 3.1 + 402: "PaymentRequired", // RFC 7231, 6.5.2 + 403: "Forbidden", // RFC 7231, 6.5.3 + 404: "NotFound", // RFC 7231, 6.5.4 + 405: "MethodNotAllowed", // RFC 7231, 6.5.5 + 406: "NotAcceptable", // RFC 7231, 6.5.6 + 407: "ProxyAuthRequired", // RFC 7235, 3.2 + 408: "RequestTimeout", // RFC 7231, 6.5.7 + 409: "Conflict", // RFC 7231, 6.5.8 + 410: "Gone", // RFC 7231, 6.5.9 + 411: "LengthRequired", // RFC 7231, 6.5.10 + 412: "PreconditionFailed", // RFC 7232, 4.2 + 413: "RequestEntityTooLarge", // RFC 7231, 6.5.11 + 414: "RequestURITooLong", // RFC 7231, 6.5.12 + 415: "UnsupportedMediaType", // RFC 7231, 6.5.13 + 416: "RequestedRangeNotSatisfiable", // RFC 7233, 4.4 + 417: "ExpectationFailed", // RFC 7231, 6.5.14 + 418: "Teapot", // RFC 7168, 2.3.3 + 421: "MisdirectedRequest", // RFC 7540, 9.1.2 + 422: "UnprocessableEntity", // RFC 4918, 11.2 + 423: "Locked", // RFC 4918, 11.3 + 424: "FailedDependency", // RFC 4918, 11.4 + 425: "TooEarly", // RFC 8470, 5.2. + 426: "UpgradeRequired", // RFC 7231, 6.5.15 + 428: "PreconditionRequired", // RFC 6585, 3 + 429: "TooManyRequests", // RFC 6585, 4 + 431: "RequestHeaderFieldsTooLarge", // RFC 6585, 5 + 451: "UnavailableForLegalReasons", // RFC 7725, 3 + 500: "InternalServerError", // RFC 7231, 6.6.1 + 501: "NotImplemented", // RFC 7231, 6.6.2 + 502: "BadGateway", // RFC 7231, 6.6.3 + 503: "ServiceUnavailable", // RFC 7231, 6.6.4 + 504: "GatewayTimeout", // RFC 7231, 6.6.5 + 505: "HTTPVersionNotSupported", // RFC 7231, 6.6.6 + 506: "VariantAlsoNegotiates", // RFC 2295, 8.1 + 507: "InsufficientStorage", // RFC 4918, 11.5 + 508: "LoopDetected", // RFC 5842, 7.2 + 510: "NotExtended", // RFC 2774, 7 + 511: "NetworkAuthenticationRequired", // RFC 6585, 6 +} + +type MsgResp struct { + Msg string `json:"msg"` +} + +func NewMsgResp(code int, msg string) (int, interface{}) { + _, ok := statusCodes[code] + if !ok { + panic(fmt.Sprintf("status code not found %d", code)) + } + return code, &MsgResp{Msg: msg} + +} + +func Resp(code int) (int, interface{}) { + msg, ok := statusCodes[code] + if !ok { + panic(fmt.Sprintf("status code not found %d", code)) + } + return code, &MsgResp{Msg: msg} + +} + +func ErrResp(c *gin.Context, code int, err error) (int, interface{}) { + _, ok := statusCodes[code] + if !ok { + panic(fmt.Sprintf("status code not found %d", code)) + } + gErr := c.Error(err) + return code, gErr.JSON() + +} diff --git a/src/idgen/idgen_interface.go b/src/idgen/idgen_interface.go new file mode 100644 index 0000000..e546c31 --- /dev/null +++ b/src/idgen/idgen_interface.go @@ -0,0 +1,5 @@ +package idgen + +type IIDGen interface { + Gen() uint64 +} diff --git a/src/idgen/simpleidgen/simple_id_gen.go b/src/idgen/simpleidgen/simple_id_gen.go new file mode 100644 index 0000000..fba093a --- /dev/null +++ b/src/idgen/simpleidgen/simple_id_gen.go @@ -0,0 +1,27 @@ +package simpleidgen + +import ( + "sync" + "time" +) + +var lastID = uint64(0) +var mux = &sync.Mutex{} + +type SimpleIDGen struct{} + +func New() *SimpleIDGen { + return &SimpleIDGen{} +} + +func (id *SimpleIDGen) Gen() uint64 { + mux.Lock() + defer mux.Unlock() + newID := uint64(time.Now().UnixNano()) + if newID != lastID { + lastID = newID + return lastID + } + lastID = newID + 1 + return lastID +} diff --git a/src/kvstore/boltdbpvd/provider.go b/src/kvstore/boltdbpvd/provider.go new file mode 100644 index 0000000..dedd97f --- /dev/null +++ b/src/kvstore/boltdbpvd/provider.go @@ -0,0 +1,225 @@ +package boltdbpvd + +import ( + "encoding/binary" + "fmt" + "math" + "path" + "time" + + "github.com/boltdb/bolt" + + "github.com/ihexxa/quickshare/src/kvstore" +) + +type BoltPvd struct { + dbPath string + db *bolt.DB + maxStrLen int +} + +func New(dbPath string, maxStrLen int) *BoltPvd { + boltPath := path.Join(path.Clean(dbPath), "quickshare.db") + db, err := bolt.Open(boltPath, 0600, &bolt.Options{Timeout: 1 * time.Second}) + if err != nil { + panic(err) + } + + buckets := []string{"bools", "ints", "int64s", "floats", "strings", "locks"} + for _, bucketName := range buckets { + db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(bucketName)) + if b != nil { + return nil + } + + _, err := tx.CreateBucket([]byte(bucketName)) + if err != nil { + panic(err) + } + return nil + }) + } + + return &BoltPvd{ + dbPath: dbPath, + db: db, + maxStrLen: maxStrLen, + } +} + +func (bp *BoltPvd) Close() error { + return bp.db.Close() +} + +func (bp *BoltPvd) GetBool(key string) (bool, bool) { + buf, ok := make([]byte, 1), false + + bp.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("bools")) + v := b.Get([]byte(key)) + copy(buf, v) + ok = v != nil + return nil + }) + + // 1 means true, 0 means false + return buf[0] == 1, ok +} + +func (bp *BoltPvd) SetBool(key string, val bool) error { + var bVal byte = 0 + if val { + bVal = 1 + } + return bp.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("bools")) + return b.Put([]byte(key), []byte{bVal}) + }) +} + +func (bp *BoltPvd) DelBool(key string) error { + return bp.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("bools")) + return b.Delete([]byte(key)) + }) +} + +func (bp *BoltPvd) GetInt(key string) (int, bool) { + x, ok := bp.GetInt64(key) + return int(x), ok +} + +func (bp *BoltPvd) SetInt(key string, val int) error { + return bp.SetInt64(key, int64(val)) +} + +func (bp *BoltPvd) DelInt(key string) error { + return bp.DelInt64(key) +} + +func (bp *BoltPvd) GetInt64(key string) (int64, bool) { + buf, ok := make([]byte, binary.MaxVarintLen64), false + + bp.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("int64s")) + v := b.Get([]byte(key)) + copy(buf, v) + ok = v != nil + return nil + }) + + if !ok { + return 0, false + } + x, n := binary.Varint(buf) + if n < 0 { + return 0, false + } + return x, true +} + +func (bp *BoltPvd) SetInt64(key string, val int64) error { + buf := make([]byte, binary.MaxVarintLen64) + n := binary.PutVarint(buf, val) + + return bp.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("int64s")) + return b.Put([]byte(key), buf[:n]) + }) +} +func (bp *BoltPvd) DelInt64(key string) error { + return bp.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("int64s")) + return b.Delete([]byte(key)) + }) +} + +func float64ToBytes(num float64) []byte { + buf := make([]byte, 64) + binary.PutUvarint(buf, math.Float64bits(num)) + return buf +} + +func bytesToFloat64(buf []byte) float64 { + uintVal, _ := binary.Uvarint(buf[:64]) + return math.Float64frombits(uintVal) +} + +func (bp *BoltPvd) GetFloat(key string) (float64, bool) { + buf, ok := make([]byte, 64), false + bp.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("floats")) + v := b.Get([]byte(key)) + copy(buf, v) + ok = v != nil + return nil + }) + if !ok { + return 0.0, false + } + return bytesToFloat64(buf), true +} + +func (bp *BoltPvd) SetFloat(key string, val float64) error { + return bp.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("floats")) + return b.Put([]byte(key), float64ToBytes(val)) + }) +} +func (bp *BoltPvd) DelFloat(key string) error { + return bp.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("floats")) + return b.Delete([]byte(key)) + }) +} + +func (bp *BoltPvd) GetString(key string) (string, bool) { + buf, ok, length := make([]byte, bp.maxStrLen), false, bp.maxStrLen + bp.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("strings")) + v := b.Get([]byte(key)) + length = copy(buf, v) + ok = v != nil + return nil + }) + return string(buf[:length]), ok +} + +func (bp *BoltPvd) SetString(key string, val string) error { + if len(val) > bp.maxStrLen { + return fmt.Errorf("can not set string value longer than %d", bp.maxStrLen) + } + + return bp.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("strings")) + return b.Put([]byte(key), []byte(val)) + }) +} + +func (bp *BoltPvd) DelString(key string) error { + return bp.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("strings")) + return b.Delete([]byte(key)) + }) +} + +func (bp *BoltPvd) TryLock(key string) error { + return bp.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("locks")) + if b.Get([]byte(key)) != nil { + return kvstore.ErrLocked + } + return b.Put([]byte(key), []byte{}) + }) +} + +func (bp *BoltPvd) Unlock(key string) error { + return bp.db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("locks")) + if b.Get([]byte(key)) != nil { + return b.Delete([]byte(key)) + } + return kvstore.ErrNoLock + }) +} diff --git a/src/kvstore/kvstore_interface.go b/src/kvstore/kvstore_interface.go new file mode 100644 index 0000000..cc99397 --- /dev/null +++ b/src/kvstore/kvstore_interface.go @@ -0,0 +1,26 @@ +package kvstore + +import "errors" + +var ErrLocked = errors.New("already locked") +var ErrNoLock = errors.New("no lock to unlock") + +type IKVStore interface { + GetBool(key string) (bool, bool) + SetBool(key string, val bool) error + DelBool(key string) error + GetInt(key string) (int, bool) + SetInt(key string, val int) error + DelInt(key string) error + GetInt64(key string) (int64, bool) + SetInt64(key string, val int64) error + DelInt64(key string) error + GetFloat(key string) (float64, bool) + SetFloat(key string, val float64) error + DelFloat(key string) error + GetString(key string) (string, bool) + SetString(key string, val string) error + DelString(key string) error + TryLock(key string) error + Unlock(key string) error +} diff --git a/src/kvstore/memstore/provider.go b/src/kvstore/memstore/provider.go new file mode 100644 index 0000000..d8ef96e --- /dev/null +++ b/src/kvstore/memstore/provider.go @@ -0,0 +1,166 @@ +package memstore + +import ( + "sync" + + "github.com/ihexxa/quickshare/src/kvstore" +) + +type MemStore struct { + bools map[string]bool + boolsMtx *sync.Mutex + ints map[string]int + intsMtx *sync.Mutex + int64s map[string]int64 + int64sMtx *sync.Mutex + floats map[string]float64 + floatsMtx *sync.Mutex + strings map[string]string + stringsMtx *sync.Mutex + locks map[string]bool + locksMtx *sync.Mutex +} + +func New() *MemStore { + return &MemStore{ + bools: map[string]bool{}, + boolsMtx: &sync.Mutex{}, + ints: map[string]int{}, + intsMtx: &sync.Mutex{}, + int64s: map[string]int64{}, + int64sMtx: &sync.Mutex{}, + floats: map[string]float64{}, + floatsMtx: &sync.Mutex{}, + strings: map[string]string{}, + stringsMtx: &sync.Mutex{}, + locks: map[string]bool{}, + locksMtx: &sync.Mutex{}, + } +} + +func (st *MemStore) GetBool(key string) (bool, bool) { + st.boolsMtx.Lock() + defer st.boolsMtx.Unlock() + val, ok := st.bools[key] + return val, ok +} + +func (st *MemStore) SetBool(key string, val bool) error { + st.boolsMtx.Lock() + defer st.boolsMtx.Unlock() + st.bools[key] = val + return nil +} + +func (st *MemStore) GetInt(key string) (int, bool) { + st.intsMtx.Lock() + defer st.intsMtx.Unlock() + val, ok := st.ints[key] + return val, ok +} + +func (st *MemStore) SetInt(key string, val int) error { + st.intsMtx.Lock() + defer st.intsMtx.Unlock() + st.ints[key] = val + return nil +} + +func (st *MemStore) GetInt64(key string) (int64, bool) { + st.int64sMtx.Lock() + defer st.int64sMtx.Unlock() + val, ok := st.int64s[key] + return val, ok +} + +func (st *MemStore) SetInt64(key string, val int64) error { + st.int64sMtx.Lock() + defer st.int64sMtx.Unlock() + st.int64s[key] = val + return nil +} + +func (st *MemStore) GetFloat(key string) (float64, bool) { + st.floatsMtx.Lock() + defer st.floatsMtx.Unlock() + val, ok := st.floats[key] + return val, ok +} + +func (st *MemStore) SetFloat(key string, val float64) error { + st.floatsMtx.Lock() + defer st.floatsMtx.Unlock() + st.floats[key] = val + return nil +} + +func (st *MemStore) GetString(key string) (string, bool) { + st.stringsMtx.Lock() + defer st.stringsMtx.Unlock() + val, ok := st.strings[key] + return val, ok +} + +func (st *MemStore) SetString(key string, val string) error { + st.stringsMtx.Lock() + defer st.stringsMtx.Unlock() + st.strings[key] = val + return nil +} + +func (st *MemStore) DelBool(key string) error { + st.boolsMtx.Lock() + defer st.boolsMtx.Unlock() + delete(st.bools, key) + return nil +} + +func (st *MemStore) DelInt(key string) error { + st.intsMtx.Lock() + defer st.intsMtx.Unlock() + delete(st.ints, key) + return nil +} + +func (st *MemStore) DelInt64(key string) error { + st.int64sMtx.Lock() + defer st.int64sMtx.Unlock() + delete(st.int64s, key) + return nil +} + +func (st *MemStore) DelFloat(key string) error { + st.floatsMtx.Lock() + defer st.floatsMtx.Unlock() + delete(st.floats, key) + return nil +} + +func (st *MemStore) DelString(key string) error { + st.stringsMtx.Lock() + defer st.stringsMtx.Unlock() + delete(st.strings, key) + return nil +} + +func (st *MemStore) TryLock(key string) error { + st.stringsMtx.Lock() + defer st.stringsMtx.Unlock() + _, ok := st.locks[key] + if ok { + return kvstore.ErrLocked + } + st.locks[key] = true + return nil +} + +func (st *MemStore) Unlock(key string) error { + st.stringsMtx.Lock() + defer st.stringsMtx.Unlock() + _, ok := st.locks[key] + if !ok { + return kvstore.ErrNoLock + } + delete(st.locks, key) + return nil +} diff --git a/src/kvstore/test/provider_test.go b/src/kvstore/test/provider_test.go new file mode 100644 index 0000000..8d3e4ec --- /dev/null +++ b/src/kvstore/test/provider_test.go @@ -0,0 +1,185 @@ +package test + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/ihexxa/quickshare/src/kvstore" + "github.com/ihexxa/quickshare/src/kvstore/boltdbpvd" + "github.com/ihexxa/quickshare/src/kvstore/memstore" +) + +func TestKVStoreProviders(t *testing.T) { + var err error + var ok bool + key, boolV, intV, int64V, floatV, stringV := "key", true, 2027, int64(2027), 3.1415, "foobar" + + kvstoreTest := func(store kvstore.IKVStore, t *testing.T) { + // test bools + _, ok = store.GetBool(key) + if ok { + t.Error("value should not exist") + } + err = store.SetBool(key, boolV) + if err != nil { + t.Errorf("there should be no error %v", err) + } + boolVGot, ok := store.GetBool(key) + if !ok { + t.Error("value should exit") + } else if boolVGot != boolV { + t.Error(fmt.Sprintln("value not equal", boolVGot, boolV)) + } + err = store.DelBool(key) + if err != nil { + t.Errorf("there should be no error %v", err) + } + _, ok = store.GetBool(key) + if ok { + t.Error("value should not exist") + } + + // test ints + _, ok = store.GetInt(key) + if ok { + t.Error("value should not exist") + } + err = store.SetInt(key, intV) + if err != nil { + t.Errorf("there should be no error %v", err) + } + intVGot, ok := store.GetInt(key) + if !ok { + t.Error("value should exit") + } else if intVGot != intV { + t.Error(fmt.Sprintln("value not equal", intVGot, intV)) + } + err = store.DelInt(key) + if err != nil { + t.Errorf("there should be no error %v", err) + } + _, ok = store.GetInt(key) + if ok { + t.Error("value should not exist") + } + + // test int64s + _, ok = store.GetInt64(key) + if ok { + t.Error("value should not exist") + } + err = store.SetInt64(key, int64V) + if err != nil { + t.Errorf("there should be no error %v", err) + } + int64VGot, ok := store.GetInt64(key) + if !ok { + t.Error("value should exit") + } else if int64VGot != int64V { + t.Error(fmt.Sprintln("value not equal", int64VGot, int64V)) + } + err = store.DelInt64(key) + if err != nil { + t.Errorf("there should be no error %v", err) + } + _, ok = store.GetInt64(key) + if ok { + t.Error("value should not exist") + } + + // test floats + _, ok = store.GetFloat(key) + if ok { + t.Error("value should not exist") + } + err = store.SetFloat(key, floatV) + if err != nil { + t.Errorf("there should be no error %v", err) + } + floatVGot, ok := store.GetFloat(key) + if !ok { + t.Error("value should exit") + } else if floatVGot != floatV { + t.Error(fmt.Sprintln("value not equal", floatVGot, floatV)) + } + err = store.DelFloat(key) + if err != nil { + t.Errorf("there should be no error %v", err) + } + _, ok = store.GetFloat(key) + if ok { + t.Error("value should not exist") + } + + // test strings + _, ok = store.GetString(key) + if ok { + t.Error("value should not exist") + } + err = store.SetString(key, stringV) + if err != nil { + t.Errorf("there should be no error %v", err) + } + stringVGot, ok := store.GetString(key) + if !ok { + t.Error("value should exit") + } else if stringVGot != stringV { + t.Error(fmt.Sprintln("value not equal", stringVGot, stringV)) + } + err = store.DelString(key) + if err != nil { + t.Errorf("there should be no error %v", err) + } + _, ok = store.GetString(key) + if ok { + t.Error("value should not exist") + } + + // test locks + err = store.TryLock(key) + if err != nil { + t.Errorf("there should be no error %v", err) + } + err = store.TryLock(key) + if err == nil || err != kvstore.ErrLocked { + t.Error("there should be locked") + } + err = store.TryLock("key2") + if err != nil { + t.Errorf("there should be no error %v", err) + } + err = store.Unlock(key) + if err != nil { + t.Errorf("there should be no error %v", err) + } + err = store.Unlock("key2") + if err != nil { + t.Errorf("there should be no error %v", err) + } + } + + t.Run("test bolt provider", func(t *testing.T) { + rootPath, err := ioutil.TempDir("./", "quickshare_kvstore_test_") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(rootPath) + + store := boltdbpvd.New(rootPath, 1024) + defer store.Close() + kvstoreTest(store, t) + }) + + t.Run("test in-memory provider", func(t *testing.T) { + rootPath, err := ioutil.TempDir("./", "quickshare_kvstore_test_") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(rootPath) + + store := memstore.New() + kvstoreTest(store, t) + }) +} diff --git a/src/logging/log_interface.go b/src/logging/log_interface.go new file mode 100644 index 0000000..19ff735 --- /dev/null +++ b/src/logging/log_interface.go @@ -0,0 +1,9 @@ +package logging + +type ILogger interface { + Debug() + Log(values ...interface{}) + Logf(pattern string, values ...interface{}) + Error(values ...interface{}) + Errorf(pattern string, values ...interface{}) +} diff --git a/src/logging/simplelog/simple_log.go b/src/logging/simplelog/simple_log.go new file mode 100644 index 0000000..25518b7 --- /dev/null +++ b/src/logging/simplelog/simple_log.go @@ -0,0 +1,32 @@ +package simplelog + +import ( + "fmt" + "log" +) + +type SimpleLogger struct { + debug bool +} + +func NewSimpleLogger() *SimpleLogger { + return &SimpleLogger{} +} + +func (l *SimpleLogger) Debug() { + l.debug = true +} + +func (l *SimpleLogger) Log(values ...interface{}) { + log.Println(values...) +} + +func (l *SimpleLogger) Logf(pattern string, values ...interface{}) { + log.Printf(pattern, values...) +} +func (l *SimpleLogger) Error(values ...interface{}) { + log.Println(append([]interface{}{"error:"}, values...)...) +} +func (l *SimpleLogger) Errorf(pattern string, values ...interface{}) { + log.Printf(fmt.Sprintf("error: %s", pattern), values...) +} diff --git a/src/server/config.go b/src/server/config.go new file mode 100644 index 0000000..0bf74df --- /dev/null +++ b/src/server/config.go @@ -0,0 +1,49 @@ +package server + +type FSConfig struct { + Root string `json:"root"` + OpensLimit int `json:"opensLimit"` + OpenTTL int `json:"openTTL"` +} + +type Secrets struct { + TokenSecret string `json:"tokenSecret" cfg:"env"` +} + +type ServerCfg struct { + ProdMode bool `json:"prodMode"` + Addr string `json:"addr"` + ReadTimeout int `json:"readTimeout"` + WriteTimeout int `json:"writeTimeout"` + MaxHeaderBytes int `json:"maxHeaderBytes"` +} + +type Config struct { + Fs *FSConfig `json:"fs"` + Secrets *Secrets `json:"secrets"` + Server *ServerCfg `json:"server"` +} + +func NewEmptyConfig() *Config { + return &Config{} +} + +func NewDefaultConfig() *Config { + return &Config{ + Fs: &FSConfig{ + Root: ".", + OpensLimit: 128, + OpenTTL: 60, // 1 min + }, + Secrets: &Secrets{ + TokenSecret: "", + }, + Server: &ServerCfg{ + ProdMode: true, + Addr: "127.0.0.1:8888", + ReadTimeout: 2000, + WriteTimeout: 2000, + MaxHeaderBytes: 512, + }, + } +} diff --git a/src/server/quickshare.db b/src/server/quickshare.db new file mode 100644 index 0000000000000000000000000000000000000000..23e620353d0829ae2bfb674cb29c51cb5abcee40 GIT binary patch literal 32768 zcmeI(K}y3w6oBDat!@;$bKxd8Xf$a|?;v;ulO~a3X~d>0_ij9eR}kE|cBKpVqDSb) z6Zpo=gCK<{)LP*Wyv|JW+DyLPn@k<2IyP)Od)c0~Outi|Shwf3(Y#H2;?*YX9X}nt zKeuO^M~eUg2q1s}0tg_000IagfB*ukBd}(tqILb(`u}q+pf=sM?Q8x2X6yf3_kOo~ z`EXTBrHcRp2q1s}0tg_000IagfB*t3BB1qs%c&}QvmS6FVd{KW!i=X9=Bg`U#@7<{ z@r{hl`hhE9uHQ-2zyB!XEeTg5DT>iV+PQ~LCLa}X*_bsPmtpXotj3&CF*s{Xnv@sA z@o8h)+y~<%0tg_000IagfB*srAb2)xD9Pe}$P$9Jqd@=x1Q0*~ z0R#|0009ILShj%f`|mi_%XfVRulxL$Js<9e00IagfB*srAbWI$!I5 z-4CGa0*k*RzlZ<=2q1s}0tg_000IagfB*srAb 0 { + t.Fatal(errs) + } else if res.StatusCode != 200 { + t.Fatal(res.StatusCode) + } + + // check uploading file + uploadFilePath := path.Join(fileshdr.UploadDir, fmt.Sprintf("%x", sha1.Sum([]byte(filePath)))) + info, err := fs.Stat(uploadFilePath) + if err != nil { + t.Fatal(err) + } else if info.Name() != filepath.Base(uploadFilePath) { + t.Fatal(info.Name(), filepath.Base(uploadFilePath)) + } + + // upload a chunk + i := 0 + contentBytes := []byte(content) + for i < len(contentBytes) { + right := i + chunkSize + if right > len(contentBytes) { + right = len(contentBytes) + } + + res, _, errs = cl.UploadChunk(filePath, string(contentBytes[i:right]), int64(i)) + i = right + if len(errs) > 0 { + t.Fatal(errs) + } else if res.StatusCode != 200 { + t.Fatal(res.StatusCode) + } + + if int64(i) != fileSize { + _, statusResp, errs := cl.UploadStatus(filePath) + if len(errs) > 0 { + t.Fatal(errs) + } else if statusResp.Path != filePath || + statusResp.IsDir || + statusResp.FileSize != fileSize || + statusResp.Uploaded != int64(i) { + t.Fatal("incorrect uploadinfo info", statusResp) + } + } + } + + // check uploaded file + fsFilePath := filepath.Join(fileshdr.FsDir, filePath) + info, err = fs.Stat(fsFilePath) + if err != nil { + t.Fatal(err) + } else if info.Name() != filepath.Base(fsFilePath) { + t.Fatal(info.Name(), filepath.Base(fsFilePath)) + } + + // metadata + _, mRes, errs := cl.Metadata(filePath) + if len(errs) > 0 { + t.Fatal(errs) + } else if mRes.Name != info.Name() || + mRes.IsDir != info.IsDir() || + mRes.Size != info.Size() { + // TODO: modTime is not checked + t.Fatal("incorrect uploaded info", mRes) + } + + // delete file + res, _, errs = cl.Delete(filePath) + if len(errs) > 0 { + t.Fatal(errs) + } else if res.StatusCode != 200 { + t.Fatal(res.StatusCode) + } + } + }) + + t.Run("test file APIs: Mkdir-Create-UploadChunk-List", func(t *testing.T) { + for dirPath, files := range map[string]map[string]string{ + "dir/path1/": map[string]string{ + "f1.md": "11111", + "f2.md": "22222222222", + }, + "dir/path1/path2": map[string]string{ + "f3.md": "3333333", + }, + } { + res, _, errs := cl.Mkdir(dirPath) + if len(errs) > 0 { + t.Fatal(errs) + } else if res.StatusCode != 200 { + t.Fatal(res.StatusCode) + } + + for fileName, content := range files { + filePath := filepath.Join(dirPath, fileName) + + fileSize := int64(len([]byte(content))) + // create a file + res, _, errs := cl.Create(filePath, fileSize) + if len(errs) > 0 { + t.Fatal(errs) + } else if res.StatusCode != 200 { + t.Fatal(res.StatusCode) + } + + res, _, errs = cl.UploadChunk(filePath, content, 0) + if len(errs) > 0 { + t.Fatal(errs) + } else if res.StatusCode != 200 { + t.Fatal(res.StatusCode) + } + } + + _, lResp, errs := cl.List(dirPath) + if len(errs) > 0 { + t.Fatal(errs) + } + for _, metadata := range lResp.Metadatas { + content, ok := files[metadata.Name] + if !ok { + t.Fatalf("%s not found", metadata.Name) + } else if int64(len(content)) != metadata.Size { + t.Fatalf("size not match %d %d \n", len(content), metadata.Size) + } + } + } + }) + + t.Run("test file APIs: Mkdir-Create-UploadChunk-Move-List", func(t *testing.T) { + srcDir := "move/src" + dstDir := "move/dst" + + for _, dirPath := range []string{srcDir, dstDir} { + res, _, errs := cl.Mkdir(dirPath) + if len(errs) > 0 { + t.Fatal(errs) + } else if res.StatusCode != 200 { + t.Fatal(res.StatusCode) + } + } + + files := map[string]string{ + "f1.md": "111", + "f2.md": "22222", + } + + for fileName, content := range files { + oldPath := filepath.Join(srcDir, fileName) + newPath := filepath.Join(dstDir, fileName) + fileSize := int64(len([]byte(content))) + + // create a file + res, _, errs := cl.Create(oldPath, fileSize) + if len(errs) > 0 { + t.Fatal(errs) + } else if res.StatusCode != 200 { + t.Fatal(res.StatusCode) + } + + res, _, errs = cl.UploadChunk(oldPath, content, 0) + if len(errs) > 0 { + t.Fatal(errs) + } else if res.StatusCode != 200 { + t.Fatal(res.StatusCode) + } + + res, _, errs = cl.Move(oldPath, newPath) + if len(errs) > 0 { + t.Fatal(errs) + } else if res.StatusCode != 200 { + t.Fatal(res.StatusCode) + } + } + + _, lResp, errs := cl.List(dstDir) + if len(errs) > 0 { + t.Fatal(errs) + } + for _, metadata := range lResp.Metadatas { + content, ok := files[metadata.Name] + if !ok { + t.Fatalf("%s not found", metadata.Name) + } else if int64(len(content)) != metadata.Size { + t.Fatalf("size not match %d %d \n", len(content), metadata.Size) + } + } + }) +} diff --git a/src/uploadmgr/mgr.go b/src/uploadmgr/mgr.go new file mode 100644 index 0000000..f5ae6e6 --- /dev/null +++ b/src/uploadmgr/mgr.go @@ -0,0 +1,110 @@ +package uploadmgr + +import ( + "errors" + "fmt" + "path" + + "github.com/ihexxa/quickshare/src/depidx" +) + +// TODO: +// uploading resumption test +// rename file after uploaded +// differetiate file and dir + +var ErrBadData = errors.New("file size or uploaded not found for a file") +var ErrUploaded = errors.New("file already uploaded") +var ErrWriteUploaded = errors.New("try to write acknowledge part") + +type UploadMgr struct { + deps *depidx.Deps +} + +func NewUploadMgr(deps *depidx.Deps) (*UploadMgr, error) { + if deps.KV() == nil { + return nil, errors.New("kvstore is not found in deps") + } + if deps.FS() == nil { + return nil, errors.New("fs is not found in deps") + } + + return &UploadMgr{ + deps: deps, + }, nil +} + +func fileSizeKey(filePath string) string { return fmt.Sprintf("%s:size", filePath) } +func fileUploadedKey(filePath string) string { return fmt.Sprintf("%s:uploaded", filePath) } + +func (mgr *UploadMgr) Create(filePath string, size int64) error { + // _, found := mgr.deps.KV().GetBool(filePath) + // if found { + // return os.ErrExist + // } + + dirPath := path.Dir(filePath) + if dirPath != "" { + err := mgr.deps.FS().MkdirAll(dirPath) + if err != nil { + return err + } + } + + err := mgr.deps.FS().Create(filePath) + if err != nil { + return err + } + + // mgr.deps.KV().SetBool(filePath, true) + // mgr.deps.KV().SetInt64(fileSizeKey(filePath), size) + // mgr.deps.KV().SetInt64(fileUploadedKey(filePath), 0) + return nil +} + +func (mgr *UploadMgr) WriteChunk(filePath string, chunk []byte, off int64) (int, error) { + // _, found := mgr.deps.KV().GetBool(filePath) + // if !found { + // return 0, os.ErrNotExist + // } + + // fileSize, ok1 := mgr.deps.KV().GetInt64(fileSizeKey(filePath)) + // uploaded, ok2 := mgr.deps.KV().GetInt64(fileUploadedKey(filePath)) + // if !ok1 || !ok2 { + // return 0, ErrBadData + // } else if uploaded == fileSize { + // return 0, ErrUploaded + // } else if off != uploaded { + // return 0, ErrWriteUploaded + // } + + wrote, err := mgr.deps.FS().WriteAt(filePath, chunk, off) + if err != nil { + return wrote, err + } + + // mgr.deps.KV().SetInt64(fileUploadedKey(filePath), off+int64(wrote)) + return wrote, nil +} + +func (mgr *UploadMgr) Status(filePath string) (int64, bool, error) { + // _, found := mgr.deps.KV().GetBool(filePath) + // if !found { + // return 0, false, os.ErrNotExist + // } + + fileSize, ok1 := mgr.deps.KV().GetInt64(fileSizeKey(filePath)) + fileUploaded, ok2 := mgr.deps.KV().GetInt64(fileUploadedKey(filePath)) + if !ok1 || !ok2 { + return 0, false, ErrBadData + } + return fileUploaded, fileSize == fileUploaded, nil +} + +func (mgr *UploadMgr) Close() error { + return mgr.deps.FS().Close() +} + +func (mgr *UploadMgr) Sync() error { + return mgr.deps.FS().Sync() +} diff --git a/src/uploadmgr/mgr_test.go b/src/uploadmgr/mgr_test.go new file mode 100644 index 0000000..54b3df6 --- /dev/null +++ b/src/uploadmgr/mgr_test.go @@ -0,0 +1,155 @@ +package uploadmgr + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "path" + "sync" + "testing" + + "github.com/ihexxa/gocfg" + "github.com/ihexxa/quickshare/src/depidx" + "github.com/ihexxa/quickshare/src/fs" + "github.com/ihexxa/quickshare/src/fs/local" + "github.com/ihexxa/quickshare/src/kvstore/memstore" + "github.com/ihexxa/quickshare/src/server" +) + +var debug = flag.Bool("d", false, "debug mode") + +// TODO: teardown after each test case + +func TestUploadMgr(t *testing.T) { + rootPath, err := ioutil.TempDir("./", "quickshare_test_") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(rootPath) + + newTestUploadMgr := func() (*UploadMgr, fs.ISimpleFS) { + cfg := gocfg.New() + err := cfg.Load(gocfg.JSONStr("{}"), server.NewEmptyConfig()) + if err != nil { + t.Fatal(err) + } + + filesystem := local.NewLocalFS(rootPath, 0660, 32, 10) + kvstore := memstore.New() + + deps := depidx.NewDeps(cfg) + deps.SetFS(filesystem) + deps.SetKV(kvstore) + + mgr, err := NewUploadMgr(deps) + if err != nil { + t.Fatal(err) + } + return mgr, filesystem + } + + t.Run("normal upload", func(t *testing.T) { + mgr, _ := newTestUploadMgr() + defer mgr.Close() + + testCases := map[string]string{ + "foo.md": "", + "bar.md": "1", + "path1/foobar.md": "1110011", + } + + for filePath, content := range testCases { + err = mgr.Create(filePath, int64(len([]byte(content)))) + if err != nil { + t.Fatal(err) + } + + bytes := []byte(content) + for i := 0; i < len(bytes); i++ { + wrote, err := mgr.WriteChunk(filePath, bytes[i:i+1], int64(i)) + if err != nil { + t.Fatal(err) + } + if wrote != 1 { + t.Fatalf("wrote(%d) != 1", wrote) + } + } + + if err = mgr.Sync(); err != nil { + t.Fatal(err) + } + + gotBytes, err := ioutil.ReadFile(path.Join(rootPath, filePath)) + if err != nil { + t.Fatal(err) + } + if string(gotBytes) != content { + t.Errorf("content not same expected(%s) got(%s)", content, string(gotBytes)) + } + } + }) + + t.Run("concurrently upload", func(t *testing.T) { + mgr, _ := newTestUploadMgr() + defer mgr.Close() + + testCases := []map[string]string{ + map[string]string{ + "file20.md": "111", + "file21.md": "2222000", + "path1/file22.md": "1010011", + "path2/file22.md": "1010011", + }, + } + + uploadWorker := func(id int, filePath, content string, wg *sync.WaitGroup) { + err = mgr.Create(filePath, int64(len([]byte(content)))) + if err != nil { + t.Fatal(err) + } + + bytes := []byte(content) + for i := 0; i < len(bytes); i++ { + wrote, err := mgr.WriteChunk(filePath, bytes[i:i+1], int64(i)) + if err != nil { + t.Fatal(err) + } + if wrote != 1 { + t.Fatalf("wrote(%d) != 1", wrote) + } + if *debug { + fmt.Printf("worker-%d wrote %s\n", id, string(bytes[i:i+1])) + } + } + + wg.Done() + } + + for _, files := range testCases { + wg := &sync.WaitGroup{} + workerID := 0 + for filePath, content := range files { + wg.Add(1) + go uploadWorker(workerID, filePath, content, wg) + workerID++ + } + + wg.Wait() + + if err = mgr.Sync(); err != nil { + t.Fatal(err) + } + + for filePath, content := range files { + gotBytes, err := ioutil.ReadFile(path.Join(rootPath, filePath)) + if err != nil { + t.Fatal(err) + } + if string(gotBytes) != content { + t.Errorf("content not same expected(%s) got(%s)", content, string(gotBytes)) + } + } + } + }) +}