!1 Merge back to master

Merge pull request !1 from dev branch
This commit is contained in:
hekk 2018-05-27 21:32:55 +08:00
parent 30c963a5f0
commit 61a1c93f0f
89 changed files with 15859 additions and 2 deletions

View file

@ -0,0 +1,13 @@
package qtube
import (
"net/http"
)
import (
"quickshare/server/libs/fileidx"
)
type Downloader interface {
ServeFile(res http.ResponseWriter, req *http.Request, fileInfo *fileidx.FileInfo) error
}

280
server/libs/qtube/qtube.go Normal file
View file

@ -0,0 +1,280 @@
package qtube
import (
"errors"
"fmt"
"io"
"net/http"
"path/filepath"
"strconv"
"strings"
"time"
)
import (
"quickshare/server/libs/fileidx"
)
var (
ErrCopy = errors.New("ServeFile: copy error")
ErrUnknown = errors.New("ServeFile: unknown error")
)
type httpRange struct {
start, length int64
}
func (ra *httpRange) GetStart() int64 {
return ra.start
}
func (ra *httpRange) GetLength() int64 {
return ra.length
}
func (ra *httpRange) SetStart(start int64) {
ra.start = start
}
func (ra *httpRange) SetLength(length int64) {
ra.length = length
}
func NewQTube(root string, copySpeed, maxRangeLen int64, filer FileReadSeekCloser) Downloader {
return &QTube{
Root: root,
BytesPerSec: copySpeed,
MaxRangeLen: maxRangeLen,
Filer: filer,
}
}
type QTube struct {
Root string
BytesPerSec int64
MaxRangeLen int64
Filer FileReadSeekCloser
}
type FileReadSeekCloser interface {
Open(filePath string) (ReadSeekCloser, error)
}
type ReadSeekCloser interface {
io.Reader
io.Seeker
io.Closer
}
const (
ErrorInvalidRange = "ServeFile: invalid Range"
ErrorInvalidSize = "ServeFile: invalid Range total size"
)
func (tb *QTube) ServeFile(res http.ResponseWriter, req *http.Request, fileInfo *fileidx.FileInfo) error {
headerRange := req.Header.Get("Range")
switch {
case req.Method == http.MethodHead:
res.Header().Set("Accept-Ranges", "bytes")
res.Header().Set("Content-Length", fmt.Sprintf("%d", fileInfo.Uploaded))
res.Header().Set("Content-Type", "application/octet-stream")
res.WriteHeader(http.StatusOK)
return nil
case headerRange == "":
return tb.serveAll(res, fileInfo)
default:
return tb.serveRanges(res, headerRange, fileInfo)
}
}
func (tb *QTube) serveAll(res http.ResponseWriter, fileInfo *fileidx.FileInfo) error {
res.Header().Set("Accept-Ranges", "bytes")
res.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filepath.Base(fileInfo.PathLocal)))
res.Header().Set("Content-Length", fmt.Sprintf("%d", fileInfo.Uploaded))
res.Header().Set("Content-Type", "application/octet-stream")
res.Header().Set("Last-Modified", time.Unix(fileInfo.ModTime, 0).UTC().Format(http.TimeFormat))
res.WriteHeader(http.StatusOK)
// TODO: need verify path
file, openErr := tb.Filer.Open(filepath.Join(tb.Root, fileInfo.PathLocal))
defer file.Close()
if openErr != nil {
return openErr
}
copyErr := tb.throttledCopyN(res, file, fileInfo.Uploaded)
if copyErr != nil && copyErr != io.EOF {
return copyErr
}
return nil
}
func (tb *QTube) serveRanges(res http.ResponseWriter, headerRange string, fileInfo *fileidx.FileInfo) error {
ranges, rangeErr := getRanges(headerRange, fileInfo.Uploaded)
if rangeErr != nil {
http.Error(res, rangeErr.Error(), http.StatusRequestedRangeNotSatisfiable)
return errors.New(rangeErr.Error())
}
switch {
case len(ranges) == 1 || len(ranges) > 1:
if tb.copyRange(res, ranges[0], fileInfo) != nil {
return ErrCopy
}
default:
// TODO: add support for multiple ranges
return ErrUnknown
}
return nil
}
func getRanges(headerRange string, size int64) ([]httpRange, error) {
ranges, raParseErr := parseRange(headerRange, size)
// TODO: check max number of ranges, range start end
if len(ranges) <= 0 || raParseErr != nil {
return nil, errors.New(ErrorInvalidRange)
}
if sumRangesSize(ranges) > size {
return nil, errors.New(ErrorInvalidSize)
}
return ranges, nil
}
func (tb *QTube) copyRange(res http.ResponseWriter, ra httpRange, fileInfo *fileidx.FileInfo) error {
// TODO: comfirm this wont cause problem
if ra.GetLength() > tb.MaxRangeLen {
ra.SetLength(tb.MaxRangeLen)
}
// TODO: add headers(ETag): https://tools.ietf.org/html/rfc7233#section-4.1 p11 2nd paragraph
res.Header().Set("Accept-Ranges", "bytes")
res.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filepath.Base(fileInfo.PathLocal)))
res.Header().Set("Content-Type", "application/octet-stream")
res.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", ra.start, ra.start+ra.length-1, fileInfo.Uploaded))
res.Header().Set("Content-Length", strconv.FormatInt(ra.GetLength(), 10))
res.Header().Set("Last-Modified", time.Unix(fileInfo.ModTime, 0).UTC().Format(http.TimeFormat))
res.WriteHeader(http.StatusPartialContent)
// TODO: need verify path
file, openErr := tb.Filer.Open(filepath.Join(tb.Root, fileInfo.PathLocal))
defer file.Close()
if openErr != nil {
return openErr
}
if _, seekErr := file.Seek(ra.start, io.SeekStart); seekErr != nil {
return seekErr
}
copyErr := tb.throttledCopyN(res, file, ra.length)
if copyErr != nil && copyErr != io.EOF {
return copyErr
}
return nil
}
func (tb *QTube) throttledCopyN(dst io.Writer, src io.Reader, length int64) error {
sum := int64(0)
timeSlot := time.Duration(1 * time.Second)
for sum < length {
start := time.Now()
chunkSize := length - sum
if length-sum > tb.BytesPerSec {
chunkSize = tb.BytesPerSec
}
copied, err := io.CopyN(dst, src, chunkSize)
if err != nil {
return err
}
sum += copied
end := time.Now()
if end.Before(start.Add(timeSlot)) {
time.Sleep(start.Add(timeSlot).Sub(end))
}
}
return nil
}
func parseRange(headerRange string, size int64) ([]httpRange, error) {
if headerRange == "" {
return nil, nil // header not present
}
const keyByte = "bytes="
if !strings.HasPrefix(headerRange, keyByte) {
return nil, errors.New("byte= not found")
}
var ranges []httpRange
noOverlap := false
for _, ra := range strings.Split(headerRange[len(keyByte):], ",") {
ra = strings.TrimSpace(ra)
if ra == "" {
continue
}
i := strings.Index(ra, "-")
if i < 0 {
return nil, errors.New("- not found")
}
start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:])
var r httpRange
if start == "" {
i, err := strconv.ParseInt(end, 10, 64)
if err != nil {
return nil, errors.New("invalid range")
}
if i > size {
i = size
}
r.start = size - i
r.length = size - r.start
} else {
i, err := strconv.ParseInt(start, 10, 64)
if err != nil || i < 0 {
return nil, errors.New("invalid range")
}
if i >= size {
// If the range begins after the size of the content,
// then it does not overlap.
noOverlap = true
continue
}
r.start = i
if end == "" {
// If no end is specified, range extends to end of the file.
r.length = size - r.start
} else {
i, err := strconv.ParseInt(end, 10, 64)
if err != nil || r.start > i {
return nil, errors.New("invalid range")
}
if i >= size {
i = size - 1
}
r.length = i - r.start + 1
}
}
ranges = append(ranges, r)
}
if noOverlap && len(ranges) == 0 {
// The specified ranges did not overlap with the content.
return nil, errors.New("parseRanges: no overlap")
}
return ranges, nil
}
func sumRangesSize(ranges []httpRange) (size int64) {
for _, ra := range ranges {
size += ra.length
}
return
}

View file

@ -0,0 +1,354 @@
package qtube
import (
"bytes"
"fmt"
"net/http"
"strings"
"testing"
"time"
)
import (
"quickshare/server/libs/fileidx"
)
// Range format examples:
// Range: <unit>=<range-start>-
// Range: <unit>=<range-start>-<range-end>
// Range: <unit>=<range-start>-<range-end>, <range-start>-<range-end>
// Range: <unit>=<range-start>-<range-end>, <range-start>-<range-end>, <range-start>-<range-end>
func TestGetRanges(t *testing.T) {
type Input struct {
HeaderRange string
Size int64
}
type Output struct {
Ranges []httpRange
ErrorMsg string
}
type testCase struct {
Desc string
Input
Output
}
testCases := []testCase{
testCase{
Desc: "invalid range",
Input: Input{
HeaderRange: "bytes=start-invalid end",
Size: 0,
},
Output: Output{
ErrorMsg: ErrorInvalidRange,
},
},
testCase{
Desc: "invalid range total size",
Input: Input{
HeaderRange: "bytes=0-1, 2-3, 0-1, 0-2",
Size: 3,
},
Output: Output{
ErrorMsg: ErrorInvalidSize,
},
},
testCase{
Desc: "range ok",
Input: Input{
HeaderRange: "bytes=0-1, 2-3",
Size: 4,
},
Output: Output{
Ranges: []httpRange{
httpRange{start: 0, length: 2},
httpRange{start: 2, length: 2},
},
ErrorMsg: "",
},
},
}
for _, tCase := range testCases {
ranges, err := getRanges(tCase.HeaderRange, tCase.Size)
if err != nil {
if err.Error() != tCase.ErrorMsg || len(tCase.Ranges) != 0 {
t.Fatalf("getRanges: incorrect errorMsg want: %v got: %v", tCase.ErrorMsg, err.Error())
} else {
continue
}
} else {
for id, ra := range ranges {
if ra.GetStart() != tCase.Ranges[id].GetStart() {
t.Fatalf("getRanges: incorrect range start, got: %v want: %v", ra.GetStart(), tCase.Ranges[id])
}
if ra.GetLength() != tCase.Ranges[id].GetLength() {
t.Fatalf("getRanges: incorrect range length, got: %v want: %v", ra.GetLength(), tCase.Ranges[id])
}
}
}
}
}
func TestThrottledCopyN(t *testing.T) {
type Init struct {
BytesPerSec int64
MaxRangeLen int64
}
type Input struct {
Src string
Length int64
}
// after starting throttledCopyN by DstAtTime.AtMs millisecond,
// copied valueshould equal to DstAtTime.Dst.
type DstAtTime struct {
AtMS int
Dst string
}
type Output struct {
ExpectDsts []DstAtTime
}
type testCase struct {
Desc string
Init
Input
Output
}
verifyDsts := func(dst *bytes.Buffer, expectDsts []DstAtTime) {
for _, expectDst := range expectDsts {
// fmt.Printf("sleep: %d\n", time.Now().UnixNano())
time.Sleep(time.Duration(expectDst.AtMS) * time.Millisecond)
dstStr := string(dst.Bytes())
// fmt.Printf("check: %d\n", time.Now().UnixNano())
if dstStr != expectDst.Dst {
panic(
fmt.Sprintf(
"throttledCopyN want: <%s> | got: <%s> | at: %d",
expectDst.Dst,
dstStr,
expectDst.AtMS,
),
)
}
}
}
testCases := []testCase{
testCase{
Desc: "4 byte per sec",
Init: Init{
BytesPerSec: 5,
MaxRangeLen: 10,
},
Input: Input{
Src: "aaaa_aaaa_",
Length: 10,
},
Output: Output{
ExpectDsts: []DstAtTime{
DstAtTime{AtMS: 200, Dst: "aaaa_"},
DstAtTime{AtMS: 200, Dst: "aaaa_"},
DstAtTime{AtMS: 200, Dst: "aaaa_"},
DstAtTime{AtMS: 600, Dst: "aaaa_aaaa_"},
DstAtTime{AtMS: 200, Dst: "aaaa_aaaa_"},
DstAtTime{AtMS: 200, Dst: "aaaa_aaaa_"},
},
},
},
}
for _, tCase := range testCases {
tb := NewQTube("", tCase.BytesPerSec, tCase.MaxRangeLen, &stubFiler{}).(*QTube)
dst := bytes.NewBuffer(make([]byte, len(tCase.Src)))
dst.Reset()
go verifyDsts(dst, tCase.ExpectDsts)
tb.throttledCopyN(dst, strings.NewReader(tCase.Src), tCase.Length)
}
}
// TODO: using same stub with testhelper
type stubWriter struct {
Headers http.Header
Response []byte
StatusCode int
}
func (w *stubWriter) Header() http.Header {
return w.Headers
}
func (w *stubWriter) Write(body []byte) (int, error) {
w.Response = append(w.Response, body...)
return len(body), nil
}
func (w *stubWriter) WriteHeader(statusCode int) {
w.StatusCode = statusCode
}
func TestCopyRange(t *testing.T) {
type Init struct {
Content string
}
type Input struct {
Range httpRange
Info fileidx.FileInfo
}
type Output struct {
StatusCode int
Headers map[string][]string
Body string
}
type testCase struct {
Desc string
Init
Input
Output
}
testCases := []testCase{
testCase{
Desc: "copy ok",
Init: Init{
Content: "abcd_abcd_",
},
Input: Input{
Range: httpRange{
start: 6,
length: 3,
},
Info: fileidx.FileInfo{
ModTime: 0,
Uploaded: 10,
PathLocal: "filename.jpg",
},
},
Output: Output{
StatusCode: 206,
Headers: map[string][]string{
"Accept-Ranges": []string{"bytes"},
"Content-Disposition": []string{`attachment; filename="filename.jpg"`},
"Content-Type": []string{"application/octet-stream"},
"Content-Range": []string{"bytes 6-8/10"},
"Content-Length": []string{"3"},
"Last-Modified": []string{time.Unix(0, 0).UTC().Format(http.TimeFormat)},
},
Body: "abc",
},
},
}
for _, tCase := range testCases {
filer := &stubFiler{
&StubFile{
Content: tCase.Content,
Offset: 0,
},
}
tb := NewQTube("", 100, 100, filer).(*QTube)
res := &stubWriter{
Headers: make(map[string][]string),
Response: make([]byte, 0),
}
err := tb.copyRange(res, tCase.Range, &tCase.Info)
if err != nil {
t.Fatalf("copyRange: %v", err)
}
if res.StatusCode != tCase.Output.StatusCode {
t.Fatalf("copyRange: statusCode not match got: %v want: %v", res.StatusCode, tCase.Output.StatusCode)
}
if string(res.Response) != tCase.Output.Body {
t.Fatalf("copyRange: body not match \ngot: %v \nwant: %v", string(res.Response), tCase.Output.Body)
}
for key, vals := range tCase.Output.Headers {
if res.Header().Get(key) != vals[0] {
t.Fatalf("copyRange: header not match %v got: %v want: %v", key, res.Header().Get(key), vals[0])
}
}
if res.StatusCode != tCase.Output.StatusCode {
t.Fatalf("copyRange: statusCodes are not match %v", res.StatusCode, tCase.Output.StatusCode)
}
}
}
func TestServeAll(t *testing.T) {
type Init struct {
Content string
}
type Input struct {
Info fileidx.FileInfo
}
type Output struct {
StatusCode int
Headers map[string][]string
Body string
}
type testCase struct {
Desc string
Init
Input
Output
}
testCases := []testCase{
testCase{
Desc: "copy ok",
Init: Init{
Content: "abcd_abcd_",
},
Input: Input{
Info: fileidx.FileInfo{
ModTime: 0,
Uploaded: 10,
PathLocal: "filename.jpg",
},
},
Output: Output{
StatusCode: 200,
Headers: map[string][]string{
"Accept-Ranges": []string{"bytes"},
"Content-Disposition": []string{`attachment; filename="filename.jpg"`},
"Content-Type": []string{"application/octet-stream"},
"Content-Length": []string{"10"},
"Last-Modified": []string{time.Unix(0, 0).UTC().Format(http.TimeFormat)},
},
Body: "abcd_abcd_",
},
},
}
for _, tCase := range testCases {
filer := &stubFiler{
&StubFile{
Content: tCase.Content,
Offset: 0,
},
}
tb := NewQTube("", 100, 100, filer).(*QTube)
res := &stubWriter{
Headers: make(map[string][]string),
Response: make([]byte, 0),
}
err := tb.serveAll(res, &tCase.Info)
if err != nil {
t.Fatalf("serveAll: %v", err)
}
if res.StatusCode != tCase.Output.StatusCode {
t.Fatalf("serveAll: statusCode not match got: %v want: %v", res.StatusCode, tCase.Output.StatusCode)
}
if string(res.Response) != tCase.Output.Body {
t.Fatalf("serveAll: body not match \ngot: %v \nwant: %v", string(res.Response), tCase.Output.Body)
}
for key, vals := range tCase.Output.Headers {
if res.Header().Get(key) != vals[0] {
t.Fatalf("serveAll: header not match %v got: %v want: %v", key, res.Header().Get(key), vals[0])
}
}
if res.StatusCode != tCase.Output.StatusCode {
t.Fatalf("serveAll: statusCodes are not match %v", res.StatusCode, tCase.Output.StatusCode)
}
}
}

View file

@ -0,0 +1,28 @@
package qtube
type StubFile struct {
Content string
Offset int64
}
func (file *StubFile) Read(p []byte) (int, error) {
copied := copy(p[:], []byte(file.Content)[:len(p)])
return copied, nil
}
func (file *StubFile) Seek(offset int64, whence int) (int64, error) {
file.Offset = offset
return offset, nil
}
func (file *StubFile) Close() error {
return nil
}
type stubFiler struct {
file *StubFile
}
func (filer *stubFiler) Open(filePath string) (ReadSeekCloser, error) {
return filer.file, nil
}