diff --git a/src/search/fileindex/fsearch.go b/src/search/fileindex/fsearch.go index f282432..228ed59 100644 --- a/src/search/fileindex/fsearch.go +++ b/src/search/fileindex/fsearch.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "os" "strings" "github.com/ihexxa/fsearch" @@ -21,6 +20,7 @@ type IFileIndex interface { WriteTo(pathname string) error ReadFrom(pathname string) error Reset() error + String() string } type FileTreeIndex struct { @@ -66,8 +66,12 @@ func (idx *FileTreeIndex) MovePath(pathname, dstParentPath string) error { func (idx *FileTreeIndex) WriteTo(pathname string) error { rowsChan := idx.index.Marshal() - err := idx.fs.Create(pathname) - if err != nil && !errors.Is(err, os.ErrExist) { + err := idx.fs.Remove(pathname) + if err != nil { + return err + } + err = idx.fs.Create(pathname) + if err != nil { return err } @@ -78,17 +82,20 @@ func (idx *FileTreeIndex) WriteTo(pathname string) error { batch := []string{} for { row, ok = <-rowsChan + if ok { + batch = append(batch, row+"\n") + } if !ok || len(batch) > 1024 { - wrote, err = idx.fs.WriteAt(pathname, []byte(strings.Join(batch, "\n")), offset) + wrote, err = idx.fs.WriteAt(pathname, []byte(strings.Join(batch, "")), offset) if err != nil { return err } offset += int64(wrote) + batch = batch[:0] } if !ok { break } - batch = append(batch, row) } return idx.index.Error() @@ -105,19 +112,37 @@ func (idx *FileTreeIndex) ReadFrom(pathname string) error { rowSeparator := byte('\n') reader := bufio.NewReader(f) rowsChan := make(chan string, 1024) - defer close(rowsChan) - for { - row, err = reader.ReadString(rowSeparator) - if err != nil { - if errors.Is(err, io.EOF) { - rowsChan <- row // some content may still read - break - } else { - return err + + var workerErr error + readWorker := func() { + defer close(rowsChan) + for { + row, err = reader.ReadString(rowSeparator) + if err != nil { + if errors.Is(err, io.EOF) { + if row != "" { + rowsChan <- row + } + break + } else { + workerErr = err + break + } + } + if row != "" { + rowsChan <- row } } - rowsChan <- row } + go readWorker() + idx.index.Unmarshal(rowsChan) + if workerErr != nil { + return err + } return idx.index.Error() } + +func (idx *FileTreeIndex) String() string { + return idx.index.String() +} diff --git a/src/search/fileindex/fsearch_test.go b/src/search/fileindex/fsearch_test.go new file mode 100644 index 0000000..6c19a13 --- /dev/null +++ b/src/search/fileindex/fsearch_test.go @@ -0,0 +1,63 @@ +package fileindex + +import ( + "math/rand" + "os" + "strings" + "testing" + "time" + + "github.com/ihexxa/quickshare/src/fs/local" + "github.com/ihexxa/quickshare/src/idgen/simpleidgen" + "github.com/ihexxa/randstr" +) + +func TestFileSearch(t *testing.T) { + dirPath := "tmp" + err := os.MkdirAll(dirPath, 0700) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dirPath) + + makePaths := func(maxPathLen, count int) map[string]bool { + rand.Seed(time.Now().UnixNano()) + randStr := randstr.New([]string{}) + + paths := map[string]bool{} + for i := 0; i < count; i++ { + pathLen := rand.Intn(maxPathLen) + 1 + pathParts := []string{} + for j := 0; j < pathLen; j++ { + pathParts = append(pathParts, randStr.Alnums()) + } + paths[strings.Join(pathParts, "/")] = true + } + + return paths + } + + ider := simpleidgen.New() + fs := local.NewLocalFS(dirPath, 0660, 1024, 60, 60, ider) + fileIndex := NewFileTreeIndex(fs, "/", 0) + + paths := makePaths(8, 256) + for pathname := range paths { + err := fileIndex.AddPath(pathname) + if err != nil { + t.Fatal(err) + } + } + + indexPath := "/fileindex" + err = fileIndex.WriteTo(indexPath) + if err != nil { + t.Fatal(err) + } + + fileIndex2 := NewFileTreeIndex(fs, "/", 0) + err = fileIndex2.ReadFrom(indexPath) + if err != nil { + t.Fatal(err) + } +}