diff --git a/src/db/common.go b/src/db/common.go index 63b0b7e..fdc0d78 100644 --- a/src/db/common.go +++ b/src/db/common.go @@ -113,11 +113,12 @@ var ( ) type FileInfo struct { - IsDir bool `json:"isDir" yaml:"isDir"` // deprecated - Shared bool `json:"shared" yaml:"shared"` // deprecated - ShareID string `json:"shareID" yaml:"shareID"` // deprecated + Id uint64 `json:"id" yaml:"id"` + IsDir bool `json:"isDir" yaml:"isDir"` + Shared bool `json:"shared" yaml:"shared"` + ShareID string `json:"shareID" yaml:"shareID"` Sha1 string `json:"sha1" yaml:"sha1"` - Size int64 `json:"size" yaml:"size"` // deprecated + Size int64 `json:"size" yaml:"size"` } type UserCfg struct { diff --git a/src/db/interface.go b/src/db/interfaces.go similarity index 89% rename from src/db/interface.go rename to src/db/interfaces.go index 8831866..4645f82 100644 --- a/src/db/interface.go +++ b/src/db/interfaces.go @@ -70,7 +70,7 @@ type IFilesFunctions interface { } type IFileDB interface { - AddFileInfo(ctx context.Context, userId uint64, itemPath string, info *FileInfo) error + AddFileInfo(ctx context.Context, infoId, userId uint64, itemPath string, info *FileInfo) error DelFileInfo(ctx context.Context, userId uint64, itemPath string) error GetFileInfo(ctx context.Context, itemPath string) (*FileInfo, error) SetSha1(ctx context.Context, itemPath, sign string) error @@ -78,9 +78,9 @@ type IFileDB interface { ListFileInfos(ctx context.Context, itemPaths []string) (map[string]*FileInfo, error) } type IUploadDB interface { - AddUploadInfos(ctx context.Context, userId uint64, tmpPath, filePath string, info *FileInfo) error + AddUploadInfos(ctx context.Context, uploadId, userId uint64, tmpPath, filePath string, info *FileInfo) error DelUploadingInfos(ctx context.Context, userId uint64, realPath string) error - MoveUploadingInfos(ctx context.Context, userId uint64, uploadPath, itemPath string) error + MoveUploadingInfos(ctx context.Context, uploadId, userId uint64, uploadPath, itemPath string) error SetUploadInfo(ctx context.Context, user uint64, filePath string, newUploaded int64) error GetUploadInfo(ctx context.Context, userId uint64, filePath string) (string, int64, int64, error) ListUploadInfos(ctx context.Context, user uint64) ([]*UploadInfo, error) @@ -89,7 +89,7 @@ type IUploadDB interface { type ISharingDB interface { IsSharing(ctx context.Context, dirPath string) (bool, error) GetSharingDir(ctx context.Context, hashID string) (string, error) - AddSharing(ctx context.Context, userId uint64, dirPath string) error + AddSharing(ctx context.Context, infoId, userId uint64, dirPath string) error DelSharing(ctx context.Context, userId uint64, dirPath string) error ListSharingsByLocation(ctx context.Context, location string) (map[string]string, error) } diff --git a/src/db/migration_tests/testdata/quickshare_v1.db b/src/db/migration_tests/testdata/quickshare_v1.db deleted file mode 100644 index 6790eb2..0000000 Binary files a/src/db/migration_tests/testdata/quickshare_v1.db and /dev/null differ diff --git a/src/db/rdb/sqlite/files.go b/src/db/rdb/sqlite/files.go index 7b0ae2c..2388845 100644 --- a/src/db/rdb/sqlite/files.go +++ b/src/db/rdb/sqlite/files.go @@ -15,16 +15,18 @@ import ( func (st *SQLiteStore) getFileInfo(ctx context.Context, itemPath string) (*db.FileInfo, error) { var infoStr string fInfo := &db.FileInfo{} + var id uint64 var isDir bool var size int64 var shareId string err := st.db.QueryRowContext( ctx, - `select is_dir, size, share_id, info + `select id, is_dir, size, share_id, info from t_file_info where path=?`, itemPath, ).Scan( + &id, &isDir, &size, &shareId, @@ -41,6 +43,7 @@ func (st *SQLiteStore) getFileInfo(ctx context.Context, itemPath string) (*db.Fi if err != nil { return nil, err } + fInfo.Id = id fInfo.IsDir = isDir fInfo.Size = size fInfo.ShareID = shareId @@ -69,7 +72,7 @@ func (st *SQLiteStore) ListFileInfos(ctx context.Context, itemPaths []string) (m rows, err := st.db.QueryContext( ctx, fmt.Sprintf( - `select path, is_dir, size, share_id, info + `select id, path, is_dir, size, share_id, info from t_file_info where path in (%s) `, @@ -85,11 +88,12 @@ func (st *SQLiteStore) ListFileInfos(ctx context.Context, itemPaths []string) (m var fInfoStr, itemPath, shareId string var isDir bool var size int64 + var id uint64 fInfos := map[string]*db.FileInfo{} for rows.Next() { fInfo := &db.FileInfo{} - err = rows.Scan(&itemPath, &isDir, &size, &shareId, &fInfoStr) + err = rows.Scan(&id, &itemPath, &isDir, &size, &shareId, &fInfoStr) if err != nil { return nil, err } @@ -98,6 +102,7 @@ func (st *SQLiteStore) ListFileInfos(ctx context.Context, itemPaths []string) (m if err != nil { return nil, err } + fInfo.Id = id fInfo.IsDir = isDir fInfo.Size = size fInfo.ShareID = shareId @@ -111,7 +116,7 @@ func (st *SQLiteStore) ListFileInfos(ctx context.Context, itemPaths []string) (m return fInfos, nil } -func (st *SQLiteStore) addFileInfo(ctx context.Context, userId uint64, itemPath string, info *db.FileInfo) error { +func (st *SQLiteStore) addFileInfo(ctx context.Context, infoId, userId uint64, itemPath string, info *db.FileInfo) error { infoStr, err := json.Marshal(info) if err != nil { return err @@ -126,24 +131,24 @@ func (st *SQLiteStore) addFileInfo(ctx context.Context, userId uint64, itemPath _, err = st.db.ExecContext( ctx, `insert into t_file_info ( - path, user, location, parent, name, + id, path, user, location, parent, name, is_dir, size, share_id, info ) values ( - ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )`, - itemPath, userId, location, dirPath, itemName, + infoId, itemPath, userId, location, dirPath, itemName, info.IsDir, info.Size, info.ShareID, infoStr, ) return err } -func (st *SQLiteStore) AddFileInfo(ctx context.Context, userId uint64, itemPath string, info *db.FileInfo) error { +func (st *SQLiteStore) AddFileInfo(ctx context.Context, infoId, userId uint64, itemPath string, info *db.FileInfo) error { st.Lock() defer st.Unlock() - err := st.addFileInfo(ctx, userId, itemPath, info) + err := st.addFileInfo(ctx, infoId, userId, itemPath, info) if err != nil { return err } @@ -260,7 +265,7 @@ func (st *SQLiteStore) MoveFileInfo(ctx context.Context, userId uint64, oldPath, if err != nil { return err } - return st.addFileInfo(ctx, userId, newPath, info) + return st.addFileInfo(ctx, info.Id, userId, newPath, info) } func getLocation(itemPath string) (string, error) { diff --git a/src/db/rdb/sqlite/files_sharings.go b/src/db/rdb/sqlite/files_sharings.go index e40c246..019f192 100644 --- a/src/db/rdb/sqlite/files_sharings.go +++ b/src/db/rdb/sqlite/files_sharings.go @@ -78,7 +78,7 @@ func (st *SQLiteStore) GetSharingDir(ctx context.Context, hashID string) (string return sharedPath, nil } -func (st *SQLiteStore) AddSharing(ctx context.Context, userId uint64, dirPath string) error { +func (st *SQLiteStore) AddSharing(ctx context.Context, infoId, userId uint64, dirPath string) error { st.Lock() defer st.Unlock() @@ -109,14 +109,17 @@ func (st *SQLiteStore) AddSharing(ctx context.Context, userId uint64, dirPath st _, err = st.db.ExecContext( ctx, `insert into t_file_info ( - path, user, location, parent, name, + id, path, user, + location, parent, name, is_dir, size, share_id, info ) values ( - ?, ?, ?, ?, ?, + ?, ?, ?, + ?, ?, ?, ?, ?, ?, ? )`, - dirPath, userId, location, parentPath, name, + infoId, dirPath, userId, + location, parentPath, name, true, 0, shareID, infoStr, ) return err diff --git a/src/db/rdb/sqlite/files_uploadings.go b/src/db/rdb/sqlite/files_uploadings.go index 86d38fb..7650520 100644 --- a/src/db/rdb/sqlite/files_uploadings.go +++ b/src/db/rdb/sqlite/files_uploadings.go @@ -8,21 +8,21 @@ import ( "github.com/ihexxa/quickshare/src/db" ) -func (st *SQLiteStore) addUploadInfoOnly(ctx context.Context, userId uint64, tmpPath, filePath string, fileSize int64) error { +func (st *SQLiteStore) addUploadInfoOnly(ctx context.Context, uploadId, userId uint64, tmpPath, filePath string, fileSize int64) error { _, err := st.db.ExecContext( ctx, `insert into t_file_uploading ( - real_path, tmp_path, user, size, uploaded + id, real_path, tmp_path, user, size, uploaded ) values ( - ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ? )`, - filePath, tmpPath, userId, fileSize, 0, + uploadId, filePath, tmpPath, userId, fileSize, 0, ) return err } -func (st *SQLiteStore) AddUploadInfos(ctx context.Context, userId uint64, tmpPath, filePath string, info *db.FileInfo) error { +func (st *SQLiteStore) AddUploadInfos(ctx context.Context, uploadId, userId uint64, tmpPath, filePath string, info *db.FileInfo) error { st.Lock() defer st.Unlock() @@ -46,7 +46,7 @@ func (st *SQLiteStore) AddUploadInfos(ctx context.Context, userId uint64, tmpPat return err } - return st.addUploadInfoOnly(ctx, userId, tmpPath, filePath, info.Size) + return st.addUploadInfoOnly(ctx, uploadId, userId, tmpPath, filePath, info.Size) } func (st *SQLiteStore) DelUploadingInfos(ctx context.Context, userId uint64, realPath string) error { @@ -86,7 +86,7 @@ func (st *SQLiteStore) delUploadInfoOnly(ctx context.Context, userId uint64, fil return err } -func (st *SQLiteStore) MoveUploadingInfos(ctx context.Context, userId uint64, uploadPath, itemPath string) error { +func (st *SQLiteStore) MoveUploadingInfos(ctx context.Context, infoId, userId uint64, uploadPath, itemPath string) error { st.Lock() defer st.Unlock() @@ -98,7 +98,7 @@ func (st *SQLiteStore) MoveUploadingInfos(ctx context.Context, userId uint64, up if err != nil { return err } - return st.addFileInfo(ctx, userId, itemPath, &db.FileInfo{ + return st.addFileInfo(ctx, infoId, userId, itemPath, &db.FileInfo{ Size: size, }) } diff --git a/src/db/rdb/sqlite/sqlite.go b/src/db/rdb/sqlite/init.go similarity index 93% rename from src/db/rdb/sqlite/sqlite.go rename to src/db/rdb/sqlite/init.go index d3c5605..60b08fe 100644 --- a/src/db/rdb/sqlite/sqlite.go +++ b/src/db/rdb/sqlite/init.go @@ -20,7 +20,7 @@ type SQLite struct { } func NewSQLite(dbPath string) (*SQLite, error) { - db, err := sql.Open("sqlite3", fmt.Sprintf("%s?_synchronous=FULL", dbPath)) + db, err := sql.Open("sqlite3", fmt.Sprintf("%s?_sync=FULL&_locking=EXCLUSIVE", dbPath)) if err != nil { return nil, err } @@ -148,6 +148,7 @@ func (st *SQLiteStore) InitFileTables(ctx context.Context) error { _, err := st.db.ExecContext( ctx, `create table if not exists t_file_info ( + id bigint not null, path varchar not null, user bigint not null, location varchar not null, @@ -157,13 +158,21 @@ func (st *SQLiteStore) InitFileTables(ctx context.Context) error { size bigint not null, share_id varchar not null, info varchar not null, - primary key(path) + primary key(id) )`, ) if err != nil { return err } + _, err = st.db.ExecContext( + ctx, + `create index if not exists t_file_path on t_file_info (path, location)`, + ) + if err != nil { + return err + } + _, err = st.db.ExecContext( ctx, `create index if not exists t_file_share on t_file_info (share_id, location)`, @@ -175,12 +184,13 @@ func (st *SQLiteStore) InitFileTables(ctx context.Context) error { _, err = st.db.ExecContext( ctx, `create table if not exists t_file_uploading ( + id bigint not null, real_path varchar not null, tmp_path varchar not null unique, user bigint not null, size bigint not null, uploaded bigint not null, - primary key(real_path) + primary key(id) )`, ) if err != nil { diff --git a/src/db/tests/files_test.go b/src/db/tests/files_test.go index 864e67b..a342224 100644 --- a/src/db/tests/files_test.go +++ b/src/db/tests/files_test.go @@ -53,7 +53,8 @@ func testSharingMethods(t *testing.T, store db.IDBQuickshare) { adminId := uint64(0) // add some of paths... - err = store.AddFileInfo(ctx, adminId, "admin/path1", &db.FileInfo{ + infoId := uint64(0) + err = store.AddFileInfo(ctx, adminId, infoId, "admin/path1", &db.FileInfo{ // Shared: false, // deprecated IsDir: false, Size: int64(0), @@ -65,8 +66,9 @@ func testSharingMethods(t *testing.T, store db.IDBQuickshare) { } // add sharings - for _, dirPath := range dirPaths { - err = store.AddSharing(ctx, adminId, dirPath) + for i, dirPath := range dirPaths { + infoId := uint64(i) + err = store.AddSharing(ctx, infoId, adminId, dirPath) if err != nil { t.Fatal(err) } @@ -80,7 +82,7 @@ func testSharingMethods(t *testing.T, store db.IDBQuickshare) { t.Fatalf("share size not match %d %d", len(dirToID), len(dirPaths)) } - for _, sharingDir := range dirPaths { + for i, sharingDir := range dirPaths { if _, ok := dirToID[sharingDir]; !ok { t.Fatalf("sharing(%s) not found", sharingDir) } @@ -96,6 +98,8 @@ func testSharingMethods(t *testing.T, store db.IDBQuickshare) { t.Fatal(err) } else if len(info.ShareID) != 7 { t.Fatalf("incorrect ShareID %s", info.ShareID) + } else if info.Id != uint64(i) { + t.Fatalf("incorrect file info ID %d", info.Id) } gotSharingDir, err := store.GetSharingDir(ctx, info.ShareID) @@ -178,6 +182,10 @@ func testUploadingMethods(t *testing.T, store db.IDBQuickshare) { }, } var err error + pathnames := []string{} + for pathname := range pathInfos { + pathnames = append(pathnames, pathname) + } adminId := uint64(0) ctx := context.TODO() @@ -187,10 +195,12 @@ func testUploadingMethods(t *testing.T, store db.IDBQuickshare) { usedSpaceAfterDeleting := int64(0) itemPaths := []string{} pathToTmpPath := map[string]string{} - for itemPath, info := range pathInfos { + for i, itemPath := range pathnames { + infoId := uint64(i) + info := pathInfos[itemPath] tmpPath := strings.ReplaceAll(itemPath, "origin", "uploads") pathToTmpPath[itemPath] = tmpPath - err = store.AddUploadInfos(ctx, adminId, tmpPath, itemPath, info) + err = store.AddUploadInfos(ctx, infoId, adminId, tmpPath, itemPath, info) if err != nil { t.Fatal(err) } @@ -299,6 +309,7 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) { pathInfos := map[string]*db.FileInfo{ "admin/origin/item1": &db.FileInfo{ // Shared: false, // deprecated + Id: 100, IsDir: false, Size: int64(7), ShareID: "", @@ -306,6 +317,7 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) { }, "admin/origin/item2": &db.FileInfo{ // Shared: false, // deprecated + Id: 101, IsDir: false, Size: int64(3), ShareID: "", @@ -313,6 +325,7 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) { }, "admin/origin/dir": &db.FileInfo{ // Shared: true, // deprecated + Id: 102, IsDir: true, Size: int64(0), ShareID: "mockedShareID", @@ -323,11 +336,12 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) { adminId := uint64(0) ctx := context.TODO() + // add infos usedSpace := int64(0) itemPaths := []string{} for itemPath, info := range pathInfos { - err = store.AddFileInfo(ctx, adminId, itemPath, info) + err = store.AddFileInfo(ctx, info.Id, adminId, itemPath, info) if err != nil { t.Fatal(err) } @@ -344,7 +358,8 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) { if info.ShareID != expected.ShareID || info.IsDir != expected.IsDir || info.Sha1 != expected.Sha1 || - info.Size != expected.Size { + info.Size != expected.Size || + info.Id != expected.Id { t.Fatalf("info not equaled (%v) (%v)", expected, info) } } @@ -356,15 +371,18 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) { } else if len(pathToInfo) != len(pathInfos) { t.Fatalf("list result size not match (%v) (%d)", pathToInfo, len(pathInfos)) } + for pathname, info := range pathInfos { gotInfo, ok := pathToInfo[pathname] + if !ok { t.Fatalf("path not found (%s)", pathname) } if info.ShareID != gotInfo.ShareID || info.IsDir != gotInfo.IsDir || info.Sha1 != gotInfo.Sha1 || - info.Size != gotInfo.Size { + info.Size != gotInfo.Size || + info.Id != gotInfo.Id { t.Fatalf("info not equaled (%v) (%v)", gotInfo, info) } } @@ -386,23 +404,40 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) { } // move paths - newPaths := []string{} + newPaths := map[string]string{} + newPathsList := []string{} for itemPath, info := range pathInfos { newItemPath := strings.ReplaceAll(itemPath, "origin", "new") err = store.MoveFileInfo(ctx, adminId, itemPath, newItemPath, info.IsDir) if err != nil { t.Fatal(err) } - newPaths = append(newPaths, newItemPath) + newPaths[newItemPath] = itemPath + newPathsList = append(newPathsList, newItemPath) } // list infos - pathToInfo, err = store.ListFileInfos(ctx, newPaths) + pathToInfo, err = store.ListFileInfos(ctx, newPathsList) if err != nil { t.Fatal(err) } else if len(pathToInfo) != len(pathInfos) { t.Fatalf("list result size not match (%v) (%d)", pathToInfo, len(pathInfos)) } + for newPathname, oldPathname := range newPaths { + info := pathInfos[oldPathname] + gotInfo, ok := pathToInfo[newPathname] + + if !ok { + t.Fatalf("path not found (%s)", newPathname) + } + if info.ShareID != gotInfo.ShareID || + info.IsDir != gotInfo.IsDir || + testSha1 != gotInfo.Sha1 || // sha is already updated + info.Size != gotInfo.Size || + info.Id != gotInfo.Id { + t.Fatalf("info not equaled (%v) (%v)", gotInfo, info) + } + } // check used space adminInfo, err := store.GetUser(ctx, adminId) @@ -413,7 +448,7 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) { } // del info - for _, itemPath := range newPaths { + for itemPath := range newPaths { err = store.DelFileInfo(ctx, adminId, itemPath) if err != nil { t.Fatal(err)