doesnm/quickshare
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(rdb): add id column for file info, sharing info tables

Browse source
This commit is contained in:
hexxa 2022-10-08 11:28:04 +08:00 • committed by Hexxa
parent 033c2bc316
commit 2b50c84273
8 changed files with 100 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/db/common.go
View file

@ -113,11 +113,12 @@ var (
)
type FileInfo struct {
IsDir bool `json:"isDir" yaml:"isDir"` // deprecated
Shared bool `json:"shared" yaml:"shared"` // deprecated
ShareID string `json:"shareID" yaml:"shareID"` // deprecated
Id uint64 `json:"id" yaml:"id"`
IsDir bool `json:"isDir" yaml:"isDir"`
Shared bool `json:"shared" yaml:"shared"`
ShareID string `json:"shareID" yaml:"shareID"`
Sha1 string `json:"sha1" yaml:"sha1"`
Size int64 `json:"size" yaml:"size"` // deprecated
Size int64 `json:"size" yaml:"size"`
}
type UserCfg struct {

8
src/db/interface.go → src/db/interfaces.go
View file

@ -70,7 +70,7 @@ type IFilesFunctions interface {
}
type IFileDB interface {
AddFileInfo(ctx context.Context, userId uint64, itemPath string, info *FileInfo) error
AddFileInfo(ctx context.Context, infoId, userId uint64, itemPath string, info *FileInfo) error
DelFileInfo(ctx context.Context, userId uint64, itemPath string) error
GetFileInfo(ctx context.Context, itemPath string) (*FileInfo, error)
SetSha1(ctx context.Context, itemPath, sign string) error
@ -78,9 +78,9 @@ type IFileDB interface {
ListFileInfos(ctx context.Context, itemPaths []string) (map[string]*FileInfo, error)
}
type IUploadDB interface {
AddUploadInfos(ctx context.Context, userId uint64, tmpPath, filePath string, info *FileInfo) error
AddUploadInfos(ctx context.Context, uploadId, userId uint64, tmpPath, filePath string, info *FileInfo) error
DelUploadingInfos(ctx context.Context, userId uint64, realPath string) error
MoveUploadingInfos(ctx context.Context, userId uint64, uploadPath, itemPath string) error
MoveUploadingInfos(ctx context.Context, uploadId, userId uint64, uploadPath, itemPath string) error
SetUploadInfo(ctx context.Context, user uint64, filePath string, newUploaded int64) error
GetUploadInfo(ctx context.Context, userId uint64, filePath string) (string, int64, int64, error)
ListUploadInfos(ctx context.Context, user uint64) ([]*UploadInfo, error)
@ -89,7 +89,7 @@ type IUploadDB interface {
type ISharingDB interface {
IsSharing(ctx context.Context, dirPath string) (bool, error)
GetSharingDir(ctx context.Context, hashID string) (string, error)
AddSharing(ctx context.Context, userId uint64, dirPath string) error
AddSharing(ctx context.Context, infoId, userId uint64, dirPath string) error
DelSharing(ctx context.Context, userId uint64, dirPath string) error
ListSharingsByLocation(ctx context.Context, location string) (map[string]string, error)
}

BIN
src/db/migration_tests/testdata/quickshare_v1.db vendored
View file

Binary file not shown.

25
src/db/rdb/sqlite/files.go
View file

@ -15,16 +15,18 @@ import (
func (st *SQLiteStore) getFileInfo(ctx context.Context, itemPath string) (*db.FileInfo, error) {
var infoStr string
fInfo := &db.FileInfo{}
var id uint64
var isDir bool
var size int64
var shareId string
err := st.db.QueryRowContext(
ctx,
`select is_dir, size, share_id, info
`select id, is_dir, size, share_id, info
from t_file_info
where path=?`,
itemPath,
).Scan(
&id,
&isDir,
&size,
&shareId,
@ -41,6 +43,7 @@ func (st *SQLiteStore) getFileInfo(ctx context.Context, itemPath string) (*db.Fi
if err != nil {
return nil, err
}
fInfo.Id = id
fInfo.IsDir = isDir
fInfo.Size = size
fInfo.ShareID = shareId
@ -69,7 +72,7 @@ func (st *SQLiteStore) ListFileInfos(ctx context.Context, itemPaths []string) (m
rows, err := st.db.QueryContext(
ctx,
fmt.Sprintf(
`select path, is_dir, size, share_id, info
`select id, path, is_dir, size, share_id, info
from t_file_info
where path in (%s)
`,
@ -85,11 +88,12 @@ func (st *SQLiteStore) ListFileInfos(ctx context.Context, itemPaths []string) (m
var fInfoStr, itemPath, shareId string
var isDir bool
var size int64
var id uint64
fInfos := map[string]*db.FileInfo{}
for rows.Next() {
fInfo := &db.FileInfo{}
err = rows.Scan(&itemPath, &isDir, &size, &shareId, &fInfoStr)
err = rows.Scan(&id, &itemPath, &isDir, &size, &shareId, &fInfoStr)
if err != nil {
return nil, err
}
@ -98,6 +102,7 @@ func (st *SQLiteStore) ListFileInfos(ctx context.Context, itemPaths []string) (m
if err != nil {
return nil, err
}
fInfo.Id = id
fInfo.IsDir = isDir
fInfo.Size = size
fInfo.ShareID = shareId
@ -111,7 +116,7 @@ func (st *SQLiteStore) ListFileInfos(ctx context.Context, itemPaths []string) (m
return fInfos, nil
}
func (st *SQLiteStore) addFileInfo(ctx context.Context, userId uint64, itemPath string, info *db.FileInfo) error {
func (st *SQLiteStore) addFileInfo(ctx context.Context, infoId, userId uint64, itemPath string, info *db.FileInfo) error {
infoStr, err := json.Marshal(info)
if err != nil {
return err
@ -126,24 +131,24 @@ func (st *SQLiteStore) addFileInfo(ctx context.Context, userId uint64, itemPath
_, err = st.db.ExecContext(
ctx,
`insert into t_file_info (
path, user, location, parent, name,
id, path, user, location, parent, name,
is_dir, size, share_id, info
)
values (
?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?,
?, ?, ?, ?
)`,
itemPath, userId, location, dirPath, itemName,
infoId, itemPath, userId, location, dirPath, itemName,
info.IsDir, info.Size, info.ShareID, infoStr,
)
return err
}
func (st *SQLiteStore) AddFileInfo(ctx context.Context, userId uint64, itemPath string, info *db.FileInfo) error {
func (st *SQLiteStore) AddFileInfo(ctx context.Context, infoId, userId uint64, itemPath string, info *db.FileInfo) error {
st.Lock()
defer st.Unlock()
err := st.addFileInfo(ctx, userId, itemPath, info)
err := st.addFileInfo(ctx, infoId, userId, itemPath, info)
if err != nil {
return err
}
@ -260,7 +265,7 @@ func (st *SQLiteStore) MoveFileInfo(ctx context.Context, userId uint64, oldPath,
if err != nil {
return err
}
return st.addFileInfo(ctx, userId, newPath, info)
return st.addFileInfo(ctx, info.Id, userId, newPath, info)
}
func getLocation(itemPath string) (string, error) {

11
src/db/rdb/sqlite/files_sharings.go
View file

@ -78,7 +78,7 @@ func (st *SQLiteStore) GetSharingDir(ctx context.Context, hashID string) (string
return sharedPath, nil
}
func (st *SQLiteStore) AddSharing(ctx context.Context, userId uint64, dirPath string) error {
func (st *SQLiteStore) AddSharing(ctx context.Context, infoId, userId uint64, dirPath string) error {
st.Lock()
defer st.Unlock()
@ -109,14 +109,17 @@ func (st *SQLiteStore) AddSharing(ctx context.Context, userId uint64, dirPath st
_, err = st.db.ExecContext(
ctx,
`insert into t_file_info (
path, user, location, parent, name,
id, path, user,
location, parent, name,
is_dir, size, share_id, info
)
values (
?, ?, ?, ?, ?,
?, ?, ?,
?, ?, ?,
?, ?, ?, ?
)`,
dirPath, userId, location, parentPath, name,
infoId, dirPath, userId,
location, parentPath, name,
true, 0, shareID, infoStr,
)
return err

16
src/db/rdb/sqlite/files_uploadings.go
View file

@ -8,21 +8,21 @@ import (
"github.com/ihexxa/quickshare/src/db"
)
func (st *SQLiteStore) addUploadInfoOnly(ctx context.Context, userId uint64, tmpPath, filePath string, fileSize int64) error {
func (st *SQLiteStore) addUploadInfoOnly(ctx context.Context, uploadId, userId uint64, tmpPath, filePath string, fileSize int64) error {
_, err := st.db.ExecContext(
ctx,
`insert into t_file_uploading (
real_path, tmp_path, user, size, uploaded
id, real_path, tmp_path, user, size, uploaded
)
values (
?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?
)`,
filePath, tmpPath, userId, fileSize, 0,
uploadId, filePath, tmpPath, userId, fileSize, 0,
)
return err
}
func (st *SQLiteStore) AddUploadInfos(ctx context.Context, userId uint64, tmpPath, filePath string, info *db.FileInfo) error {
func (st *SQLiteStore) AddUploadInfos(ctx context.Context, uploadId, userId uint64, tmpPath, filePath string, info *db.FileInfo) error {
st.Lock()
defer st.Unlock()
@ -46,7 +46,7 @@ func (st *SQLiteStore) AddUploadInfos(ctx context.Context, userId uint64, tmpPat
return err
}
return st.addUploadInfoOnly(ctx, userId, tmpPath, filePath, info.Size)
return st.addUploadInfoOnly(ctx, uploadId, userId, tmpPath, filePath, info.Size)
}
func (st *SQLiteStore) DelUploadingInfos(ctx context.Context, userId uint64, realPath string) error {
@ -86,7 +86,7 @@ func (st *SQLiteStore) delUploadInfoOnly(ctx context.Context, userId uint64, fil
return err
}
func (st *SQLiteStore) MoveUploadingInfos(ctx context.Context, userId uint64, uploadPath, itemPath string) error {
func (st *SQLiteStore) MoveUploadingInfos(ctx context.Context, infoId, userId uint64, uploadPath, itemPath string) error {
st.Lock()
defer st.Unlock()
@ -98,7 +98,7 @@ func (st *SQLiteStore) MoveUploadingInfos(ctx context.Context, userId uint64, up
if err != nil {
return err
}
return st.addFileInfo(ctx, userId, itemPath, &db.FileInfo{
return st.addFileInfo(ctx, infoId, userId, itemPath, &db.FileInfo{
Size: size,
})
}

16
src/db/rdb/sqlite/sqlite.go → src/db/rdb/sqlite/init.go
View file

@ -20,7 +20,7 @@ type SQLite struct {
}
func NewSQLite(dbPath string) (*SQLite, error) {
db, err := sql.Open("sqlite3", fmt.Sprintf("%s?_synchronous=FULL", dbPath))
db, err := sql.Open("sqlite3", fmt.Sprintf("%s?_sync=FULL&_locking=EXCLUSIVE", dbPath))
if err != nil {
return nil, err
}
@ -148,6 +148,7 @@ func (st *SQLiteStore) InitFileTables(ctx context.Context) error {
_, err := st.db.ExecContext(
ctx,
`create table if not exists t_file_info (
id bigint not null,
path varchar not null,
user bigint not null,
location varchar not null,
@ -157,13 +158,21 @@ func (st *SQLiteStore) InitFileTables(ctx context.Context) error {
size bigint not null,
share_id varchar not null,
info varchar not null,
primary key(path)
primary key(id)
)`,
)
if err != nil {
return err
}
_, err = st.db.ExecContext(
ctx,
`create index if not exists t_file_path on t_file_info (path, location)`,
)
if err != nil {
return err
}
_, err = st.db.ExecContext(
ctx,
`create index if not exists t_file_share on t_file_info (share_id, location)`,
@ -175,12 +184,13 @@ func (st *SQLiteStore) InitFileTables(ctx context.Context) error {
_, err = st.db.ExecContext(
ctx,
`create table if not exists t_file_uploading (
id bigint not null,
real_path varchar not null,
tmp_path varchar not null unique,
user bigint not null,
size bigint not null,
uploaded bigint not null,
primary key(real_path)
primary key(id)
)`,
)
if err != nil {

61
src/db/tests/files_test.go
View file

@ -53,7 +53,8 @@ func testSharingMethods(t *testing.T, store db.IDBQuickshare) {
adminId := uint64(0)
// add some of paths...
err = store.AddFileInfo(ctx, adminId, "admin/path1", &db.FileInfo{
infoId := uint64(0)
err = store.AddFileInfo(ctx, adminId, infoId, "admin/path1", &db.FileInfo{
// Shared: false, // deprecated
IsDir: false,
Size: int64(0),
@ -65,8 +66,9 @@ func testSharingMethods(t *testing.T, store db.IDBQuickshare) {
}
// add sharings
for _, dirPath := range dirPaths {
err = store.AddSharing(ctx, adminId, dirPath)
for i, dirPath := range dirPaths {
infoId := uint64(i)
err = store.AddSharing(ctx, infoId, adminId, dirPath)
if err != nil {
t.Fatal(err)
}
@ -80,7 +82,7 @@ func testSharingMethods(t *testing.T, store db.IDBQuickshare) {
t.Fatalf("share size not match %d %d", len(dirToID), len(dirPaths))
}
for _, sharingDir := range dirPaths {
for i, sharingDir := range dirPaths {
if _, ok := dirToID[sharingDir]; !ok {
t.Fatalf("sharing(%s) not found", sharingDir)
}
@ -96,6 +98,8 @@ func testSharingMethods(t *testing.T, store db.IDBQuickshare) {
t.Fatal(err)
} else if len(info.ShareID) != 7 {
t.Fatalf("incorrect ShareID %s", info.ShareID)
} else if info.Id != uint64(i) {
t.Fatalf("incorrect file info ID %d", info.Id)
}
gotSharingDir, err := store.GetSharingDir(ctx, info.ShareID)
@ -178,6 +182,10 @@ func testUploadingMethods(t *testing.T, store db.IDBQuickshare) {
},
}
var err error
pathnames := []string{}
for pathname := range pathInfos {
pathnames = append(pathnames, pathname)
}
adminId := uint64(0)
ctx := context.TODO()
@ -187,10 +195,12 @@ func testUploadingMethods(t *testing.T, store db.IDBQuickshare) {
usedSpaceAfterDeleting := int64(0)
itemPaths := []string{}
pathToTmpPath := map[string]string{}
for itemPath, info := range pathInfos {
for i, itemPath := range pathnames {
infoId := uint64(i)
info := pathInfos[itemPath]
tmpPath := strings.ReplaceAll(itemPath, "origin", "uploads")
pathToTmpPath[itemPath] = tmpPath
err = store.AddUploadInfos(ctx, adminId, tmpPath, itemPath, info)
err = store.AddUploadInfos(ctx, infoId, adminId, tmpPath, itemPath, info)
if err != nil {
t.Fatal(err)
}
@ -299,6 +309,7 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) {
pathInfos := map[string]*db.FileInfo{
"admin/origin/item1": &db.FileInfo{
// Shared: false, // deprecated
Id: 100,
IsDir: false,
Size: int64(7),
ShareID: "",
@ -306,6 +317,7 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) {
},
"admin/origin/item2": &db.FileInfo{
// Shared: false, // deprecated
Id: 101,
IsDir: false,
Size: int64(3),
ShareID: "",
@ -313,6 +325,7 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) {
},
"admin/origin/dir": &db.FileInfo{
// Shared: true, // deprecated
Id: 102,
IsDir: true,
Size: int64(0),
ShareID: "mockedShareID",
@ -323,11 +336,12 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) {
adminId := uint64(0)
ctx := context.TODO()
// add infos
usedSpace := int64(0)
itemPaths := []string{}
for itemPath, info := range pathInfos {
err = store.AddFileInfo(ctx, adminId, itemPath, info)
err = store.AddFileInfo(ctx, info.Id, adminId, itemPath, info)
if err != nil {
t.Fatal(err)
}
@ -344,7 +358,8 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) {
if info.ShareID != expected.ShareID ||
info.IsDir != expected.IsDir ||
info.Sha1 != expected.Sha1 ||
info.Size != expected.Size {
info.Size != expected.Size ||
info.Id != expected.Id {
t.Fatalf("info not equaled (%v) (%v)", expected, info)
}
}
@ -356,15 +371,18 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) {
} else if len(pathToInfo) != len(pathInfos) {
t.Fatalf("list result size not match (%v) (%d)", pathToInfo, len(pathInfos))
}
for pathname, info := range pathInfos {
gotInfo, ok := pathToInfo[pathname]
if !ok {
t.Fatalf("path not found (%s)", pathname)
}
if info.ShareID != gotInfo.ShareID ||
info.IsDir != gotInfo.IsDir ||
info.Sha1 != gotInfo.Sha1 ||
info.Size != gotInfo.Size {
info.Size != gotInfo.Size ||
info.Id != gotInfo.Id {
t.Fatalf("info not equaled (%v) (%v)", gotInfo, info)
}
}
@ -386,23 +404,40 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) {
}
// move paths
newPaths := []string{}
newPaths := map[string]string{}
newPathsList := []string{}
for itemPath, info := range pathInfos {
newItemPath := strings.ReplaceAll(itemPath, "origin", "new")
err = store.MoveFileInfo(ctx, adminId, itemPath, newItemPath, info.IsDir)
if err != nil {
t.Fatal(err)
}
newPaths = append(newPaths, newItemPath)
newPaths[newItemPath] = itemPath
newPathsList = append(newPathsList, newItemPath)
}
// list infos
pathToInfo, err = store.ListFileInfos(ctx, newPaths)
pathToInfo, err = store.ListFileInfos(ctx, newPathsList)
if err != nil {
t.Fatal(err)
} else if len(pathToInfo) != len(pathInfos) {
t.Fatalf("list result size not match (%v) (%d)", pathToInfo, len(pathInfos))
}
for newPathname, oldPathname := range newPaths {
info := pathInfos[oldPathname]
gotInfo, ok := pathToInfo[newPathname]
if !ok {
t.Fatalf("path not found (%s)", newPathname)
}
if info.ShareID != gotInfo.ShareID ||
info.IsDir != gotInfo.IsDir ||
testSha1 != gotInfo.Sha1 || // sha is already updated
info.Size != gotInfo.Size ||
info.Id != gotInfo.Id {
t.Fatalf("info not equaled (%v) (%v)", gotInfo, info)
}
}
// check used space
adminInfo, err := store.GetUser(ctx, adminId)
@ -413,7 +448,7 @@ func testFileInfoMethods(t *testing.T, store db.IDBQuickshare) {
}
// del info
for _, itemPath := range newPaths {
for itemPath := range newPaths {
err = store.DelFileInfo(ctx, adminId, itemPath)
if err != nil {
t.Fatal(err)