mirror of
https://github.com/pocket-id/pocket-id.git
synced 2026-02-11 18:44:01 +00:00
feat: add database storage backend (#1091)
Co-authored-by: Elias Schneider <login@eliasschneider.com>
This commit is contained in:
Alessandro (Ale) Segala
committed by
GitHub
GitHub
parent
12125713a2
commit
29a1d3b778
226
backend/internal/storage/database.go
Normal file
226
backend/internal/storage/database.go
Normal file
@@ -0,0 +1,226 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pocket-id/pocket-id/backend/internal/model"
|
||||
datatype "github.com/pocket-id/pocket-id/backend/internal/model/types"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
var TypeDatabase = "database"
|
||||
|
||||
type databaseStorage struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
// NewDatabaseStorage creates a new database storage provider
|
||||
func NewDatabaseStorage(db *gorm.DB) (FileStorage, error) {
|
||||
if db == nil {
|
||||
return nil, errors.New("database connection is required")
|
||||
}
|
||||
return &databaseStorage{db: db}, nil
|
||||
}
|
||||
|
||||
func (s *databaseStorage) Type() string {
|
||||
return TypeDatabase
|
||||
}
|
||||
|
||||
func (s *databaseStorage) Save(ctx context.Context, relativePath string, data io.Reader) error {
|
||||
// Normalize the path
|
||||
relativePath = filepath.ToSlash(filepath.Clean(relativePath))
|
||||
|
||||
// Read all data into memory
|
||||
b, err := io.ReadAll(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read data: %w", err)
|
||||
}
|
||||
|
||||
now := datatype.DateTime(time.Now())
|
||||
storage := model.Storage{
|
||||
Path: relativePath,
|
||||
Data: b,
|
||||
Size: int64(len(b)),
|
||||
ModTime: now,
|
||||
CreatedAt: now,
|
||||
}
|
||||
|
||||
// Use upsert: insert or update on conflict
|
||||
result := s.db.
|
||||
WithContext(ctx).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "path"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"data", "size", "mod_time"}),
|
||||
}).
|
||||
Create(&storage)
|
||||
|
||||
if result.Error != nil {
|
||||
return fmt.Errorf("failed to save file to database: %w", result.Error)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *databaseStorage) Open(ctx context.Context, relativePath string) (io.ReadCloser, int64, error) {
|
||||
relativePath = filepath.ToSlash(filepath.Clean(relativePath))
|
||||
|
||||
var storage model.Storage
|
||||
result := s.db.
|
||||
WithContext(ctx).
|
||||
Where("path = ?", relativePath).
|
||||
First(&storage)
|
||||
|
||||
if result.Error != nil {
|
||||
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
|
||||
return nil, 0, os.ErrNotExist
|
||||
}
|
||||
return nil, 0, fmt.Errorf("failed to read file from database: %w", result.Error)
|
||||
}
|
||||
|
||||
reader := io.NopCloser(bytes.NewReader(storage.Data))
|
||||
return reader, storage.Size, nil
|
||||
}
|
||||
|
||||
func (s *databaseStorage) Delete(ctx context.Context, relativePath string) error {
|
||||
relativePath = filepath.ToSlash(filepath.Clean(relativePath))
|
||||
|
||||
result := s.db.
|
||||
WithContext(ctx).
|
||||
Where("path = ?", relativePath).
|
||||
Delete(&model.Storage{})
|
||||
if result.Error != nil {
|
||||
return fmt.Errorf("failed to delete file from database: %w", result.Error)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *databaseStorage) DeleteAll(ctx context.Context, prefix string) error {
|
||||
prefix = filepath.ToSlash(filepath.Clean(prefix))
|
||||
|
||||
// If empty prefix, delete all
|
||||
if isRootPath(prefix) {
|
||||
result := s.db.
|
||||
WithContext(ctx).
|
||||
Where("1 = 1"). // Delete everything
|
||||
Delete(&model.Storage{})
|
||||
if result.Error != nil {
|
||||
return fmt.Errorf("failed to delete all files from database: %w", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ensure prefix ends with / for proper prefix matching
|
||||
if !strings.HasSuffix(prefix, "/") {
|
||||
prefix += "/"
|
||||
}
|
||||
|
||||
query := s.db.WithContext(ctx)
|
||||
query = addPathPrefixClause(s.db.Name(), query, prefix)
|
||||
result := query.Delete(&model.Storage{})
|
||||
if result.Error != nil {
|
||||
return fmt.Errorf("failed to delete files with prefix '%s' from database: %w", prefix, result.Error)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *databaseStorage) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||
prefix = filepath.ToSlash(filepath.Clean(prefix))
|
||||
|
||||
var storageItems []model.Storage
|
||||
query := s.db.WithContext(ctx)
|
||||
|
||||
if !isRootPath(prefix) {
|
||||
// Ensure prefix matching
|
||||
if !strings.HasSuffix(prefix, "/") {
|
||||
prefix += "/"
|
||||
}
|
||||
query = addPathPrefixClause(s.db.Name(), query, prefix)
|
||||
}
|
||||
|
||||
result := query.
|
||||
Select("path", "size", "mod_time").
|
||||
Find(&storageItems)
|
||||
if result.Error != nil {
|
||||
return nil, fmt.Errorf("failed to list files from database: %w", result.Error)
|
||||
}
|
||||
|
||||
objects := make([]ObjectInfo, 0, len(storageItems))
|
||||
for _, item := range storageItems {
|
||||
// Filter out directory-like paths (those that contain additional slashes after the prefix)
|
||||
relativePath := strings.TrimPrefix(item.Path, prefix)
|
||||
if strings.ContainsRune(relativePath, '/') {
|
||||
continue
|
||||
}
|
||||
|
||||
objects = append(objects, ObjectInfo{
|
||||
Path: item.Path,
|
||||
Size: item.Size,
|
||||
ModTime: time.Time(item.ModTime),
|
||||
})
|
||||
}
|
||||
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
func (s *databaseStorage) Walk(ctx context.Context, root string, fn func(ObjectInfo) error) error {
|
||||
root = filepath.ToSlash(filepath.Clean(root))
|
||||
|
||||
var storageItems []model.Storage
|
||||
query := s.db.WithContext(ctx)
|
||||
|
||||
if !isRootPath(root) {
|
||||
// Ensure root matching
|
||||
if !strings.HasSuffix(root, "/") {
|
||||
root += "/"
|
||||
}
|
||||
query = addPathPrefixClause(s.db.Name(), query, root)
|
||||
}
|
||||
|
||||
result := query.
|
||||
Select("path", "size", "mod_time").
|
||||
Find(&storageItems)
|
||||
if result.Error != nil {
|
||||
return fmt.Errorf("failed to walk files from database: %w", result.Error)
|
||||
}
|
||||
|
||||
for _, item := range storageItems {
|
||||
err := fn(ObjectInfo{
|
||||
Path: item.Path,
|
||||
Size: item.Size,
|
||||
ModTime: time.Time(item.ModTime),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isRootPath(path string) bool {
|
||||
return path == "" || path == "/" || path == "."
|
||||
}
|
||||
|
||||
func addPathPrefixClause(dialect string, query *gorm.DB, prefix string) *gorm.DB {
|
||||
// In SQLite, we use "GLOB" which can use the index
|
||||
switch dialect {
|
||||
case "sqlite":
|
||||
return query.Where("path GLOB ?", prefix+"*")
|
||||
case "postgres":
|
||||
return query.Where("path LIKE ?", prefix+"%")
|
||||
default:
|
||||
// Indicates a development-time error
|
||||
panic(fmt.Errorf("unsupported database dialect: %s", dialect))
|
||||
}
|
||||
}
|
||||
148
backend/internal/storage/database_test.go
Normal file
148
backend/internal/storage/database_test.go
Normal file
@@ -0,0 +1,148 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
testingutil "github.com/pocket-id/pocket-id/backend/internal/utils/testing"
|
||||
)
|
||||
|
||||
func TestDatabaseStorageOperations(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
db := testingutil.NewDatabaseForTest(t)
|
||||
store, err := NewDatabaseStorage(db)
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("type should be database", func(t *testing.T) {
|
||||
assert.Equal(t, TypeDatabase, store.Type())
|
||||
})
|
||||
|
||||
t.Run("save, open and list files", func(t *testing.T) {
|
||||
err := store.Save(ctx, "images/logo.png", bytes.NewBufferString("logo-data"))
|
||||
require.NoError(t, err)
|
||||
|
||||
reader, size, err := store.Open(ctx, "images/logo.png")
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
|
||||
contents, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []byte("logo-data"), contents)
|
||||
assert.Equal(t, int64(len(contents)), size)
|
||||
|
||||
err = store.Save(ctx, "images/nested/child.txt", bytes.NewBufferString("child"))
|
||||
require.NoError(t, err)
|
||||
|
||||
files, err := store.List(ctx, "images")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, files, 1)
|
||||
assert.Equal(t, "images/logo.png", files[0].Path)
|
||||
assert.Equal(t, int64(len("logo-data")), files[0].Size)
|
||||
})
|
||||
|
||||
t.Run("save should update existing file", func(t *testing.T) {
|
||||
err := store.Save(ctx, "test/update.txt", bytes.NewBufferString("original"))
|
||||
require.NoError(t, err)
|
||||
|
||||
err = store.Save(ctx, "test/update.txt", bytes.NewBufferString("updated"))
|
||||
require.NoError(t, err)
|
||||
|
||||
reader, size, err := store.Open(ctx, "test/update.txt")
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
|
||||
contents, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []byte("updated"), contents)
|
||||
assert.Equal(t, int64(len("updated")), size)
|
||||
})
|
||||
|
||||
t.Run("delete files individually", func(t *testing.T) {
|
||||
err := store.Save(ctx, "images/delete-me.txt", bytes.NewBufferString("temp"))
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, store.Delete(ctx, "images/delete-me.txt"))
|
||||
_, _, err = store.Open(ctx, "images/delete-me.txt")
|
||||
require.Error(t, err)
|
||||
assert.True(t, IsNotExist(err))
|
||||
})
|
||||
|
||||
t.Run("delete missing file should not error", func(t *testing.T) {
|
||||
require.NoError(t, store.Delete(ctx, "images/missing.txt"))
|
||||
})
|
||||
|
||||
t.Run("delete all files", func(t *testing.T) {
|
||||
require.NoError(t, store.Save(ctx, "cleanup/a.txt", bytes.NewBufferString("a")))
|
||||
require.NoError(t, store.Save(ctx, "cleanup/b.txt", bytes.NewBufferString("b")))
|
||||
require.NoError(t, store.Save(ctx, "cleanup/nested/c.txt", bytes.NewBufferString("c")))
|
||||
require.NoError(t, store.DeleteAll(ctx, "/"))
|
||||
|
||||
_, _, err := store.Open(ctx, "cleanup/a.txt")
|
||||
require.Error(t, err)
|
||||
assert.True(t, IsNotExist(err))
|
||||
|
||||
_, _, err = store.Open(ctx, "cleanup/b.txt")
|
||||
require.Error(t, err)
|
||||
assert.True(t, IsNotExist(err))
|
||||
|
||||
_, _, err = store.Open(ctx, "cleanup/nested/c.txt")
|
||||
require.Error(t, err)
|
||||
assert.True(t, IsNotExist(err))
|
||||
})
|
||||
|
||||
t.Run("delete all files under a prefix", func(t *testing.T) {
|
||||
require.NoError(t, store.Save(ctx, "cleanup/a.txt", bytes.NewBufferString("a")))
|
||||
require.NoError(t, store.Save(ctx, "cleanup/b.txt", bytes.NewBufferString("b")))
|
||||
require.NoError(t, store.Save(ctx, "cleanup/nested/c.txt", bytes.NewBufferString("c")))
|
||||
require.NoError(t, store.DeleteAll(ctx, "cleanup"))
|
||||
|
||||
_, _, err := store.Open(ctx, "cleanup/a.txt")
|
||||
require.Error(t, err)
|
||||
assert.True(t, IsNotExist(err))
|
||||
|
||||
_, _, err = store.Open(ctx, "cleanup/b.txt")
|
||||
require.Error(t, err)
|
||||
assert.True(t, IsNotExist(err))
|
||||
|
||||
_, _, err = store.Open(ctx, "cleanup/nested/c.txt")
|
||||
require.Error(t, err)
|
||||
assert.True(t, IsNotExist(err))
|
||||
})
|
||||
|
||||
t.Run("walk files", func(t *testing.T) {
|
||||
require.NoError(t, store.Save(ctx, "walk/file1.txt", bytes.NewBufferString("1")))
|
||||
require.NoError(t, store.Save(ctx, "walk/file2.txt", bytes.NewBufferString("2")))
|
||||
require.NoError(t, store.Save(ctx, "walk/nested/file3.txt", bytes.NewBufferString("3")))
|
||||
|
||||
var paths []string
|
||||
err := store.Walk(ctx, "walk", func(info ObjectInfo) error {
|
||||
paths = append(paths, info.Path)
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, paths, 3)
|
||||
assert.Contains(t, paths, "walk/file1.txt")
|
||||
assert.Contains(t, paths, "walk/file2.txt")
|
||||
assert.Contains(t, paths, "walk/nested/file3.txt")
|
||||
})
|
||||
}
|
||||
|
||||
func TestNewDatabaseStorage(t *testing.T) {
|
||||
t.Run("should return error with nil database", func(t *testing.T) {
|
||||
_, err := NewDatabaseStorage(nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "database connection is required")
|
||||
})
|
||||
|
||||
t.Run("should create storage with valid database", func(t *testing.T) {
|
||||
db := testingutil.NewDatabaseForTest(t)
|
||||
store, err := NewDatabaseStorage(db)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, store)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user