feat: add support for S3 storage backend (#1080)

Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com>

backend/internal/bootstrap/app_images_bootstrap.go

@@ -2,68 +2,76 @@ package bootstrap
 
 import (
 	"bytes"
+	"context"
+	"crypto/sha256"
 	"encoding/hex"
+	"errors"
 	"fmt"
+	"io"
+	"io/fs"
 	"log/slog"
 	"os"
 	"path"
 
-	"github.com/pocket-id/pocket-id/backend/internal/common"
+	"github.com/pocket-id/pocket-id/backend/internal/storage"
 	"github.com/pocket-id/pocket-id/backend/internal/utils"
 	"github.com/pocket-id/pocket-id/backend/resources"
 )
 
-// initApplicationImages copies the images from the images directory to the application-images directory
+// initApplicationImages copies the images from the embedded directory to the storage backend
 // and returns a map containing the detected file extensions in the application-images directory.
-func initApplicationImages() (map[string]string, error) {
+func initApplicationImages(ctx context.Context, fileStorage storage.FileStorage) (map[string]string, error) {
 	// Previous versions of images
 	// If these are found, they are deleted
 	legacyImageHashes := imageHashMap{
 		"background.jpg": mustDecodeHex("138d510030ed845d1d74de34658acabff562d306476454369a60ab8ade31933f"),
 	}
 
-	dirPath := common.EnvConfig.UploadPath + "/application-images"
-
 	sourceFiles, err := resources.FS.ReadDir("images")
 	if err != nil && !os.IsNotExist(err) {
 		return nil, fmt.Errorf("failed to read directory: %w", err)
 	}
 
-	destinationFiles, err := os.ReadDir(dirPath)
-	if err != nil && !os.IsNotExist(err) {
-		return nil, fmt.Errorf("failed to read directory: %w", err)
+	destinationFiles, err := fileStorage.List(ctx, "application-images")
+	if err != nil {
+		if storage.IsNotExist(err) {
+			destinationFiles = []storage.ObjectInfo{}
+		} else {
+			return nil, fmt.Errorf("failed to list application images: %w", err)
+		}
+
 	}
 	dstNameToExt := make(map[string]string, len(destinationFiles))
 	for _, f := range destinationFiles {
-		if f.IsDir() {
-			continue
-		}
-		name := f.Name()
-		nameWithoutExt, ext := utils.SplitFileName(name)
-		destFilePath := path.Join(dirPath, name)
-
 		// Skip directories
-		if f.IsDir() {
+		_, name := path.Split(f.Path)
+		if name == "" {
 			continue
 		}
-
-		h, err := utils.CreateSha256FileHash(destFilePath)
+		nameWithoutExt, ext := utils.SplitFileName(name)
+		reader, _, err := fileStorage.Open(ctx, f.Path)
 		if err != nil {
-			slog.Warn("Failed to get hash for file", slog.String("name", name), slog.Any("error", err))
+			if errors.Is(err, fs.ErrNotExist) {
+				continue
+			}
+			slog.Warn("Failed to open application image for hashing", slog.String("name", name), slog.Any("error", err))
+			continue
+		}
+		hash, err := hashStream(reader)
+		reader.Close()
+		if err != nil {
+			slog.Warn("Failed to hash application image", slog.String("name", name), slog.Any("error", err))
 			continue
 		}
 
 		// Check if the file is a legacy one - if so, delete it
-		if legacyImageHashes.Contains(h) {
+		if legacyImageHashes.Contains(hash) {
 			slog.Info("Found legacy application image that will be removed", slog.String("name", name))
-			err = os.Remove(destFilePath)
-			if err != nil {
+			if err := fileStorage.Delete(ctx, f.Path); err != nil {
 				return nil, fmt.Errorf("failed to remove legacy file '%s': %w", name, err)
 			}
 			continue
 		}
-
-		// Track existing files
 		dstNameToExt[nameWithoutExt] = ext
 	}
 
@@ -76,21 +84,21 @@ func initApplicationImages() (map[string]string, error) {
 		name := sourceFile.Name()
 		nameWithoutExt, ext := utils.SplitFileName(name)
 		srcFilePath := path.Join("images", name)
-		destFilePath := path.Join(dirPath, name)
 
-		// Skip if there's already an image at the path
-		// We do not check the extension because users could have uploaded a different one
 		if _, exists := dstNameToExt[nameWithoutExt]; exists {
 			continue
 		}
 
 		slog.Info("Writing new application image", slog.String("name", name))
-		err := utils.CopyEmbeddedFileToDisk(srcFilePath, destFilePath)
+		srcFile, err := resources.FS.Open(srcFilePath)
 		if err != nil {
-			return nil, fmt.Errorf("failed to copy file: %w", err)
+			return nil, fmt.Errorf("failed to open embedded file '%s': %w", name, err)
 		}
-
-		// Track the newly copied file so it can be included in the extensions map later
+		if err := fileStorage.Save(ctx, path.Join("application-images", name), srcFile); err != nil {
+			srcFile.Close()
+			return nil, fmt.Errorf("failed to store application image '%s': %w", name, err)
+		}
+		srcFile.Close()
 		dstNameToExt[nameWithoutExt] = ext
 	}
 
@@ -118,3 +126,11 @@ func mustDecodeHex(str string) []byte {
 	}
 	return b
 }
+
+func hashStream(r io.Reader) ([]byte, error) {
+	h := sha256.New()
+	if _, err := io.Copy(h, r); err != nil {
+		return nil, err
+	}
+	return h.Sum(nil), nil
+}

backend/internal/bootstrap/bootstrap.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/pocket-id/pocket-id/backend/internal/common"
 	"github.com/pocket-id/pocket-id/backend/internal/job"
+	"github.com/pocket-id/pocket-id/backend/internal/storage"
 	"github.com/pocket-id/pocket-id/backend/internal/utils"
 )
 
@@ -21,7 +22,31 @@ func Bootstrap(ctx context.Context) error {
 	}
 	slog.InfoContext(ctx, "Pocket ID is starting")
 
-	imageExtensions, err := initApplicationImages()
+	// Initialize the file storage backend
+	var fileStorage storage.FileStorage
+
+	switch common.EnvConfig.FileBackend {
+	case storage.TypeFileSystem:
+		fileStorage, err = storage.NewFilesystemStorage(common.EnvConfig.UploadPath)
+	case storage.TypeS3:
+		s3Cfg := storage.S3Config{
+			Bucket:          common.EnvConfig.S3Bucket,
+			Region:          common.EnvConfig.S3Region,
+			Endpoint:        common.EnvConfig.S3Endpoint,
+			AccessKeyID:     common.EnvConfig.S3AccessKeyID,
+			SecretAccessKey: common.EnvConfig.S3SecretAccessKey,
+			ForcePathStyle:  common.EnvConfig.S3ForcePathStyle,
+			Root:            common.EnvConfig.UploadPath,
+		}
+		fileStorage, err = storage.NewS3Storage(ctx, s3Cfg)
+	default:
+		err = fmt.Errorf("unknown file storage backend: %s", common.EnvConfig.FileBackend)
+	}
+	if err != nil {
+		return fmt.Errorf("failed to initialize file storage: %w", err)
+	}
+
+	imageExtensions, err := initApplicationImages(ctx, fileStorage)
 	if err != nil {
 		return fmt.Errorf("failed to initialize application images: %w", err)
 	}
@@ -33,7 +58,7 @@ func Bootstrap(ctx context.Context) error {
 	}
 
 	// Create all services
-	svc, err := initServices(ctx, db, httpClient, imageExtensions)
+	svc, err := initServices(ctx, db, httpClient, imageExtensions, fileStorage)
 	if err != nil {
 		return fmt.Errorf("failed to initialize services: %w", err)
 	}

backend/internal/bootstrap/e2etest_router_bootstrap.go

@@ -17,7 +17,7 @@ import (
 func init() {
 	registerTestControllers = []func(apiGroup *gin.RouterGroup, db *gorm.DB, svc *services){
 		func(apiGroup *gin.RouterGroup, db *gorm.DB, svc *services) {
-			testService, err := service.NewTestService(db, svc.appConfigService, svc.jwtService, svc.ldapService)
+			testService, err := service.NewTestService(db, svc.appConfigService, svc.jwtService, svc.ldapService, svc.fileStorage)
 			if err != nil {
 				slog.Error("Failed to initialize test service", slog.Any("error", err))
 				os.Exit(1)

backend/internal/bootstrap/scheduler_bootstrap.go

@@ -23,7 +23,7 @@ func registerScheduledJobs(ctx context.Context, db *gorm.DB, svc *services, http
 	if err != nil {
 		return fmt.Errorf("failed to register DB cleanup jobs in scheduler: %w", err)
 	}
-	err = scheduler.RegisterFileCleanupJobs(ctx, db)
+	err = scheduler.RegisterFileCleanupJobs(ctx, db, svc.fileStorage)
 	if err != nil {
 		return fmt.Errorf("failed to register file cleanup jobs in scheduler: %w", err)
 	}

backend/internal/bootstrap/services_bootstrap.go

@@ -8,6 +8,7 @@ import (
 	"gorm.io/gorm"
 
 	"github.com/pocket-id/pocket-id/backend/internal/service"
+	"github.com/pocket-id/pocket-id/backend/internal/storage"
 )
 
 type services struct {
@@ -25,10 +26,11 @@ type services struct {
 	ldapService        *service.LdapService
 	apiKeyService      *service.ApiKeyService
 	versionService     *service.VersionService
+	fileStorage        storage.FileStorage
 }
 
 // Initializes all services
-func initServices(ctx context.Context, db *gorm.DB, httpClient *http.Client, imageExtensions map[string]string) (svc *services, err error) {
+func initServices(ctx context.Context, db *gorm.DB, httpClient *http.Client, imageExtensions map[string]string, fileStorage storage.FileStorage) (svc *services, err error) {
 	svc = &services{}
 
 	svc.appConfigService, err = service.NewAppConfigService(ctx, db)
@@ -36,7 +38,8 @@ func initServices(ctx context.Context, db *gorm.DB, httpClient *http.Client, ima
 		return nil, fmt.Errorf("failed to create app config service: %w", err)
 	}
 
-	svc.appImagesService = service.NewAppImagesService(imageExtensions)
+	svc.fileStorage = fileStorage
+	svc.appImagesService = service.NewAppImagesService(imageExtensions, fileStorage)
 
 	svc.emailService, err = service.NewEmailService(db, svc.appConfigService)
 	if err != nil {
@@ -56,13 +59,13 @@ func initServices(ctx context.Context, db *gorm.DB, httpClient *http.Client, ima
 		return nil, fmt.Errorf("failed to create WebAuthn service: %w", err)
 	}
 
-	svc.oidcService, err = service.NewOidcService(ctx, db, svc.jwtService, svc.appConfigService, svc.auditLogService, svc.customClaimService, svc.webauthnService, httpClient)
+	svc.oidcService, err = service.NewOidcService(ctx, db, svc.jwtService, svc.appConfigService, svc.auditLogService, svc.customClaimService, svc.webauthnService, httpClient, fileStorage)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create OIDC service: %w", err)
 	}
 
 	svc.userGroupService = service.NewUserGroupService(db, svc.appConfigService)
-	svc.userService = service.NewUserService(db, svc.jwtService, svc.auditLogService, svc.emailService, svc.appConfigService, svc.customClaimService, svc.appImagesService)
+	svc.userService = service.NewUserService(db, svc.jwtService, svc.auditLogService, svc.emailService, svc.appConfigService, svc.customClaimService, svc.appImagesService, fileStorage)
 	svc.ldapService = service.NewLdapService(db, httpClient, svc.appConfigService, svc.userService, svc.userGroupService)
 	svc.apiKeyService = service.NewApiKeyService(db, svc.emailService)