mirror of
https://github.com/pocket-id/pocket-id.git
synced 2026-02-14 15:52:31 +00:00
feat: add CLI command for importing and exporting Pocket ID data (#998)
Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/pocket-id/pocket-id/backend/internal/common"
|
||||
"github.com/pocket-id/pocket-id/backend/internal/job"
|
||||
@@ -15,6 +16,16 @@ import (
|
||||
)
|
||||
|
||||
func Bootstrap(ctx context.Context) error {
|
||||
var shutdownFns []utils.Service
|
||||
defer func() { //nolint:contextcheck
|
||||
// Invoke all shutdown functions on exit
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := utils.NewServiceRunner(shutdownFns...).Run(shutdownCtx); err != nil {
|
||||
slog.Error("Error during graceful shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Initialize the observability stack, including the logger, distributed tracing, and metrics
|
||||
shutdownFns, httpClient, err := initObservability(ctx, common.EnvConfig.MetricsEnabled, common.EnvConfig.TracingEnabled)
|
||||
if err != nil {
|
||||
@@ -22,15 +33,80 @@ func Bootstrap(ctx context.Context) error {
|
||||
}
|
||||
slog.InfoContext(ctx, "Pocket ID is starting")
|
||||
|
||||
// Connect to the database
|
||||
db, err := NewDatabase()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize database: %w", err)
|
||||
}
|
||||
|
||||
// Initialize the file storage backend
|
||||
var fileStorage storage.FileStorage
|
||||
fileStorage, err := InitStorage(ctx, db)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize file storage (backend: %s): %w", common.EnvConfig.FileBackend, err)
|
||||
}
|
||||
|
||||
imageExtensions, err := initApplicationImages(ctx, fileStorage)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize application images: %w", err)
|
||||
}
|
||||
|
||||
// Create all services
|
||||
svc, err := initServices(ctx, db, httpClient, imageExtensions, fileStorage)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize services: %w", err)
|
||||
}
|
||||
|
||||
waitUntil, err := svc.appLockService.Acquire(ctx, false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to acquire application lock: %w", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Until(waitUntil)):
|
||||
}
|
||||
|
||||
shutdownFn := func(shutdownCtx context.Context) error {
|
||||
sErr := svc.appLockService.Release(shutdownCtx)
|
||||
if sErr != nil {
|
||||
return fmt.Errorf("failed to release application lock: %w", sErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
shutdownFns = append(shutdownFns, shutdownFn)
|
||||
|
||||
// Init the job scheduler
|
||||
scheduler, err := job.NewScheduler()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create job scheduler: %w", err)
|
||||
}
|
||||
err = registerScheduledJobs(ctx, db, svc, httpClient, scheduler)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to register scheduled jobs: %w", err)
|
||||
}
|
||||
|
||||
// Init the router
|
||||
router, err := initRouter(db, svc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize router: %w", err)
|
||||
}
|
||||
|
||||
// Run all background services
|
||||
// This call blocks until the context is canceled
|
||||
services := []utils.Service{svc.appLockService.RunRenewal, router}
|
||||
|
||||
if common.EnvConfig.AppEnv != "test" {
|
||||
services = append(services, scheduler.Run)
|
||||
}
|
||||
|
||||
err = utils.NewServiceRunner(services...).Run(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to run services: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func InitStorage(ctx context.Context, db *gorm.DB) (fileStorage storage.FileStorage, err error) {
|
||||
switch common.EnvConfig.FileBackend {
|
||||
case storage.TypeFileSystem:
|
||||
fileStorage, err = storage.NewFilesystemStorage(common.EnvConfig.UploadPath)
|
||||
@@ -52,53 +128,8 @@ func Bootstrap(ctx context.Context) error {
|
||||
err = fmt.Errorf("unknown file storage backend: %s", common.EnvConfig.FileBackend)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize file storage (backend: %s): %w", common.EnvConfig.FileBackend, err)
|
||||
return fileStorage, err
|
||||
}
|
||||
|
||||
imageExtensions, err := initApplicationImages(ctx, fileStorage)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize application images: %w", err)
|
||||
}
|
||||
|
||||
// Create all services
|
||||
svc, err := initServices(ctx, db, httpClient, imageExtensions, fileStorage)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize services: %w", err)
|
||||
}
|
||||
|
||||
// Init the job scheduler
|
||||
scheduler, err := job.NewScheduler()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create job scheduler: %w", err)
|
||||
}
|
||||
err = registerScheduledJobs(ctx, db, svc, httpClient, scheduler)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to register scheduled jobs: %w", err)
|
||||
}
|
||||
|
||||
// Init the router
|
||||
router := initRouter(db, svc)
|
||||
|
||||
// Run all background services
|
||||
// This call blocks until the context is canceled
|
||||
err = utils.
|
||||
NewServiceRunner(router, scheduler.Run).
|
||||
Run(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to run services: %w", err)
|
||||
}
|
||||
|
||||
// Invoke all shutdown functions
|
||||
// We give these a timeout of 5s
|
||||
// Note: we use a background context because the run context has been canceled already
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer shutdownCancel()
|
||||
err = utils.
|
||||
NewServiceRunner(shutdownFns...).
|
||||
Run(shutdownCtx) //nolint:contextcheck
|
||||
if err != nil {
|
||||
slog.Error("Error shutting down services", slog.Any("error", err))
|
||||
}
|
||||
|
||||
return nil
|
||||
return fileStorage, nil
|
||||
}
|
||||
|
||||
@@ -12,12 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
postgresMigrate "github.com/golang-migrate/migrate/v4/database/postgres"
|
||||
sqliteMigrate "github.com/golang-migrate/migrate/v4/database/sqlite3"
|
||||
_ "github.com/golang-migrate/migrate/v4/source/github"
|
||||
"github.com/golang-migrate/migrate/v4/source/iofs"
|
||||
slogGorm "github.com/orandin/slog-gorm"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
@@ -26,11 +21,10 @@ import (
|
||||
"github.com/pocket-id/pocket-id/backend/internal/common"
|
||||
"github.com/pocket-id/pocket-id/backend/internal/utils"
|
||||
sqliteutil "github.com/pocket-id/pocket-id/backend/internal/utils/sqlite"
|
||||
"github.com/pocket-id/pocket-id/backend/resources"
|
||||
)
|
||||
|
||||
func NewDatabase() (db *gorm.DB, err error) {
|
||||
db, err = connectDatabase()
|
||||
db, err = ConnectDatabase()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to database: %w", err)
|
||||
}
|
||||
@@ -39,105 +33,15 @@ func NewDatabase() (db *gorm.DB, err error) {
|
||||
return nil, fmt.Errorf("failed to get sql.DB: %w", err)
|
||||
}
|
||||
|
||||
// Choose the correct driver for the database provider
|
||||
var driver database.Driver
|
||||
switch common.EnvConfig.DbProvider {
|
||||
case common.DbProviderSqlite:
|
||||
driver, err = sqliteMigrate.WithInstance(sqlDb, &sqliteMigrate.Config{
|
||||
NoTxWrap: true,
|
||||
})
|
||||
case common.DbProviderPostgres:
|
||||
driver, err = postgresMigrate.WithInstance(sqlDb, &postgresMigrate.Config{})
|
||||
default:
|
||||
// Should never happen at this point
|
||||
return nil, fmt.Errorf("unsupported database provider: %s", common.EnvConfig.DbProvider)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create migration driver: %w", err)
|
||||
}
|
||||
|
||||
// Run migrations
|
||||
if err := migrateDatabase(driver); err != nil {
|
||||
if err := utils.MigrateDatabase(sqlDb); err != nil {
|
||||
return nil, fmt.Errorf("failed to run migrations: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func migrateDatabase(driver database.Driver) error {
|
||||
// Embedded migrations via iofs
|
||||
path := "migrations/" + string(common.EnvConfig.DbProvider)
|
||||
source, err := iofs.New(resources.FS, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create embedded migration source: %w", err)
|
||||
}
|
||||
|
||||
m, err := migrate.NewWithInstance("iofs", source, "pocket-id", driver)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create migration instance: %w", err)
|
||||
}
|
||||
|
||||
requiredVersion, err := getRequiredMigrationVersion(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get last migration version: %w", err)
|
||||
}
|
||||
|
||||
currentVersion, _, _ := m.Version()
|
||||
if currentVersion > requiredVersion {
|
||||
slog.Warn("Database version is newer than the application supports, possible downgrade detected", slog.Uint64("db_version", uint64(currentVersion)), slog.Uint64("app_version", uint64(requiredVersion)))
|
||||
if !common.EnvConfig.AllowDowngrade {
|
||||
return fmt.Errorf("database version (%d) is newer than application version (%d), downgrades are not allowed (set ALLOW_DOWNGRADE=true to enable)", currentVersion, requiredVersion)
|
||||
}
|
||||
slog.Info("Fetching migrations from GitHub to handle possible downgrades")
|
||||
return migrateDatabaseFromGitHub(driver, requiredVersion)
|
||||
}
|
||||
|
||||
if err := m.Migrate(requiredVersion); err != nil && !errors.Is(err, migrate.ErrNoChange) {
|
||||
return fmt.Errorf("failed to apply embedded migrations: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateDatabaseFromGitHub(driver database.Driver, version uint) error {
|
||||
srcURL := "github://pocket-id/pocket-id/backend/resources/migrations/" + string(common.EnvConfig.DbProvider)
|
||||
|
||||
m, err := migrate.NewWithDatabaseInstance(srcURL, "pocket-id", driver)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create GitHub migration instance: %w", err)
|
||||
}
|
||||
|
||||
if err := m.Migrate(version); err != nil && !errors.Is(err, migrate.ErrNoChange) {
|
||||
return fmt.Errorf("failed to apply GitHub migrations: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getRequiredMigrationVersion reads the embedded migration files and returns the highest version number found.
|
||||
func getRequiredMigrationVersion(path string) (uint, error) {
|
||||
entries, err := resources.FS.ReadDir(path)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to read migration directory: %w", err)
|
||||
}
|
||||
|
||||
var maxVersion uint
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
var version uint
|
||||
n, err := fmt.Sscanf(name, "%d_", &version)
|
||||
if err == nil && n == 1 {
|
||||
if version > maxVersion {
|
||||
maxVersion = version
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return maxVersion, nil
|
||||
}
|
||||
|
||||
func connectDatabase() (db *gorm.DB, err error) {
|
||||
func ConnectDatabase() (db *gorm.DB, err error) {
|
||||
var dialector gorm.Dialector
|
||||
|
||||
// Choose the correct database provider
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
func init() {
|
||||
registerTestControllers = []func(apiGroup *gin.RouterGroup, db *gorm.DB, svc *services){
|
||||
func(apiGroup *gin.RouterGroup, db *gorm.DB, svc *services) {
|
||||
testService, err := service.NewTestService(db, svc.appConfigService, svc.jwtService, svc.ldapService, svc.fileStorage)
|
||||
testService, err := service.NewTestService(db, svc.appConfigService, svc.jwtService, svc.ldapService, svc.appLockService, svc.fileStorage)
|
||||
if err != nil {
|
||||
slog.Error("Failed to initialize test service", slog.Any("error", err))
|
||||
os.Exit(1)
|
||||
|
||||
@@ -29,16 +29,7 @@ import (
|
||||
// This is used to register additional controllers for tests
|
||||
var registerTestControllers []func(apiGroup *gin.RouterGroup, db *gorm.DB, svc *services)
|
||||
|
||||
func initRouter(db *gorm.DB, svc *services) utils.Service {
|
||||
runner, err := initRouterInternal(db, svc)
|
||||
if err != nil {
|
||||
slog.Error("Failed to init router", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return runner
|
||||
}
|
||||
|
||||
func initRouterInternal(db *gorm.DB, svc *services) (utils.Service, error) {
|
||||
func initRouter(db *gorm.DB, svc *services) (utils.Service, error) {
|
||||
// Set the appropriate Gin mode based on the environment
|
||||
switch common.EnvConfig.AppEnv {
|
||||
case common.AppEnvProduction:
|
||||
|
||||
@@ -27,6 +27,7 @@ type services struct {
|
||||
apiKeyService *service.ApiKeyService
|
||||
versionService *service.VersionService
|
||||
fileStorage storage.FileStorage
|
||||
appLockService *service.AppLockService
|
||||
}
|
||||
|
||||
// Initializes all services
|
||||
@@ -40,6 +41,7 @@ func initServices(ctx context.Context, db *gorm.DB, httpClient *http.Client, ima
|
||||
|
||||
svc.fileStorage = fileStorage
|
||||
svc.appImagesService = service.NewAppImagesService(imageExtensions, fileStorage)
|
||||
svc.appLockService = service.NewAppLockService(db)
|
||||
|
||||
svc.emailService, err = service.NewEmailService(db, svc.appConfigService)
|
||||
if err != nil {
|
||||
|
||||
70
backend/internal/cmds/export.go
Normal file
70
backend/internal/cmds/export.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package cmds
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/pocket-id/pocket-id/backend/internal/bootstrap"
|
||||
"github.com/pocket-id/pocket-id/backend/internal/service"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
type exportFlags struct {
|
||||
Path string
|
||||
}
|
||||
|
||||
func init() {
|
||||
var flags exportFlags
|
||||
|
||||
exportCmd := &cobra.Command{
|
||||
Use: "export",
|
||||
Short: "Exports all data of Pocket ID into a ZIP file",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runExport(cmd.Context(), flags)
|
||||
},
|
||||
}
|
||||
|
||||
exportCmd.Flags().StringVarP(&flags.Path, "path", "p", "pocket-id-export.zip", "Path to the ZIP file to export the data to, or '-' to write to stdout")
|
||||
|
||||
rootCmd.AddCommand(exportCmd)
|
||||
}
|
||||
|
||||
// runExport orchestrates the export flow
|
||||
func runExport(ctx context.Context, flags exportFlags) error {
|
||||
db, err := bootstrap.NewDatabase()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to database: %w", err)
|
||||
}
|
||||
|
||||
storage, err := bootstrap.InitStorage(ctx, db)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize storage: %w", err)
|
||||
}
|
||||
|
||||
exportService := service.NewExportService(db, storage)
|
||||
|
||||
var w io.Writer
|
||||
if flags.Path == "-" {
|
||||
w = os.Stdout
|
||||
} else {
|
||||
file, err := os.Create(flags.Path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create export file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
w = file
|
||||
}
|
||||
|
||||
if err := exportService.ExportToZip(ctx, w); err != nil {
|
||||
return fmt.Errorf("failed to export data: %w", err)
|
||||
}
|
||||
|
||||
if flags.Path != "-" {
|
||||
fmt.Printf("Exported data to %s\n", flags.Path)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
191
backend/internal/cmds/import.go
Normal file
191
backend/internal/cmds/import.go
Normal file
@@ -0,0 +1,191 @@
|
||||
package cmds
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/pocket-id/pocket-id/backend/internal/bootstrap"
|
||||
"github.com/pocket-id/pocket-id/backend/internal/common"
|
||||
"github.com/pocket-id/pocket-id/backend/internal/service"
|
||||
"github.com/pocket-id/pocket-id/backend/internal/utils"
|
||||
)
|
||||
|
||||
type importFlags struct {
|
||||
Path string
|
||||
Yes bool
|
||||
ForcefullyAcquireLock bool
|
||||
}
|
||||
|
||||
func init() {
|
||||
var flags importFlags
|
||||
|
||||
importCmd := &cobra.Command{
|
||||
Use: "import",
|
||||
Short: "Imports all data of Pocket ID from a ZIP file",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runImport(cmd.Context(), flags)
|
||||
},
|
||||
}
|
||||
|
||||
importCmd.Flags().StringVarP(&flags.Path, "path", "p", "pocket-id-export.zip", "Path to the ZIP file to import the data from, or '-' to read from stdin")
|
||||
importCmd.Flags().BoolVarP(&flags.Yes, "yes", "y", false, "Skip confirmation prompts")
|
||||
importCmd.Flags().BoolVarP(&flags.ForcefullyAcquireLock, "forcefully-acquire-lock", "", false, "Forcefully acquire the application lock by terminating the Pocket ID instance")
|
||||
|
||||
rootCmd.AddCommand(importCmd)
|
||||
}
|
||||
|
||||
// runImport handles the high-level orchestration of the import process
|
||||
func runImport(ctx context.Context, flags importFlags) error {
|
||||
if !flags.Yes {
|
||||
ok, err := askForConfirmation()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get confirmation: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
fmt.Println("Aborted")
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
zipReader *zip.ReadCloser
|
||||
cleanup func()
|
||||
err error
|
||||
)
|
||||
|
||||
if flags.Path == "-" {
|
||||
zipReader, cleanup, err = readZipFromStdin()
|
||||
defer cleanup()
|
||||
} else {
|
||||
zipReader, err = zip.OpenReader(flags.Path)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open zip: %w", err)
|
||||
}
|
||||
defer zipReader.Close()
|
||||
|
||||
db, err := bootstrap.ConnectDatabase()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = acquireImportLock(ctx, db, flags.ForcefullyAcquireLock)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
storage, err := bootstrap.InitStorage(ctx, db)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize storage: %w", err)
|
||||
}
|
||||
|
||||
importService := service.NewImportService(db, storage)
|
||||
err = importService.ImportFromZip(ctx, &zipReader.Reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to import data from zip: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("Import completed successfully.")
|
||||
return nil
|
||||
}
|
||||
|
||||
func acquireImportLock(ctx context.Context, db *gorm.DB, force bool) error {
|
||||
// Check if the kv table exists, in case we are starting from an empty database
|
||||
exists, err := utils.DBTableExists(db, "kv")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check if kv table exists: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
// This either means the database is empty, or the import is into an old version of PocketID that doesn't support locks
|
||||
// In either case, there's no lock to acquire
|
||||
fmt.Println("Could not acquire a lock because the 'kv' table does not exist. This is fine if you're importing into a new database, but make sure that there isn't an instance of Pocket ID currently running and using the same database.")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Note that we do not call a deferred Release if the data was imported
|
||||
// This is because we are overriding the contents of the database, so the lock is automatically lost
|
||||
appLockService := service.NewAppLockService(db)
|
||||
|
||||
opCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
waitUntil, err := appLockService.Acquire(opCtx, force)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrLockUnavailable) {
|
||||
//nolint:staticcheck
|
||||
return errors.New("Pocket ID must be stopped before importing data; please stop the running instance or run with --forcefully-acquire-lock to terminate the other instance")
|
||||
}
|
||||
return fmt.Errorf("failed to acquire application lock: %w", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Until(waitUntil)):
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func askForConfirmation() (bool, error) {
|
||||
fmt.Println("WARNING: This feature is experimental and may not work correctly. Please create a backup before proceeding and report any issues you encounter.")
|
||||
fmt.Println()
|
||||
fmt.Println("WARNING: Import will erase all existing data at the following locations:")
|
||||
fmt.Printf("Database: %s\n", absolutePathOrOriginal(common.EnvConfig.DbConnectionString))
|
||||
fmt.Printf("Uploads Path: %s\n", absolutePathOrOriginal(common.EnvConfig.UploadPath))
|
||||
|
||||
ok, err := utils.PromptForConfirmation("Do you want to continue?")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
// absolutePathOrOriginal returns the absolute path of the given path, or the original if it fails
|
||||
func absolutePathOrOriginal(path string) string {
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return path
|
||||
}
|
||||
return abs
|
||||
}
|
||||
|
||||
func readZipFromStdin() (*zip.ReadCloser, func(), error) {
|
||||
tmpFile, err := os.CreateTemp("", "pocket-id-import-*.zip")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create temporary file: %w", err)
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
_ = os.Remove(tmpFile.Name())
|
||||
}
|
||||
|
||||
if _, err := io.Copy(tmpFile, os.Stdin); err != nil {
|
||||
tmpFile.Close()
|
||||
cleanup()
|
||||
return nil, nil, fmt.Errorf("failed to read data from stdin: %w", err)
|
||||
}
|
||||
|
||||
if err := tmpFile.Close(); err != nil {
|
||||
cleanup()
|
||||
return nil, nil, fmt.Errorf("failed to close temporary file: %w", err)
|
||||
}
|
||||
|
||||
r, err := zip.OpenReader(tmpFile.Name())
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return r, cleanup, nil
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/lestrrat-go/jwx/v3/jwa"
|
||||
@@ -78,7 +79,7 @@ func keyRotate(ctx context.Context, flags keyRotateFlags, db *gorm.DB, envConfig
|
||||
}
|
||||
if !ok {
|
||||
fmt.Println("Aborted")
|
||||
return nil
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,9 +12,10 @@ import (
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "pocket-id",
|
||||
Short: "A simple and easy-to-use OIDC provider that allows users to authenticate with their passkeys to your services.",
|
||||
Long: "By default, this command starts the pocket-id server.",
|
||||
Use: "pocket-id",
|
||||
Short: "A simple and easy-to-use OIDC provider that allows users to authenticate with their passkeys to your services.",
|
||||
Long: "By default, this command starts the pocket-id server.",
|
||||
SilenceUsage: true,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
// Start the server
|
||||
err := bootstrap.Bootstrap(cmd.Context())
|
||||
|
||||
@@ -40,6 +40,11 @@ func (tc *TestController) resetAndSeedHandler(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := tc.TestService.ResetLock(c.Request.Context()); err != nil {
|
||||
_ = c.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := tc.TestService.ResetApplicationImages(c.Request.Context()); err != nil {
|
||||
_ = c.Error(err)
|
||||
return
|
||||
@@ -69,8 +74,6 @@ func (tc *TestController) resetAndSeedHandler(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
tc.TestService.SetJWTKeys()
|
||||
|
||||
c.Status(http.StatusNoContent)
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,15 @@ import (
|
||||
// DateTime custom type for time.Time to store date as unix timestamp for sqlite and as date for postgres
|
||||
type DateTime time.Time //nolint:recvcheck
|
||||
|
||||
func DateTimeFromString(str string) (DateTime, error) {
|
||||
t, err := time.Parse(time.RFC3339Nano, str)
|
||||
if err != nil {
|
||||
return DateTime{}, fmt.Errorf("failed to parse date string: %w", err)
|
||||
}
|
||||
|
||||
return DateTime(t), nil
|
||||
}
|
||||
|
||||
func (date *DateTime) Scan(value any) (err error) {
|
||||
switch v := value.(type) {
|
||||
case time.Time:
|
||||
|
||||
296
backend/internal/service/app_lock_service.go
Normal file
296
backend/internal/service/app_lock_service.go
Normal file
@@ -0,0 +1,296 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/pocket-id/pocket-id/backend/internal/model"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrLockUnavailable = errors.New("lock is already held by another process")
|
||||
ErrLockLost = errors.New("lock ownership lost")
|
||||
)
|
||||
|
||||
const (
|
||||
ttl = 30 * time.Second
|
||||
renewInterval = 20 * time.Second
|
||||
renewRetries = 3
|
||||
lockKey = "application_lock"
|
||||
)
|
||||
|
||||
type AppLockService struct {
|
||||
db *gorm.DB
|
||||
lockID string
|
||||
processID int64
|
||||
hostID string
|
||||
}
|
||||
|
||||
func NewAppLockService(db *gorm.DB) *AppLockService {
|
||||
host, err := os.Hostname()
|
||||
if err != nil || host == "" {
|
||||
host = "unknown-host"
|
||||
}
|
||||
|
||||
return &AppLockService{
|
||||
db: db,
|
||||
processID: int64(os.Getpid()),
|
||||
hostID: host,
|
||||
lockID: uuid.NewString(),
|
||||
}
|
||||
}
|
||||
|
||||
type lockValue struct {
|
||||
ProcessID int64 `json:"process_id"`
|
||||
HostID string `json:"host_id"`
|
||||
LockID string `json:"lock_id"`
|
||||
ExpiresAt int64 `json:"expires_at"`
|
||||
}
|
||||
|
||||
func (lv *lockValue) Marshal() (string, error) {
|
||||
data, err := json.Marshal(lv)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func (lv *lockValue) Unmarshal(raw string) error {
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal([]byte(raw), lv)
|
||||
}
|
||||
|
||||
// Acquire obtains the lock. When force is true, the lock is stolen from any existing owner.
|
||||
// If the lock is forcefully acquired, it blocks until the previous lock has expired.
|
||||
func (s *AppLockService) Acquire(ctx context.Context, force bool) (waitUntil time.Time, err error) {
|
||||
tx := s.db.Begin()
|
||||
defer func() {
|
||||
tx.Rollback()
|
||||
}()
|
||||
|
||||
var prevLockRaw string
|
||||
err = tx.
|
||||
WithContext(ctx).
|
||||
Model(&model.KV{}).
|
||||
Where("key = ?", lockKey).
|
||||
Clauses(clause.Locking{Strength: "UPDATE"}).
|
||||
Select("value").
|
||||
Scan(&prevLockRaw).
|
||||
Error
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("query existing lock: %w", err)
|
||||
}
|
||||
|
||||
var prevLock lockValue
|
||||
if prevLockRaw != "" {
|
||||
if err := prevLock.Unmarshal(prevLockRaw); err != nil {
|
||||
return time.Time{}, fmt.Errorf("decode existing lock value: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
nowUnix := now.Unix()
|
||||
|
||||
value := lockValue{
|
||||
ProcessID: s.processID,
|
||||
HostID: s.hostID,
|
||||
LockID: s.lockID,
|
||||
ExpiresAt: now.Add(ttl).Unix(),
|
||||
}
|
||||
raw, err := value.Marshal()
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("encode lock value: %w", err)
|
||||
}
|
||||
|
||||
var query string
|
||||
switch s.db.Name() {
|
||||
case "sqlite":
|
||||
query = `
|
||||
INSERT INTO kv (key, value)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
value = excluded.value
|
||||
WHERE (json_extract(kv.value, '$.expires_at') < ?) OR ?
|
||||
`
|
||||
case "postgres":
|
||||
query = `
|
||||
INSERT INTO kv (key, value)
|
||||
VALUES ($1, $2)
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
value = excluded.value
|
||||
WHERE ((kv.value::json->>'expires_at')::bigint < $3) OR ($4::boolean IS TRUE)
|
||||
`
|
||||
default:
|
||||
return time.Time{}, fmt.Errorf("unsupported database dialect: %s", s.db.Name())
|
||||
}
|
||||
|
||||
res := tx.WithContext(ctx).Exec(query, lockKey, raw, nowUnix, force)
|
||||
if res.Error != nil {
|
||||
return time.Time{}, fmt.Errorf("lock acquisition failed: %w", res.Error)
|
||||
}
|
||||
|
||||
if err := tx.Commit().Error; err != nil {
|
||||
return time.Time{}, fmt.Errorf("commit lock acquisition: %w", err)
|
||||
}
|
||||
|
||||
// If there is a lock that is not expired and force is false, no rows will be affected
|
||||
if res.RowsAffected == 0 {
|
||||
return time.Time{}, ErrLockUnavailable
|
||||
}
|
||||
|
||||
if force && prevLock.ExpiresAt > nowUnix && prevLock.LockID != s.lockID {
|
||||
waitUntil = time.Unix(prevLock.ExpiresAt, 0)
|
||||
}
|
||||
|
||||
attrs := []any{
|
||||
slog.Int64("process_id", s.processID),
|
||||
slog.String("host_id", s.hostID),
|
||||
}
|
||||
if wait := time.Until(waitUntil); wait > 0 {
|
||||
attrs = append(attrs, slog.Duration("wait_before_proceeding", wait))
|
||||
}
|
||||
slog.Info("Acquired application lock", attrs...)
|
||||
|
||||
return waitUntil, nil
|
||||
}
|
||||
|
||||
// RunRenewal keeps renewing the lock until the context is canceled.
|
||||
func (s *AppLockService) RunRenewal(ctx context.Context) error {
|
||||
ticker := time.NewTicker(renewInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
if err := s.renew(ctx); err != nil {
|
||||
return fmt.Errorf("renew lock: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Release releases the lock if it is held by this process.
|
||||
func (s *AppLockService) Release(ctx context.Context) error {
|
||||
opCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var query string
|
||||
switch s.db.Name() {
|
||||
case "sqlite":
|
||||
query = `
|
||||
DELETE FROM kv
|
||||
WHERE key = ?
|
||||
AND json_extract(value, '$.lock_id') = ?
|
||||
`
|
||||
case "postgres":
|
||||
query = `
|
||||
DELETE FROM kv
|
||||
WHERE key = $1
|
||||
AND value::json->>'lock_id' = $2
|
||||
`
|
||||
default:
|
||||
return fmt.Errorf("unsupported database dialect: %s", s.db.Name())
|
||||
}
|
||||
|
||||
res := s.db.WithContext(opCtx).Exec(query, lockKey, s.lockID)
|
||||
if res.Error != nil {
|
||||
return fmt.Errorf("release lock failed: %w", res.Error)
|
||||
}
|
||||
|
||||
if res.RowsAffected == 0 {
|
||||
slog.Warn("Application lock not held by this process, cannot release",
|
||||
slog.Int64("process_id", s.processID),
|
||||
slog.String("host_id", s.hostID),
|
||||
)
|
||||
}
|
||||
|
||||
slog.Info("Released application lock",
|
||||
slog.Int64("process_id", s.processID),
|
||||
slog.String("host_id", s.hostID),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// renew tries to renew the lock, retrying up to renewRetries times (sleeping 1s between attempts).
|
||||
func (s *AppLockService) renew(ctx context.Context) error {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= renewRetries; attempt++ {
|
||||
now := time.Now()
|
||||
nowUnix := now.Unix()
|
||||
expiresAt := now.Add(ttl).Unix()
|
||||
|
||||
value := lockValue{
|
||||
LockID: s.lockID,
|
||||
ProcessID: s.processID,
|
||||
HostID: s.hostID,
|
||||
ExpiresAt: expiresAt,
|
||||
}
|
||||
raw, err := value.Marshal()
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode lock value: %w", err)
|
||||
}
|
||||
|
||||
var query string
|
||||
switch s.db.Name() {
|
||||
case "sqlite":
|
||||
query = `
|
||||
UPDATE kv
|
||||
SET value = ?
|
||||
WHERE key = ?
|
||||
AND json_extract(value, '$.lock_id') = ?
|
||||
AND json_extract(value, '$.expires_at') > ?
|
||||
`
|
||||
case "postgres":
|
||||
query = `
|
||||
UPDATE kv
|
||||
SET value = $1
|
||||
WHERE key = $2
|
||||
AND value::json->>'lock_id' = $3
|
||||
AND ((value::json->>'expires_at')::bigint > $4)
|
||||
`
|
||||
default:
|
||||
return fmt.Errorf("unsupported database dialect: %s", s.db.Name())
|
||||
}
|
||||
|
||||
opCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
res := s.db.WithContext(opCtx).Exec(query, raw, lockKey, s.lockID, nowUnix)
|
||||
cancel()
|
||||
|
||||
switch {
|
||||
case res.Error != nil:
|
||||
lastErr = fmt.Errorf("lock renewal failed: %w", res.Error)
|
||||
case res.RowsAffected == 0:
|
||||
// Must be after checking res.Error
|
||||
return ErrLockLost
|
||||
default:
|
||||
slog.Debug("Renewed application lock",
|
||||
slog.Int64("process_id", s.processID),
|
||||
slog.String("host_id", s.hostID),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Wait before next attempt or cancel if context is done
|
||||
if attempt < renewRetries {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(1 * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
189
backend/internal/service/app_lock_service_test.go
Normal file
189
backend/internal/service/app_lock_service_test.go
Normal file
@@ -0,0 +1,189 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/pocket-id/pocket-id/backend/internal/model"
|
||||
testutils "github.com/pocket-id/pocket-id/backend/internal/utils/testing"
|
||||
)
|
||||
|
||||
func newTestAppLockService(t *testing.T, db *gorm.DB) *AppLockService {
|
||||
t.Helper()
|
||||
|
||||
return &AppLockService{
|
||||
db: db,
|
||||
processID: 1,
|
||||
hostID: "test-host",
|
||||
lockID: "a13c7673-c7ae-49f1-9112-2cd2d0d4b0c1",
|
||||
}
|
||||
}
|
||||
|
||||
func insertLock(t *testing.T, db *gorm.DB, value lockValue) {
|
||||
t.Helper()
|
||||
|
||||
raw, err := value.Marshal()
|
||||
require.NoError(t, err)
|
||||
|
||||
err = db.Create(&model.KV{Key: lockKey, Value: &raw}).Error
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func readLockValue(t *testing.T, db *gorm.DB) lockValue {
|
||||
t.Helper()
|
||||
|
||||
var row model.KV
|
||||
err := db.Take(&row, "key = ?", lockKey).Error
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NotNil(t, row.Value)
|
||||
|
||||
var value lockValue
|
||||
err = value.Unmarshal(*row.Value)
|
||||
require.NoError(t, err)
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
func TestAppLockServiceAcquire(t *testing.T) {
|
||||
t.Run("creates new lock when none exists", func(t *testing.T) {
|
||||
db := testutils.NewDatabaseForTest(t)
|
||||
service := newTestAppLockService(t, db)
|
||||
|
||||
_, err := service.Acquire(context.Background(), false)
|
||||
require.NoError(t, err)
|
||||
|
||||
stored := readLockValue(t, db)
|
||||
require.Equal(t, service.processID, stored.ProcessID)
|
||||
require.Equal(t, service.hostID, stored.HostID)
|
||||
require.Greater(t, stored.ExpiresAt, time.Now().Unix())
|
||||
})
|
||||
|
||||
t.Run("returns ErrLockUnavailable when lock held by another process", func(t *testing.T) {
|
||||
db := testutils.NewDatabaseForTest(t)
|
||||
service := newTestAppLockService(t, db)
|
||||
|
||||
existing := lockValue{
|
||||
ProcessID: 99,
|
||||
HostID: "other-host",
|
||||
ExpiresAt: time.Now().Add(ttl).Unix(),
|
||||
}
|
||||
insertLock(t, db, existing)
|
||||
|
||||
_, err := service.Acquire(context.Background(), false)
|
||||
require.ErrorIs(t, err, ErrLockUnavailable)
|
||||
|
||||
current := readLockValue(t, db)
|
||||
require.Equal(t, existing, current)
|
||||
})
|
||||
|
||||
t.Run("force acquisition steals lock", func(t *testing.T) {
|
||||
db := testutils.NewDatabaseForTest(t)
|
||||
service := newTestAppLockService(t, db)
|
||||
|
||||
insertLock(t, db, lockValue{
|
||||
ProcessID: 99,
|
||||
HostID: "other-host",
|
||||
ExpiresAt: time.Now().Unix(),
|
||||
})
|
||||
|
||||
_, err := service.Acquire(context.Background(), true)
|
||||
require.NoError(t, err)
|
||||
|
||||
stored := readLockValue(t, db)
|
||||
require.Equal(t, service.processID, stored.ProcessID)
|
||||
require.Equal(t, service.hostID, stored.HostID)
|
||||
require.Greater(t, stored.ExpiresAt, time.Now().Unix())
|
||||
})
|
||||
}
|
||||
|
||||
func TestAppLockServiceRelease(t *testing.T) {
|
||||
t.Run("removes owned lock", func(t *testing.T) {
|
||||
db := testutils.NewDatabaseForTest(t)
|
||||
service := newTestAppLockService(t, db)
|
||||
|
||||
_, err := service.Acquire(context.Background(), false)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = service.Release(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
var row model.KV
|
||||
err = db.Take(&row, "key = ?", lockKey).Error
|
||||
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
|
||||
})
|
||||
|
||||
t.Run("ignores lock held by another owner", func(t *testing.T) {
|
||||
db := testutils.NewDatabaseForTest(t)
|
||||
service := newTestAppLockService(t, db)
|
||||
|
||||
existing := lockValue{
|
||||
ProcessID: 2,
|
||||
HostID: "other-host",
|
||||
ExpiresAt: time.Now().Add(ttl).Unix(),
|
||||
}
|
||||
insertLock(t, db, existing)
|
||||
|
||||
err := service.Release(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
stored := readLockValue(t, db)
|
||||
require.Equal(t, existing, stored)
|
||||
})
|
||||
}
|
||||
|
||||
func TestAppLockServiceRenew(t *testing.T) {
|
||||
t.Run("extends expiration when lock is still owned", func(t *testing.T) {
|
||||
db := testutils.NewDatabaseForTest(t)
|
||||
service := newTestAppLockService(t, db)
|
||||
|
||||
_, err := service.Acquire(context.Background(), false)
|
||||
require.NoError(t, err)
|
||||
|
||||
before := readLockValue(t, db)
|
||||
|
||||
err = service.renew(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
||||
after := readLockValue(t, db)
|
||||
require.Equal(t, service.processID, after.ProcessID)
|
||||
require.Equal(t, service.hostID, after.HostID)
|
||||
require.GreaterOrEqual(t, after.ExpiresAt, before.ExpiresAt)
|
||||
})
|
||||
|
||||
t.Run("returns ErrLockLost when lock is missing", func(t *testing.T) {
|
||||
db := testutils.NewDatabaseForTest(t)
|
||||
service := newTestAppLockService(t, db)
|
||||
|
||||
err := service.renew(context.Background())
|
||||
require.ErrorIs(t, err, ErrLockLost)
|
||||
})
|
||||
|
||||
t.Run("returns ErrLockLost when ownership changed", func(t *testing.T) {
|
||||
db := testutils.NewDatabaseForTest(t)
|
||||
service := newTestAppLockService(t, db)
|
||||
|
||||
_, err := service.Acquire(context.Background(), false)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Simulate a different process taking the lock.
|
||||
newOwner := lockValue{
|
||||
ProcessID: 9,
|
||||
HostID: "stolen-host",
|
||||
ExpiresAt: time.Now().Add(ttl).Unix(),
|
||||
}
|
||||
raw, marshalErr := newOwner.Marshal()
|
||||
require.NoError(t, marshalErr)
|
||||
updateErr := db.Model(&model.KV{}).
|
||||
Where("key = ?", lockKey).
|
||||
Update("value", raw).Error
|
||||
require.NoError(t, updateErr)
|
||||
|
||||
err = service.renew(context.Background())
|
||||
require.ErrorIs(t, err, ErrLockLost)
|
||||
})
|
||||
}
|
||||
@@ -7,14 +7,12 @@ import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/fxamacker/cbor/v2"
|
||||
"github.com/go-webauthn/webauthn/protocol"
|
||||
"github.com/lestrrat-go/jwx/v3/jwa"
|
||||
"github.com/lestrrat-go/jwx/v3/jwk"
|
||||
@@ -36,15 +34,17 @@ type TestService struct {
|
||||
appConfigService *AppConfigService
|
||||
ldapService *LdapService
|
||||
fileStorage storage.FileStorage
|
||||
appLockService *AppLockService
|
||||
externalIdPKey jwk.Key
|
||||
}
|
||||
|
||||
func NewTestService(db *gorm.DB, appConfigService *AppConfigService, jwtService *JwtService, ldapService *LdapService, fileStorage storage.FileStorage) (*TestService, error) {
|
||||
func NewTestService(db *gorm.DB, appConfigService *AppConfigService, jwtService *JwtService, ldapService *LdapService, appLockService *AppLockService, fileStorage storage.FileStorage) (*TestService, error) {
|
||||
s := &TestService{
|
||||
db: db,
|
||||
appConfigService: appConfigService,
|
||||
jwtService: jwtService,
|
||||
ldapService: ldapService,
|
||||
appLockService: appLockService,
|
||||
fileStorage: fileStorage,
|
||||
}
|
||||
err := s.initExternalIdP()
|
||||
@@ -288,8 +288,8 @@ func (s *TestService) SeedDatabase(baseURL string) error {
|
||||
// openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 | \
|
||||
// openssl pkcs8 -topk8 -nocrypt | tee >(openssl pkey -pubout)
|
||||
|
||||
publicKeyPasskey1, _ := s.getCborPublicKey("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEwcOo5KV169KR67QEHrcYkeXE3CCxv2BgwnSq4VYTQxyLtdmKxegexa8JdwFKhKXa2BMI9xaN15BoL6wSCRFJhg==")
|
||||
publicKeyPasskey2, _ := s.getCborPublicKey("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEj4qA0PrZzg8Co1C27nyUbzrp8Ewjr7eOlGI2LfrzmbL5nPhZRAdJ3hEaqrHMSnJBhfMqtQGKwDYpaLIQFAKLhw==")
|
||||
publicKeyPasskey1, _ := base64.StdEncoding.DecodeString("pQMmIAEhWCDBw6jkpXXr0pHrtAQetxiR5cTcILG/YGDCdKrhVhNDHCJYIIu12YrF6B7Frwl3AUqEpdrYEwj3Fo3XkGgvrBIJEUmGAQI=")
|
||||
publicKeyPasskey2, _ := base64.StdEncoding.DecodeString("pSJYIPmc+FlEB0neERqqscxKckGF8yq1AYrANiloshAUAouHAQIDJiABIVggj4qA0PrZzg8Co1C27nyUbzrp8Ewjr7eOlGI2LfrzmbI=")
|
||||
webauthnCredentials := []model.WebauthnCredential{
|
||||
{
|
||||
Name: "Passkey 1",
|
||||
@@ -318,6 +318,10 @@ func (s *TestService) SeedDatabase(baseURL string) error {
|
||||
Challenge: "challenge",
|
||||
ExpiresAt: datatype.DateTime(time.Now().Add(1 * time.Hour)),
|
||||
UserVerification: "preferred",
|
||||
CredentialParams: model.CredentialParameters{
|
||||
{Type: "public-key", Algorithm: -7},
|
||||
{Type: "public-key", Algorithm: -257},
|
||||
},
|
||||
}
|
||||
if err := tx.Create(&webauthnSession).Error; err != nil {
|
||||
return err
|
||||
@@ -327,9 +331,10 @@ func (s *TestService) SeedDatabase(baseURL string) error {
|
||||
Base: model.Base{
|
||||
ID: "5f1fa856-c164-4295-961e-175a0d22d725",
|
||||
},
|
||||
Name: "Test API Key",
|
||||
Key: "6c34966f57ef2bb7857649aff0e7ab3ad67af93c846342ced3f5a07be8706c20",
|
||||
UserID: users[0].ID,
|
||||
Name: "Test API Key",
|
||||
Key: "6c34966f57ef2bb7857649aff0e7ab3ad67af93c846342ced3f5a07be8706c20",
|
||||
UserID: users[0].ID,
|
||||
ExpiresAt: datatype.DateTime(time.Now().Add(30 * 24 * time.Hour)),
|
||||
}
|
||||
if err := tx.Create(&apiKey).Error; err != nil {
|
||||
return err
|
||||
@@ -379,6 +384,20 @@ func (s *TestService) SeedDatabase(baseURL string) error {
|
||||
}
|
||||
}
|
||||
|
||||
keyValues := []model.KV{
|
||||
{
|
||||
Key: jwkutils.PrivateKeyDBKey,
|
||||
// {"alg":"RS256","d":"mvMDWSdPPvcum0c0iEHE2gbqtV2NKMmLwrl9E6K7g8lTV95SePLnW_bwyMPV7EGp7PQk3l17I5XRhFjze7GqTnFIOgKzMianPs7jv2ELtBMGK0xOPATgu1iGb70xZ6vcvuEfRyY3dJ0zr4jpUdVuXwKmx9rK4IdZn2dFCKfvSuspqIpz11RhF1ALrqDLkxGVv7ZwNh0_VhJZU9hcjG5l6xc7rQEKpPRkZp0IdjkGS8Z0FskoVaiRIWAbZuiVFB9WCW8k1czC4HQTPLpII01bUQx2ludbm0UlXRgVU9ptUUbU7GAImQqTOW8LfPGklEvcgzlIlR_oqw4P9yBxLi-yMQ","dp":"pvNCSnnhbo8Igw9psPR-DicxFnkXlu_ix4gpy6efTrxA-z1VDFDioJ814vKQNioYDzpyAP1gfMPhRkvG_q0hRZsJah3Sb9dfA-WkhSWY7lURQP4yIBTMU0PF_rEATuS7lRciYk1SOx5fqXZd3m_LP0vpBC4Ujlq6NAq6CIjCnms","dq":"TtUVGCCkPNgfOLmkYXu7dxxUCV5kB01-xAEK2OY0n0pG8vfDophH4_D_ZC7nvJ8J9uDhs_3JStexq1lIvaWtG99RNTChIEDzpdn6GH9yaVcb_eB4uJjrNm64FhF8PGCCwxA-xMCZMaARKwhMB2_IOMkxUbWboL3gnhJ2rDO_QO0","e":"AQAB","kid":"8uHDw3M6rf8","kty":"RSA","n":"yaeEL0VKoPBXIAaWXsUgmu05lAvEIIdJn0FX9lHh4JE5UY9B83C5sCNdhs9iSWzpeP11EVjWp8i3Yv2CF7c7u50BXnVBGtxpZpFC-585UXacoJ0chUmarL9GRFJcM1nPHBTFu68aRrn1rIKNHUkNaaxFo0NFGl_4EDDTO8HwawTjwkPoQlRzeByhlvGPVvwgB3Fn93B8QJ_cZhXKxJvjjrC_8Pk76heC_ntEMru71Ix77BoC3j2TuyiN7m9RNBW8BU5q6lKoIdvIeZfTFLzi37iufyfvMrJTixp9zhNB1NxlLCeOZl2MXegtiGqd2H3cbAyqoOiv9ihUWTfXj7SxJw","p":"_Yylc9e07CKdqNRD2EosMC2mrhrEa9j5oY_l00Qyy4-jmCA59Q9viyqvveRo0U7cRvFA5BWgWN6GGLh1DG3X-QBqVr0dnk3uzbobb55RYUXyPLuBZI2q6w2oasbiDwPdY7KpkVv_H-bpITQlyDvO8hhucA6rUV7F6KTQVz8M3Ms","q":"y5p3hch-7jJ21TkAhp_Vk1fLCAuD4tbErwQs2of9ja8sB4iJOs5Wn6HD3P7Mc8Plye7qaLHvzc8I5g0tPKWvC0DPd_FLPXiWwMVAzee3NUX_oGeJNOQp11y1w_KqdO9qZqHSEPZ3NcFL_SZMFgggxhM1uzRiPzsVN0lnD_6prZU","qi":"2Grt6uXHm61ji3xSdkBWNtUnj19vS1-7rFJp5SoYztVQVThf_W52BAiXKBdYZDRVoItC_VS2NvAOjeJjhYO_xQ_q3hK7MdtuXfEPpLnyXKkmWo3lrJ26wbeF6l05LexCkI7ShsOuSt-dsyaTJTszuKDIA6YOfWvfo3aVZmlWRaI","use":"sig"}
|
||||
Value: utils.Ptr("7d/5hl7diJ2rnFL14hEAQf9tzpu29aqXQ8jpJ2iqqKUNFZpdOkEpud0CmRv4H3r8yyk2u/Gqqj9klSy58DJkYXGF5PAYgLyoBIb7L3JXWRbxg4cQ3QJCug13l2OTmpAKoVc+rmX8c3j3h1sNqyJ+7Ql5sS0jSeyiYgIsFNCdnK5alBDyvtcpe/QDpklmP4JCeVpvmf2rLGplk3g5UO5ydJ8UiDXxfDmi+gF6NKJvrGnnah8Ar3G/x88z+tTJtp0DIQFwxXwUM2XZqzEVGm8K2r0w5o9/Keh6bBBaiuH2C78ZOaijGV3DovhR+e9J0cYUYGwT42MZMx9fSWQ/lvWGGnf+Uq3MXJfjWSREfhkp8KTQwR9F7+dnVJWswOEk7jPR8I7hCWTMxJyvaFX3wgAXIVmhrgXZQQbYOqTt56IoqUl0xOJku8dA8opg2UcLlmmuOh6+hfkXKsiiS/H/9c1BVIGj1fCOiT6IePh4wKKSTbwJnPD5EKmdJpgTsUpjcDnXQKY4ReO0UpdRdKxwRDDLeQuG6j+ljGxR9GPudCU9Nmci6rFVI6n5LWYkQxBA1O73RpmXRZPDzntDfpXMEonkmSvOoxaCK2Id7CRKMdqvR0kEouwnhk5WSFtsfi3sA0pkXzPFxwZeWM8vFtbffZOZzXaOhxCOfcj1NClZohlZhyc4jvkxmrpY7PSaAzih0AmHI7y0LYFi6fZu/K4EheVa1+KF55nWZ8ARikHMWKAKkyExkTak7xyN884TDmzURRaPlQg4jzQte5WMNjAG/hlHibdMBNvgwiYd49ZxteJ8ABdbiXVRl+2JGbdjl2ubpQZwOn7bJKlqO56bIwsZ+e4+pXsuOGdBahkHrUjtMEmH3DZbGc6CJLbcmdhdpApLQRRcLAazxJhzAwJ47FRYsHsj57LnYNvmcKdIxw8rxCdLUuzz95uw0T3ankEO5J9sjem+HMEuKdwXK1UcuOn2rjR8Sd/BuvQmeso27dFbPXqXYNS90Ml45YyTvcKSiopD181oZR703TFUSpR7dsiqROMr+p/2jN9h6a8WbQ8xpksyclaQByY/M77AssbXnG6wfhRsntNIINCZLbBnjXOyz6ZHIC5K4tSTdcnWaiYPeRPQmnw9UUvHAcNU2yMWsy0eU377yDS0WstTxOdQutTdkczl8kv5Lo26JiEK7mSIuRK19ffF9Zz8FG8+eKv5zdyIPjyQRDYBysUoDv5huKe2eoxJu/MWS2Pql/ZtUGeD6Ozm3mCvh0vQ9ceagBkY6Ocm3du0ziAKP29Ri0mjg4DizVorbLzsh+EQH/s2Pi9MnjUZDlEmuLl2Xfp7/w4j/8u0N0tVR70VDFuGdKpTjFY3vS8EJrPtyMTM51x1D9rb8gIql8aR/rJw4YF+huxg1mv5n6+tGVqg5msbPmF12eJijP4lkmaRwIpLW5pJTtaDkUj7uOeu1mm4k+Dt5nh0/0jPHzrv6bcTCcbV7UjMHDoTXXqEpFAAJ66rHR7zdAJu+YKsnTIZyLmOpcowq7LL8G9qTvV0OSpyQWUIavRSgbDHFqEqRs+JU94jAzkq8nCY5MTd9m5sIv9InfdT3k+pwpsE/FKge8nghFLtbUrafGkzTky8SE2druvVcIvbfXMfLIKRUYjJgnWc0gQzF5J6pzXM7D2r/RG6JDzASqjlbURq6v9bhNerlOVdMujWKEEVcKWIzlbt4RkihRjM8AUqIZQOyicGQ+4yfIjAHw5viuABONYs3OIWULnFqJxdvS9rNKhfxSjIq9cfqyzevq2xrRoMXEonobh6M3bD2Vang8OAeVeD1OXWPERi4pepCYFS9RJ/Xa/UWxptsqSNuGcb3fAzQSmLpXLGdWRoKXvSe7EYgc0bGcLOjSTu5RURKo+EF9i4KT9EJauf6VXw5dTf/CCIJRXE1bWzXhSCFYntohYhX2ldOCDYpi/jFBC6Vtkw0ud3/xq8Nmhd5gUk+SpngByCZH3Pm3H+jvlbMpiqkDkm1v74hDX13Xhrcw2eWyuqKBVoRCCniUvwpYNbGvBfjC6Hcizv0Aybciwj+4nybt5EPoEUm6S6Gs7fG7QpPdvrzpAxX70MlmdkF/gwyuhbEeJhLK+WL7qAsN5CvHPzVbsIf90x+nGTtMJPgpxVr0tJMj+vprXV4WxutfARBiOnqe58MhA857sd+MzKBgKnoLOBRTiC3qc/0/ULwbG2HCCD7nmwzz7M4nUuMvo8rgS7z0BF68OClT8X3JwSXbL5Wg=="),
|
||||
},
|
||||
}
|
||||
|
||||
for _, kv := range keyValues {
|
||||
if err := tx.Create(&kv).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -464,47 +483,29 @@ func (s *TestService) ResetAppConfig(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Manually set instance ID
|
||||
err = s.appConfigService.UpdateAppConfigValues(ctx, "instanceId", "test-instance-id")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Reload the app config from the database after resetting the values
|
||||
return s.appConfigService.LoadDbConfig(ctx)
|
||||
err = s.appConfigService.LoadDbConfig(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Reload the JWK
|
||||
if err := s.jwtService.LoadOrGenerateKey(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *TestService) SetJWTKeys() {
|
||||
const privateKeyString = `{"alg":"RS256","d":"mvMDWSdPPvcum0c0iEHE2gbqtV2NKMmLwrl9E6K7g8lTV95SePLnW_bwyMPV7EGp7PQk3l17I5XRhFjze7GqTnFIOgKzMianPs7jv2ELtBMGK0xOPATgu1iGb70xZ6vcvuEfRyY3dJ0zr4jpUdVuXwKmx9rK4IdZn2dFCKfvSuspqIpz11RhF1ALrqDLkxGVv7ZwNh0_VhJZU9hcjG5l6xc7rQEKpPRkZp0IdjkGS8Z0FskoVaiRIWAbZuiVFB9WCW8k1czC4HQTPLpII01bUQx2ludbm0UlXRgVU9ptUUbU7GAImQqTOW8LfPGklEvcgzlIlR_oqw4P9yBxLi-yMQ","dp":"pvNCSnnhbo8Igw9psPR-DicxFnkXlu_ix4gpy6efTrxA-z1VDFDioJ814vKQNioYDzpyAP1gfMPhRkvG_q0hRZsJah3Sb9dfA-WkhSWY7lURQP4yIBTMU0PF_rEATuS7lRciYk1SOx5fqXZd3m_LP0vpBC4Ujlq6NAq6CIjCnms","dq":"TtUVGCCkPNgfOLmkYXu7dxxUCV5kB01-xAEK2OY0n0pG8vfDophH4_D_ZC7nvJ8J9uDhs_3JStexq1lIvaWtG99RNTChIEDzpdn6GH9yaVcb_eB4uJjrNm64FhF8PGCCwxA-xMCZMaARKwhMB2_IOMkxUbWboL3gnhJ2rDO_QO0","e":"AQAB","kid":"8uHDw3M6rf8","kty":"RSA","n":"yaeEL0VKoPBXIAaWXsUgmu05lAvEIIdJn0FX9lHh4JE5UY9B83C5sCNdhs9iSWzpeP11EVjWp8i3Yv2CF7c7u50BXnVBGtxpZpFC-585UXacoJ0chUmarL9GRFJcM1nPHBTFu68aRrn1rIKNHUkNaaxFo0NFGl_4EDDTO8HwawTjwkPoQlRzeByhlvGPVvwgB3Fn93B8QJ_cZhXKxJvjjrC_8Pk76heC_ntEMru71Ix77BoC3j2TuyiN7m9RNBW8BU5q6lKoIdvIeZfTFLzi37iufyfvMrJTixp9zhNB1NxlLCeOZl2MXegtiGqd2H3cbAyqoOiv9ihUWTfXj7SxJw","p":"_Yylc9e07CKdqNRD2EosMC2mrhrEa9j5oY_l00Qyy4-jmCA59Q9viyqvveRo0U7cRvFA5BWgWN6GGLh1DG3X-QBqVr0dnk3uzbobb55RYUXyPLuBZI2q6w2oasbiDwPdY7KpkVv_H-bpITQlyDvO8hhucA6rUV7F6KTQVz8M3Ms","q":"y5p3hch-7jJ21TkAhp_Vk1fLCAuD4tbErwQs2of9ja8sB4iJOs5Wn6HD3P7Mc8Plye7qaLHvzc8I5g0tPKWvC0DPd_FLPXiWwMVAzee3NUX_oGeJNOQp11y1w_KqdO9qZqHSEPZ3NcFL_SZMFgggxhM1uzRiPzsVN0lnD_6prZU","qi":"2Grt6uXHm61ji3xSdkBWNtUnj19vS1-7rFJp5SoYztVQVThf_W52BAiXKBdYZDRVoItC_VS2NvAOjeJjhYO_xQ_q3hK7MdtuXfEPpLnyXKkmWo3lrJ26wbeF6l05LexCkI7ShsOuSt-dsyaTJTszuKDIA6YOfWvfo3aVZmlWRaI","use":"sig"}`
|
||||
|
||||
privateKey, _ := jwk.ParseKey([]byte(privateKeyString))
|
||||
_ = s.jwtService.SetKey(privateKey)
|
||||
}
|
||||
|
||||
// getCborPublicKey decodes a Base64 encoded public key and returns the CBOR encoded COSE key
|
||||
func (s *TestService) getCborPublicKey(base64PublicKey string) ([]byte, error) {
|
||||
decodedKey, err := base64.StdEncoding.DecodeString(base64PublicKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode base64 key: %w", err)
|
||||
}
|
||||
pubKey, err := x509.ParsePKIXPublicKey(decodedKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse public key: %w", err)
|
||||
}
|
||||
|
||||
ecdsaPubKey, ok := pubKey.(*ecdsa.PublicKey)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("not an ECDSA public key")
|
||||
}
|
||||
|
||||
coseKey := map[int]interface{}{
|
||||
1: 2, // Key type: EC2
|
||||
3: -7, // Algorithm: ECDSA with SHA-256
|
||||
-1: 1, // Curve: P-256
|
||||
-2: ecdsaPubKey.X.Bytes(), // X coordinate
|
||||
-3: ecdsaPubKey.Y.Bytes(), // Y coordinate
|
||||
}
|
||||
|
||||
cborPublicKey, err := cbor.Marshal(coseKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal COSE key: %w", err)
|
||||
}
|
||||
|
||||
return cborPublicKey, nil
|
||||
func (s *TestService) ResetLock(ctx context.Context) error {
|
||||
_, err := s.appLockService.Acquire(ctx, true)
|
||||
return err
|
||||
}
|
||||
|
||||
// SyncLdap triggers an LDAP synchronization
|
||||
|
||||
217
backend/internal/service/export_service.go
Normal file
217
backend/internal/service/export_service.go
Normal file
@@ -0,0 +1,217 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
datatype "github.com/pocket-id/pocket-id/backend/internal/model/types"
|
||||
"github.com/pocket-id/pocket-id/backend/internal/storage"
|
||||
"github.com/pocket-id/pocket-id/backend/internal/utils"
|
||||
)
|
||||
|
||||
// ExportService handles exporting Pocket ID data into a ZIP archive.
|
||||
type ExportService struct {
|
||||
db *gorm.DB
|
||||
storage storage.FileStorage
|
||||
}
|
||||
|
||||
func NewExportService(db *gorm.DB, storage storage.FileStorage) *ExportService {
|
||||
return &ExportService{
|
||||
db: db,
|
||||
storage: storage,
|
||||
}
|
||||
}
|
||||
|
||||
// ExportToZip performs the full export process and writes the ZIP data to the given writer.
|
||||
func (s *ExportService) ExportToZip(ctx context.Context, w io.Writer) error {
|
||||
dbData, err := s.extractDatabase()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.writeExportZipStream(ctx, w, dbData)
|
||||
}
|
||||
|
||||
// extractDatabase reads all tables into a DatabaseExport struct
|
||||
func (s *ExportService) extractDatabase() (DatabaseExport, error) {
|
||||
schema, err := utils.LoadDBSchemaTypes(s.db)
|
||||
if err != nil {
|
||||
return DatabaseExport{}, fmt.Errorf("failed to load schema types: %w", err)
|
||||
}
|
||||
|
||||
version, err := s.schemaVersion()
|
||||
if err != nil {
|
||||
return DatabaseExport{}, err
|
||||
}
|
||||
|
||||
out := DatabaseExport{
|
||||
Provider: s.db.Name(),
|
||||
Version: version,
|
||||
Tables: map[string][]map[string]any{},
|
||||
// These tables need to be inserted in a specific order because of foreign key constraints
|
||||
// Not all tables are listed here, because not all tables are order-dependent
|
||||
TableOrder: []string{"users", "user_groups", "oidc_clients"},
|
||||
}
|
||||
|
||||
for table := range schema {
|
||||
if table == "storage" || table == "schema_migrations" {
|
||||
continue
|
||||
}
|
||||
err = s.dumpTable(table, schema[table], &out)
|
||||
if err != nil {
|
||||
return DatabaseExport{}, err
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *ExportService) schemaVersion() (uint, error) {
|
||||
var version uint
|
||||
if err := s.db.Raw("SELECT version FROM schema_migrations").Row().Scan(&version); err != nil {
|
||||
return 0, fmt.Errorf("failed to query schema version: %w", err)
|
||||
}
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// dumpTable selects all rows from a table and appends them to out.Tables
|
||||
func (s *ExportService) dumpTable(table string, types utils.DBSchemaTableTypes, out *DatabaseExport) error {
|
||||
rows, err := s.db.Raw("SELECT * FROM " + table).Rows()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read table %s: %w", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
cols, _ := rows.Columns()
|
||||
if len(cols) != len(types) {
|
||||
// Should never happen...
|
||||
return fmt.Errorf("mismatched columns in table (%d) and schema (%d)", len(cols), len(types))
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
vals := s.getScanValuesForTable(cols, types)
|
||||
err = rows.Scan(vals...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to scan row in table %s: %w", table, err)
|
||||
}
|
||||
|
||||
rowMap := make(map[string]any, len(cols))
|
||||
for i, col := range cols {
|
||||
rowMap[col] = vals[i]
|
||||
}
|
||||
|
||||
// Skip the app lock row in the kv table
|
||||
if table == "kv" {
|
||||
if keyPtr, ok := rowMap["key"].(*string); ok && keyPtr != nil && *keyPtr == lockKey {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
out.Tables[table] = append(out.Tables[table], rowMap)
|
||||
}
|
||||
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func (s *ExportService) getScanValuesForTable(cols []string, types utils.DBSchemaTableTypes) []any {
|
||||
res := make([]any, len(cols))
|
||||
for i, col := range cols {
|
||||
// Store a pointer
|
||||
// Note: don't create a helper function for this switch, because it would return type "any" and mess everything up
|
||||
// If the column is nullable, we need a pointer to a pointer!
|
||||
switch types[col].Name {
|
||||
case "boolean", "bool":
|
||||
var x bool
|
||||
if types[col].Nullable {
|
||||
res[i] = utils.Ptr(utils.Ptr(x))
|
||||
} else {
|
||||
res[i] = utils.Ptr(x)
|
||||
}
|
||||
case "blob", "bytea", "jsonb":
|
||||
// Treat jsonb columns as binary too
|
||||
var x []byte
|
||||
if types[col].Nullable {
|
||||
res[i] = utils.Ptr(utils.Ptr(x))
|
||||
} else {
|
||||
res[i] = utils.Ptr(x)
|
||||
}
|
||||
case "timestamp", "timestamptz", "timestamp with time zone", "datetime":
|
||||
var x datatype.DateTime
|
||||
if types[col].Nullable {
|
||||
res[i] = utils.Ptr(utils.Ptr(x))
|
||||
} else {
|
||||
res[i] = utils.Ptr(x)
|
||||
}
|
||||
case "integer", "int", "bigint":
|
||||
var x int64
|
||||
if types[col].Nullable {
|
||||
res[i] = utils.Ptr(utils.Ptr(x))
|
||||
} else {
|
||||
res[i] = utils.Ptr(x)
|
||||
}
|
||||
default:
|
||||
// Treat everything else as a string (including the "numeric" type)
|
||||
var x string
|
||||
if types[col].Nullable {
|
||||
res[i] = utils.Ptr(utils.Ptr(x))
|
||||
} else {
|
||||
res[i] = utils.Ptr(x)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func (s *ExportService) writeExportZipStream(ctx context.Context, w io.Writer, dbData DatabaseExport) error {
|
||||
zipWriter := zip.NewWriter(w)
|
||||
|
||||
// Add database.json
|
||||
jsonWriter, err := zipWriter.Create("database.json")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create database.json in zip: %w", err)
|
||||
}
|
||||
|
||||
jsonEncoder := json.NewEncoder(jsonWriter)
|
||||
jsonEncoder.SetEscapeHTML(false)
|
||||
|
||||
if err := jsonEncoder.Encode(dbData); err != nil {
|
||||
return fmt.Errorf("failed to encode database.json: %w", err)
|
||||
}
|
||||
|
||||
// Add uploaded files
|
||||
if err := s.addUploadsToZip(ctx, zipWriter); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return zipWriter.Close()
|
||||
}
|
||||
|
||||
// addUploadsToZip adds all files from the storage to the ZIP archive under the "uploads/" directory
|
||||
func (s *ExportService) addUploadsToZip(ctx context.Context, zipWriter *zip.Writer) error {
|
||||
return s.storage.Walk(ctx, "/", func(p storage.ObjectInfo) error {
|
||||
zipPath := filepath.Join("uploads", p.Path)
|
||||
|
||||
w, err := zipWriter.Create(zipPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create zip entry for %s: %w", zipPath, err)
|
||||
}
|
||||
|
||||
f, _, err := s.storage.Open(ctx, p.Path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open file %s: %w", zipPath, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if _, err := io.Copy(w, f); err != nil {
|
||||
return fmt.Errorf("failed to copy file %s into zip: %w", zipPath, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
264
backend/internal/service/import_service.go
Normal file
264
backend/internal/service/import_service.go
Normal file
@@ -0,0 +1,264 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
datatype "github.com/pocket-id/pocket-id/backend/internal/model/types"
|
||||
"github.com/pocket-id/pocket-id/backend/internal/storage"
|
||||
"github.com/pocket-id/pocket-id/backend/internal/utils"
|
||||
)
|
||||
|
||||
// ImportService handles importing Pocket ID data from an exported ZIP archive.
|
||||
type ImportService struct {
|
||||
db *gorm.DB
|
||||
storage storage.FileStorage
|
||||
}
|
||||
|
||||
type DatabaseExport struct {
|
||||
Provider string `json:"provider"`
|
||||
Version uint `json:"version"`
|
||||
Tables map[string][]map[string]any `json:"tables"`
|
||||
TableOrder []string `json:"tableOrder"`
|
||||
}
|
||||
|
||||
func NewImportService(db *gorm.DB, storage storage.FileStorage) *ImportService {
|
||||
return &ImportService{
|
||||
db: db,
|
||||
storage: storage,
|
||||
}
|
||||
}
|
||||
|
||||
// ImportFromZip performs the full import process from the given ZIP reader.
|
||||
func (s *ImportService) ImportFromZip(ctx context.Context, r *zip.Reader) error {
|
||||
dbData, err := processZipDatabaseJson(r.File)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = s.ImportDatabase(dbData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = s.importUploads(ctx, r.File)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ImportDatabase only imports the database data from the given DatabaseExport struct.
|
||||
func (s *ImportService) ImportDatabase(dbData DatabaseExport) error {
|
||||
err := s.resetSchema(dbData.Version, dbData.Provider)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = s.insertData(dbData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// processZipDatabaseJson extracts database.json from the ZIP archive
|
||||
func processZipDatabaseJson(files []*zip.File) (dbData DatabaseExport, err error) {
|
||||
for _, f := range files {
|
||||
if f.Name == "database.json" {
|
||||
return parseDatabaseJsonStream(f)
|
||||
}
|
||||
}
|
||||
return dbData, errors.New("database.json not found in the ZIP file")
|
||||
}
|
||||
|
||||
func parseDatabaseJsonStream(f *zip.File) (dbData DatabaseExport, err error) {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return dbData, fmt.Errorf("failed to open database.json: %w", err)
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
err = json.NewDecoder(rc).Decode(&dbData)
|
||||
if err != nil {
|
||||
return dbData, fmt.Errorf("failed to decode database.json: %w", err)
|
||||
}
|
||||
|
||||
return dbData, nil
|
||||
}
|
||||
|
||||
// importUploads imports files from the uploads/ directory in the ZIP archive
|
||||
func (s *ImportService) importUploads(ctx context.Context, files []*zip.File) error {
|
||||
const maxFileSize = 50 << 20 // 50 MiB
|
||||
const uploadsPrefix = "uploads/"
|
||||
|
||||
for _, f := range files {
|
||||
if !strings.HasPrefix(f.Name, uploadsPrefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
if f.UncompressedSize64 > maxFileSize {
|
||||
return fmt.Errorf("file %s too large (%d bytes)", f.Name, f.UncompressedSize64)
|
||||
}
|
||||
|
||||
targetPath := strings.TrimPrefix(f.Name, uploadsPrefix)
|
||||
if strings.HasSuffix(f.Name, "/") || targetPath == "" {
|
||||
continue // Skip directories
|
||||
}
|
||||
|
||||
err := s.storage.DeleteAll(ctx, targetPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete existing file %s: %w", targetPath, err)
|
||||
}
|
||||
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
buf, err := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("read file %s: %w", f.Name, err)
|
||||
}
|
||||
|
||||
err = s.storage.Save(ctx, targetPath, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save file %s: %w", targetPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// resetSchema drops the existing schema and migrates to the target version
|
||||
func (s *ImportService) resetSchema(targetVersion uint, exportDbProvider string) error {
|
||||
sqlDb, err := s.db.DB()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get sql.DB: %w", err)
|
||||
}
|
||||
|
||||
m, err := utils.GetEmbeddedMigrateInstance(sqlDb)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get migrate instance: %w", err)
|
||||
}
|
||||
|
||||
err = m.Drop()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to drop existing schema: %w", err)
|
||||
}
|
||||
|
||||
// Needs to be called again to re-create the schema_migrations table
|
||||
m, err = utils.GetEmbeddedMigrateInstance(sqlDb)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get migrate instance: %w", err)
|
||||
}
|
||||
|
||||
err = m.Migrate(targetVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("migration failed: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// insertData populates the DB with the imported data
|
||||
func (s *ImportService) insertData(dbData DatabaseExport) error {
|
||||
schema, err := utils.LoadDBSchemaTypes(s.db)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load schema types: %w", err)
|
||||
}
|
||||
|
||||
return s.db.Transaction(func(tx *gorm.DB) error {
|
||||
// Iterate through all tables
|
||||
// Some tables need to be processed in order
|
||||
tables := make([]string, 0, len(dbData.Tables))
|
||||
tables = append(tables, dbData.TableOrder...)
|
||||
|
||||
for t := range dbData.Tables {
|
||||
// Skip tables already present where the order matters
|
||||
// Also skip the schema_migrations table
|
||||
if slices.Contains(dbData.TableOrder, t) || t == "schema_migrations" {
|
||||
continue
|
||||
}
|
||||
tables = append(tables, t)
|
||||
}
|
||||
|
||||
// Insert rows
|
||||
for _, table := range tables {
|
||||
for _, row := range dbData.Tables[table] {
|
||||
err = normalizeRowWithSchema(row, table, schema)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to normalize row for table '%s': %w", table, err)
|
||||
}
|
||||
err = tx.Table(table).Create(row).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed inserting into table '%s': %w", table, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// normalizeRowWithSchema converts row values based on the DB schema
|
||||
func normalizeRowWithSchema(row map[string]any, table string, schema utils.DBSchemaTypes) error {
|
||||
if schema[table] == nil {
|
||||
return fmt.Errorf("schema not found for table '%s'", table)
|
||||
}
|
||||
|
||||
for col, val := range row {
|
||||
if val == nil {
|
||||
// If the value is nil, skip the column
|
||||
continue
|
||||
}
|
||||
|
||||
colType := schema[table][col]
|
||||
|
||||
switch colType.Name {
|
||||
case "timestamp", "timestamptz", "timestamp with time zone", "datetime":
|
||||
// Dates are stored as strings
|
||||
str, ok := val.(string)
|
||||
if !ok {
|
||||
return fmt.Errorf("value for column '%s/%s' was expected to be a string, but was '%T'", table, col, val)
|
||||
}
|
||||
d, err := datatype.DateTimeFromString(str)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decode value for column '%s/%s' as timestamp: %w", table, col, err)
|
||||
}
|
||||
row[col] = d
|
||||
|
||||
case "blob", "bytea", "jsonb":
|
||||
// Binary data and jsonb data is stored in the file as base64-encoded string
|
||||
str, ok := val.(string)
|
||||
if !ok {
|
||||
return fmt.Errorf("value for column '%s/%s' was expected to be a string, but was '%T'", table, col, val)
|
||||
}
|
||||
b, err := base64.StdEncoding.DecodeString(str)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decode value for column '%s/%s' from base64: %w", table, col, err)
|
||||
}
|
||||
|
||||
// For jsonb, we additionally cast to json.RawMessage
|
||||
if colType.Name == "jsonb" {
|
||||
row[col] = json.RawMessage(b)
|
||||
} else {
|
||||
row[col] = b
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -48,6 +48,7 @@ const (
|
||||
)
|
||||
|
||||
type JwtService struct {
|
||||
db *gorm.DB
|
||||
envConfig *common.EnvConfigSchema
|
||||
privateKey jwk.Key
|
||||
keyId string
|
||||
@@ -58,7 +59,6 @@ type JwtService struct {
|
||||
func NewJwtService(db *gorm.DB, appConfigService *AppConfigService) (*JwtService, error) {
|
||||
service := &JwtService{}
|
||||
|
||||
// Ensure keys are generated or loaded
|
||||
err := service.init(db, appConfigService, &common.EnvConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -70,14 +70,15 @@ func NewJwtService(db *gorm.DB, appConfigService *AppConfigService) (*JwtService
|
||||
func (s *JwtService) init(db *gorm.DB, appConfigService *AppConfigService, envConfig *common.EnvConfigSchema) (err error) {
|
||||
s.appConfigService = appConfigService
|
||||
s.envConfig = envConfig
|
||||
s.db = db
|
||||
|
||||
// Ensure keys are generated or loaded
|
||||
return s.loadOrGenerateKey(db)
|
||||
return s.LoadOrGenerateKey()
|
||||
}
|
||||
|
||||
func (s *JwtService) loadOrGenerateKey(db *gorm.DB) error {
|
||||
func (s *JwtService) LoadOrGenerateKey() error {
|
||||
// Get the key provider
|
||||
keyProvider, err := jwkutils.GetKeyProvider(db, s.envConfig, s.appConfigService.GetDbConfig().InstanceID.Value)
|
||||
keyProvider, err := jwkutils.GetKeyProvider(s.db, s.envConfig, s.appConfigService.GetDbConfig().InstanceID.Value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get key provider: %w", err)
|
||||
}
|
||||
|
||||
130
backend/internal/utils/db_migration_util.go
Normal file
130
backend/internal/utils/db_migration_util.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
postgresMigrate "github.com/golang-migrate/migrate/v4/database/postgres"
|
||||
sqliteMigrate "github.com/golang-migrate/migrate/v4/database/sqlite3"
|
||||
"github.com/golang-migrate/migrate/v4/source/iofs"
|
||||
"github.com/pocket-id/pocket-id/backend/internal/common"
|
||||
"github.com/pocket-id/pocket-id/backend/resources"
|
||||
)
|
||||
|
||||
// MigrateDatabase applies database migrations using embedded migration files or fetches them from GitHub if a downgrade is detected.
|
||||
func MigrateDatabase(sqlDb *sql.DB) error {
|
||||
m, err := GetEmbeddedMigrateInstance(sqlDb)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get migrate instance: %w", err)
|
||||
}
|
||||
|
||||
path := "migrations/" + string(common.EnvConfig.DbProvider)
|
||||
requiredVersion, err := getRequiredMigrationVersion(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get last migration version: %w", err)
|
||||
}
|
||||
|
||||
currentVersion, _, _ := m.Version()
|
||||
if currentVersion > requiredVersion {
|
||||
slog.Warn("Database version is newer than the application supports, possible downgrade detected", slog.Uint64("db_version", uint64(currentVersion)), slog.Uint64("app_version", uint64(requiredVersion)))
|
||||
if !common.EnvConfig.AllowDowngrade {
|
||||
return fmt.Errorf("database version (%d) is newer than application version (%d), downgrades are not allowed (set ALLOW_DOWNGRADE=true to enable)", currentVersion, requiredVersion)
|
||||
}
|
||||
slog.Info("Fetching migrations from GitHub to handle possible downgrades")
|
||||
return migrateDatabaseFromGitHub(sqlDb, requiredVersion)
|
||||
}
|
||||
|
||||
if err := m.Migrate(requiredVersion); err != nil && !errors.Is(err, migrate.ErrNoChange) {
|
||||
return fmt.Errorf("failed to apply embedded migrations: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetEmbeddedMigrateInstance creates a migrate.Migrate instance using embedded migration files.
|
||||
func GetEmbeddedMigrateInstance(sqlDb *sql.DB) (*migrate.Migrate, error) {
|
||||
path := "migrations/" + string(common.EnvConfig.DbProvider)
|
||||
source, err := iofs.New(resources.FS, path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create embedded migration source: %w", err)
|
||||
}
|
||||
|
||||
driver, err := newMigrationDriver(sqlDb, common.EnvConfig.DbProvider)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create migration driver: %w", err)
|
||||
}
|
||||
|
||||
m, err := migrate.NewWithInstance("iofs", source, "pocket-id", driver)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create migration instance: %w", err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// newMigrationDriver creates a database.Driver instance based on the given database provider.
|
||||
func newMigrationDriver(sqlDb *sql.DB, dbProvider common.DbProvider) (driver database.Driver, err error) {
|
||||
switch dbProvider {
|
||||
case common.DbProviderSqlite:
|
||||
driver, err = sqliteMigrate.WithInstance(sqlDb, &sqliteMigrate.Config{
|
||||
NoTxWrap: true,
|
||||
})
|
||||
case common.DbProviderPostgres:
|
||||
driver, err = postgresMigrate.WithInstance(sqlDb, &postgresMigrate.Config{})
|
||||
default:
|
||||
// Should never happen at this point
|
||||
return nil, fmt.Errorf("unsupported database provider: %s", common.EnvConfig.DbProvider)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create migration driver: %w", err)
|
||||
}
|
||||
|
||||
return driver, nil
|
||||
}
|
||||
|
||||
// migrateDatabaseFromGitHub applies database migrations fetched from GitHub to handle downgrades.
|
||||
func migrateDatabaseFromGitHub(sqlDb *sql.DB, version uint) error {
|
||||
srcURL := "github://pocket-id/pocket-id/backend/resources/migrations/" + string(common.EnvConfig.DbProvider)
|
||||
|
||||
driver, err := newMigrationDriver(sqlDb, common.EnvConfig.DbProvider)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create migration driver: %w", err)
|
||||
}
|
||||
|
||||
m, err := migrate.NewWithDatabaseInstance(srcURL, "pocket-id", driver)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create GitHub migration instance: %w", err)
|
||||
}
|
||||
|
||||
if err := m.Migrate(version); err != nil && !errors.Is(err, migrate.ErrNoChange) {
|
||||
return fmt.Errorf("failed to apply GitHub migrations: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getRequiredMigrationVersion reads the embedded migration files and returns the highest version number found.
|
||||
func getRequiredMigrationVersion(path string) (uint, error) {
|
||||
entries, err := resources.FS.ReadDir(path)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to read migration directory: %w", err)
|
||||
}
|
||||
|
||||
var maxVersion uint
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
var version uint
|
||||
n, err := fmt.Sscanf(name, "%d_", &version)
|
||||
if err == nil && n == 1 {
|
||||
if version > maxVersion {
|
||||
maxVersion = version
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return maxVersion, nil
|
||||
}
|
||||
116
backend/internal/utils/db_util.go
Normal file
116
backend/internal/utils/db_util.go
Normal file
@@ -0,0 +1,116 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// DBTableExists checks if a table exists in the database
|
||||
func DBTableExists(db *gorm.DB, tableName string) (exists bool, err error) {
|
||||
switch db.Name() {
|
||||
case "postgres":
|
||||
query := `SELECT EXISTS (
|
||||
SELECT FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = ?
|
||||
)`
|
||||
err = db.Raw(query, tableName).Scan(&exists).Error
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
case "sqlite":
|
||||
query := `SELECT COUNT(*) > 0 FROM sqlite_master WHERE type='table' AND name=?`
|
||||
err = db.Raw(query, tableName).Scan(&exists).Error
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
default:
|
||||
return false, fmt.Errorf("unsupported database dialect: %s", db.Name())
|
||||
}
|
||||
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
type DBSchemaColumn struct {
|
||||
Name string
|
||||
Nullable bool
|
||||
}
|
||||
type DBSchemaTableTypes = map[string]DBSchemaColumn
|
||||
type DBSchemaTypes = map[string]DBSchemaTableTypes
|
||||
|
||||
// LoadDBSchemaTypes retrieves the column types for all tables in the DB
|
||||
// Result is a map of "table --> column --> {name: column type name, nullable: boolean}"
|
||||
func LoadDBSchemaTypes(db *gorm.DB) (result DBSchemaTypes, err error) {
|
||||
result = make(DBSchemaTypes)
|
||||
|
||||
switch db.Name() {
|
||||
case "postgres":
|
||||
var rows []struct {
|
||||
TableName string
|
||||
ColumnName string
|
||||
DataType string
|
||||
Nullable bool
|
||||
}
|
||||
err := db.
|
||||
Raw(`
|
||||
SELECT table_name, column_name, data_type, is_nullable = 'YES' AS nullable
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public';
|
||||
`).
|
||||
Scan(&rows).
|
||||
Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, r := range rows {
|
||||
t := strings.ToLower(r.DataType)
|
||||
if result[r.TableName] == nil {
|
||||
result[r.TableName] = make(map[string]DBSchemaColumn)
|
||||
}
|
||||
result[r.TableName][r.ColumnName] = DBSchemaColumn{
|
||||
Name: strings.ToLower(t),
|
||||
Nullable: r.Nullable,
|
||||
}
|
||||
}
|
||||
|
||||
case "sqlite":
|
||||
var tables []string
|
||||
err = db.
|
||||
Raw(`SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%';`).
|
||||
Scan(&tables).
|
||||
Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, table := range tables {
|
||||
var cols []struct {
|
||||
Name string
|
||||
Type string
|
||||
Notnull bool
|
||||
}
|
||||
err := db.
|
||||
Raw(`PRAGMA table_info("` + table + `");`).
|
||||
Scan(&cols).
|
||||
Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range cols {
|
||||
if result[table] == nil {
|
||||
result[table] = make(map[string]DBSchemaColumn)
|
||||
}
|
||||
result[table][c.Name] = DBSchemaColumn{
|
||||
Name: strings.ToLower(c.Type),
|
||||
Nullable: !c.Notnull,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported database dialect: %s", db.Name())
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
@@ -38,6 +38,7 @@ func (r *ServiceRunner) Run(ctx context.Context) error {
|
||||
|
||||
// Ignore context canceled errors here as they generally indicate that the service is stopping
|
||||
if rErr != nil && !errors.Is(rErr, context.Canceled) {
|
||||
cancel()
|
||||
errCh <- rErr
|
||||
return
|
||||
}
|
||||
|
||||
@@ -61,6 +61,26 @@ func TestServiceRunner_Run(t *testing.T) {
|
||||
require.ErrorIs(t, err, expectedErr)
|
||||
})
|
||||
|
||||
t.Run("service error cancels others", func(t *testing.T) {
|
||||
expectedErr := errors.New("boom")
|
||||
errorService := func(ctx context.Context) error {
|
||||
return expectedErr
|
||||
}
|
||||
waitingService := func(ctx context.Context) error {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
runner := NewServiceRunner(errorService, waitingService)
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := runner.Run(ctx)
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, expectedErr)
|
||||
})
|
||||
|
||||
t.Run("context canceled", func(t *testing.T) {
|
||||
// Create a service that waits until context is canceled
|
||||
waitingService := func(ctx context.Context) error {
|
||||
|
||||
Reference in New Issue
Block a user