mirrors/pocket-id
1
0
Fork 0
mirror of https://github.com/pocket-id/pocket-id.git synced 2026-02-15 07:00:06 +00:00
Code Activity

feat: add CLI command for importing and exporting Pocket ID data (#998)

Browse Source 
Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Elias Schneider
2025-11-26 10:38:15 +01:00
parent f0144584af
commit 3420a00073
56 changed files with 3178 additions and 643 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
backend/internal/bootstrap/bootstrap.go
View File

@@ -7,6 +7,7 @@ import (
"time"
_ "github.com/golang-migrate/migrate/v4/source/file"
"gorm.io/gorm"
"github.com/pocket-id/pocket-id/backend/internal/common"
"github.com/pocket-id/pocket-id/backend/internal/job"
@@ -15,6 +16,16 @@ import (
)
func Bootstrap(ctx context.Context) error {
var shutdownFns []utils.Service
defer func() { //nolint:contextcheck
// Invoke all shutdown functions on exit
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := utils.NewServiceRunner(shutdownFns...).Run(shutdownCtx); err != nil {
slog.Error("Error during graceful shutdown", "error", err)
}
}()
// Initialize the observability stack, including the logger, distributed tracing, and metrics
shutdownFns, httpClient, err := initObservability(ctx, common.EnvConfig.MetricsEnabled, common.EnvConfig.TracingEnabled)
if err != nil {
@@ -22,15 +33,80 @@ func Bootstrap(ctx context.Context) error {
}
slog.InfoContext(ctx, "Pocket ID is starting")
// Connect to the database
db, err := NewDatabase()
if err != nil {
return fmt.Errorf("failed to initialize database: %w", err)
}
// Initialize the file storage backend
var fileStorage storage.FileStorage
fileStorage, err := InitStorage(ctx, db)
if err != nil {
return fmt.Errorf("failed to initialize file storage (backend: %s): %w", common.EnvConfig.FileBackend, err)
}
imageExtensions, err := initApplicationImages(ctx, fileStorage)
if err != nil {
return fmt.Errorf("failed to initialize application images: %w", err)
}
// Create all services
svc, err := initServices(ctx, db, httpClient, imageExtensions, fileStorage)
if err != nil {
return fmt.Errorf("failed to initialize services: %w", err)
}
waitUntil, err := svc.appLockService.Acquire(ctx, false)
if err != nil {
return fmt.Errorf("failed to acquire application lock: %w", err)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Until(waitUntil)):
}
shutdownFn := func(shutdownCtx context.Context) error {
sErr := svc.appLockService.Release(shutdownCtx)
if sErr != nil {
return fmt.Errorf("failed to release application lock: %w", sErr)
}
return nil
}
shutdownFns = append(shutdownFns, shutdownFn)
// Init the job scheduler
scheduler, err := job.NewScheduler()
if err != nil {
return fmt.Errorf("failed to create job scheduler: %w", err)
}
err = registerScheduledJobs(ctx, db, svc, httpClient, scheduler)
if err != nil {
return fmt.Errorf("failed to register scheduled jobs: %w", err)
}
// Init the router
router, err := initRouter(db, svc)
if err != nil {
return fmt.Errorf("failed to initialize router: %w", err)
}
// Run all background services
// This call blocks until the context is canceled
services := []utils.Service{svc.appLockService.RunRenewal, router}
if common.EnvConfig.AppEnv != "test" {
services = append(services, scheduler.Run)
}
err = utils.NewServiceRunner(services...).Run(ctx)
if err != nil {
return fmt.Errorf("failed to run services: %w", err)
}
return nil
}
func InitStorage(ctx context.Context, db *gorm.DB) (fileStorage storage.FileStorage, err error) {
switch common.EnvConfig.FileBackend {
case storage.TypeFileSystem:
fileStorage, err = storage.NewFilesystemStorage(common.EnvConfig.UploadPath)
@@ -52,53 +128,8 @@ func Bootstrap(ctx context.Context) error {
err = fmt.Errorf("unknown file storage backend: %s", common.EnvConfig.FileBackend)
}
if err != nil {
return fmt.Errorf("failed to initialize file storage (backend: %s): %w", common.EnvConfig.FileBackend, err)
return fileStorage, err
}
imageExtensions, err := initApplicationImages(ctx, fileStorage)
if err != nil {
return fmt.Errorf("failed to initialize application images: %w", err)
}
// Create all services
svc, err := initServices(ctx, db, httpClient, imageExtensions, fileStorage)
if err != nil {
return fmt.Errorf("failed to initialize services: %w", err)
}
// Init the job scheduler
scheduler, err := job.NewScheduler()
if err != nil {
return fmt.Errorf("failed to create job scheduler: %w", err)
}
err = registerScheduledJobs(ctx, db, svc, httpClient, scheduler)
if err != nil {
return fmt.Errorf("failed to register scheduled jobs: %w", err)
}
// Init the router
router := initRouter(db, svc)
// Run all background services
// This call blocks until the context is canceled
err = utils.
NewServiceRunner(router, scheduler.Run).
Run(ctx)
if err != nil {
return fmt.Errorf("failed to run services: %w", err)
}
// Invoke all shutdown functions
// We give these a timeout of 5s
// Note: we use a background context because the run context has been canceled already
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer shutdownCancel()
err = utils.
NewServiceRunner(shutdownFns...).
Run(shutdownCtx) //nolint:contextcheck
if err != nil {
slog.Error("Error shutting down services", slog.Any("error", err))
}
return nil
return fileStorage, nil
}

102
backend/internal/bootstrap/db_bootstrap.go
View File

@@ -12,12 +12,7 @@ import (
"time"
"github.com/glebarez/sqlite"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database"
postgresMigrate "github.com/golang-migrate/migrate/v4/database/postgres"
sqliteMigrate "github.com/golang-migrate/migrate/v4/database/sqlite3"
_ "github.com/golang-migrate/migrate/v4/source/github"
"github.com/golang-migrate/migrate/v4/source/iofs"
slogGorm "github.com/orandin/slog-gorm"
"gorm.io/driver/postgres"
"gorm.io/gorm"
@@ -26,11 +21,10 @@ import (
"github.com/pocket-id/pocket-id/backend/internal/common"
"github.com/pocket-id/pocket-id/backend/internal/utils"
sqliteutil "github.com/pocket-id/pocket-id/backend/internal/utils/sqlite"
"github.com/pocket-id/pocket-id/backend/resources"
)
func NewDatabase() (db *gorm.DB, err error) {
db, err = connectDatabase()
db, err = ConnectDatabase()
if err != nil {
return nil, fmt.Errorf("failed to connect to database: %w", err)
}
@@ -39,105 +33,15 @@ func NewDatabase() (db *gorm.DB, err error) {
return nil, fmt.Errorf("failed to get sql.DB: %w", err)
}
// Choose the correct driver for the database provider
var driver database.Driver
switch common.EnvConfig.DbProvider {
case common.DbProviderSqlite:
driver, err = sqliteMigrate.WithInstance(sqlDb, &sqliteMigrate.Config{
NoTxWrap: true,
})
case common.DbProviderPostgres:
driver, err = postgresMigrate.WithInstance(sqlDb, &postgresMigrate.Config{})
default:
// Should never happen at this point
return nil, fmt.Errorf("unsupported database provider: %s", common.EnvConfig.DbProvider)
}
if err != nil {
return nil, fmt.Errorf("failed to create migration driver: %w", err)
}
// Run migrations
if err := migrateDatabase(driver); err != nil {
if err := utils.MigrateDatabase(sqlDb); err != nil {
return nil, fmt.Errorf("failed to run migrations: %w", err)
}
return db, nil
}
func migrateDatabase(driver database.Driver) error {
// Embedded migrations via iofs
path := "migrations/" + string(common.EnvConfig.DbProvider)
source, err := iofs.New(resources.FS, path)
if err != nil {
return fmt.Errorf("failed to create embedded migration source: %w", err)
}
m, err := migrate.NewWithInstance("iofs", source, "pocket-id", driver)
if err != nil {
return fmt.Errorf("failed to create migration instance: %w", err)
}
requiredVersion, err := getRequiredMigrationVersion(path)
if err != nil {
return fmt.Errorf("failed to get last migration version: %w", err)
}
currentVersion, _, _ := m.Version()
if currentVersion > requiredVersion {
slog.Warn("Database version is newer than the application supports, possible downgrade detected", slog.Uint64("db_version", uint64(currentVersion)), slog.Uint64("app_version", uint64(requiredVersion)))
if !common.EnvConfig.AllowDowngrade {
return fmt.Errorf("database version (%d) is newer than application version (%d), downgrades are not allowed (set ALLOW_DOWNGRADE=true to enable)", currentVersion, requiredVersion)
}
slog.Info("Fetching migrations from GitHub to handle possible downgrades")
return migrateDatabaseFromGitHub(driver, requiredVersion)
}
if err := m.Migrate(requiredVersion); err != nil && !errors.Is(err, migrate.ErrNoChange) {
return fmt.Errorf("failed to apply embedded migrations: %w", err)
}
return nil
}
func migrateDatabaseFromGitHub(driver database.Driver, version uint) error {
srcURL := "github://pocket-id/pocket-id/backend/resources/migrations/" + string(common.EnvConfig.DbProvider)
m, err := migrate.NewWithDatabaseInstance(srcURL, "pocket-id", driver)
if err != nil {
return fmt.Errorf("failed to create GitHub migration instance: %w", err)
}
if err := m.Migrate(version); err != nil && !errors.Is(err, migrate.ErrNoChange) {
return fmt.Errorf("failed to apply GitHub migrations: %w", err)
}
return nil
}
// getRequiredMigrationVersion reads the embedded migration files and returns the highest version number found.
func getRequiredMigrationVersion(path string) (uint, error) {
entries, err := resources.FS.ReadDir(path)
if err != nil {
return 0, fmt.Errorf("failed to read migration directory: %w", err)
}
var maxVersion uint
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
var version uint
n, err := fmt.Sscanf(name, "%d_", &version)
if err == nil && n == 1 {
if version > maxVersion {
maxVersion = version
}
}
}
return maxVersion, nil
}
func connectDatabase() (db *gorm.DB, err error) {
func ConnectDatabase() (db *gorm.DB, err error) {
var dialector gorm.Dialector
// Choose the correct database provider

2
backend/internal/bootstrap/e2etest_router_bootstrap.go
View File

@@ -17,7 +17,7 @@ import (
func init() {
registerTestControllers = []func(apiGroup *gin.RouterGroup, db *gorm.DB, svc *services){
func(apiGroup *gin.RouterGroup, db *gorm.DB, svc *services) {
testService, err := service.NewTestService(db, svc.appConfigService, svc.jwtService, svc.ldapService, svc.fileStorage)
testService, err := service.NewTestService(db, svc.appConfigService, svc.jwtService, svc.ldapService, svc.appLockService, svc.fileStorage)
if err != nil {
slog.Error("Failed to initialize test service", slog.Any("error", err))
os.Exit(1)

11
backend/internal/bootstrap/router_bootstrap.go
View File

@@ -29,16 +29,7 @@ import (
// This is used to register additional controllers for tests
var registerTestControllers []func(apiGroup *gin.RouterGroup, db *gorm.DB, svc *services)
func initRouter(db *gorm.DB, svc *services) utils.Service {
runner, err := initRouterInternal(db, svc)
if err != nil {
slog.Error("Failed to init router", "error", err)
os.Exit(1)
}
return runner
}
func initRouterInternal(db *gorm.DB, svc *services) (utils.Service, error) {
func initRouter(db *gorm.DB, svc *services) (utils.Service, error) {
// Set the appropriate Gin mode based on the environment
switch common.EnvConfig.AppEnv {
case common.AppEnvProduction:

2
backend/internal/bootstrap/services_bootstrap.go
View File

@@ -27,6 +27,7 @@ type services struct {
apiKeyService *service.ApiKeyService
versionService *service.VersionService
fileStorage storage.FileStorage
appLockService *service.AppLockService
}
// Initializes all services
@@ -40,6 +41,7 @@ func initServices(ctx context.Context, db *gorm.DB, httpClient *http.Client, ima
svc.fileStorage = fileStorage
svc.appImagesService = service.NewAppImagesService(imageExtensions, fileStorage)
svc.appLockService = service.NewAppLockService(db)
svc.emailService, err = service.NewEmailService(db, svc.appConfigService)
if err != nil {