1
0
mirror of https://github.com/pocket-id/pocket-id.git synced 2026-02-15 08:10:13 +00:00

feat: add CLI command for importing and exporting Pocket ID data (#998)

Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Elias Schneider
2025-11-26 10:38:15 +01:00
committed by GitHub
parent e306d6eb58
commit 39c1f93756
56 changed files with 3276 additions and 823 deletions

View File

@@ -0,0 +1,296 @@
package service
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"os"
"time"
"github.com/google/uuid"
"github.com/pocket-id/pocket-id/backend/internal/model"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
var (
ErrLockUnavailable = errors.New("lock is already held by another process")
ErrLockLost = errors.New("lock ownership lost")
)
const (
ttl = 30 * time.Second
renewInterval = 20 * time.Second
renewRetries = 3
lockKey = "application_lock"
)
type AppLockService struct {
db *gorm.DB
lockID string
processID int64
hostID string
}
func NewAppLockService(db *gorm.DB) *AppLockService {
host, err := os.Hostname()
if err != nil || host == "" {
host = "unknown-host"
}
return &AppLockService{
db: db,
processID: int64(os.Getpid()),
hostID: host,
lockID: uuid.NewString(),
}
}
type lockValue struct {
ProcessID int64 `json:"process_id"`
HostID string `json:"host_id"`
LockID string `json:"lock_id"`
ExpiresAt int64 `json:"expires_at"`
}
func (lv *lockValue) Marshal() (string, error) {
data, err := json.Marshal(lv)
if err != nil {
return "", err
}
return string(data), nil
}
func (lv *lockValue) Unmarshal(raw string) error {
if raw == "" {
return nil
}
return json.Unmarshal([]byte(raw), lv)
}
// Acquire obtains the lock. When force is true, the lock is stolen from any existing owner.
// If the lock is forcefully acquired, it blocks until the previous lock has expired.
func (s *AppLockService) Acquire(ctx context.Context, force bool) (waitUntil time.Time, err error) {
tx := s.db.Begin()
defer func() {
tx.Rollback()
}()
var prevLockRaw string
err = tx.
WithContext(ctx).
Model(&model.KV{}).
Where("key = ?", lockKey).
Clauses(clause.Locking{Strength: "UPDATE"}).
Select("value").
Scan(&prevLockRaw).
Error
if err != nil {
return time.Time{}, fmt.Errorf("query existing lock: %w", err)
}
var prevLock lockValue
if prevLockRaw != "" {
if err := prevLock.Unmarshal(prevLockRaw); err != nil {
return time.Time{}, fmt.Errorf("decode existing lock value: %w", err)
}
}
now := time.Now()
nowUnix := now.Unix()
value := lockValue{
ProcessID: s.processID,
HostID: s.hostID,
LockID: s.lockID,
ExpiresAt: now.Add(ttl).Unix(),
}
raw, err := value.Marshal()
if err != nil {
return time.Time{}, fmt.Errorf("encode lock value: %w", err)
}
var query string
switch s.db.Name() {
case "sqlite":
query = `
INSERT INTO kv (key, value)
VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET
value = excluded.value
WHERE (json_extract(kv.value, '$.expires_at') < ?) OR ?
`
case "postgres":
query = `
INSERT INTO kv (key, value)
VALUES ($1, $2)
ON CONFLICT(key) DO UPDATE SET
value = excluded.value
WHERE ((kv.value::json->>'expires_at')::bigint < $3) OR ($4::boolean IS TRUE)
`
default:
return time.Time{}, fmt.Errorf("unsupported database dialect: %s", s.db.Name())
}
res := tx.WithContext(ctx).Exec(query, lockKey, raw, nowUnix, force)
if res.Error != nil {
return time.Time{}, fmt.Errorf("lock acquisition failed: %w", res.Error)
}
if err := tx.Commit().Error; err != nil {
return time.Time{}, fmt.Errorf("commit lock acquisition: %w", err)
}
// If there is a lock that is not expired and force is false, no rows will be affected
if res.RowsAffected == 0 {
return time.Time{}, ErrLockUnavailable
}
if force && prevLock.ExpiresAt > nowUnix && prevLock.LockID != s.lockID {
waitUntil = time.Unix(prevLock.ExpiresAt, 0)
}
attrs := []any{
slog.Int64("process_id", s.processID),
slog.String("host_id", s.hostID),
}
if wait := time.Until(waitUntil); wait > 0 {
attrs = append(attrs, slog.Duration("wait_before_proceeding", wait))
}
slog.Info("Acquired application lock", attrs...)
return waitUntil, nil
}
// RunRenewal keeps renewing the lock until the context is canceled.
func (s *AppLockService) RunRenewal(ctx context.Context) error {
ticker := time.NewTicker(renewInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return nil
case <-ticker.C:
if err := s.renew(ctx); err != nil {
return fmt.Errorf("renew lock: %w", err)
}
}
}
}
// Release releases the lock if it is held by this process.
func (s *AppLockService) Release(ctx context.Context) error {
opCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
var query string
switch s.db.Name() {
case "sqlite":
query = `
DELETE FROM kv
WHERE key = ?
AND json_extract(value, '$.lock_id') = ?
`
case "postgres":
query = `
DELETE FROM kv
WHERE key = $1
AND value::json->>'lock_id' = $2
`
default:
return fmt.Errorf("unsupported database dialect: %s", s.db.Name())
}
res := s.db.WithContext(opCtx).Exec(query, lockKey, s.lockID)
if res.Error != nil {
return fmt.Errorf("release lock failed: %w", res.Error)
}
if res.RowsAffected == 0 {
slog.Warn("Application lock not held by this process, cannot release",
slog.Int64("process_id", s.processID),
slog.String("host_id", s.hostID),
)
}
slog.Info("Released application lock",
slog.Int64("process_id", s.processID),
slog.String("host_id", s.hostID),
)
return nil
}
// renew tries to renew the lock, retrying up to renewRetries times (sleeping 1s between attempts).
func (s *AppLockService) renew(ctx context.Context) error {
var lastErr error
for attempt := 1; attempt <= renewRetries; attempt++ {
now := time.Now()
nowUnix := now.Unix()
expiresAt := now.Add(ttl).Unix()
value := lockValue{
LockID: s.lockID,
ProcessID: s.processID,
HostID: s.hostID,
ExpiresAt: expiresAt,
}
raw, err := value.Marshal()
if err != nil {
return fmt.Errorf("encode lock value: %w", err)
}
var query string
switch s.db.Name() {
case "sqlite":
query = `
UPDATE kv
SET value = ?
WHERE key = ?
AND json_extract(value, '$.lock_id') = ?
AND json_extract(value, '$.expires_at') > ?
`
case "postgres":
query = `
UPDATE kv
SET value = $1
WHERE key = $2
AND value::json->>'lock_id' = $3
AND ((value::json->>'expires_at')::bigint > $4)
`
default:
return fmt.Errorf("unsupported database dialect: %s", s.db.Name())
}
opCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
res := s.db.WithContext(opCtx).Exec(query, raw, lockKey, s.lockID, nowUnix)
cancel()
switch {
case res.Error != nil:
lastErr = fmt.Errorf("lock renewal failed: %w", res.Error)
case res.RowsAffected == 0:
// Must be after checking res.Error
return ErrLockLost
default:
slog.Debug("Renewed application lock",
slog.Int64("process_id", s.processID),
slog.String("host_id", s.hostID),
)
return nil
}
// Wait before next attempt or cancel if context is done
if attempt < renewRetries {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(1 * time.Second):
}
}
}
return lastErr
}

View File

@@ -0,0 +1,189 @@
package service
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
"github.com/pocket-id/pocket-id/backend/internal/model"
testutils "github.com/pocket-id/pocket-id/backend/internal/utils/testing"
)
func newTestAppLockService(t *testing.T, db *gorm.DB) *AppLockService {
t.Helper()
return &AppLockService{
db: db,
processID: 1,
hostID: "test-host",
lockID: "a13c7673-c7ae-49f1-9112-2cd2d0d4b0c1",
}
}
func insertLock(t *testing.T, db *gorm.DB, value lockValue) {
t.Helper()
raw, err := value.Marshal()
require.NoError(t, err)
err = db.Create(&model.KV{Key: lockKey, Value: &raw}).Error
require.NoError(t, err)
}
func readLockValue(t *testing.T, db *gorm.DB) lockValue {
t.Helper()
var row model.KV
err := db.Take(&row, "key = ?", lockKey).Error
require.NoError(t, err)
require.NotNil(t, row.Value)
var value lockValue
err = value.Unmarshal(*row.Value)
require.NoError(t, err)
return value
}
func TestAppLockServiceAcquire(t *testing.T) {
t.Run("creates new lock when none exists", func(t *testing.T) {
db := testutils.NewDatabaseForTest(t)
service := newTestAppLockService(t, db)
_, err := service.Acquire(context.Background(), false)
require.NoError(t, err)
stored := readLockValue(t, db)
require.Equal(t, service.processID, stored.ProcessID)
require.Equal(t, service.hostID, stored.HostID)
require.Greater(t, stored.ExpiresAt, time.Now().Unix())
})
t.Run("returns ErrLockUnavailable when lock held by another process", func(t *testing.T) {
db := testutils.NewDatabaseForTest(t)
service := newTestAppLockService(t, db)
existing := lockValue{
ProcessID: 99,
HostID: "other-host",
ExpiresAt: time.Now().Add(ttl).Unix(),
}
insertLock(t, db, existing)
_, err := service.Acquire(context.Background(), false)
require.ErrorIs(t, err, ErrLockUnavailable)
current := readLockValue(t, db)
require.Equal(t, existing, current)
})
t.Run("force acquisition steals lock", func(t *testing.T) {
db := testutils.NewDatabaseForTest(t)
service := newTestAppLockService(t, db)
insertLock(t, db, lockValue{
ProcessID: 99,
HostID: "other-host",
ExpiresAt: time.Now().Unix(),
})
_, err := service.Acquire(context.Background(), true)
require.NoError(t, err)
stored := readLockValue(t, db)
require.Equal(t, service.processID, stored.ProcessID)
require.Equal(t, service.hostID, stored.HostID)
require.Greater(t, stored.ExpiresAt, time.Now().Unix())
})
}
func TestAppLockServiceRelease(t *testing.T) {
t.Run("removes owned lock", func(t *testing.T) {
db := testutils.NewDatabaseForTest(t)
service := newTestAppLockService(t, db)
_, err := service.Acquire(context.Background(), false)
require.NoError(t, err)
err = service.Release(context.Background())
require.NoError(t, err)
var row model.KV
err = db.Take(&row, "key = ?", lockKey).Error
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
})
t.Run("ignores lock held by another owner", func(t *testing.T) {
db := testutils.NewDatabaseForTest(t)
service := newTestAppLockService(t, db)
existing := lockValue{
ProcessID: 2,
HostID: "other-host",
ExpiresAt: time.Now().Add(ttl).Unix(),
}
insertLock(t, db, existing)
err := service.Release(context.Background())
require.NoError(t, err)
stored := readLockValue(t, db)
require.Equal(t, existing, stored)
})
}
func TestAppLockServiceRenew(t *testing.T) {
t.Run("extends expiration when lock is still owned", func(t *testing.T) {
db := testutils.NewDatabaseForTest(t)
service := newTestAppLockService(t, db)
_, err := service.Acquire(context.Background(), false)
require.NoError(t, err)
before := readLockValue(t, db)
err = service.renew(context.Background())
require.NoError(t, err)
after := readLockValue(t, db)
require.Equal(t, service.processID, after.ProcessID)
require.Equal(t, service.hostID, after.HostID)
require.GreaterOrEqual(t, after.ExpiresAt, before.ExpiresAt)
})
t.Run("returns ErrLockLost when lock is missing", func(t *testing.T) {
db := testutils.NewDatabaseForTest(t)
service := newTestAppLockService(t, db)
err := service.renew(context.Background())
require.ErrorIs(t, err, ErrLockLost)
})
t.Run("returns ErrLockLost when ownership changed", func(t *testing.T) {
db := testutils.NewDatabaseForTest(t)
service := newTestAppLockService(t, db)
_, err := service.Acquire(context.Background(), false)
require.NoError(t, err)
// Simulate a different process taking the lock.
newOwner := lockValue{
ProcessID: 9,
HostID: "stolen-host",
ExpiresAt: time.Now().Add(ttl).Unix(),
}
raw, marshalErr := newOwner.Marshal()
require.NoError(t, marshalErr)
updateErr := db.Model(&model.KV{}).
Where("key = ?", lockKey).
Update("value", raw).Error
require.NoError(t, updateErr)
err = service.renew(context.Background())
require.ErrorIs(t, err, ErrLockLost)
})
}

View File

@@ -7,14 +7,12 @@ import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"encoding/base64"
"fmt"
"log/slog"
"path"
"time"
"github.com/fxamacker/cbor/v2"
"github.com/go-webauthn/webauthn/protocol"
"github.com/lestrrat-go/jwx/v3/jwa"
"github.com/lestrrat-go/jwx/v3/jwk"
@@ -36,15 +34,17 @@ type TestService struct {
appConfigService *AppConfigService
ldapService *LdapService
fileStorage storage.FileStorage
appLockService *AppLockService
externalIdPKey jwk.Key
}
func NewTestService(db *gorm.DB, appConfigService *AppConfigService, jwtService *JwtService, ldapService *LdapService, fileStorage storage.FileStorage) (*TestService, error) {
func NewTestService(db *gorm.DB, appConfigService *AppConfigService, jwtService *JwtService, ldapService *LdapService, appLockService *AppLockService, fileStorage storage.FileStorage) (*TestService, error) {
s := &TestService{
db: db,
appConfigService: appConfigService,
jwtService: jwtService,
ldapService: ldapService,
appLockService: appLockService,
fileStorage: fileStorage,
}
err := s.initExternalIdP()
@@ -288,8 +288,8 @@ func (s *TestService) SeedDatabase(baseURL string) error {
// openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 | \
// openssl pkcs8 -topk8 -nocrypt | tee >(openssl pkey -pubout)
publicKeyPasskey1, _ := s.getCborPublicKey("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEwcOo5KV169KR67QEHrcYkeXE3CCxv2BgwnSq4VYTQxyLtdmKxegexa8JdwFKhKXa2BMI9xaN15BoL6wSCRFJhg==")
publicKeyPasskey2, _ := s.getCborPublicKey("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEj4qA0PrZzg8Co1C27nyUbzrp8Ewjr7eOlGI2LfrzmbL5nPhZRAdJ3hEaqrHMSnJBhfMqtQGKwDYpaLIQFAKLhw==")
publicKeyPasskey1, _ := base64.StdEncoding.DecodeString("pQMmIAEhWCDBw6jkpXXr0pHrtAQetxiR5cTcILG/YGDCdKrhVhNDHCJYIIu12YrF6B7Frwl3AUqEpdrYEwj3Fo3XkGgvrBIJEUmGAQI=")
publicKeyPasskey2, _ := base64.StdEncoding.DecodeString("pSJYIPmc+FlEB0neERqqscxKckGF8yq1AYrANiloshAUAouHAQIDJiABIVggj4qA0PrZzg8Co1C27nyUbzrp8Ewjr7eOlGI2LfrzmbI=")
webauthnCredentials := []model.WebauthnCredential{
{
Name: "Passkey 1",
@@ -318,6 +318,10 @@ func (s *TestService) SeedDatabase(baseURL string) error {
Challenge: "challenge",
ExpiresAt: datatype.DateTime(time.Now().Add(1 * time.Hour)),
UserVerification: "preferred",
CredentialParams: model.CredentialParameters{
{Type: "public-key", Algorithm: -7},
{Type: "public-key", Algorithm: -257},
},
}
if err := tx.Create(&webauthnSession).Error; err != nil {
return err
@@ -327,9 +331,10 @@ func (s *TestService) SeedDatabase(baseURL string) error {
Base: model.Base{
ID: "5f1fa856-c164-4295-961e-175a0d22d725",
},
Name: "Test API Key",
Key: "6c34966f57ef2bb7857649aff0e7ab3ad67af93c846342ced3f5a07be8706c20",
UserID: users[0].ID,
Name: "Test API Key",
Key: "6c34966f57ef2bb7857649aff0e7ab3ad67af93c846342ced3f5a07be8706c20",
UserID: users[0].ID,
ExpiresAt: datatype.DateTime(time.Now().Add(30 * 24 * time.Hour)),
}
if err := tx.Create(&apiKey).Error; err != nil {
return err
@@ -379,6 +384,20 @@ func (s *TestService) SeedDatabase(baseURL string) error {
}
}
keyValues := []model.KV{
{
Key: jwkutils.PrivateKeyDBKey,
// {"alg":"RS256","d":"mvMDWSdPPvcum0c0iEHE2gbqtV2NKMmLwrl9E6K7g8lTV95SePLnW_bwyMPV7EGp7PQk3l17I5XRhFjze7GqTnFIOgKzMianPs7jv2ELtBMGK0xOPATgu1iGb70xZ6vcvuEfRyY3dJ0zr4jpUdVuXwKmx9rK4IdZn2dFCKfvSuspqIpz11RhF1ALrqDLkxGVv7ZwNh0_VhJZU9hcjG5l6xc7rQEKpPRkZp0IdjkGS8Z0FskoVaiRIWAbZuiVFB9WCW8k1czC4HQTPLpII01bUQx2ludbm0UlXRgVU9ptUUbU7GAImQqTOW8LfPGklEvcgzlIlR_oqw4P9yBxLi-yMQ","dp":"pvNCSnnhbo8Igw9psPR-DicxFnkXlu_ix4gpy6efTrxA-z1VDFDioJ814vKQNioYDzpyAP1gfMPhRkvG_q0hRZsJah3Sb9dfA-WkhSWY7lURQP4yIBTMU0PF_rEATuS7lRciYk1SOx5fqXZd3m_LP0vpBC4Ujlq6NAq6CIjCnms","dq":"TtUVGCCkPNgfOLmkYXu7dxxUCV5kB01-xAEK2OY0n0pG8vfDophH4_D_ZC7nvJ8J9uDhs_3JStexq1lIvaWtG99RNTChIEDzpdn6GH9yaVcb_eB4uJjrNm64FhF8PGCCwxA-xMCZMaARKwhMB2_IOMkxUbWboL3gnhJ2rDO_QO0","e":"AQAB","kid":"8uHDw3M6rf8","kty":"RSA","n":"yaeEL0VKoPBXIAaWXsUgmu05lAvEIIdJn0FX9lHh4JE5UY9B83C5sCNdhs9iSWzpeP11EVjWp8i3Yv2CF7c7u50BXnVBGtxpZpFC-585UXacoJ0chUmarL9GRFJcM1nPHBTFu68aRrn1rIKNHUkNaaxFo0NFGl_4EDDTO8HwawTjwkPoQlRzeByhlvGPVvwgB3Fn93B8QJ_cZhXKxJvjjrC_8Pk76heC_ntEMru71Ix77BoC3j2TuyiN7m9RNBW8BU5q6lKoIdvIeZfTFLzi37iufyfvMrJTixp9zhNB1NxlLCeOZl2MXegtiGqd2H3cbAyqoOiv9ihUWTfXj7SxJw","p":"_Yylc9e07CKdqNRD2EosMC2mrhrEa9j5oY_l00Qyy4-jmCA59Q9viyqvveRo0U7cRvFA5BWgWN6GGLh1DG3X-QBqVr0dnk3uzbobb55RYUXyPLuBZI2q6w2oasbiDwPdY7KpkVv_H-bpITQlyDvO8hhucA6rUV7F6KTQVz8M3Ms","q":"y5p3hch-7jJ21TkAhp_Vk1fLCAuD4tbErwQs2of9ja8sB4iJOs5Wn6HD3P7Mc8Plye7qaLHvzc8I5g0tPKWvC0DPd_FLPXiWwMVAzee3NUX_oGeJNOQp11y1w_KqdO9qZqHSEPZ3NcFL_SZMFgggxhM1uzRiPzsVN0lnD_6prZU","qi":"2Grt6uXHm61ji3xSdkBWNtUnj19vS1-7rFJp5SoYztVQVThf_W52BAiXKBdYZDRVoItC_VS2NvAOjeJjhYO_xQ_q3hK7MdtuXfEPpLnyXKkmWo3lrJ26wbeF6l05LexCkI7ShsOuSt-dsyaTJTszuKDIA6YOfWvfo3aVZmlWRaI","use":"sig"}
Value: utils.Ptr("7d/5hl7diJ2rnFL14hEAQf9tzpu29aqXQ8jpJ2iqqKUNFZpdOkEpud0CmRv4H3r8yyk2u/Gqqj9klSy58DJkYXGF5PAYgLyoBIb7L3JXWRbxg4cQ3QJCug13l2OTmpAKoVc+rmX8c3j3h1sNqyJ+7Ql5sS0jSeyiYgIsFNCdnK5alBDyvtcpe/QDpklmP4JCeVpvmf2rLGplk3g5UO5ydJ8UiDXxfDmi+gF6NKJvrGnnah8Ar3G/x88z+tTJtp0DIQFwxXwUM2XZqzEVGm8K2r0w5o9/Keh6bBBaiuH2C78ZOaijGV3DovhR+e9J0cYUYGwT42MZMx9fSWQ/lvWGGnf+Uq3MXJfjWSREfhkp8KTQwR9F7+dnVJWswOEk7jPR8I7hCWTMxJyvaFX3wgAXIVmhrgXZQQbYOqTt56IoqUl0xOJku8dA8opg2UcLlmmuOh6+hfkXKsiiS/H/9c1BVIGj1fCOiT6IePh4wKKSTbwJnPD5EKmdJpgTsUpjcDnXQKY4ReO0UpdRdKxwRDDLeQuG6j+ljGxR9GPudCU9Nmci6rFVI6n5LWYkQxBA1O73RpmXRZPDzntDfpXMEonkmSvOoxaCK2Id7CRKMdqvR0kEouwnhk5WSFtsfi3sA0pkXzPFxwZeWM8vFtbffZOZzXaOhxCOfcj1NClZohlZhyc4jvkxmrpY7PSaAzih0AmHI7y0LYFi6fZu/K4EheVa1+KF55nWZ8ARikHMWKAKkyExkTak7xyN884TDmzURRaPlQg4jzQte5WMNjAG/hlHibdMBNvgwiYd49ZxteJ8ABdbiXVRl+2JGbdjl2ubpQZwOn7bJKlqO56bIwsZ+e4+pXsuOGdBahkHrUjtMEmH3DZbGc6CJLbcmdhdpApLQRRcLAazxJhzAwJ47FRYsHsj57LnYNvmcKdIxw8rxCdLUuzz95uw0T3ankEO5J9sjem+HMEuKdwXK1UcuOn2rjR8Sd/BuvQmeso27dFbPXqXYNS90Ml45YyTvcKSiopD181oZR703TFUSpR7dsiqROMr+p/2jN9h6a8WbQ8xpksyclaQByY/M77AssbXnG6wfhRsntNIINCZLbBnjXOyz6ZHIC5K4tSTdcnWaiYPeRPQmnw9UUvHAcNU2yMWsy0eU377yDS0WstTxOdQutTdkczl8kv5Lo26JiEK7mSIuRK19ffF9Zz8FG8+eKv5zdyIPjyQRDYBysUoDv5huKe2eoxJu/MWS2Pql/ZtUGeD6Ozm3mCvh0vQ9ceagBkY6Ocm3du0ziAKP29Ri0mjg4DizVorbLzsh+EQH/s2Pi9MnjUZDlEmuLl2Xfp7/w4j/8u0N0tVR70VDFuGdKpTjFY3vS8EJrPtyMTM51x1D9rb8gIql8aR/rJw4YF+huxg1mv5n6+tGVqg5msbPmF12eJijP4lkmaRwIpLW5pJTtaDkUj7uOeu1mm4k+Dt5nh0/0jPHzrv6bcTCcbV7UjMHDoTXXqEpFAAJ66rHR7zdAJu+YKsnTIZyLmOpcowq7LL8G9qTvV0OSpyQWUIavRSgbDHFqEqRs+JU94jAzkq8nCY5MTd9m5sIv9InfdT3k+pwpsE/FKge8nghFLtbUrafGkzTky8SE2druvVcIvbfXMfLIKRUYjJgnWc0gQzF5J6pzXM7D2r/RG6JDzASqjlbURq6v9bhNerlOVdMujWKEEVcKWIzlbt4RkihRjM8AUqIZQOyicGQ+4yfIjAHw5viuABONYs3OIWULnFqJxdvS9rNKhfxSjIq9cfqyzevq2xrRoMXEonobh6M3bD2Vang8OAeVeD1OXWPERi4pepCYFS9RJ/Xa/UWxptsqSNuGcb3fAzQSmLpXLGdWRoKXvSe7EYgc0bGcLOjSTu5RURKo+EF9i4KT9EJauf6VXw5dTf/CCIJRXE1bWzXhSCFYntohYhX2ldOCDYpi/jFBC6Vtkw0ud3/xq8Nmhd5gUk+SpngByCZH3Pm3H+jvlbMpiqkDkm1v74hDX13Xhrcw2eWyuqKBVoRCCniUvwpYNbGvBfjC6Hcizv0Aybciwj+4nybt5EPoEUm6S6Gs7fG7QpPdvrzpAxX70MlmdkF/gwyuhbEeJhLK+WL7qAsN5CvHPzVbsIf90x+nGTtMJPgpxVr0tJMj+vprXV4WxutfARBiOnqe58MhA857sd+MzKBgKnoLOBRTiC3qc/0/ULwbG2HCCD7nmwzz7M4nUuMvo8rgS7z0BF68OClT8X3JwSXbL5Wg=="),
},
}
for _, kv := range keyValues {
if err := tx.Create(&kv).Error; err != nil {
return err
}
}
return nil
})
@@ -464,47 +483,29 @@ func (s *TestService) ResetAppConfig(ctx context.Context) error {
return err
}
// Manually set instance ID
err = s.appConfigService.UpdateAppConfigValues(ctx, "instanceId", "test-instance-id")
if err != nil {
return err
}
// Reload the app config from the database after resetting the values
return s.appConfigService.LoadDbConfig(ctx)
err = s.appConfigService.LoadDbConfig(ctx)
if err != nil {
return err
}
// Reload the JWK
if err := s.jwtService.LoadOrGenerateKey(); err != nil {
return err
}
return nil
}
func (s *TestService) SetJWTKeys() {
const privateKeyString = `{"alg":"RS256","d":"mvMDWSdPPvcum0c0iEHE2gbqtV2NKMmLwrl9E6K7g8lTV95SePLnW_bwyMPV7EGp7PQk3l17I5XRhFjze7GqTnFIOgKzMianPs7jv2ELtBMGK0xOPATgu1iGb70xZ6vcvuEfRyY3dJ0zr4jpUdVuXwKmx9rK4IdZn2dFCKfvSuspqIpz11RhF1ALrqDLkxGVv7ZwNh0_VhJZU9hcjG5l6xc7rQEKpPRkZp0IdjkGS8Z0FskoVaiRIWAbZuiVFB9WCW8k1czC4HQTPLpII01bUQx2ludbm0UlXRgVU9ptUUbU7GAImQqTOW8LfPGklEvcgzlIlR_oqw4P9yBxLi-yMQ","dp":"pvNCSnnhbo8Igw9psPR-DicxFnkXlu_ix4gpy6efTrxA-z1VDFDioJ814vKQNioYDzpyAP1gfMPhRkvG_q0hRZsJah3Sb9dfA-WkhSWY7lURQP4yIBTMU0PF_rEATuS7lRciYk1SOx5fqXZd3m_LP0vpBC4Ujlq6NAq6CIjCnms","dq":"TtUVGCCkPNgfOLmkYXu7dxxUCV5kB01-xAEK2OY0n0pG8vfDophH4_D_ZC7nvJ8J9uDhs_3JStexq1lIvaWtG99RNTChIEDzpdn6GH9yaVcb_eB4uJjrNm64FhF8PGCCwxA-xMCZMaARKwhMB2_IOMkxUbWboL3gnhJ2rDO_QO0","e":"AQAB","kid":"8uHDw3M6rf8","kty":"RSA","n":"yaeEL0VKoPBXIAaWXsUgmu05lAvEIIdJn0FX9lHh4JE5UY9B83C5sCNdhs9iSWzpeP11EVjWp8i3Yv2CF7c7u50BXnVBGtxpZpFC-585UXacoJ0chUmarL9GRFJcM1nPHBTFu68aRrn1rIKNHUkNaaxFo0NFGl_4EDDTO8HwawTjwkPoQlRzeByhlvGPVvwgB3Fn93B8QJ_cZhXKxJvjjrC_8Pk76heC_ntEMru71Ix77BoC3j2TuyiN7m9RNBW8BU5q6lKoIdvIeZfTFLzi37iufyfvMrJTixp9zhNB1NxlLCeOZl2MXegtiGqd2H3cbAyqoOiv9ihUWTfXj7SxJw","p":"_Yylc9e07CKdqNRD2EosMC2mrhrEa9j5oY_l00Qyy4-jmCA59Q9viyqvveRo0U7cRvFA5BWgWN6GGLh1DG3X-QBqVr0dnk3uzbobb55RYUXyPLuBZI2q6w2oasbiDwPdY7KpkVv_H-bpITQlyDvO8hhucA6rUV7F6KTQVz8M3Ms","q":"y5p3hch-7jJ21TkAhp_Vk1fLCAuD4tbErwQs2of9ja8sB4iJOs5Wn6HD3P7Mc8Plye7qaLHvzc8I5g0tPKWvC0DPd_FLPXiWwMVAzee3NUX_oGeJNOQp11y1w_KqdO9qZqHSEPZ3NcFL_SZMFgggxhM1uzRiPzsVN0lnD_6prZU","qi":"2Grt6uXHm61ji3xSdkBWNtUnj19vS1-7rFJp5SoYztVQVThf_W52BAiXKBdYZDRVoItC_VS2NvAOjeJjhYO_xQ_q3hK7MdtuXfEPpLnyXKkmWo3lrJ26wbeF6l05LexCkI7ShsOuSt-dsyaTJTszuKDIA6YOfWvfo3aVZmlWRaI","use":"sig"}`
privateKey, _ := jwk.ParseKey([]byte(privateKeyString))
_ = s.jwtService.SetKey(privateKey)
}
// getCborPublicKey decodes a Base64 encoded public key and returns the CBOR encoded COSE key
func (s *TestService) getCborPublicKey(base64PublicKey string) ([]byte, error) {
decodedKey, err := base64.StdEncoding.DecodeString(base64PublicKey)
if err != nil {
return nil, fmt.Errorf("failed to decode base64 key: %w", err)
}
pubKey, err := x509.ParsePKIXPublicKey(decodedKey)
if err != nil {
return nil, fmt.Errorf("failed to parse public key: %w", err)
}
ecdsaPubKey, ok := pubKey.(*ecdsa.PublicKey)
if !ok {
return nil, fmt.Errorf("not an ECDSA public key")
}
coseKey := map[int]interface{}{
1: 2, // Key type: EC2
3: -7, // Algorithm: ECDSA with SHA-256
-1: 1, // Curve: P-256
-2: ecdsaPubKey.X.Bytes(), // X coordinate
-3: ecdsaPubKey.Y.Bytes(), // Y coordinate
}
cborPublicKey, err := cbor.Marshal(coseKey)
if err != nil {
return nil, fmt.Errorf("failed to marshal COSE key: %w", err)
}
return cborPublicKey, nil
func (s *TestService) ResetLock(ctx context.Context) error {
_, err := s.appLockService.Acquire(ctx, true)
return err
}
// SyncLdap triggers an LDAP synchronization

View File

@@ -0,0 +1,217 @@
package service
import (
"archive/zip"
"context"
"encoding/json"
"fmt"
"io"
"path/filepath"
"gorm.io/gorm"
datatype "github.com/pocket-id/pocket-id/backend/internal/model/types"
"github.com/pocket-id/pocket-id/backend/internal/storage"
"github.com/pocket-id/pocket-id/backend/internal/utils"
)
// ExportService handles exporting Pocket ID data into a ZIP archive.
type ExportService struct {
db *gorm.DB
storage storage.FileStorage
}
func NewExportService(db *gorm.DB, storage storage.FileStorage) *ExportService {
return &ExportService{
db: db,
storage: storage,
}
}
// ExportToZip performs the full export process and writes the ZIP data to the given writer.
func (s *ExportService) ExportToZip(ctx context.Context, w io.Writer) error {
dbData, err := s.extractDatabase()
if err != nil {
return err
}
return s.writeExportZipStream(ctx, w, dbData)
}
// extractDatabase reads all tables into a DatabaseExport struct
func (s *ExportService) extractDatabase() (DatabaseExport, error) {
schema, err := utils.LoadDBSchemaTypes(s.db)
if err != nil {
return DatabaseExport{}, fmt.Errorf("failed to load schema types: %w", err)
}
version, err := s.schemaVersion()
if err != nil {
return DatabaseExport{}, err
}
out := DatabaseExport{
Provider: s.db.Name(),
Version: version,
Tables: map[string][]map[string]any{},
// These tables need to be inserted in a specific order because of foreign key constraints
// Not all tables are listed here, because not all tables are order-dependent
TableOrder: []string{"users", "user_groups", "oidc_clients"},
}
for table := range schema {
if table == "storage" || table == "schema_migrations" {
continue
}
err = s.dumpTable(table, schema[table], &out)
if err != nil {
return DatabaseExport{}, err
}
}
return out, nil
}
func (s *ExportService) schemaVersion() (uint, error) {
var version uint
if err := s.db.Raw("SELECT version FROM schema_migrations").Row().Scan(&version); err != nil {
return 0, fmt.Errorf("failed to query schema version: %w", err)
}
return version, nil
}
// dumpTable selects all rows from a table and appends them to out.Tables
func (s *ExportService) dumpTable(table string, types utils.DBSchemaTableTypes, out *DatabaseExport) error {
rows, err := s.db.Raw("SELECT * FROM " + table).Rows()
if err != nil {
return fmt.Errorf("failed to read table %s: %w", table, err)
}
defer rows.Close()
cols, _ := rows.Columns()
if len(cols) != len(types) {
// Should never happen...
return fmt.Errorf("mismatched columns in table (%d) and schema (%d)", len(cols), len(types))
}
for rows.Next() {
vals := s.getScanValuesForTable(cols, types)
err = rows.Scan(vals...)
if err != nil {
return fmt.Errorf("failed to scan row in table %s: %w", table, err)
}
rowMap := make(map[string]any, len(cols))
for i, col := range cols {
rowMap[col] = vals[i]
}
// Skip the app lock row in the kv table
if table == "kv" {
if keyPtr, ok := rowMap["key"].(*string); ok && keyPtr != nil && *keyPtr == lockKey {
continue
}
}
out.Tables[table] = append(out.Tables[table], rowMap)
}
return rows.Err()
}
func (s *ExportService) getScanValuesForTable(cols []string, types utils.DBSchemaTableTypes) []any {
res := make([]any, len(cols))
for i, col := range cols {
// Store a pointer
// Note: don't create a helper function for this switch, because it would return type "any" and mess everything up
// If the column is nullable, we need a pointer to a pointer!
switch types[col].Name {
case "boolean", "bool":
var x bool
if types[col].Nullable {
res[i] = utils.Ptr(utils.Ptr(x))
} else {
res[i] = utils.Ptr(x)
}
case "blob", "bytea", "jsonb":
// Treat jsonb columns as binary too
var x []byte
if types[col].Nullable {
res[i] = utils.Ptr(utils.Ptr(x))
} else {
res[i] = utils.Ptr(x)
}
case "timestamp", "timestamptz", "timestamp with time zone", "datetime":
var x datatype.DateTime
if types[col].Nullable {
res[i] = utils.Ptr(utils.Ptr(x))
} else {
res[i] = utils.Ptr(x)
}
case "integer", "int", "bigint":
var x int64
if types[col].Nullable {
res[i] = utils.Ptr(utils.Ptr(x))
} else {
res[i] = utils.Ptr(x)
}
default:
// Treat everything else as a string (including the "numeric" type)
var x string
if types[col].Nullable {
res[i] = utils.Ptr(utils.Ptr(x))
} else {
res[i] = utils.Ptr(x)
}
}
}
return res
}
func (s *ExportService) writeExportZipStream(ctx context.Context, w io.Writer, dbData DatabaseExport) error {
zipWriter := zip.NewWriter(w)
// Add database.json
jsonWriter, err := zipWriter.Create("database.json")
if err != nil {
return fmt.Errorf("failed to create database.json in zip: %w", err)
}
jsonEncoder := json.NewEncoder(jsonWriter)
jsonEncoder.SetEscapeHTML(false)
if err := jsonEncoder.Encode(dbData); err != nil {
return fmt.Errorf("failed to encode database.json: %w", err)
}
// Add uploaded files
if err := s.addUploadsToZip(ctx, zipWriter); err != nil {
return err
}
return zipWriter.Close()
}
// addUploadsToZip adds all files from the storage to the ZIP archive under the "uploads/" directory
func (s *ExportService) addUploadsToZip(ctx context.Context, zipWriter *zip.Writer) error {
return s.storage.Walk(ctx, "/", func(p storage.ObjectInfo) error {
zipPath := filepath.Join("uploads", p.Path)
w, err := zipWriter.Create(zipPath)
if err != nil {
return fmt.Errorf("failed to create zip entry for %s: %w", zipPath, err)
}
f, _, err := s.storage.Open(ctx, p.Path)
if err != nil {
return fmt.Errorf("failed to open file %s: %w", zipPath, err)
}
defer f.Close()
if _, err := io.Copy(w, f); err != nil {
return fmt.Errorf("failed to copy file %s into zip: %w", zipPath, err)
}
return nil
})
}

View File

@@ -0,0 +1,264 @@
package service
import (
"archive/zip"
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"slices"
"strings"
"gorm.io/gorm"
datatype "github.com/pocket-id/pocket-id/backend/internal/model/types"
"github.com/pocket-id/pocket-id/backend/internal/storage"
"github.com/pocket-id/pocket-id/backend/internal/utils"
)
// ImportService handles importing Pocket ID data from an exported ZIP archive.
type ImportService struct {
db *gorm.DB
storage storage.FileStorage
}
type DatabaseExport struct {
Provider string `json:"provider"`
Version uint `json:"version"`
Tables map[string][]map[string]any `json:"tables"`
TableOrder []string `json:"tableOrder"`
}
func NewImportService(db *gorm.DB, storage storage.FileStorage) *ImportService {
return &ImportService{
db: db,
storage: storage,
}
}
// ImportFromZip performs the full import process from the given ZIP reader.
func (s *ImportService) ImportFromZip(ctx context.Context, r *zip.Reader) error {
dbData, err := processZipDatabaseJson(r.File)
if err != nil {
return err
}
err = s.ImportDatabase(dbData)
if err != nil {
return err
}
err = s.importUploads(ctx, r.File)
if err != nil {
return err
}
return nil
}
// ImportDatabase only imports the database data from the given DatabaseExport struct.
func (s *ImportService) ImportDatabase(dbData DatabaseExport) error {
err := s.resetSchema(dbData.Version, dbData.Provider)
if err != nil {
return err
}
err = s.insertData(dbData)
if err != nil {
return err
}
return nil
}
// processZipDatabaseJson extracts database.json from the ZIP archive
func processZipDatabaseJson(files []*zip.File) (dbData DatabaseExport, err error) {
for _, f := range files {
if f.Name == "database.json" {
return parseDatabaseJsonStream(f)
}
}
return dbData, errors.New("database.json not found in the ZIP file")
}
func parseDatabaseJsonStream(f *zip.File) (dbData DatabaseExport, err error) {
rc, err := f.Open()
if err != nil {
return dbData, fmt.Errorf("failed to open database.json: %w", err)
}
defer rc.Close()
err = json.NewDecoder(rc).Decode(&dbData)
if err != nil {
return dbData, fmt.Errorf("failed to decode database.json: %w", err)
}
return dbData, nil
}
// importUploads imports files from the uploads/ directory in the ZIP archive
func (s *ImportService) importUploads(ctx context.Context, files []*zip.File) error {
const maxFileSize = 50 << 20 // 50 MiB
const uploadsPrefix = "uploads/"
for _, f := range files {
if !strings.HasPrefix(f.Name, uploadsPrefix) {
continue
}
if f.UncompressedSize64 > maxFileSize {
return fmt.Errorf("file %s too large (%d bytes)", f.Name, f.UncompressedSize64)
}
targetPath := strings.TrimPrefix(f.Name, uploadsPrefix)
if strings.HasSuffix(f.Name, "/") || targetPath == "" {
continue // Skip directories
}
err := s.storage.DeleteAll(ctx, targetPath)
if err != nil {
return fmt.Errorf("failed to delete existing file %s: %w", targetPath, err)
}
rc, err := f.Open()
if err != nil {
return err
}
buf, err := io.ReadAll(rc)
rc.Close()
if err != nil {
return fmt.Errorf("read file %s: %w", f.Name, err)
}
err = s.storage.Save(ctx, targetPath, bytes.NewReader(buf))
if err != nil {
return fmt.Errorf("failed to save file %s: %w", targetPath, err)
}
}
return nil
}
// resetSchema drops the existing schema and migrates to the target version
func (s *ImportService) resetSchema(targetVersion uint, exportDbProvider string) error {
sqlDb, err := s.db.DB()
if err != nil {
return fmt.Errorf("failed to get sql.DB: %w", err)
}
m, err := utils.GetEmbeddedMigrateInstance(sqlDb)
if err != nil {
return fmt.Errorf("failed to get migrate instance: %w", err)
}
err = m.Drop()
if err != nil {
return fmt.Errorf("failed to drop existing schema: %w", err)
}
// Needs to be called again to re-create the schema_migrations table
m, err = utils.GetEmbeddedMigrateInstance(sqlDb)
if err != nil {
return fmt.Errorf("failed to get migrate instance: %w", err)
}
err = m.Migrate(targetVersion)
if err != nil {
return fmt.Errorf("migration failed: %w", err)
}
return nil
}
// insertData populates the DB with the imported data
func (s *ImportService) insertData(dbData DatabaseExport) error {
schema, err := utils.LoadDBSchemaTypes(s.db)
if err != nil {
return fmt.Errorf("failed to load schema types: %w", err)
}
return s.db.Transaction(func(tx *gorm.DB) error {
// Iterate through all tables
// Some tables need to be processed in order
tables := make([]string, 0, len(dbData.Tables))
tables = append(tables, dbData.TableOrder...)
for t := range dbData.Tables {
// Skip tables already present where the order matters
// Also skip the schema_migrations table
if slices.Contains(dbData.TableOrder, t) || t == "schema_migrations" {
continue
}
tables = append(tables, t)
}
// Insert rows
for _, table := range tables {
for _, row := range dbData.Tables[table] {
err = normalizeRowWithSchema(row, table, schema)
if err != nil {
return fmt.Errorf("failed to normalize row for table '%s': %w", table, err)
}
err = tx.Table(table).Create(row).Error
if err != nil {
return fmt.Errorf("failed inserting into table '%s': %w", table, err)
}
}
}
return nil
})
}
// normalizeRowWithSchema converts row values based on the DB schema
func normalizeRowWithSchema(row map[string]any, table string, schema utils.DBSchemaTypes) error {
if schema[table] == nil {
return fmt.Errorf("schema not found for table '%s'", table)
}
for col, val := range row {
if val == nil {
// If the value is nil, skip the column
continue
}
colType := schema[table][col]
switch colType.Name {
case "timestamp", "timestamptz", "timestamp with time zone", "datetime":
// Dates are stored as strings
str, ok := val.(string)
if !ok {
return fmt.Errorf("value for column '%s/%s' was expected to be a string, but was '%T'", table, col, val)
}
d, err := datatype.DateTimeFromString(str)
if err != nil {
return fmt.Errorf("failed to decode value for column '%s/%s' as timestamp: %w", table, col, err)
}
row[col] = d
case "blob", "bytea", "jsonb":
// Binary data and jsonb data is stored in the file as base64-encoded string
str, ok := val.(string)
if !ok {
return fmt.Errorf("value for column '%s/%s' was expected to be a string, but was '%T'", table, col, val)
}
b, err := base64.StdEncoding.DecodeString(str)
if err != nil {
return fmt.Errorf("failed to decode value for column '%s/%s' from base64: %w", table, col, err)
}
// For jsonb, we additionally cast to json.RawMessage
if colType.Name == "jsonb" {
row[col] = json.RawMessage(b)
} else {
row[col] = b
}
}
}
return nil
}

View File

@@ -48,6 +48,7 @@ const (
)
type JwtService struct {
db *gorm.DB
envConfig *common.EnvConfigSchema
privateKey jwk.Key
keyId string
@@ -58,7 +59,6 @@ type JwtService struct {
func NewJwtService(db *gorm.DB, appConfigService *AppConfigService) (*JwtService, error) {
service := &JwtService{}
// Ensure keys are generated or loaded
err := service.init(db, appConfigService, &common.EnvConfig)
if err != nil {
return nil, err
@@ -70,14 +70,15 @@ func NewJwtService(db *gorm.DB, appConfigService *AppConfigService) (*JwtService
func (s *JwtService) init(db *gorm.DB, appConfigService *AppConfigService, envConfig *common.EnvConfigSchema) (err error) {
s.appConfigService = appConfigService
s.envConfig = envConfig
s.db = db
// Ensure keys are generated or loaded
return s.loadOrGenerateKey(db)
return s.LoadOrGenerateKey()
}
func (s *JwtService) loadOrGenerateKey(db *gorm.DB) error {
func (s *JwtService) LoadOrGenerateKey() error {
// Get the key provider
keyProvider, err := jwkutils.GetKeyProvider(db, s.envConfig, s.appConfigService.GetDbConfig().InstanceID.Value)
keyProvider, err := jwkutils.GetKeyProvider(s.db, s.envConfig, s.appConfigService.GetDbConfig().InstanceID.Value)
if err != nil {
return fmt.Errorf("failed to get key provider: %w", err)
}