feat: add CLI command for importing and exporting Pocket ID data (#998)

Co-authored-by: Alessandro (Ale) Segala <43508+ItalyPaleAle@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

backend/resources/e2e-tests/database.json

@@ -0,0 +1 @@
+../../../tests/database.json

backend/resources/files_test.go

@@ -0,0 +1,72 @@
+package resources
+
+import (
+	"embed"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// This test is meant to enforce that for every new migration added, a file with the same migration number exists for all supported databases
+// This is necessary to ensure import/export works correctly
+// Note: if a migration is not needed for a database, ensure there's a file with an empty (no-op) migration (e.g. even just a comment)
+func TestMigrationsMatchingVersions(t *testing.T) {
+	// We can ignore migrations with version below 20251115000000
+	const ignoreBefore = 20251115000000
+
+	// Scan postgres migrations
+	postgresMigrations := scanMigrations(t, FS, "migrations/postgres", ignoreBefore)
+
+	// Scan sqlite migrations
+	sqliteMigrations := scanMigrations(t, FS, "migrations/sqlite", ignoreBefore)
+
+	// Sort both lists for consistent comparison
+	slices.Sort(postgresMigrations)
+	slices.Sort(sqliteMigrations)
+
+	// Compare the lists
+	assert.Equal(t, postgresMigrations, sqliteMigrations, "Migration versions must match between Postgres and SQLite")
+}
+
+// scanMigrations scans a directory for migration files and returns a list of versions
+func scanMigrations(t *testing.T, fs embed.FS, dir string, ignoreBefore int64) []int64 {
+	t.Helper()
+
+	entries, err := fs.ReadDir(dir)
+	require.NoErrorf(t, err, "Failed to read directory '%s'", dir)
+
+	// Divide by 2 because of up and down files
+	versions := make([]int64, 0, len(entries)/2)
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+
+		filename := entry.Name()
+
+		// Only consider .up.sql files
+		if !strings.HasSuffix(filename, ".up.sql") {
+			continue
+		}
+
+		// Extract version from filename (format: <version>_<anything>.up.sql)
+		versionString, _, ok := strings.Cut(filename, "_")
+		require.Truef(t, ok, "Migration file has unexpected format: %s", filename)
+
+		version, err := strconv.ParseInt(versionString, 10, 64)
+		require.NoErrorf(t, err, "Failed to parse version from filename '%s'", filename)
+
+		// Exclude migrations with version below ignoreBefore
+		if version < ignoreBefore {
+			continue
+		}
+
+		versions = append(versions, version)
+	}
+
+	return versions
+}

backend/resources/migrations/postgres/20251117141000_export_normalization.down.sql

@@ -0,0 +1 @@
+-- No-op in Postgres

backend/resources/migrations/postgres/20251117141000_export_normalization.up.sql

@@ -0,0 +1 @@
+-- No-op in Postgres

backend/resources/migrations/sqlite/20251117141000_export_normalization.down.sql

@@ -0,0 +1,133 @@
+PRAGMA foreign_keys = OFF;
+
+BEGIN;
+
+CREATE TABLE users_old
+(
+    id           TEXT                   NOT NULL PRIMARY KEY,
+    created_at   DATETIME,
+    username     TEXT COLLATE NOCASE    NOT NULL UNIQUE,
+    email        TEXT                   NOT NULL UNIQUE,
+    first_name   TEXT,
+    last_name    TEXT                   NOT NULL,
+    display_name TEXT                   NOT NULL,
+    is_admin     NUMERIC DEFAULT 0      NOT NULL,
+    ldap_id      TEXT,
+    locale       TEXT,
+    disabled     NUMERIC DEFAULT 0      NOT NULL
+);
+
+INSERT INTO users_old (
+    id,
+    created_at,
+    username,
+    email,
+    first_name,
+    last_name,
+    display_name,
+    is_admin,
+    ldap_id,
+    locale,
+    disabled
+)
+SELECT
+    id,
+    created_at,
+    username,
+    email,
+    first_name,
+    last_name,
+    display_name,
+    CASE WHEN is_admin THEN 1 ELSE 0 END,
+    ldap_id,
+    locale,
+    CASE WHEN disabled THEN 1 ELSE 0 END
+FROM users;
+
+DROP TABLE users;
+
+ALTER TABLE users_old RENAME TO users;
+
+CREATE UNIQUE INDEX users_ldap_id ON users (ldap_id);
+
+
+
+CREATE TABLE webauthn_credentials_old
+(
+    id               TEXT                   PRIMARY KEY,
+    created_at       DATETIME               NOT NULL,
+    name             TEXT                   NOT NULL,
+    credential_id    TEXT                   NOT NULL UNIQUE,
+    public_key       BLOB                   NOT NULL,
+    attestation_type TEXT                   NOT NULL,
+    transport        BLOB                   NOT NULL,
+    user_id          TEXT REFERENCES users ON DELETE CASCADE,
+    backup_eligible  NUMERIC DEFAULT 0      NOT NULL,
+    backup_state     NUMERIC DEFAULT 0      NOT NULL
+);
+
+INSERT INTO webauthn_credentials_old (
+    id,
+    created_at,
+    name,
+    credential_id,
+    public_key,
+    attestation_type,
+    transport,
+    user_id,
+    backup_eligible,
+    backup_state
+)
+SELECT
+    id,
+    created_at,
+    name,
+    credential_id,
+    public_key,
+    attestation_type,
+    transport,
+    user_id,
+    CASE WHEN backup_eligible THEN 1 ELSE 0 END,
+    CASE WHEN backup_state THEN 1 ELSE 0 END
+FROM webauthn_credentials;
+
+DROP TABLE webauthn_credentials;
+
+ALTER TABLE webauthn_credentials_old RENAME TO webauthn_credentials;
+
+
+
+CREATE TABLE webauthn_sessions_old
+(
+    id                TEXT              NOT NULL PRIMARY KEY,
+    created_at        DATETIME,
+    challenge         TEXT              NOT NULL UNIQUE,
+    expires_at        DATETIME          NOT NULL,
+    user_verification TEXT              NOT NULL,
+    credential_params TEXT DEFAULT '[]' NOT NULL
+);
+
+INSERT INTO webauthn_sessions_old (
+    id,
+    created_at,
+    challenge,
+    expires_at,
+    user_verification,
+    credential_params
+)
+SELECT
+    id,
+    created_at,
+    challenge,
+    expires_at,
+    user_verification,
+    credential_params
+FROM webauthn_sessions;
+
+DROP TABLE webauthn_sessions;
+
+ALTER TABLE webauthn_sessions_old RENAME TO webauthn_sessions;
+
+COMMIT;
+
+PRAGMA foreign_keys = ON;

backend/resources/migrations/sqlite/20251117141000_export_normalization.up.sql

@@ -0,0 +1,144 @@
+PRAGMA foreign_keys = OFF;
+
+BEGIN;
+
+-- 1. Create a new table with BOOLEAN columns
+CREATE TABLE users_new
+(
+    id           TEXT                  NOT NULL PRIMARY KEY,
+    created_at   DATETIME,
+    username     TEXT COLLATE NOCASE   NOT NULL UNIQUE,
+    email        TEXT                  NOT NULL UNIQUE,
+    first_name   TEXT,
+    last_name    TEXT                  NOT NULL,
+    display_name TEXT                  NOT NULL,
+    is_admin     BOOLEAN DEFAULT FALSE NOT NULL,
+    ldap_id      TEXT,
+    locale       TEXT,
+    disabled     BOOLEAN DEFAULT FALSE NOT NULL
+);
+
+-- 2. Copy all existing data, converting numeric bools to real booleans
+INSERT INTO users_new (
+    id,
+    created_at,
+    username,
+    email,
+    first_name,
+    last_name,
+    display_name,
+    is_admin,
+    ldap_id,
+    locale,
+    disabled
+)
+SELECT
+    id,
+    created_at,
+    username,
+    email,
+    first_name,
+    last_name,
+    display_name,
+    CASE WHEN is_admin != 0 THEN TRUE ELSE FALSE END,
+    ldap_id,
+    locale,
+    CASE WHEN disabled != 0 THEN TRUE ELSE FALSE END
+FROM users;
+
+-- 3. Drop old table
+DROP TABLE users;
+
+-- 4. Rename new table to original name
+ALTER TABLE users_new RENAME TO users;
+
+-- 5. Recreate index
+CREATE UNIQUE INDEX users_ldap_id ON users (ldap_id);
+
+-- 6. Create temporary table with changed credential_id type to BLOB
+CREATE TABLE webauthn_credentials_dg_tmp
+(
+    id               TEXT PRIMARY KEY,
+    created_at       DATETIME             NOT NULL,
+    name             TEXT                 NOT NULL,
+    credential_id    BLOB                 NOT NULL UNIQUE,
+    public_key       BLOB                 NOT NULL,
+    attestation_type TEXT                 NOT NULL,
+    transport        BLOB                 NOT NULL,
+    user_id          TEXT REFERENCES users ON DELETE CASCADE,
+    backup_eligible  BOOLEAN DEFAULT FALSE NOT NULL,
+    backup_state     BOOLEAN DEFAULT FALSE NOT NULL
+);
+
+-- 7. Copy existing data into the temporary table
+INSERT INTO webauthn_credentials_dg_tmp (
+    id,
+    created_at,
+    name,
+    credential_id,
+    public_key,
+    attestation_type,
+    transport,
+    user_id,
+    backup_eligible,
+    backup_state
+)
+SELECT
+    id,
+    created_at,
+    name,
+    credential_id,
+    public_key,
+    attestation_type,
+    transport,
+    user_id,
+    backup_eligible,
+    backup_state
+FROM webauthn_credentials;
+
+-- 8. Drop old table
+DROP TABLE webauthn_credentials;
+
+-- 9. Rename temporary table to original name
+ALTER TABLE webauthn_credentials_dg_tmp
+    RENAME TO webauthn_credentials;
+
+-- 10. Create temporary table with credential_params type changed to BLOB
+CREATE TABLE webauthn_sessions_dg_tmp
+(
+    id                TEXT              NOT NULL PRIMARY KEY,
+    created_at        DATETIME,
+    challenge         TEXT              NOT NULL UNIQUE,
+    expires_at        DATETIME          NOT NULL,
+    user_verification TEXT              NOT NULL,
+    credential_params BLOB DEFAULT '[]' NOT NULL
+);
+
+-- 11. Copy existing data into the temporary sessions table
+INSERT INTO webauthn_sessions_dg_tmp (
+    id,
+    created_at,
+    challenge,
+    expires_at,
+    user_verification,
+    credential_params
+)
+SELECT
+    id,
+    created_at,
+    challenge,
+    expires_at,
+    user_verification,
+    credential_params
+FROM webauthn_sessions;
+
+-- 12. Drop old table
+DROP TABLE webauthn_sessions;
+
+-- 13. Rename temporary sessions table to original name
+ALTER TABLE webauthn_sessions_dg_tmp
+    RENAME TO webauthn_sessions;
+
+COMMIT;
+
+PRAGMA foreign_keys = ON;