From 7143a36c1976f028cd565f629d73db839e129808 Mon Sep 17 00:00:00 2001 From: R3D347HR4Y Date: Sat, 13 Jun 2026 12:47:08 +0200 Subject: [PATCH] feat(mail): integrate Stalwart hosted mail and migration features - Added configuration options for Stalwart hosted mail in .env.example. - Updated Docker Compose to include Stalwart service with health checks. - Introduced new API endpoints for managing mail domains and migration projects. - Enhanced Authentik blueprints for user enrollment and post-migration security. - Updated OAuth handling for Google and Microsoft migration processes. - Improved error handling and response structures in the mail API. - Added integration tests for email claiming and migration workflows. --- .env.example | 49 ++ deploy/authentik/README.md | 7 +- .../blueprints/01-ulti-enrollment.yaml | 76 +- .../04-ulti-post-migration-security.yaml | 82 ++ deploy/docker-compose.yml | 17 + deploy/nginx/default.conf.template | 22 + go.mod | 3 +- go.sum | 6 +- internal/api/admin/handlers.go | 11 + internal/api/admin/handlers_mail_domains.go | 390 ++++++++++ internal/api/admin/service.go | 22 +- internal/api/mail/handlers.go | 1 + internal/api/mail/handlers_hosted.go | 48 ++ internal/api/mail/service.go | 10 + internal/api/migration/handlers.go | 257 +++++++ internal/auth/oidc.go | 54 +- internal/config/config.go | 62 ++ internal/integrationtest/harness.go | 50 +- .../migration/claim_email_test.go | 129 ++++ .../integrationtest/migration/delta_test.go | 562 ++++++++++++++ .../migration/domain_claim_test.go | 159 ++++ .../integrationtest/migration/helpers_test.go | 117 +++ .../migration/migration_test.go | 388 ++++++++++ .../integrationtest/migration/suite_test.go | 14 + internal/integrationtest/oidc.go | 8 +- internal/mail/hosted/dns_verify.go | 166 ++++ internal/mail/hosted/dns_verify_test.go | 24 + internal/mail/hosted/service.go | 435 +++++++++++ internal/mail/stalwart/client.go | 233 ++++++ internal/mail/stalwart/client_test.go | 28 + internal/migration/admin_jobs.go | 161 ++++ internal/migration/auth_modes.go | 36 + internal/migration/batch_config.go | 46 ++ internal/migration/batch_config_test.go | 25 + internal/migration/calendar_delta_test.go | 70 ++ internal/migration/calendar_import.go | 542 +++++++++++++ internal/migration/claim_email_match.go | 150 ++++ internal/migration/claim_email_match_test.go | 92 +++ internal/migration/contacts_delta_test.go | 54 ++ internal/migration/contacts_import.go | 476 ++++++++++++ internal/migration/cutover_dns.go | 142 ++++ internal/migration/cutover_dns_test.go | 37 + internal/migration/drive_delta.go | 425 ++++++++++ internal/migration/drive_delta_test.go | 110 +++ internal/migration/drive_helpers.go | 87 +++ internal/migration/drive_helpers_test.go | 95 +++ internal/migration/drive_import.go | 405 ++++++++++ internal/migration/gmail_attachments.go | 285 +++++++ internal/migration/gmail_attachments_test.go | 55 ++ internal/migration/gmail_delta_test.go | 30 + internal/migration/gmail_import.go | 727 ++++++++++++++++++ internal/migration/gmail_import_test.go | 57 ++ internal/migration/google_dwd.go | 77 ++ internal/migration/google_dwd_test.go | 37 + internal/migration/graph_import.go | 531 +++++++++++++ internal/migration/graph_import_test.go | 49 ++ internal/migration/graph_user.go | 22 + internal/migration/http_retry.go | 186 +++++ internal/migration/http_retry_test.go | 103 +++ internal/migration/httpmock_test.go | 59 ++ internal/migration/import_helpers.go | 142 ++++ internal/migration/import_helpers_test.go | 35 + internal/migration/imported_items.go | 206 +++++ internal/migration/imported_items_test.go | 109 +++ internal/migration/job_audit.go | 158 ++++ internal/migration/microsoft_admin_consent.go | 143 ++++ .../migration/microsoft_admin_consent_test.go | 17 + internal/migration/microsoft_app.go | 103 +++ internal/migration/microsoft_app_test.go | 141 ++++ internal/migration/oauth.go | 228 ++++++ internal/migration/oauth_admin_test.go | 61 ++ internal/migration/oauth_refresh.go | 78 ++ internal/migration/onboarding.go | 93 +++ internal/migration/onboarding_test.go | 75 ++ internal/migration/pkce.go | 30 + internal/migration/project_columns.go | 94 +++ internal/migration/project_columns_test.go | 32 + internal/migration/service.go | 499 ++++++++++++ internal/migration/test_exports.go | 11 + internal/migration/worker.go | 285 +++++++ internal/migration/worker_test.go | 26 + internal/observability/metrics.go | 40 + internal/provision/authentik.go | 75 ++ internal/provision/handler.go | 151 ++++ internal/server/bootstrap.go | 102 ++- internal/users/provision.go | 2 + .../000040_hosted_mail_migration.down.sql | 6 + .../000040_hosted_mail_migration.up.sql | 96 +++ .../000041_migration_auth_mode.down.sql | 2 + migrations/000041_migration_auth_mode.up.sql | 7 + .../000042_migration_imported_items.down.sql | 1 + .../000042_migration_imported_items.up.sql | 11 + .../000043_migration_cutover_dns.down.sql | 1 + .../000043_migration_cutover_dns.up.sql | 3 + ...migration_microsoft_admin_consent.down.sql | 6 + ...4_migration_microsoft_admin_consent.up.sql | 20 + ...migration_invite_alternate_emails.down.sql | 2 + ...5_migration_invite_alternate_emails.up.sql | 2 + .../000046_migration_item_audit.down.sql | 8 + migrations/000046_migration_item_audit.up.sql | 11 + ...0047_migration_microsoft_app_auth.down.sql | 5 + ...000047_migration_microsoft_app_auth.up.sql | 6 + project-plan/checklist-execution.md | 28 + 103 files changed, 11698 insertions(+), 54 deletions(-) create mode 100644 deploy/authentik/blueprints/04-ulti-post-migration-security.yaml create mode 100644 internal/api/admin/handlers_mail_domains.go create mode 100644 internal/api/mail/handlers_hosted.go create mode 100644 internal/api/migration/handlers.go create mode 100644 internal/integrationtest/migration/claim_email_test.go create mode 100644 internal/integrationtest/migration/delta_test.go create mode 100644 internal/integrationtest/migration/domain_claim_test.go create mode 100644 internal/integrationtest/migration/helpers_test.go create mode 100644 internal/integrationtest/migration/migration_test.go create mode 100644 internal/integrationtest/migration/suite_test.go create mode 100644 internal/mail/hosted/dns_verify.go create mode 100644 internal/mail/hosted/dns_verify_test.go create mode 100644 internal/mail/hosted/service.go create mode 100644 internal/mail/stalwart/client.go create mode 100644 internal/mail/stalwart/client_test.go create mode 100644 internal/migration/admin_jobs.go create mode 100644 internal/migration/auth_modes.go create mode 100644 internal/migration/batch_config.go create mode 100644 internal/migration/batch_config_test.go create mode 100644 internal/migration/calendar_delta_test.go create mode 100644 internal/migration/calendar_import.go create mode 100644 internal/migration/claim_email_match.go create mode 100644 internal/migration/claim_email_match_test.go create mode 100644 internal/migration/contacts_delta_test.go create mode 100644 internal/migration/contacts_import.go create mode 100644 internal/migration/cutover_dns.go create mode 100644 internal/migration/cutover_dns_test.go create mode 100644 internal/migration/drive_delta.go create mode 100644 internal/migration/drive_delta_test.go create mode 100644 internal/migration/drive_helpers.go create mode 100644 internal/migration/drive_helpers_test.go create mode 100644 internal/migration/drive_import.go create mode 100644 internal/migration/gmail_attachments.go create mode 100644 internal/migration/gmail_attachments_test.go create mode 100644 internal/migration/gmail_delta_test.go create mode 100644 internal/migration/gmail_import.go create mode 100644 internal/migration/gmail_import_test.go create mode 100644 internal/migration/google_dwd.go create mode 100644 internal/migration/google_dwd_test.go create mode 100644 internal/migration/graph_import.go create mode 100644 internal/migration/graph_import_test.go create mode 100644 internal/migration/graph_user.go create mode 100644 internal/migration/http_retry.go create mode 100644 internal/migration/http_retry_test.go create mode 100644 internal/migration/httpmock_test.go create mode 100644 internal/migration/import_helpers.go create mode 100644 internal/migration/import_helpers_test.go create mode 100644 internal/migration/imported_items.go create mode 100644 internal/migration/imported_items_test.go create mode 100644 internal/migration/job_audit.go create mode 100644 internal/migration/microsoft_admin_consent.go create mode 100644 internal/migration/microsoft_admin_consent_test.go create mode 100644 internal/migration/microsoft_app.go create mode 100644 internal/migration/microsoft_app_test.go create mode 100644 internal/migration/oauth.go create mode 100644 internal/migration/oauth_admin_test.go create mode 100644 internal/migration/oauth_refresh.go create mode 100644 internal/migration/onboarding.go create mode 100644 internal/migration/onboarding_test.go create mode 100644 internal/migration/pkce.go create mode 100644 internal/migration/project_columns.go create mode 100644 internal/migration/project_columns_test.go create mode 100644 internal/migration/service.go create mode 100644 internal/migration/test_exports.go create mode 100644 internal/migration/worker.go create mode 100644 internal/migration/worker_test.go create mode 100644 internal/provision/authentik.go create mode 100644 internal/provision/handler.go create mode 100644 migrations/000040_hosted_mail_migration.down.sql create mode 100644 migrations/000040_hosted_mail_migration.up.sql create mode 100644 migrations/000041_migration_auth_mode.down.sql create mode 100644 migrations/000041_migration_auth_mode.up.sql create mode 100644 migrations/000042_migration_imported_items.down.sql create mode 100644 migrations/000042_migration_imported_items.up.sql create mode 100644 migrations/000043_migration_cutover_dns.down.sql create mode 100644 migrations/000043_migration_cutover_dns.up.sql create mode 100644 migrations/000044_migration_microsoft_admin_consent.down.sql create mode 100644 migrations/000044_migration_microsoft_admin_consent.up.sql create mode 100644 migrations/000045_migration_invite_alternate_emails.down.sql create mode 100644 migrations/000045_migration_invite_alternate_emails.up.sql create mode 100644 migrations/000046_migration_item_audit.down.sql create mode 100644 migrations/000046_migration_item_audit.up.sql create mode 100644 migrations/000047_migration_microsoft_app_auth.down.sql create mode 100644 migrations/000047_migration_microsoft_app_auth.up.sql diff --git a/.env.example b/.env.example index d52c93a..ed2325b 100644 --- a/.env.example +++ b/.env.example @@ -259,6 +259,55 @@ MAIL_MICROSOFT_OAUTH_TENANT=common MAIL_OAUTH_REDIRECT_URL= MAIL_APP_URL=http://localhost/mail # Cible nginx → suite frontend unifié mail+drive (dev: Next sur l'hôte :3004 ; prod: suite-frontend:3000) + +# ----------------------------------------------------------------------------- +# Stalwart hosted mail (optional — enable for @ultisuite.fr / custom domains) +# ----------------------------------------------------------------------------- +STALWART_ENABLED=false +STALWART_API_URL=http://stalwart:8080 +# STALWART_API_KEY — API key from Stalwart webadmin (management JMAP API) +STALWART_IMAP_HOST=stalwart +STALWART_IMAP_PORT=993 +STALWART_IMAP_TLS=true +STALWART_SMTP_HOST=stalwart +STALWART_SMTP_PORT=587 +STALWART_SMTP_TLS=true +STALWART_RECOVERY_ADMIN=admin:changeme-stalwart-admin +PLATFORM_MAIL_DOMAIN=ultisuite.fr +# PROVISION_WEBHOOK_SECRET — shared secret for Authentik enrollment webhook → ultid +PROVISION_WEBHOOK_SECRET=changeme-provision-webhook + +# ----------------------------------------------------------------------------- +# Migration OAuth (Google Workspace / Microsoft 365 import) +# Falls back to MAIL_* OAuth credentials when unset. +# ----------------------------------------------------------------------------- +MIGRATION_GOOGLE_OAUTH_CLIENT_ID= +MIGRATION_GOOGLE_OAUTH_CLIENT_SECRET= +MIGRATION_MICROSOFT_OAUTH_CLIENT_ID= +MIGRATION_MICROSOFT_OAUTH_CLIENT_SECRET= +MIGRATION_MICROSOFT_OAUTH_TENANT=common +MIGRATION_OAUTH_REDIRECT_URL= +MIGRATION_WORKER_INTERVAL=30s +# Worker picks up to JOB_LIMIT pending jobs per tick (0 = concurrency*3, min 5). +MIGRATION_WORKER_CONCURRENCY=2 +MIGRATION_WORKER_JOB_LIMIT=0 +# Items processed per worker tick (mail/contacts/calendar vs drive file downloads). +MIGRATION_IMPORT_BATCH_SIZE=25 +MIGRATION_DRIVE_BATCH_SIZE=10 +# Rate-limit backoff for Google/Microsoft migration API calls (429 + Retry-After). +MIGRATION_RATE_LIMIT_MAX_RETRIES=6 +MIGRATION_RATE_LIMIT_BASE_DELAY=2s +MIGRATION_RATE_LIMIT_MAX_DELAY=2m +# Microsoft 365 app-only (client credentials) uses MIGRATION_MICROSOFT_OAUTH_* above. +# Requires Azure AD application permissions (Mail.Read, Calendars.Read, Contacts.Read, Files.Read.All) +# and tenant admin consent; per-project tenant id is stored after admin consent redirect. +# Google Workspace domain-wide delegation (service account JSON, one line) +MIGRATION_GOOGLE_SERVICE_ACCOUNT_JSON= +# MX hosts expected at cutover (comma-separated). Defaults: mail.{PLATFORM_MAIL_DOMAIN}, then STALWART_IMAP_HOST if FQDN. +MIGRATION_CUTOVER_MX_HOSTS= +# Block cutover when live MX does not match expected hosts (requires domain_id on project). +MIGRATION_CUTOVER_REQUIRE_MX=false + MAIL_FRONTEND_UPSTREAM=host.docker.internal:3004 MAIL_WEBHOOK_SHARED_SECRET_ROTATED_AT=2026-01-01T00:00:00Z diff --git a/deploy/authentik/README.md b/deploy/authentik/README.md index 0cc56e0..8d8ea31 100644 --- a/deploy/authentik/README.md +++ b/deploy/authentik/README.md @@ -4,9 +4,10 @@ Blueprints in `blueprints/` are mounted into Authentik at `/blueprints/custom` a | Fichier | Rôle | |---------|------| -| `01-ulti-enrollment.yaml` | Inscription self-service (`ulti-enrollment`) | +| `01-ulti-enrollment.yaml` | Inscription self-service (`ulti-enrollment`, @ultisuite.fr) | | `02-ulti-brand.yaml` | Branding UltiSuite + lien « Créer un compte » sur login | | `03-ulti-suite-groups.yaml` | Claim OIDC `groups` (RBAC contacts/calendar/drive/photos) | +| `04-ulti-post-migration-security.yaml` | Flow WebAuthn/TOTP post-migration (`ulti-post-migration-security`) | | `ulti-oidc.yaml` | App OIDC Ultimail | | `nextcloud-oidc.yaml` | App OIDC Nextcloud | | `onlyoffice-oidc.yaml` | App OIDC OnlyOffice | @@ -44,6 +45,10 @@ Flow public : `http://localhost/auth/if/flow/ulti-enrollment/` 2. Nom et prénom, téléphone (optionnel), avatar (optionnel) 3. Création du compte + connexion automatique +L'email d'inscription est construit comme `username@ultisuite.fr`. ultid peut provisionner la boîte Stalwart via `POST /internal/provision/user` (header `X-Provision-Secret: $PROVISION_WEBHOOK_SECRET`). + +Flow post-migration (WebAuthn/TOTP) : `/auth/if/flow/ulti-post-migration-security/` + Sur la page de connexion Authentik, lien **« Besoin d'un compte ? S'inscrire »** (identification stage). ## Branding diff --git a/deploy/authentik/blueprints/01-ulti-enrollment.yaml b/deploy/authentik/blueprints/01-ulti-enrollment.yaml index 2d0686b..4631e0f 100644 --- a/deploy/authentik/blueprints/01-ulti-enrollment.yaml +++ b/deploy/authentik/blueprints/01-ulti-enrollment.yaml @@ -22,12 +22,26 @@ entries: attrs: field_key: username label: Adresse e-mail - type: email + type: text required: true - placeholder: vous@exemple.com + placeholder: prenom.nom placeholder_expression: false order: 0 + - model: authentik_stages_prompt.prompt + id: ulti-enroll-field-domain-hint + identifiers: + name: ulti-enrollment-field-domain-hint + attrs: + field_key: domain_hint + label: Votre adresse sera + type: static + required: false + initial_value: "@ultisuite.fr" + initial_value_expression: false + placeholder_expression: false + order: 1 + - model: authentik_stages_prompt.prompt id: ulti-enroll-field-email-sync identifiers: @@ -37,7 +51,7 @@ entries: label: E-mail type: hidden required: true - initial_value: "{{ prompt_data.username }}" + initial_value: "{{ prompt_data.username }}@ultisuite.fr" initial_value_expression: true placeholder_expression: false order: 1 @@ -114,6 +128,7 @@ entries: attrs: fields: - !KeyOf ulti-enroll-field-email + - !KeyOf ulti-enroll-field-domain-hint - !KeyOf ulti-enroll-field-email-sync - !KeyOf ulti-enroll-field-password - !KeyOf ulti-enroll-field-password-repeat @@ -141,6 +156,55 @@ entries: identifiers: name: ulti-enrollment-user-login + - model: authentik_policies_expression.expressionpolicy + id: ulti-enroll-policy-username-available + identifiers: + name: ulti-enrollment-username-available + attrs: + name: Ultimail — adresse disponible + expression: | + import json + from urllib.request import urlopen + local = (request.context.get("prompt_data") or {}).get("username", "").strip().lower() + if not local or len(local) < 2: + return False + url = f"http://ultid:8080/api/v1/mail/addresses/check?local={local}&domain=ultisuite.fr" + try: + with urlopen(url, timeout=5) as resp: + data = json.loads(resp.read().decode("utf-8")) + return data.get("available") is True + except Exception: + return False + + - model: authentik_policies.policybinding + identifiers: + order: 0 + target: !KeyOf ulti-enroll-prompt-credentials + policy: !KeyOf ulti-enroll-policy-username-available + attrs: + enabled: true + timeout: 10 + failure_result: false + + - model: authentik_stages_webhook.webhookstage + id: ulti-enroll-provision-webhook + identifiers: + name: ulti-enrollment-provision-webhook + attrs: + url: http://ultid:8080/internal/provision/user?secret=changeme-provision-webhook + method: POST + headers: + X-Provision-Secret: changeme-provision-webhook + Content-Type: application/json + body: | + { + "email": "{{ prompt_data.email }}", + "username": "{{ prompt_data.username }}", + "password": "{{ prompt_data.password }}", + "name": "{{ prompt_data.name }}", + "external_id": "{{ user.uuid }}" + } + - model: authentik_flows.flowstagebinding identifiers: target: !KeyOf ulti-enrollment-flow @@ -159,6 +223,12 @@ entries: stage: !KeyOf ulti-enroll-user-write order: 30 + - model: authentik_flows.flowstagebinding + identifiers: + target: !KeyOf ulti-enrollment-flow + stage: !KeyOf ulti-enroll-provision-webhook + order: 40 + - model: authentik_flows.flowstagebinding identifiers: target: !KeyOf ulti-enrollment-flow diff --git a/deploy/authentik/blueprints/04-ulti-post-migration-security.yaml b/deploy/authentik/blueprints/04-ulti-post-migration-security.yaml new file mode 100644 index 0000000..f5c7677 --- /dev/null +++ b/deploy/authentik/blueprints/04-ulti-post-migration-security.yaml @@ -0,0 +1,82 @@ +# Post-migration — encourage WebAuthn / TOTP before disabling legacy IdP +version: 1 +metadata: + name: Ulti post-migration security + labels: + blueprints.goauthentik.io/instantiate: "true" +entries: + - model: authentik_flows.flow + id: ulti-post-migration-flow + identifiers: + slug: ulti-post-migration-security + attrs: + name: UltiSuite — Sécuriser votre compte + title: Sécuriser votre compte UltiSuite + designation: stage_configuration + authentication: require_authenticated + + - model: authentik_stages_prompt.prompt + id: ulti-post-mig-info + identifiers: + name: ulti-post-migration-info + attrs: + field_key: info + label: Information + type: static + required: false + initial_value: > + Votre migration est en cours ou terminée. Enregistrez une clé de sécurité (WebAuthn) + ou une application TOTP pour vous connecter sans dépendre de Google Workspace ou Microsoft 365. + initial_value_expression: false + placeholder_expression: false + order: 0 + + - model: authentik_stages_prompt.promptstage + id: ulti-post-mig-prompt + identifiers: + name: ulti-post-migration-prompt + attrs: + fields: + - !KeyOf ulti-post-mig-info + + - model: authentik_stages_authenticator_webauthn.authenticatorwebauthnstage + id: ulti-post-mig-webauthn + identifiers: + name: ulti-post-migration-webauthn + attrs: + user_verification: preferred + device_type_restrictions: no_restrictions + + - model: authentik_stages_authenticator_totp.authenticatortotpstage + id: ulti-post-mig-totp + identifiers: + name: ulti-post-migration-totp + + - model: authentik_stages_user_login.userloginstage + id: ulti-post-mig-done + identifiers: + name: ulti-post-migration-done + + - model: authentik_flows.flowstagebinding + identifiers: + target: !KeyOf ulti-post-migration-flow + stage: !KeyOf ulti-post-mig-prompt + order: 10 + + - model: authentik_flows.flowstagebinding + identifiers: + target: !KeyOf ulti-post-migration-flow + stage: !KeyOf ulti-post-mig-webauthn + order: 20 + + - model: authentik_flows.flowstagebinding + identifiers: + target: !KeyOf ulti-post-migration-flow + stage: !KeyOf ulti-post-mig-totp + order: 30 + + - model: authentik_flows.flowstagebinding + identifiers: + target: !KeyOf ulti-post-migration-flow + stage: !KeyOf ulti-post-mig-done + order: 100 diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 08c08a9..37faef0 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -219,6 +219,22 @@ services: depends_on: - ultid + stalwart: + image: stalwartlabs/stalwart:v0.16 + restart: unless-stopped + environment: + STALWART_RECOVERY_ADMIN: ${STALWART_RECOVERY_ADMIN:-admin:changeme-stalwart-admin} + volumes: + - stalwart_data:/opt/stalwart + networks: + - ulti-net + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/healthz >/dev/null 2>&1 || exit 1"] + interval: 15s + timeout: 5s + retries: 8 + start_period: 30s + networks: ulti-net: driver: bridge @@ -229,3 +245,4 @@ volumes: rustfs_data: prometheus_data: grafana_data: + stalwart_data: diff --git a/deploy/nginx/default.conf.template b/deploy/nginx/default.conf.template index 6202adc..3c445c2 100644 --- a/deploy/nginx/default.conf.template +++ b/deploy/nginx/default.conf.template @@ -472,3 +472,25 @@ server { return 404 "Not found\n"; } } + +# Stalwart webadmin + JMAP (mail.${DOMAIN}) +server { + listen 80; + server_name mail.${DOMAIN}; + + client_max_body_size 100M; + + location / { + resolver 127.0.0.11 valid=10s ipv6=off; + set $stalwart_upstream stalwart:8080; + + proxy_pass http://$stalwart_upstream; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } +} diff --git a/go.mod b/go.mod index ff713a0..dccc3c3 100644 --- a/go.mod +++ b/go.mod @@ -23,12 +23,13 @@ require ( github.com/testcontainers/testcontainers-go/modules/minio v0.35.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.35.0 golang.org/x/net v0.55.0 - golang.org/x/oauth2 v0.30.0 + golang.org/x/oauth2 v0.36.0 golang.org/x/text v0.37.0 golang.org/x/time v0.15.0 ) require ( + cloud.google.com/go/compute/metadata v0.3.0 // indirect dario.cat/mergo v1.0.0 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect diff --git a/go.sum b/go.sum index 0bdd8d5..fb3fe38 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= @@ -255,8 +257,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/internal/api/admin/handlers.go b/internal/api/admin/handlers.go index 74f85e0..65c96a4 100644 --- a/internal/api/admin/handlers.go +++ b/internal/api/admin/handlers.go @@ -15,6 +15,8 @@ import ( "github.com/ultisuite/ulti-backend/internal/api/middleware" "github.com/ultisuite/ulti-backend/internal/api/query" "github.com/ultisuite/ulti-backend/internal/config" + "github.com/ultisuite/ulti-backend/internal/mail/hosted" + migr "github.com/ultisuite/ulti-backend/internal/migration" "github.com/ultisuite/ulti-backend/internal/nextcloud" "github.com/ultisuite/ulti-backend/internal/permission" "github.com/ultisuite/ulti-backend/internal/securityaudit" @@ -33,6 +35,14 @@ func NewHandler(db *pgxpool.Pool, audit *securityaudit.Logger, cfg *config.Confi } } +func (h *Handler) SetHostedService(svc *hosted.Service) { + h.svc.SetHostedService(svc) +} + +func (h *Handler) SetMigrationService(svc *migr.Service) { + h.svc.SetMigrationService(svc) +} + func (h *Handler) Routes() chi.Router { r := chi.NewRouter() read := middleware.RequireAdminScope(permission.AdminScopeRead) @@ -65,6 +75,7 @@ func (h *Handler) Routes() chi.Router { r.With(write).Post("/org/identity-providers/{providerID}/sync", h.SyncIdentityProvider) h.registerDriveAdminRoutes(r, read, write) + h.registerMailAdminRoutes(r, read, write) return r } diff --git a/internal/api/admin/handlers_mail_domains.go b/internal/api/admin/handlers_mail_domains.go new file mode 100644 index 0000000..fe65b54 --- /dev/null +++ b/internal/api/admin/handlers_mail_domains.go @@ -0,0 +1,390 @@ +package admin + +import ( + "encoding/csv" + "errors" + "io" + "net/http" + "strings" + + "github.com/go-chi/chi/v5" + + "github.com/ultisuite/ulti-backend/internal/api/apiresponse" + "github.com/ultisuite/ulti-backend/internal/api/apivalidate" + "github.com/ultisuite/ulti-backend/internal/api/query" + migr "github.com/ultisuite/ulti-backend/internal/migration" +) + +const maxAdminMailRequestBody = 1 << 20 + +func (h *Handler) registerMailAdminRoutes(r chi.Router, read, write func(http.Handler) http.Handler) { + if h.svc.hosted == nil && h.svc.migration == nil { + return + } + r.Route("/mail", func(r chi.Router) { + if h.svc.hosted != nil { + r.With(read).Get("/domains", h.ListMailDomains) + r.With(write).Post("/domains", h.CreateMailDomain) + r.With(read).Get("/domains/{domainID}", h.GetMailDomain) + r.With(write).Post("/domains/{domainID}/verify-txt", h.VerifyMailDomainTXT) + r.With(write).Post("/domains/{domainID}/verify-mx", h.VerifyMailDomainMX) + } + }) + r.Route("/migration", func(r chi.Router) { + if h.svc.migration == nil { + return + } + r.With(read).Get("/projects", h.ListMigrationProjects) + r.With(write).Post("/projects", h.CreateMigrationProject) + r.With(write).Post("/projects/{projectID}/activate", h.ActivateMigrationProject) + r.With(read).Get("/projects/{projectID}/cutover-dns", h.PreflightMigrationCutoverDNS) + r.With(write).Post("/projects/{projectID}/cutover", h.StartMigrationCutover) + r.With(write).Post("/projects/{projectID}/invites", h.CreateMigrationInvite) + r.With(write).Post("/projects/{projectID}/invites/import", h.ImportMigrationInvites) + r.With(read).Get("/projects/{projectID}/jobs", h.ListMigrationProjectJobs) + r.With(read).Get("/projects/{projectID}/jobs/{jobID}/audit", h.ListMigrationJobAudit) + r.With(read).Get("/projects/{projectID}/jobs/{jobID}/audit/summary", h.MigrationJobAuditSummary) + r.With(write).Post("/projects/{projectID}/jobs/retry-failed", h.RetryMigrationFailedJobs) + r.With(write).Post("/projects/{projectID}/jobs/{jobID}/retry", h.RetryMigrationJob) + r.With(write).Post("/projects/{projectID}/jobs/{jobID}/reset-cursor", h.ResetMigrationJobCursor) + r.With(read).Get("/microsoft/admin-consent-url", h.MicrosoftMigrationAdminConsentURL) + r.With(read).Get("/microsoft/admin-consents", h.ListMicrosoftAdminConsents) + }) +} + +func (h *Handler) ListMailDomains(w http.ResponseWriter, r *http.Request) { + rows, err := h.svc.hosted.ListDomains(r.Context()) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"domains": rows}) +} + +type createMailDomainRequest struct { + Name string `json:"name"` +} + +func (h *Handler) CreateMailDomain(w http.ResponseWriter, r *http.Request) { + var req createMailDomainRequest + if err := apivalidate.DecodeJSON(w, r, maxAdminMailRequestBody, &req); err != nil { + return + } + row, err := h.svc.hosted.CreateDomain(r.Context(), req.Name, false) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusCreated, row) +} + +func (h *Handler) GetMailDomain(w http.ResponseWriter, r *http.Request) { + row, err := h.svc.hosted.GetDomain(r.Context(), chi.URLParam(r, "domainID")) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, row) +} + +func (h *Handler) VerifyMailDomainTXT(w http.ResponseWriter, r *http.Request) { + domainID := chi.URLParam(r, "domainID") + row, report, err := h.svc.hosted.VerifyDomainTXTRecord(r.Context(), domainID) + if err != nil { + apiresponse.WriteError(w, r, http.StatusBadRequest, "dns_txt_not_verified", err.Error(), map[string]any{"dns": report}) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"domain": row, "dns": report}) +} + +func (h *Handler) VerifyMailDomainMX(w http.ResponseWriter, r *http.Request) { + domainID := chi.URLParam(r, "domainID") + expected := h.migrationCutoverConfig().ExpectedMXHosts + row, report, err := h.svc.hosted.VerifyDomainMXRecord(r.Context(), domainID, expected) + if err != nil { + apiresponse.WriteError(w, r, http.StatusBadRequest, "dns_mx_not_verified", err.Error(), map[string]any{"dns": report}) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"domain": row, "dns": report}) +} + +type createMigrationProjectRequest struct { + Name string `json:"name"` + SourceProvider string `json:"source_provider"` + DomainID string `json:"domain_id"` + AuthMode string `json:"auth_mode"` +} + +func (h *Handler) CreateMigrationProject(w http.ResponseWriter, r *http.Request) { + var req createMigrationProjectRequest + if err := apivalidate.DecodeJSON(w, r, maxAdminMailRequestBody, &req); err != nil { + return + } + row, err := h.svc.migration.CreateProject(r.Context(), req.Name, req.SourceProvider, req.DomainID, req.AuthMode) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusCreated, row) +} + +func (h *Handler) ListMigrationProjects(w http.ResponseWriter, r *http.Request) { + rows, err := h.svc.migration.ListProjects(r.Context()) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"projects": rows}) +} + +func (h *Handler) ActivateMigrationProject(w http.ResponseWriter, r *http.Request) { + row, err := h.svc.migration.ActivateProject(r.Context(), chi.URLParam(r, "projectID")) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, row) +} + +func (h *Handler) PreflightMigrationCutoverDNS(w http.ResponseWriter, r *http.Request) { + if h.svc.migration == nil { + apivalidate.WriteInternal(w, r) + return + } + report, err := h.svc.migration.PreflightCutoverDNS(r.Context(), chi.URLParam(r, "projectID"), h.migrationCutoverConfig()) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"dns": report}) +} + +func (h *Handler) StartMigrationCutover(w http.ResponseWriter, r *http.Request) { + result, err := h.svc.migration.StartCutover(r.Context(), chi.URLParam(r, "projectID")) + if errors.Is(err, migr.ErrCutoverMXNotReady) { + apiresponse.WriteError(w, r, http.StatusConflict, "migration_cutover_mx_not_ready", err.Error(), map[string]any{ + "dns": result.DNS, + "project": result.Project, + }) + return + } + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, result) +} + +func (h *Handler) migrationCutoverConfig() migr.CutoverConfig { + if h.svc.cfg == nil { + return migr.CutoverConfig{} + } + return migr.CutoverConfig{ + ExpectedMXHosts: migr.ParseCutoverMXHosts( + h.svc.cfg.MigrationCutoverMXHosts, + h.svc.cfg.PlatformMailDomain, + h.svc.cfg.StalwartIMAPHost, + ), + RequireMX: h.svc.cfg.MigrationCutoverRequireMX, + } +} + +type createMigrationInviteRequest struct { + Email string `json:"email"` + AlternateEmails []string `json:"alternate_emails,omitempty"` +} + +func (h *Handler) CreateMigrationInvite(w http.ResponseWriter, r *http.Request) { + var req createMigrationInviteRequest + if err := apivalidate.DecodeJSON(w, r, maxAdminMailRequestBody, &req); err != nil { + return + } + row, err := h.svc.migration.CreateInvite(r.Context(), chi.URLParam(r, "projectID"), req.Email, req.AlternateEmails) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusCreated, row) +} + +func (h *Handler) ImportMigrationInvites(w http.ResponseWriter, r *http.Request) { + var emails []string + contentType := r.Header.Get("Content-Type") + if strings.Contains(contentType, "multipart/form-data") { + file, _, err := r.FormFile("file") + if err == nil { + defer file.Close() + reader := csv.NewReader(file) + for { + record, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + apivalidate.WriteValidationError(w, r, apivalidate.NewValidationError(apivalidate.FieldDetail{ + Field: "file", Message: "invalid csv", + })) + return + } + if len(record) > 0 { + emails = append(emails, record[0]) + } + } + } + } + if len(emails) == 0 { + var body struct { + Emails []string `json:"emails"` + } + if err := apivalidate.DecodeJSON(w, r, maxAdminMailRequestBody, &body); err != nil { + return + } + emails = body.Emails + } + count, err := h.svc.migration.ImportInvites(r.Context(), chi.URLParam(r, "projectID"), emails) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"imported": count}) +} + +func (h *Handler) MicrosoftMigrationAdminConsentURL(w http.ResponseWriter, r *http.Request) { + if h.svc.migration == nil { + apivalidate.WriteInternal(w, r) + return + } + consentURL, err := h.svc.migration.MicrosoftAdminConsentURL( + r.URL.Query().Get("tenant"), + r.URL.Query().Get("project_id"), + ) + if err != nil { + apiresponse.WriteError(w, r, http.StatusBadRequest, "admin_consent_unavailable", err.Error(), nil) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"url": consentURL}) +} + +func (h *Handler) ListMicrosoftAdminConsents(w http.ResponseWriter, r *http.Request) { + if h.svc.migration == nil { + apivalidate.WriteInternal(w, r) + return + } + rows, err := h.svc.migration.ListMicrosoftAdminConsents(r.Context()) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"consents": rows}) +} + +func (h *Handler) ListMigrationProjectJobs(w http.ResponseWriter, r *http.Request) { + if h.svc.migration == nil { + apivalidate.WriteInternal(w, r) + return + } + rows, err := h.svc.migration.ListProjectJobs(r.Context(), chi.URLParam(r, "projectID")) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"jobs": rows}) +} + +func (h *Handler) RetryMigrationJob(w http.ResponseWriter, r *http.Request) { + if h.svc.migration == nil { + apivalidate.WriteInternal(w, r) + return + } + row, err := h.svc.migration.RetryJob(r.Context(), chi.URLParam(r, "projectID"), chi.URLParam(r, "jobID")) + if err != nil { + apiresponse.WriteError(w, r, http.StatusNotFound, "migration_job_not_retryable", err.Error(), nil) + return + } + apiresponse.WriteJSON(w, http.StatusOK, row) +} + +func (h *Handler) ResetMigrationJobCursor(w http.ResponseWriter, r *http.Request) { + if h.svc.migration == nil { + apivalidate.WriteInternal(w, r) + return + } + row, err := h.svc.migration.ResetJobCursor(r.Context(), chi.URLParam(r, "projectID"), chi.URLParam(r, "jobID")) + if err != nil { + status := http.StatusNotFound + code := "migration_job_not_resettable" + if strings.Contains(err.Error(), "running") { + status = http.StatusConflict + } else if strings.Contains(err.Error(), "not found") { + code = "migration_job_not_found" + } + apiresponse.WriteError(w, r, status, code, err.Error(), nil) + return + } + apiresponse.WriteJSON(w, http.StatusOK, row) +} + +func (h *Handler) RetryMigrationFailedJobs(w http.ResponseWriter, r *http.Request) { + if h.svc.migration == nil { + apivalidate.WriteInternal(w, r) + return + } + count, err := h.svc.migration.RetryFailedJobs(r.Context(), chi.URLParam(r, "projectID")) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"retried": count}) +} + +func (h *Handler) ListMigrationJobAudit(w http.ResponseWriter, r *http.Request) { + if h.svc.migration == nil { + apivalidate.WriteInternal(w, r) + return + } + params, err := query.ParseListRequest(r) + if err != nil { + apivalidate.WriteQueryError(w, r, err) + return + } + items, pagination, err := h.svc.migration.ListJobAudit( + r.Context(), + chi.URLParam(r, "projectID"), + chi.URLParam(r, "jobID"), + r.URL.Query().Get("status"), + params, + ) + if err != nil { + if strings.Contains(err.Error(), "not found") { + apiresponse.WriteError(w, r, http.StatusNotFound, "migration_job_not_found", err.Error(), nil) + return + } + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{ + "items": items, + "pagination": pagination, + }) +} + +func (h *Handler) MigrationJobAuditSummary(w http.ResponseWriter, r *http.Request) { + if h.svc.migration == nil { + apivalidate.WriteInternal(w, r) + return + } + summary, err := h.svc.migration.JobAuditSummary( + r.Context(), + chi.URLParam(r, "projectID"), + chi.URLParam(r, "jobID"), + ) + if err != nil { + if strings.Contains(err.Error(), "not found") { + apiresponse.WriteError(w, r, http.StatusNotFound, "migration_job_not_found", err.Error(), nil) + return + } + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, summary) +} diff --git a/internal/api/admin/service.go b/internal/api/admin/service.go index b155093..4940aa7 100644 --- a/internal/api/admin/service.go +++ b/internal/api/admin/service.go @@ -18,6 +18,8 @@ import ( "github.com/ultisuite/ulti-backend/internal/api/query" "github.com/ultisuite/ulti-backend/internal/config" + "github.com/ultisuite/ulti-backend/internal/mail/hosted" + migr "github.com/ultisuite/ulti-backend/internal/migration" "github.com/ultisuite/ulti-backend/internal/nextcloud" "github.com/ultisuite/ulti-backend/internal/permission" "github.com/ultisuite/ulti-backend/internal/securityaudit" @@ -27,11 +29,13 @@ import ( var ErrNotFound = errors.New("not found") type Service struct { - db *pgxpool.Pool - audit *securityaudit.Logger - cfg *config.Config - nc *nextcloud.Client - logger *slog.Logger + db *pgxpool.Pool + audit *securityaudit.Logger + cfg *config.Config + nc *nextcloud.Client + hosted *hosted.Service + migration *migr.Service + logger *slog.Logger } func NewService(db *pgxpool.Pool, audit *securityaudit.Logger, cfg *config.Config, nc *nextcloud.Client) *Service { @@ -44,6 +48,14 @@ func NewService(db *pgxpool.Pool, audit *securityaudit.Logger, cfg *config.Confi } } +func (s *Service) SetHostedService(hostedSvc *hosted.Service) { + s.hosted = hostedSvc +} + +func (s *Service) SetMigrationService(migrationSvc *migr.Service) { + s.migration = migrationSvc +} + type UsersList struct { Users []map[string]any `json:"users"` Pagination query.PaginationMeta `json:"pagination,omitempty"` diff --git a/internal/api/mail/handlers.go b/internal/api/mail/handlers.go index 730fe14..c503586 100644 --- a/internal/api/mail/handlers.go +++ b/internal/api/mail/handlers.go @@ -95,6 +95,7 @@ func (h *Handler) Routes() chi.Router { r.Post("/accounts/{accountID}/test", h.TestStoredAccountConnection) r.Get("/accounts/oauth/providers", h.ListOAuthProviders) r.Post("/accounts/oauth/start", h.StartOAuthAccount) + r.Get("/addresses/check", h.CheckAddressAvailability) r.Get("/accounts/{accountID}", h.GetAccount) r.Put("/accounts/{accountID}", h.UpdateAccount) r.Delete("/accounts/{accountID}", h.DeleteAccount) diff --git a/internal/api/mail/handlers_hosted.go b/internal/api/mail/handlers_hosted.go new file mode 100644 index 0000000..d51d91c --- /dev/null +++ b/internal/api/mail/handlers_hosted.go @@ -0,0 +1,48 @@ +package mail + +import ( + "net/http" + "strings" + + "github.com/ultisuite/ulti-backend/internal/api/apiresponse" + "github.com/ultisuite/ulti-backend/internal/api/apivalidate" + "github.com/ultisuite/ulti-backend/internal/mail/hosted" +) + +func (h *Handler) SetHostedService(svc *hosted.Service) { + if s, ok := h.svc.(*Service); ok { + s.SetHostedService(svc) + } +} + +func (h *Handler) CheckAddressAvailability(w http.ResponseWriter, r *http.Request) { + svc := h.hostedService() + if svc == nil { + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"available": true, "reason": "hosted_mail_disabled"}) + return + } + local := strings.TrimSpace(r.URL.Query().Get("local")) + domain := strings.TrimSpace(r.URL.Query().Get("domain")) + if domain == "" { + domain = strings.TrimSpace(r.URL.Query().Get("domain_name")) + } + if local == "" || domain == "" { + apivalidate.WriteValidationError(w, r, apivalidate.NewValidationError(apivalidate.FieldDetail{ + Field: "local", Message: "local and domain required", + })) + return + } + available, err := svc.IsAddressAvailable(r.Context(), domain, local) + if err != nil { + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"available": false, "reason": err.Error()}) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"available": available}) +} + +func (h *Handler) hostedService() *hosted.Service { + if s, ok := h.svc.(*Service); ok { + return s.HostedService() + } + return nil +} diff --git a/internal/api/mail/service.go b/internal/api/mail/service.go index 0f97ab7..3547739 100644 --- a/internal/api/mail/service.go +++ b/internal/api/mail/service.go @@ -13,6 +13,7 @@ import ( "github.com/ultisuite/ulti-backend/internal/api/query" "github.com/ultisuite/ulti-backend/internal/filescan" "github.com/ultisuite/ulti-backend/internal/mail/credentials" + "github.com/ultisuite/ulti-backend/internal/mail/hosted" "github.com/ultisuite/ulti-backend/internal/mail/imap" "github.com/ultisuite/ulti-backend/internal/mail/sanitize" "github.com/ultisuite/ulti-backend/internal/mail/storage" @@ -40,6 +41,7 @@ type Service struct { attachmentsBucket string driveUploader DriveUploader scanner *filescan.Scanner + hosted *hosted.Service logger *slog.Logger } @@ -54,6 +56,14 @@ func NewService(db *pgxpool.Pool, audit *securityaudit.Logger, credentialManager } } +func (s *Service) SetHostedService(svc *hosted.Service) { + s.hosted = svc +} + +func (s *Service) HostedService() *hosted.Service { + return s.hosted +} + func (s *Service) DB() *pgxpool.Pool { return s.db } diff --git a/internal/api/migration/handlers.go b/internal/api/migration/handlers.go new file mode 100644 index 0000000..7e25f2b --- /dev/null +++ b/internal/api/migration/handlers.go @@ -0,0 +1,257 @@ +package migrationapi + +import ( + "log/slog" + "net/http" + "net/url" + "strings" + + "github.com/go-chi/chi/v5" + + "github.com/ultisuite/ulti-backend/internal/api/apiresponse" + "github.com/ultisuite/ulti-backend/internal/api/apivalidate" + "github.com/ultisuite/ulti-backend/internal/api/middleware" + migr "github.com/ultisuite/ulti-backend/internal/migration" +) + +const maxMigrationRequestBody = 1 << 20 + +type Handler struct { + svc *migr.Service + oauth *migr.OAuthService + appURL string + logger *slog.Logger +} + +func NewHandler(svc *migr.Service, oauth *migr.OAuthService, appURL string) *Handler { + return &Handler{ + svc: svc, + oauth: oauth, + appURL: strings.TrimRight(strings.TrimSpace(appURL), "/"), + logger: slog.Default().With("component", "migration-api"), + } +} + +func (h *Handler) Routes() chi.Router { + r := chi.NewRouter() + r.Post("/claim", h.ClaimInvite) + r.Get("/status", h.GetStatus) + r.Get("/oauth/providers", h.ListOAuthProviders) + r.Post("/oauth/start", h.StartOAuth) + return r +} + +func (h *Handler) OAuthCallback(w http.ResponseWriter, r *http.Request) { + if handled, err := h.handleMicrosoftAdminConsentCallback(w, r); handled { + if err != nil { + h.logger.Error("microsoft admin consent", "error", err) + } + return + } + state := r.URL.Query().Get("state") + code := r.URL.Query().Get("code") + if state == "" || code == "" { + http.Redirect(w, r, h.appURL+"/onboard/migration?oauth=error", http.StatusFound) + return + } + pending, token, scopes, err := h.oauth.Exchange(r.Context(), state, code) + if err != nil { + h.logger.Error("oauth exchange", "error", err) + http.Redirect(w, r, h.appURL+"/onboard/migration?oauth=error", http.StatusFound) + return + } + if err := h.svc.StoreMigrationToken(r.Context(), pending.UserID, pending.ProjectID, pending.Provider, token, scopes); err != nil { + h.logger.Error("store migration token", "error", err) + http.Redirect(w, r, h.appURL+"/onboard/migration?oauth=error", http.StatusFound) + return + } + redirect := h.appURL + "/onboard/migration?oauth=success" + if pending.InviteToken != "" { + redirect = h.appURL + "/onboard/migration?oauth=success&token=" + pending.InviteToken + } + http.Redirect(w, r, redirect, http.StatusFound) +} + +func (h *Handler) handleMicrosoftAdminConsentCallback(w http.ResponseWriter, r *http.Request) (bool, error) { + q := r.URL.Query() + projectID := migr.ParseAdminConsentProjectID(q.Get("state")) + granted := strings.EqualFold(q.Get("admin_consent"), "True") + oauthErr := strings.TrimSpace(q.Get("error")) + if !granted && (projectID == "" || oauthErr == "") { + return false, nil + } + if h.oauth == nil { + return true, migr.ErrProviderDisabled + } + record := migr.MicrosoftAdminConsentRecord{ + TenantID: q.Get("tenant"), + ClientID: h.oauth.MicrosoftClientID(), + ProjectID: projectID, + Granted: granted, + ErrorCode: oauthErr, + ErrorDescription: q.Get("error_description"), + } + if err := h.svc.RecordMicrosoftAdminConsent(r.Context(), record); err != nil { + redirect := h.appURL + "/admin/settings/mail-domains?microsoft_admin_consent=error" + http.Redirect(w, r, redirect, http.StatusFound) + return true, err + } + redirect := h.appURL + "/admin/settings/mail-domains?microsoft_admin_consent=success" + if !granted { + redirect = h.appURL + "/admin/settings/mail-domains?microsoft_admin_consent=error" + } + if tenant := strings.TrimSpace(record.TenantID); tenant != "" { + redirect += "&tenant=" + url.QueryEscape(tenant) + } + if projectID != "" { + redirect += "&project_id=" + url.QueryEscape(projectID) + } + http.Redirect(w, r, redirect, http.StatusFound) + return true, nil +} + +func (h *Handler) GetInvite(w http.ResponseWriter, r *http.Request) { + token := strings.TrimSpace(r.URL.Query().Get("token")) + if token == "" { + apivalidate.WriteValidationError(w, r, apivalidate.NewValidationError(apivalidate.FieldDetail{ + Field: "token", Message: "token required", + })) + return + } + inv, proj, err := h.svc.GetInviteByToken(r.Context(), token) + if err != nil { + apiresponse.WriteError(w, r, http.StatusNotFound, "invite_not_found", err.Error(), nil) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{ + "invite": inv, + "project": proj, + "onboarding": h.svc.BuildInviteOnboardingHints(proj, inv), + }) +} + +type claimRequest struct { + Token string `json:"token"` + Password string `json:"password"` +} + +func (h *Handler) ClaimInvite(w http.ResponseWriter, r *http.Request) { + claims := middleware.ClaimsFromContext(r.Context()) + if claims == nil { + apiresponse.WriteError(w, r, http.StatusUnauthorized, "unauthorized", "authentication required", nil) + return + } + var req claimRequest + if err := apivalidate.DecodeJSON(w, r, maxMigrationRequestBody, &req); err != nil { + return + } + userID, err := h.svc.LookupUserID(r.Context(), claims.Sub) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + status, err := h.svc.ClaimInvite(r.Context(), req.Token, userID, migr.ClaimIdentityFromAuth(claims), claims.Name, req.Password) + if err != nil { + code := http.StatusBadRequest + errCode := "claim_failed" + switch { + case err == migr.ErrInviteNotFound: + code = http.StatusNotFound + errCode = "invite_not_found" + case err == migr.ErrInviteClaimed: + errCode = "invite_already_claimed" + case err == migr.ErrEmailMismatch: + errCode = "email_mismatch" + case err == migr.ErrMigrationDomainNotActive: + errCode = "migration_domain_not_active" + case err == migr.ErrMigrationDomainMismatch: + errCode = "migration_domain_mismatch" + } + apiresponse.WriteError(w, r, code, errCode, err.Error(), nil) + return + } + apiresponse.WriteJSON(w, http.StatusOK, status) +} + +func (h *Handler) GetStatus(w http.ResponseWriter, r *http.Request) { + claims := middleware.ClaimsFromContext(r.Context()) + if claims == nil { + apiresponse.WriteError(w, r, http.StatusUnauthorized, "unauthorized", "authentication required", nil) + return + } + userID, err := h.svc.LookupUserID(r.Context(), claims.Sub) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + status, err := h.svc.GetActiveUserStatus(r.Context(), userID) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + apiresponse.WriteJSON(w, http.StatusOK, status) +} + +func (h *Handler) ListOAuthProviders(w http.ResponseWriter, r *http.Request) { + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"providers": h.oauth.EnabledProviders()}) +} + +type startOAuthRequest struct { + Provider string `json:"provider"` + InviteToken string `json:"invite_token"` + ProjectID string `json:"project_id"` +} + +func (h *Handler) StartOAuth(w http.ResponseWriter, r *http.Request) { + claims := middleware.ClaimsFromContext(r.Context()) + if claims == nil { + apiresponse.WriteError(w, r, http.StatusUnauthorized, "unauthorized", "authentication required", nil) + return + } + var req startOAuthRequest + if err := apivalidate.DecodeJSON(w, r, maxMigrationRequestBody, &req); err != nil { + return + } + userID, err := h.svc.LookupUserID(r.Context(), claims.Sub) + if err != nil { + apivalidate.WriteInternal(w, r) + return + } + projectID := req.ProjectID + if projectID == "" && req.InviteToken != "" { + inv, proj, err := h.svc.GetInviteByToken(r.Context(), req.InviteToken) + if err != nil { + apiresponse.WriteError(w, r, http.StatusNotFound, "invite_not_found", err.Error(), nil) + return + } + if inv.UserID != "" && inv.UserID != userID { + apiresponse.WriteError(w, r, http.StatusForbidden, "forbidden", "invite belongs to another user", nil) + return + } + projectID = proj.ID + } + if projectID == "" { + st, err := h.svc.GetActiveUserStatus(r.Context(), userID) + if err != nil || st.Project.ID == "" { + apivalidate.WriteValidationError(w, r, apivalidate.NewValidationError(apivalidate.FieldDetail{ + Field: "project_id", Message: "project_id required", + })) + return + } + projectID = st.Project.ID + } + provider := migr.Provider(strings.ToLower(strings.TrimSpace(req.Provider))) + if provider == "" { + provider = migr.ProviderGoogle + } + authURL, _, err := h.oauth.Start(r.Context(), migr.PendingOAuth{ + UserID: userID, + ProjectID: projectID, + InviteToken: req.InviteToken, + }, provider) + if err != nil { + apiresponse.WriteError(w, r, http.StatusBadRequest, "oauth_start_failed", err.Error(), nil) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"auth_url": authURL}) +} diff --git a/internal/auth/oidc.go b/internal/auth/oidc.go index 3a8250f..433c2f5 100644 --- a/internal/auth/oidc.go +++ b/internal/auth/oidc.go @@ -12,14 +12,16 @@ import ( ) type Claims struct { - Sub string - Email string - Name string - Groups []string - Source string - HD string - TID string - Org string + Sub string + Email string + PreferredUsername string + UPN string + Name string + Groups []string + Source string + HD string + TID string + Org string } type Verifier struct { @@ -96,27 +98,31 @@ func (v *Verifier) Verify(ctx context.Context, rawToken string) (*Claims, error) } var claims struct { - Sub string `json:"sub"` - Email string `json:"email"` - Name string `json:"name"` - Groups []string `json:"groups"` - HD string `json:"hd"` - TID string `json:"tid"` - Org string `json:"org"` - Source string `json:"ak-source"` + Sub string `json:"sub"` + Email string `json:"email"` + PreferredUsername string `json:"preferred_username"` + UPN string `json:"upn"` + Name string `json:"name"` + Groups []string `json:"groups"` + HD string `json:"hd"` + TID string `json:"tid"` + Org string `json:"org"` + Source string `json:"ak-source"` } if err := token.Claims(&claims); err != nil { return nil, err } return &Claims{ - Sub: claims.Sub, - Email: claims.Email, - Name: claims.Name, - Groups: claims.Groups, - HD: claims.HD, - TID: claims.TID, - Org: claims.Org, - Source: claims.Source, + Sub: claims.Sub, + Email: claims.Email, + PreferredUsername: claims.PreferredUsername, + UPN: claims.UPN, + Name: claims.Name, + Groups: claims.Groups, + HD: claims.HD, + TID: claims.TID, + Org: claims.Org, + Source: claims.Source, }, nil } diff --git a/internal/config/config.go b/internal/config/config.go index ea4dcbf..de1a0b5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -115,6 +115,38 @@ type Config struct { MailOAuthRedirectURL string MailAppURL string + // Stalwart hosted mail + StalwartEnabled bool + StalwartAPIURL string + StalwartAPIKey string + StalwartIMAPHost string + StalwartIMAPPort int + StalwartIMAPTLS bool + StalwartSMTPHost string + StalwartSMTPPort int + StalwartSMTPTLS bool + PlatformMailDomain string + ProvisionWebhookSecret string + + // Migration OAuth (Google/Microsoft bulk import) + MigrationGoogleOAuthClientID string + MigrationGoogleOAuthClientSecret string + MigrationMicrosoftOAuthClientID string + MigrationMicrosoftOAuthSecret string + MigrationMicrosoftOAuthTenant string + MigrationOAuthRedirectURL string + MigrationWorkerInterval time.Duration + MigrationGoogleServiceAccountJSON string + MigrationRateLimitMaxRetries int + MigrationRateLimitBaseDelay time.Duration + MigrationRateLimitMaxDelay time.Duration + MigrationWorkerConcurrency int + MigrationWorkerJobLimit int + MigrationImportBatchSize int + MigrationDriveBatchSize int + MigrationCutoverMXHosts string + MigrationCutoverRequireMX bool + // Secret rotation policy SecretRotationMaxAge time.Duration OIDCSecretRotatedAt time.Time @@ -236,6 +268,36 @@ func Load() (*Config, error) { MailOAuthRedirectURL: os.Getenv("MAIL_OAUTH_REDIRECT_URL"), MailAppURL: envOrDefault("MAIL_APP_URL", envOrDefault("NEXT_PUBLIC_APP_URL", "http://localhost:3004")), + StalwartEnabled: envBool("STALWART_ENABLED", false), + StalwartAPIURL: envOrDefault("STALWART_API_URL", "http://stalwart:8080"), + StalwartAPIKey: secrets.Env("STALWART_API_KEY"), + StalwartIMAPHost: envOrDefault("STALWART_IMAP_HOST", "stalwart"), + StalwartIMAPPort: envInt("STALWART_IMAP_PORT", 993), + StalwartIMAPTLS: envBool("STALWART_IMAP_TLS", true), + StalwartSMTPHost: envOrDefault("STALWART_SMTP_HOST", "stalwart"), + StalwartSMTPPort: envInt("STALWART_SMTP_PORT", 587), + StalwartSMTPTLS: envBool("STALWART_SMTP_TLS", true), + PlatformMailDomain: envOrDefault("PLATFORM_MAIL_DOMAIN", "ultisuite.fr"), + ProvisionWebhookSecret: secrets.Env("PROVISION_WEBHOOK_SECRET"), + + MigrationGoogleOAuthClientID: os.Getenv("MIGRATION_GOOGLE_OAUTH_CLIENT_ID"), + MigrationGoogleOAuthClientSecret: secrets.Env("MIGRATION_GOOGLE_OAUTH_CLIENT_SECRET"), + MigrationMicrosoftOAuthClientID: os.Getenv("MIGRATION_MICROSOFT_OAUTH_CLIENT_ID"), + MigrationMicrosoftOAuthSecret: secrets.Env("MIGRATION_MICROSOFT_OAUTH_CLIENT_SECRET"), + MigrationMicrosoftOAuthTenant: envOrDefault("MIGRATION_MICROSOFT_OAUTH_TENANT", "common"), + MigrationOAuthRedirectURL: os.Getenv("MIGRATION_OAUTH_REDIRECT_URL"), + MigrationWorkerInterval: envDuration("MIGRATION_WORKER_INTERVAL", 30*time.Second), + MigrationGoogleServiceAccountJSON: secrets.Env("MIGRATION_GOOGLE_SERVICE_ACCOUNT_JSON"), + MigrationRateLimitMaxRetries: envInt("MIGRATION_RATE_LIMIT_MAX_RETRIES", 6), + MigrationRateLimitBaseDelay: envDuration("MIGRATION_RATE_LIMIT_BASE_DELAY", 2*time.Second), + MigrationRateLimitMaxDelay: envDuration("MIGRATION_RATE_LIMIT_MAX_DELAY", 2*time.Minute), + MigrationWorkerConcurrency: envInt("MIGRATION_WORKER_CONCURRENCY", 2), + MigrationWorkerJobLimit: envInt("MIGRATION_WORKER_JOB_LIMIT", 0), + MigrationImportBatchSize: envInt("MIGRATION_IMPORT_BATCH_SIZE", 25), + MigrationDriveBatchSize: envInt("MIGRATION_DRIVE_BATCH_SIZE", 10), + MigrationCutoverMXHosts: os.Getenv("MIGRATION_CUTOVER_MX_HOSTS"), + MigrationCutoverRequireMX: envBool("MIGRATION_CUTOVER_REQUIRE_MX", false), + SecretRotationMaxAge: envDuration("SECRET_ROTATION_MAX_AGE", 90*24*time.Hour), OIDCSecretRotatedAt: envTime("ULTID_OIDC_CLIENT_SECRET_ROTATED_AT"), SMTPCredentialKeyRotatedAt: envTime("MAIL_CREDENTIAL_KEY_ROTATED_AT"), diff --git a/internal/integrationtest/harness.go b/internal/integrationtest/harness.go index 45ff005..3686be8 100644 --- a/internal/integrationtest/harness.go +++ b/internal/integrationtest/harness.go @@ -5,6 +5,7 @@ package integrationtest import ( "context" "fmt" + "log/slog" "net/http/httptest" "os" "sync" @@ -13,23 +14,28 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/redis/go-redis/v9" "github.com/ultisuite/ulti-backend/internal/auth" "github.com/ultisuite/ulti-backend/internal/config" "github.com/ultisuite/ulti-backend/internal/dbmigrate" "github.com/ultisuite/ulti-backend/internal/server" + mailstorage "github.com/ultisuite/ulti-backend/internal/mail/storage" ) // Harness is the shared integration test environment. type Harness struct { - Env Env - Infra *infra - OIDC *OIDCServer - App *server.App - Server *httptest.Server - Pool *pgxpool.Pool - Redis *redis.Client + Env Env + Infra *infra + OIDC *OIDCServer + App *server.App + Server *httptest.Server + Pool *pgxpool.Pool + Redis *redis.Client + AttachmentStorage *mailstorage.Client + AttachmentsBucket string } var ( @@ -128,6 +134,20 @@ func newHarness(ctx context.Context) (*Harness, error) { cfg := buildTestConfig(env, infra, oidc) + minioClient, err := minio.New(cfg.RustFSEndpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.RustFSAccessKey, cfg.RustFSSecretKey, ""), + Secure: cfg.RustFSUseSSL, + }) + if err != nil { + pool.Close() + _ = rdb.Close() + return nil, fmt.Errorf("rustfs client: %w", err) + } + attachmentStorage := mailstorage.NewClient(minioClient, cfg.MailAttachmentsBucket) + if err := attachmentStorage.EnsureBucket(ctx); err != nil { + slog.Warn("mail attachments bucket check failed", "error", err) + } + app, err := server.New(ctx, cfg, server.Options{ WithoutWorkers: true, SkipAuthentikProvisioner: true, @@ -144,13 +164,15 @@ func newHarness(ctx context.Context) (*Harness, error) { ts := httptest.NewServer(app.Router) return &Harness{ - Env: env, - Infra: infra, - OIDC: oidc, - App: app, - Server: ts, - Pool: pool, - Redis: rdb, + Env: env, + Infra: infra, + OIDC: oidc, + App: app, + Server: ts, + Pool: pool, + Redis: rdb, + AttachmentStorage: attachmentStorage, + AttachmentsBucket: cfg.MailAttachmentsBucket, }, nil } diff --git a/internal/integrationtest/migration/claim_email_test.go b/internal/integrationtest/migration/claim_email_test.go new file mode 100644 index 0000000..c381514 --- /dev/null +++ b/internal/integrationtest/migration/claim_email_test.go @@ -0,0 +1,129 @@ +//go:build integration + +package migration_test + +import ( + "context" + "testing" + + "github.com/google/uuid" + + "github.com/ultisuite/ulti-backend/internal/integrationtest" + "github.com/ultisuite/ulti-backend/internal/users" +) + +func TestClaimInviteFlexibleEmailMatch(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + adminClient, adminClaims := integrationtest.RequireAdminClient(t, h) + if _, err := users.EnsureUser(ctx, h.Pool, adminClaims); err != nil { + t.Fatalf("ensure admin: %v", err) + } + if err := users.GrantPlatformAdmin(ctx, h.Pool, adminClaims.Sub); err != nil { + t.Fatalf("grant admin: %v", err) + } + + createResp, err := adminClient.Post("/api/v1/admin/migration/projects", map[string]any{ + "name": "Flexible email match", + "source_provider": "microsoft", + }) + integrationtest.FailIf(err, t, "create project") + integrationtest.FailUnlessStatus(t, createResp, 201) + + var created struct { + ID string `json:"id"` + } + integrationtest.DecodeJSON(t, createResp, &created) + + actResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/activate", nil) + integrationtest.FailIf(err, t, "activate project") + integrationtest.FailUnlessStatus(t, actResp, 200) + + inviteEmail := "alice-" + uuid.NewString()[:8] + "@example.com" + ssoEmail := "alice.sso-" + uuid.NewString()[:8] + "@example.com" + inviteResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/invites", map[string]any{ + "email": inviteEmail, + "alternate_emails": []string{ssoEmail}, + }) + integrationtest.FailIf(err, t, "create invite") + integrationtest.FailUnlessStatus(t, inviteResp, 201) + + var invite struct { + Token string `json:"token"` + } + integrationtest.DecodeJSON(t, inviteResp, &invite) + + migrateeClaims := integrationtest.RegularUser(integrationtest.NewExternalID("flex-claim")) + migrateeClaims.Email = ssoEmail + migrateeClaims.PreferredUsername = inviteEmail + migrateeClient, err := h.Client(migrateeClaims) + integrationtest.FailIf(err, t, "migratee client") + + if _, err := users.EnsureUser(ctx, h.Pool, migrateeClaims); err != nil { + t.Fatalf("ensure migratee: %v", err) + } + + claimResp, err := migrateeClient.Post("/api/v1/migration/claim", map[string]string{ + "token": invite.Token, + "password": "test-password-123", + }) + integrationtest.FailIf(err, t, "claim invite") + integrationtest.FailUnlessStatus(t, claimResp, 200) +} + +func TestClaimInviteRejectsEmailMismatch(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + adminClient, adminClaims := integrationtest.RequireAdminClient(t, h) + if _, err := users.EnsureUser(ctx, h.Pool, adminClaims); err != nil { + t.Fatalf("ensure admin: %v", err) + } + if err := users.GrantPlatformAdmin(ctx, h.Pool, adminClaims.Sub); err != nil { + t.Fatalf("grant admin: %v", err) + } + + createResp, err := adminClient.Post("/api/v1/admin/migration/projects", map[string]any{ + "name": "Reject mismatch", + "source_provider": "google", + }) + integrationtest.FailIf(err, t, "create project") + integrationtest.FailUnlessStatus(t, createResp, 201) + + var created struct { + ID string `json:"id"` + } + integrationtest.DecodeJSON(t, createResp, &created) + + actResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/activate", nil) + integrationtest.FailIf(err, t, "activate project") + integrationtest.FailUnlessStatus(t, actResp, 200) + + inviteEmail := "victim-" + uuid.NewString() + "@example.com" + inviteResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/invites", map[string]string{ + "email": inviteEmail, + }) + integrationtest.FailIf(err, t, "create invite") + integrationtest.FailUnlessStatus(t, inviteResp, 201) + + var invite struct { + Token string `json:"token"` + } + integrationtest.DecodeJSON(t, inviteResp, &invite) + + attackerClaims := integrationtest.RegularUser(integrationtest.NewExternalID("flex-claim-bad")) + attackerClaims.Email = "attacker-" + uuid.NewString() + "@example.com" + attackerClient, err := h.Client(attackerClaims) + integrationtest.FailIf(err, t, "attacker client") + + if _, err := users.EnsureUser(ctx, h.Pool, attackerClaims); err != nil { + t.Fatalf("ensure attacker: %v", err) + } + + claimResp, err := attackerClient.Post("/api/v1/migration/claim", map[string]string{ + "token": invite.Token, + }) + integrationtest.FailIf(err, t, "claim invite") + integrationtest.AssertErrorCode(t, claimResp, 400, "email_mismatch") +} diff --git a/internal/integrationtest/migration/delta_test.go b/internal/integrationtest/migration/delta_test.go new file mode 100644 index 0000000..68fb304 --- /dev/null +++ b/internal/integrationtest/migration/delta_test.go @@ -0,0 +1,562 @@ +//go:build integration + +package migration_test + +import ( + "context" + "net/http" + "strings" + "testing" + + "github.com/ultisuite/ulti-backend/internal/integrationtest" + migr "github.com/ultisuite/ulti-backend/internal/migration" + "github.com/ultisuite/ulti-backend/internal/users" +) + +func TestMigrationCutoverResetsCompletedJobs(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + adminClient, adminClaims := integrationtest.RequireAdminClient(t, h) + if _, err := users.EnsureUser(ctx, h.Pool, adminClaims); err != nil { + t.Fatalf("ensure admin: %v", err) + } + if err := users.GrantPlatformAdmin(ctx, h.Pool, adminClaims.Sub); err != nil { + t.Fatalf("grant admin: %v", err) + } + + createResp, err := adminClient.Post("/api/v1/admin/migration/projects", map[string]any{ + "name": "Cutover test", + "source_provider": "google", + }) + integrationtest.FailIf(err, t, "create project") + integrationtest.FailUnlessStatus(t, createResp, 201) + + var created struct { + ID string `json:"id"` + } + integrationtest.DecodeJSON(t, createResp, &created) + + actResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/activate", nil) + integrationtest.FailIf(err, t, "activate project") + integrationtest.FailUnlessStatus(t, actResp, 200) + + migrateeEmail := "cutover-" + created.ID[:8] + "@example.com" + inviteResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/invites", map[string]string{ + "email": migrateeEmail, + }) + integrationtest.FailIf(err, t, "create invite") + integrationtest.FailUnlessStatus(t, inviteResp, 201) + + var invite struct { + Token string `json:"token"` + } + integrationtest.DecodeJSON(t, inviteResp, &invite) + + migrateeClaims := integrationtest.RegularUser(integrationtest.NewExternalID("cutover")) + migrateeClaims.Email = migrateeEmail + migrateeClient, err := h.Client(migrateeClaims) + integrationtest.FailIf(err, t, "migratee client") + + userID, err := users.EnsureUser(ctx, h.Pool, migrateeClaims) + integrationtest.FailIf(err, t, "ensure migratee") + + claimResp, err := migrateeClient.Post("/api/v1/migration/claim", map[string]string{ + "token": invite.Token, + "password": "test-password-123", + }) + integrationtest.FailIf(err, t, "claim invite") + integrationtest.FailUnlessStatus(t, claimResp, 200) + + _, err = h.Pool.Exec(ctx, ` + UPDATE migration_jobs SET status = 'completed', updated_at = NOW() + WHERE project_id = $1::uuid AND user_id = $2::uuid + `, created.ID, userID) + integrationtest.FailIf(err, t, "mark jobs completed") + + cutoverResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/cutover", nil) + integrationtest.FailIf(err, t, "cutover") + integrationtest.FailUnlessStatus(t, cutoverResp, 200) + + var cutover struct { + Project struct { + Status string `json:"status"` + DeltaMode bool `json:"delta_mode"` + CutoverAt *string `json:"cutover_at"` + } `json:"project"` + } + integrationtest.DecodeJSON(t, cutoverResp, &cutover) + project := cutover.Project + if project.Status != "cutover" || !project.DeltaMode || project.CutoverAt == nil { + t.Fatalf("cutover project: %#v", project) + } + + var pendingCount int + if err := h.Pool.QueryRow(ctx, ` + SELECT COUNT(*) FROM migration_jobs + WHERE project_id = $1::uuid AND user_id = $2::uuid AND status = 'pending' + `, created.ID, userID).Scan(&pendingCount); err != nil { + t.Fatalf("count pending jobs: %v", err) + } + if pendingCount != 4 { + t.Fatalf("pending jobs = %d, want 4", pendingCount) + } + + listResp, err := adminClient.Get("/api/v1/admin/migration/projects") + integrationtest.FailIf(err, t, "list projects after cutover") + integrationtest.FailUnlessStatus(t, listResp, 200) + + var listed struct { + Projects []struct { + ID string `json:"id"` + CutoverDNS *struct { + Warnings []string `json:"warnings"` + } `json:"cutover_dns"` + } `json:"projects"` + } + integrationtest.DecodeJSON(t, listResp, &listed) + var found bool + for _, p := range listed.Projects { + if p.ID != created.ID { + continue + } + found = true + if p.CutoverDNS == nil { + t.Fatal("expected cutover_dns on listed project") + } + } + if !found { + t.Fatal("cutover project not found in list") + } +} + +func TestGoogleContactsDeltaDeletesRemoved(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + userID, email := insertMigrationTestUser(t, h.Pool, "contacts-delta") + nc, _ := mockNextcloudClient(t, h.Pool, email) + + client := googleRewriteClient(t, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/v1/people/me/connections") { + _, _ = w.Write([]byte(`{ + "connections":[{ + "resourceName":"people/deleted-1", + "metadata":{"deleted":true} + }], + "nextSyncToken":"sync-next" + }`)) + return + } + http.NotFound(w, r) + }) + + importer := migr.NewContactsImporter(h.Pool, nc).WithHTTPClient(client) + job := &migr.Job{ + UserID: userID, + CursorJSON: map[string]any{ + "syncToken": "sync-old", + "imported_ids": map[string]any{ + "people/deleted-1": true, + }, + }, + StatsJSON: map[string]any{}, + } + + err := importer.ImportBatch(ctx, job, "token", "google", true, func(status string, cursor, stats map[string]any, jobErr string) error { + if jobErr != "" { + t.Fatalf("import error: %s", jobErr) + } + if status != "completed" { + t.Fatalf("status = %q, want completed", status) + } + deleted, _ := stats["delta_deleted"].(float64) + if deleted != 1 { + t.Fatalf("delta_deleted = %v, want 1", stats["delta_deleted"]) + } + if cursor["syncToken"] != "sync-next" { + t.Fatalf("sync token = %v", cursor["syncToken"]) + } + return nil + }) + integrationtest.FailIf(err, t, "import batch") +} + +func TestGoogleCalendarDeltaUpdatesExisting(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + userID, email := insertMigrationTestUser(t, h.Pool, "calendar-delta-update") + nc, _ := mockNextcloudClient(t, h.Pool, email) + + client := googleRewriteClient(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "/calendar/v3/users/me/calendarList"): + _, _ = w.Write([]byte(`{"items":[{"id":"primary","summary":"Primary"}]}`)) + case strings.Contains(r.URL.Path, "/calendar/v3/calendars/") && strings.Contains(r.URL.Path, "/events"): + _, _ = w.Write([]byte(`{ + "items":[{"id":"evt-1","status":"confirmed","summary":"Updated meeting","start":{"dateTime":"2026-06-13T10:00:00Z"},"end":{"dateTime":"2026-06-13T11:00:00Z"}}], + "nextSyncToken":"cal-sync-next" + }`)) + default: + http.NotFound(w, r) + } + }) + + importer := migr.NewCalendarImporter(h.Pool, nc).WithHTTPClient(client) + job := &migr.Job{ + UserID: userID, + CursorJSON: map[string]any{ + "calendarSyncTokens": map[string]any{"primary": "cal-sync-old"}, + "imported_ids": map[string]any{"primary:evt-1": true}, + }, + StatsJSON: map[string]any{}, + } + + err := importer.ImportBatch(ctx, job, "token", "google", true, func(status string, cursor, stats map[string]any, jobErr string) error { + if jobErr != "" { + t.Fatalf("import error: %s", jobErr) + } + updated, _ := stats["delta_updated"].(float64) + if updated != 1 { + t.Fatalf("delta_updated = %v, want 1", stats["delta_updated"]) + } + imported, _ := stats["delta_imported"].(float64) + if imported != 0 { + t.Fatalf("delta_imported = %v, want 0", stats["delta_imported"]) + } + return nil + }) + integrationtest.FailIf(err, t, "import batch") +} + +func TestGoogleContactsDeltaUpdatesExisting(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + userID, email := insertMigrationTestUser(t, h.Pool, "contacts-delta-update") + nc, _ := mockNextcloudClient(t, h.Pool, email) + + client := googleRewriteClient(t, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/v1/people/me/connections") { + _, _ = w.Write([]byte(`{ + "connections":[{ + "resourceName":"people/abc", + "names":[{"displayName":"Alice Updated"}], + "emailAddresses":[{"value":"alice@example.com"}] + }], + "nextSyncToken":"sync-next" + }`)) + return + } + http.NotFound(w, r) + }) + + importer := migr.NewContactsImporter(h.Pool, nc).WithHTTPClient(client) + job := &migr.Job{ + UserID: userID, + CursorJSON: map[string]any{ + "syncToken": "sync-old", + "imported_ids": map[string]any{ + "people/abc": true, + }, + }, + StatsJSON: map[string]any{}, + } + + err := importer.ImportBatch(ctx, job, "token", "google", true, func(status string, cursor, stats map[string]any, jobErr string) error { + if jobErr != "" { + t.Fatalf("import error: %s", jobErr) + } + updated, _ := stats["delta_updated"].(float64) + if updated != 1 { + t.Fatalf("delta_updated = %v, want 1", stats["delta_updated"]) + } + imported, _ := stats["delta_imported"].(float64) + if imported != 0 { + t.Fatalf("delta_imported = %v, want 0", stats["delta_imported"]) + } + return nil + }) + integrationtest.FailIf(err, t, "import batch") +} + +func TestGoogleDriveDeltaDeletesRemovedFile(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + userID, email := insertMigrationTestUser(t, h.Pool, "drive-delta") + nc, _ := mockNextcloudClient(t, h.Pool, email) + + client := googleRewriteClient(t, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/drive/v3/changes") { + _, _ = w.Write([]byte(`{ + "changes":[{"fileId":"file-removed","removed":true}], + "newStartPageToken":"token-next" + }`)) + return + } + http.NotFound(w, r) + }) + + importer := migr.NewDriveImporter(h.Pool, nc).WithHTTPClient(client) + job := &migr.Job{ + UserID: userID, + CursorJSON: map[string]any{ + "driveChangeToken": "token-old", + "imported_paths": map[string]any{ + "file-removed": "Docs/report.docx", + }, + "imported_ids": map[string]any{ + "file-removed": true, + }, + }, + StatsJSON: map[string]any{}, + } + + err := importer.ImportBatch(ctx, job, "token", "google", true, func(status string, cursor, stats map[string]any, jobErr string) error { + if jobErr != "" { + t.Fatalf("import error: %s", jobErr) + } + if status != "completed" { + t.Fatalf("status = %q, want completed", status) + } + deleted, _ := stats["delta_deleted"].(float64) + if deleted != 1 { + t.Fatalf("delta_deleted = %v, want 1", stats["delta_deleted"]) + } + if cursor["driveChangeToken"] != "token-next" { + t.Fatalf("change token = %v", cursor["driveChangeToken"]) + } + if _, ok := cursor["imported_ids"]; ok { + t.Fatal("expected imported_ids stripped from cursor") + } + if _, ok := cursor["imported_paths"]; ok { + t.Fatal("expected imported_paths stripped from cursor") + } + return nil + }) + integrationtest.FailIf(err, t, "import batch") +} + +func TestGoogleCalendarDeltaDeletesCancelled(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + userID, email := insertMigrationTestUser(t, h.Pool, "calendar-delta") + nc, _ := mockNextcloudClient(t, h.Pool, email) + + client := googleRewriteClient(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "/calendar/v3/users/me/calendarList"): + _, _ = w.Write([]byte(`{"items":[{"id":"primary","summary":"Primary"}]}`)) + case strings.Contains(r.URL.Path, "/calendar/v3/calendars/") && strings.Contains(r.URL.Path, "/events"): + _, _ = w.Write([]byte(`{ + "items":[{"id":"evt-1","status":"cancelled","summary":"Old meeting"}], + "nextSyncToken":"cal-sync-next" + }`)) + default: + http.NotFound(w, r) + } + }) + + importer := migr.NewCalendarImporter(h.Pool, nc).WithHTTPClient(client) + job := &migr.Job{ + UserID: userID, + CursorJSON: map[string]any{ + "calendarSyncTokens": map[string]any{"primary": "cal-sync-old"}, + "imported_ids": map[string]any{"primary:evt-1": true}, + }, + StatsJSON: map[string]any{}, + } + + err := importer.ImportBatch(ctx, job, "token", "google", true, func(status string, cursor, stats map[string]any, jobErr string) error { + if jobErr != "" { + t.Fatalf("import error: %s", jobErr) + } + deleted, _ := stats["delta_deleted"].(float64) + if deleted != 1 { + t.Fatalf("delta_deleted = %v, want 1", stats["delta_deleted"]) + } + return nil + }) + integrationtest.FailIf(err, t, "import batch") +} + +func TestMicrosoftContactsDeltaRemoved(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + userID, email := insertMigrationTestUser(t, h.Pool, "ms-contacts-delta") + nc, _ := mockNextcloudClient(t, h.Pool, email) + + client := graphRewriteClient(t, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/v1.0/me/contacts") { + _, _ = w.Write([]byte(`{ + "value":[{"id":"c-1","@removed":{"reason":"deleted"}}], + "@odata.deltaLink":"https://graph.microsoft.com/v1.0/me/contacts/delta?token=next" + }`)) + return + } + http.NotFound(w, r) + }) + + importer := migr.NewContactsImporter(h.Pool, nc).WithHTTPClient(client) + job := &migr.Job{ + UserID: userID, + CursorJSON: map[string]any{ + "deltaLink": "https://graph.microsoft.com/v1.0/me/contacts/delta?token=old", + "imported_ids": map[string]any{ + "c-1": true, + }, + }, + StatsJSON: map[string]any{}, + } + + err := importer.ImportBatch(ctx, job, "token", "microsoft", true, func(status string, cursor, stats map[string]any, jobErr string) error { + if jobErr != "" { + t.Fatalf("import error: %s", jobErr) + } + deleted, _ := stats["delta_deleted"].(float64) + if deleted != 1 { + t.Fatalf("delta_deleted = %v, want 1", stats["delta_deleted"]) + } + return nil + }) + integrationtest.FailIf(err, t, "import batch") +} + +func TestGraphMailDeltaDeletesRemoved(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + userID, err := users.EnsureUser(ctx, h.Pool, integrationtest.RegularUser(integrationtest.NewExternalID("graph-delta-mail"))) + integrationtest.FailIf(err, t, "ensure user") + + var accountID string + err = h.Pool.QueryRow(ctx, ` + INSERT INTO mail_accounts (user_id, email, provider, is_active) + VALUES ($1::uuid, 'graph-delta@test.local', 'hosted', true) + RETURNING id::text + `, userID).Scan(&accountID) + integrationtest.FailIf(err, t, "insert mail account") + + uid := migr.RemoteMessageUIDForTest("msg-removed-1") + _, err = h.Pool.Exec(ctx, ` + INSERT INTO messages (account_id, folder_id, uid, message_id, subject, from_addr, to_addrs, date, snippet, body_text, body_html, flags, labels) + SELECT $1::uuid, f.id, $2, '', 'To delete', '[]', '[]', NOW(), '', '', '', '{}', '{}' + FROM mail_folders f WHERE f.account_id = $1::uuid AND f.remote_name = 'INBOX' LIMIT 1 + `, accountID, uid) + integrationtest.FailIf(err, t, "seed message") + + client := graphRewriteClient(t, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/mailFolders") { + _, _ = w.Write([]byte(`{"value":[{"id":"inbox-id","displayName":"Inbox","wellKnownName":"inbox"}]}`)) + return + } + if strings.Contains(r.URL.Path, "/messages") { + _, _ = w.Write([]byte(`{ + "value":[{"id":"msg-removed-1","@removed":{"reason":"deleted"}}], + "@odata.deltaLink":"https://graph.microsoft.com/v1.0/me/messages/delta?token=next" + }`)) + return + } + http.NotFound(w, r) + }) + + importer := migr.NewGraphImporter(h.Pool).WithHTTPClient(client).WithBaseURL("https://graph.microsoft.com") + job := &migr.Job{ + UserID: userID, + CursorJSON: map[string]any{"deltaLink": "https://graph.microsoft.com/v1.0/me/messages/delta?token=old"}, + StatsJSON: map[string]any{}, + } + + err = importer.ImportBatch(ctx, job, "token", true, func(status string, cursor, stats map[string]any, jobErr string) error { + if jobErr != "" { + t.Fatalf("import error: %s", jobErr) + } + if status != "completed" { + t.Fatalf("status = %q, want completed", status) + } + deleted, _ := stats["delta_deleted"].(float64) + if deleted != 1 { + t.Fatalf("delta_deleted = %v, want 1", stats["delta_deleted"]) + } + return nil + }) + integrationtest.FailIf(err, t, "import batch") + + var count int + if err := h.Pool.QueryRow(ctx, `SELECT COUNT(*) FROM messages WHERE account_id = $1::uuid AND uid = $2`, accountID, uid).Scan(&count); err != nil { + t.Fatalf("count messages: %v", err) + } + if count != 0 { + t.Fatalf("message count = %d, want 0", count) + } +} + +func TestGmailHistoryDeltaDeletesMessage(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + userID, err := users.EnsureUser(ctx, h.Pool, integrationtest.RegularUser(integrationtest.NewExternalID("gmail-delta-mail"))) + integrationtest.FailIf(err, t, "ensure user") + + var accountID string + err = h.Pool.QueryRow(ctx, ` + INSERT INTO mail_accounts (user_id, email, provider, is_active) + VALUES ($1::uuid, 'gmail-delta@test.local', 'hosted', true) + RETURNING id::text + `, userID).Scan(&accountID) + integrationtest.FailIf(err, t, "insert mail account") + + gmailID := "abc123deleted" + uid := migr.GmailUIDForTest(gmailID) + _, err = h.Pool.Exec(ctx, ` + INSERT INTO messages (account_id, folder_id, uid, message_id, subject, from_addr, to_addrs, date, snippet, body_text, body_html, flags, labels) + SELECT $1::uuid, f.id, $2, '', 'To delete', '[]', '[]', NOW(), '', '', '', '{}', '{}' + FROM mail_folders f WHERE f.account_id = $1::uuid AND f.remote_name = 'INBOX' LIMIT 1 + `, accountID, uid) + integrationtest.FailIf(err, t, "seed message") + + client := googleRewriteClient(t, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/gmail/v1/users/me/history") { + _, _ = w.Write([]byte(`{ + "history":[{"messagesDeleted":[{"message":{"id":"` + gmailID + `"}}]}], + "historyId":"99999" + }`)) + return + } + http.NotFound(w, r) + }) + + importer := migr.NewGmailImporter(h.Pool).WithHTTPClient(client) + job := &migr.Job{ + UserID: userID, + CursorJSON: map[string]any{"historyId": "88888"}, + StatsJSON: map[string]any{}, + } + + err = importer.ImportBatch(ctx, job, "token", true, func(status string, cursor, stats map[string]any, jobErr string) error { + if jobErr != "" { + t.Fatalf("import error: %s", jobErr) + } + deleted, _ := stats["delta_deleted"].(float64) + if deleted != 1 { + t.Fatalf("delta_deleted = %v, want 1", stats["delta_deleted"]) + } + if cursor["historyId"] != "99999" { + t.Fatalf("historyId = %v", cursor["historyId"]) + } + return nil + }) + integrationtest.FailIf(err, t, "import batch") + + var count int + if err := h.Pool.QueryRow(ctx, `SELECT COUNT(*) FROM messages WHERE account_id = $1::uuid AND uid = $2`, accountID, uid).Scan(&count); err != nil { + t.Fatalf("count messages: %v", err) + } + if count != 0 { + t.Fatalf("message count = %d, want 0", count) + } +} diff --git a/internal/integrationtest/migration/domain_claim_test.go b/internal/integrationtest/migration/domain_claim_test.go new file mode 100644 index 0000000..2f7365b --- /dev/null +++ b/internal/integrationtest/migration/domain_claim_test.go @@ -0,0 +1,159 @@ +//go:build integration + +package migration_test + +import ( + "context" + "testing" + + "github.com/google/uuid" + + "github.com/ultisuite/ulti-backend/internal/integrationtest" + "github.com/ultisuite/ulti-backend/internal/users" +) + +func TestClaimInviteRequiresActiveProjectDomain(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + adminClient, adminClaims := integrationtest.RequireAdminClient(t, h) + if _, err := users.EnsureUser(ctx, h.Pool, adminClaims); err != nil { + t.Fatalf("ensure admin: %v", err) + } + if err := users.GrantPlatformAdmin(ctx, h.Pool, adminClaims.Sub); err != nil { + t.Fatalf("grant admin: %v", err) + } + + domainName := "migration-" + uuid.NewString()[:8] + ".test" + var domainID string + err := h.Pool.QueryRow(ctx, ` + INSERT INTO mail_domains (name, status, is_platform_domain) + VALUES ($1, 'pending_verification', false) + RETURNING id::text + `, domainName).Scan(&domainID) + integrationtest.FailIf(err, t, "insert domain") + + createResp, err := adminClient.Post("/api/v1/admin/migration/projects", map[string]any{ + "name": "Domain-bound migration", + "source_provider": "google", + "domain_id": domainID, + }) + integrationtest.FailIf(err, t, "create project") + integrationtest.FailUnlessStatus(t, createResp, 201) + + var created struct { + ID string `json:"id"` + } + integrationtest.DecodeJSON(t, createResp, &created) + + actResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/activate", nil) + integrationtest.FailIf(err, t, "activate project") + integrationtest.FailUnlessStatus(t, actResp, 200) + + migrateeEmail := "user@" + domainName + inviteResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/invites", map[string]string{ + "email": migrateeEmail, + }) + integrationtest.FailIf(err, t, "create invite") + integrationtest.FailUnlessStatus(t, inviteResp, 201) + + var invite struct { + Token string `json:"token"` + } + integrationtest.DecodeJSON(t, inviteResp, &invite) + + migrateeClaims := integrationtest.RegularUser(integrationtest.NewExternalID("domain-claim")) + migrateeClaims.Email = migrateeEmail + migrateeClient, err := h.Client(migrateeClaims) + integrationtest.FailIf(err, t, "migratee client") + + _, err = users.EnsureUser(ctx, h.Pool, migrateeClaims) + integrationtest.FailIf(err, t, "ensure migratee") + + claimResp, err := migrateeClient.Post("/api/v1/migration/claim", map[string]string{ + "token": invite.Token, + "password": "test-password-123", + }) + integrationtest.FailIf(err, t, "claim invite") + if claimResp.Status != 400 { + t.Fatalf("status = %d, want 400 for inactive domain; body=%s", claimResp.Status, string(claimResp.Body)) + } +} + +func TestClaimInviteWithActiveDomainSucceeds(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + adminClient, adminClaims := integrationtest.RequireAdminClient(t, h) + if _, err := users.EnsureUser(ctx, h.Pool, adminClaims); err != nil { + t.Fatalf("ensure admin: %v", err) + } + if err := users.GrantPlatformAdmin(ctx, h.Pool, adminClaims.Sub); err != nil { + t.Fatalf("grant admin: %v", err) + } + + domainName := "active-" + uuid.NewString()[:8] + ".test" + var domainID string + err := h.Pool.QueryRow(ctx, ` + INSERT INTO mail_domains (name, status, is_platform_domain) + VALUES ($1, 'active', false) + RETURNING id::text + `, domainName).Scan(&domainID) + integrationtest.FailIf(err, t, "insert domain") + + createResp, err := adminClient.Post("/api/v1/admin/migration/projects", map[string]any{ + "name": "Active domain migration", + "source_provider": "google", + "domain_id": domainID, + }) + integrationtest.FailIf(err, t, "create project") + integrationtest.FailUnlessStatus(t, createResp, 201) + + var created struct { + ID string `json:"id"` + } + integrationtest.DecodeJSON(t, createResp, &created) + + actResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/activate", nil) + integrationtest.FailIf(err, t, "activate project") + integrationtest.FailUnlessStatus(t, actResp, 200) + + migrateeEmail := "user@" + domainName + inviteResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/invites", map[string]string{ + "email": migrateeEmail, + }) + integrationtest.FailIf(err, t, "create invite") + integrationtest.FailUnlessStatus(t, inviteResp, 201) + + var invite struct { + Token string `json:"token"` + } + integrationtest.DecodeJSON(t, inviteResp, &invite) + + migrateeClaims := integrationtest.RegularUser(integrationtest.NewExternalID("domain-claim-ok")) + migrateeClaims.Email = migrateeEmail + migrateeClient, err := h.Client(migrateeClaims) + integrationtest.FailIf(err, t, "migratee client") + + userID, err := users.EnsureUser(ctx, h.Pool, migrateeClaims) + integrationtest.FailIf(err, t, "ensure migratee") + + claimResp, err := migrateeClient.Post("/api/v1/migration/claim", map[string]string{ + "token": invite.Token, + "password": "test-password-123", + }) + integrationtest.FailIf(err, t, "claim invite") + integrationtest.FailUnlessStatus(t, claimResp, 200) + + var mailboxCount int + if err := h.Pool.QueryRow(ctx, ` + SELECT COUNT(*) FROM mailboxes mb + JOIN mail_domains md ON md.id = mb.domain_id + WHERE mb.user_id = $1::uuid AND md.id = $2::uuid + `, userID, domainID).Scan(&mailboxCount); err != nil { + t.Fatalf("count mailboxes: %v", err) + } + if mailboxCount != 1 { + t.Fatalf("mailbox count = %d, want 1", mailboxCount) + } +} diff --git a/internal/integrationtest/migration/helpers_test.go b/internal/integrationtest/migration/helpers_test.go new file mode 100644 index 0000000..607211e --- /dev/null +++ b/internal/integrationtest/migration/helpers_test.go @@ -0,0 +1,117 @@ +//go:build integration + +package migration_test + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/ultisuite/ulti-backend/internal/integrationtest" + "github.com/ultisuite/ulti-backend/internal/mail/credentials" + "github.com/ultisuite/ulti-backend/internal/nextcloud" + "github.com/ultisuite/ulti-backend/internal/users" +) + +func mockNextcloudServer(t *testing.T) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPut, "MKCOL", "MKCALENDAR": + w.WriteHeader(http.StatusCreated) + case http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + case "PROPFIND", "REPORT", http.MethodGet: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(``)) + default: + w.WriteHeader(http.StatusNoContent) + } + })) +} + +func testCredentialManager(t *testing.T) *credentials.Manager { + t.Helper() + mgr, err := credentials.NewManager( + "v1:MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", + "v1", + ) + if err != nil { + t.Fatalf("credential manager: %v", err) + } + return mgr +} + +func mockNextcloudClient(t *testing.T, pool *pgxpool.Pool, email string) (*nextcloud.Client, string) { + t.Helper() + srv := mockNextcloudServer(t) + t.Cleanup(srv.Close) + + ncUserID := nextcloud.UserIDFromClaims(email, "") + store := nextcloud.NewDAVCredentialStore(pool, testCredentialManager(t)) + if err := store.SaveToken(context.Background(), ncUserID, "mock-app-password"); err != nil { + t.Fatalf("seed dav token: %v", err) + } + client := nextcloud.NewClient(srv.URL, "admin", "admin").WithDAVCredentials(store) + return client, ncUserID +} + +func insertMigrationTestUser(t *testing.T, pool *pgxpool.Pool, prefix string) (userID, email string) { + t.Helper() + claims := integrationtest.RegularUser(integrationtest.NewExternalID(prefix)) + claims.Email = prefix + "-" + uuid.NewString() + "@migration.test" + id, err := users.EnsureUser(context.Background(), pool, claims) + integrationtest.FailIf(err, t, "ensure user") + return id, claims.Email +} + +func googleRewriteClient(t *testing.T, handler http.HandlerFunc) *http.Client { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return &http.Client{ + Transport: &hostRewriteTransport{ + mockBase: srv.URL, + match: func(host string) bool { + return strings.Contains(host, "googleapis.com") + }, + }, + } +} + +func graphRewriteClient(t *testing.T, handler http.HandlerFunc) *http.Client { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return &http.Client{ + Transport: &hostRewriteTransport{ + mockBase: srv.URL, + match: func(host string) bool { + return strings.Contains(host, "graph.microsoft.com") + }, + }, + } +} + +type hostRewriteTransport struct { + mockBase string + match func(host string) bool +} + +func (rt *hostRewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if rt.match(req.URL.Host) { + mockURL, err := url.Parse(rt.mockBase) + if err != nil { + return nil, err + } + req.URL.Scheme = mockURL.Scheme + req.URL.Host = mockURL.Host + } + return http.DefaultTransport.RoundTrip(req) +} diff --git a/internal/integrationtest/migration/migration_test.go b/internal/integrationtest/migration/migration_test.go new file mode 100644 index 0000000..2ff47cf --- /dev/null +++ b/internal/integrationtest/migration/migration_test.go @@ -0,0 +1,388 @@ +//go:build integration + +package migration_test + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/google/uuid" + + "github.com/ultisuite/ulti-backend/internal/integrationtest" + migr "github.com/ultisuite/ulti-backend/internal/migration" + "github.com/ultisuite/ulti-backend/internal/users" +) + +func TestMigrationInviteClaimFlow(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + adminClient, adminClaims := integrationtest.RequireAdminClient(t, h) + if _, err := users.EnsureUser(ctx, h.Pool, adminClaims); err != nil { + t.Fatalf("ensure admin: %v", err) + } + if err := users.GrantPlatformAdmin(ctx, h.Pool, adminClaims.Sub); err != nil { + t.Fatalf("grant admin: %v", err) + } + + createResp, err := adminClient.Post("/api/v1/admin/migration/projects", map[string]any{ + "name": "Test migration", + "source_provider": "microsoft", + }) + integrationtest.FailIf(err, t, "create project") + integrationtest.FailUnlessStatus(t, createResp, 201) + + var created struct { + ID string `json:"id"` + } + integrationtest.DecodeJSON(t, createResp, &created) + if created.ID == "" { + t.Fatalf("missing project id") + } + + actResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/activate", nil) + integrationtest.FailIf(err, t, "activate project") + integrationtest.FailUnlessStatus(t, actResp, 200) + + migrateeEmail := "migratee-" + uuid.NewString() + "@example.com" + inviteResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/invites", map[string]string{ + "email": migrateeEmail, + }) + integrationtest.FailIf(err, t, "create invite") + integrationtest.FailUnlessStatus(t, inviteResp, 201) + + var invite struct { + Token string `json:"token"` + } + integrationtest.DecodeJSON(t, inviteResp, &invite) + if invite.Token == "" { + t.Fatalf("missing invite token") + } + + migrateeClaims := integrationtest.RegularUser(integrationtest.NewExternalID("migratee")) + migrateeClaims.Email = migrateeEmail + migrateeClient, err := h.Client(migrateeClaims) + integrationtest.FailIf(err, t, "migratee client") + + userID, err := users.EnsureUser(ctx, h.Pool, migrateeClaims) + integrationtest.FailIf(err, t, "ensure migratee") + + claimResp, err := migrateeClient.Post("/api/v1/migration/claim", map[string]string{ + "token": invite.Token, + "password": "test-password-123", + }) + integrationtest.FailIf(err, t, "claim invite") + integrationtest.FailUnlessStatus(t, claimResp, 200) + + var status struct { + Jobs []struct { + Service string `json:"service"` + Status string `json:"status"` + } `json:"jobs"` + } + integrationtest.DecodeJSON(t, claimResp, &status) + if len(status.Jobs) != 4 { + t.Fatalf("expected 4 jobs, got %d", len(status.Jobs)) + } + + var jobCount int + if err := h.Pool.QueryRow(ctx, ` + SELECT COUNT(*) FROM migration_jobs WHERE project_id = $1::uuid AND user_id = $2::uuid + `, created.ID, userID).Scan(&jobCount); err != nil { + t.Fatalf("count jobs: %v", err) + } + if jobCount != 4 { + t.Fatalf("db job count = %d, want 4", jobCount) + } + + jobsResp, err := adminClient.Get("/api/v1/admin/migration/projects/" + created.ID + "/jobs") + integrationtest.FailIf(err, t, "list admin jobs") + integrationtest.FailUnlessStatus(t, jobsResp, 200) + + var adminJobs struct { + Jobs []struct { + ID string `json:"id"` + Service string `json:"service"` + Status string `json:"status"` + Email string `json:"user_email"` + } `json:"jobs"` + } + integrationtest.DecodeJSON(t, jobsResp, &adminJobs) + if len(adminJobs.Jobs) != 4 { + t.Fatalf("admin jobs = %d, want 4", len(adminJobs.Jobs)) + } + for _, job := range adminJobs.Jobs { + if job.Email != migrateeEmail { + t.Fatalf("user_email = %q, want %q", job.Email, migrateeEmail) + } + } + + var mailJobID string + for _, job := range adminJobs.Jobs { + if job.Service == "mail" { + mailJobID = job.ID + break + } + } + if mailJobID == "" { + t.Fatal("mail job not found") + } + + _, err = h.Pool.Exec(ctx, ` + UPDATE migration_jobs SET status = 'failed', error = 'simulated failure', updated_at = NOW() + WHERE id = $1::uuid + `, mailJobID) + integrationtest.FailIf(err, t, "mark job failed") + + retryResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/jobs/"+mailJobID+"/retry", nil) + integrationtest.FailIf(err, t, "retry job") + integrationtest.FailUnlessStatus(t, retryResp, 200) + + var retried struct { + Status string `json:"status"` + } + integrationtest.DecodeJSON(t, retryResp, &retried) + if retried.Status != "pending" { + t.Fatalf("retried status = %q, want pending", retried.Status) + } + + _, err = h.Pool.Exec(ctx, ` + UPDATE migration_jobs + SET cursor_json = '{"historyId":"123"}'::jsonb, + stats_json = '{"imported":42}'::jsonb, + status = 'completed', + updated_at = NOW() + WHERE id = $1::uuid + `, mailJobID) + integrationtest.FailIf(err, t, "seed job cursor") + + _, err = h.Pool.Exec(ctx, ` + INSERT INTO migration_imported_items (job_id, source_id, status, reason) + VALUES ($1::uuid, 'msg-abc', 'imported', ''), + ($1::uuid, 'msg-fail', 'failed', 'upload timeout'), + ($1::uuid, 'msg-skip', 'skipped', 'file too large') + `, mailJobID) + integrationtest.FailIf(err, t, "seed imported items") + + summaryResp, err := adminClient.Get("/api/v1/admin/migration/projects/" + created.ID + "/jobs/" + mailJobID + "/audit/summary") + integrationtest.FailIf(err, t, "audit summary") + integrationtest.FailUnlessStatus(t, summaryResp, 200) + var auditSummary struct { + Imported int64 `json:"imported"` + Failed int64 `json:"failed"` + Skipped int64 `json:"skipped"` + Total int64 `json:"total"` + Service string `json:"service"` + } + integrationtest.DecodeJSON(t, summaryResp, &auditSummary) + if auditSummary.Imported != 1 || auditSummary.Failed != 1 || auditSummary.Skipped != 1 || auditSummary.Total != 3 { + t.Fatalf("audit summary = %+v, want 1 imported / 1 failed / 1 skipped", auditSummary) + } + if auditSummary.Service != "mail" { + t.Fatalf("audit service = %q, want mail", auditSummary.Service) + } + + failedResp, err := adminClient.Get("/api/v1/admin/migration/projects/" + created.ID + "/jobs/" + mailJobID + "/audit?status=failed") + integrationtest.FailIf(err, t, "audit failed list") + integrationtest.FailUnlessStatus(t, failedResp, 200) + var failedList struct { + Items []struct { + SourceID string `json:"source_id"` + Status string `json:"status"` + Reason string `json:"reason"` + } `json:"items"` + } + integrationtest.DecodeJSON(t, failedResp, &failedList) + if len(failedList.Items) != 1 || failedList.Items[0].SourceID != "msg-fail" { + t.Fatalf("failed audit items = %+v", failedList.Items) + } + + resetResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/jobs/"+mailJobID+"/reset-cursor", nil) + integrationtest.FailIf(err, t, "reset cursor") + integrationtest.FailUnlessStatus(t, resetResp, 200) + + var reset struct { + Status string `json:"status"` + CursorJSON map[string]any `json:"cursor_json"` + StatsJSON map[string]any `json:"stats_json"` + } + integrationtest.DecodeJSON(t, resetResp, &reset) + if reset.Status != "pending" { + t.Fatalf("reset status = %q, want pending", reset.Status) + } + if len(reset.CursorJSON) != 0 { + t.Fatalf("cursor not cleared: %#v", reset.CursorJSON) + } + if len(reset.StatsJSON) != 0 { + t.Fatalf("stats not cleared: %#v", reset.StatsJSON) + } + + var importedCount int + if err := h.Pool.QueryRow(ctx, ` + SELECT COUNT(*) FROM migration_imported_items WHERE job_id = $1::uuid + `, mailJobID).Scan(&importedCount); err != nil { + t.Fatalf("count imported items: %v", err) + } + if importedCount != 0 { + t.Fatalf("imported items = %d, want 0", importedCount) + } +} + +func TestGraphImportWritesMessages(t *testing.T) { + h := integrationtest.RequireHarness(t) + ctx := context.Background() + + userID, err := users.EnsureUser(ctx, h.Pool, integrationtest.RegularUser(integrationtest.NewExternalID("graph-import"))) + integrationtest.FailIf(err, t, "ensure user") + + var accountID string + err = h.Pool.QueryRow(ctx, ` + INSERT INTO mail_accounts (user_id, email, provider, is_active) + VALUES ($1::uuid, 'graph-import@test.local', 'hosted', true) + RETURNING id::text + `, userID).Scan(&accountID) + integrationtest.FailIf(err, t, "insert mail account") + + folderID := "inbox-folder-id" + messagesListed := false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "/mailFolders"): + _, _ = w.Write([]byte(`{"value":[{"id":"` + folderID + `","displayName":"Inbox","wellKnownName":"inbox"}]}`)) + case strings.Contains(r.URL.Path, "/messages"): + messagesListed = true + _, _ = w.Write([]byte(`{"value":[{ + "id":"msg-1", + "subject":"Hello Graph", + "bodyPreview":"Preview text", + "body":{"contentType":"text","content":"Body text"}, + "from":{"emailAddress":{"name":"Alice","address":"alice@example.com"}}, + "toRecipients":[{"emailAddress":{"name":"Bob","address":"bob@example.com"}}], + "receivedDateTime":"2024-05-01T10:00:00Z", + "parentFolderId":"` + folderID + `", + "isRead":true, + "internetMessageId":"" + }]}`)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + importer := migr.NewGraphImporter(h.Pool).WithBaseURL(srv.URL) + job := &migr.Job{ + UserID: userID, + CursorJSON: map[string]any{}, + StatsJSON: map[string]any{}, + } + err = importer.ImportBatch(ctx, job, "test-token", false, func(status string, cursor, stats map[string]any, jobErr string) error { + if jobErr != "" { + t.Fatalf("import error: %s", jobErr) + } + if status != "completed" { + t.Fatalf("status = %q, want completed", status) + } + return nil + }) + integrationtest.FailIf(err, t, "import batch") + if !messagesListed { + t.Fatal("graph messages endpoint not called") + } + + var count int + if err := h.Pool.QueryRow(ctx, ` + SELECT COUNT(*) FROM messages WHERE account_id = $1::uuid AND subject = 'Hello Graph' + `, accountID).Scan(&count); err != nil { + t.Fatalf("count messages: %v", err) + } + if count != 1 { + t.Fatalf("message count = %d, want 1", count) + } +} + +func TestGmailImportStoresAttachments(t *testing.T) { + h := integrationtest.RequireHarness(t) + if h.AttachmentStorage == nil { + t.Skip("attachment storage unavailable") + } + ctx := context.Background() + + userID, err := users.EnsureUser(ctx, h.Pool, integrationtest.RegularUser(integrationtest.NewExternalID("gmail-att-import"))) + integrationtest.FailIf(err, t, "ensure user") + + var accountID string + err = h.Pool.QueryRow(ctx, ` + INSERT INTO mail_accounts (user_id, email, provider, is_active) + VALUES ($1::uuid, 'gmail-att@test.local', 'hosted', true) + RETURNING id::text + `, userID).Scan(&accountID) + integrationtest.FailIf(err, t, "insert mail account") + + gmailID := "msg-with-att" + client := googleRewriteClient(t, func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "/users/me/profile"): + _, _ = w.Write([]byte(`{"historyId":"12345"}`)) + case strings.Contains(r.URL.Path, "/attachments/att-123"): + _, _ = w.Write([]byte(`{"size":5,"data":"aGVsbG8="}`)) + case strings.Contains(r.URL.Path, "/messages/"+gmailID): + _, _ = w.Write([]byte(`{ + "id":"` + gmailID + `", + "threadId":"t1", + "labelIds":["INBOX"], + "snippet":"see attached", + "payload":{ + "mimeType":"multipart/mixed", + "headers":[{"name":"Subject","value":"With attachment"}], + "parts":[ + {"mimeType":"text/plain","body":{"data":"dGV4dA=="}}, + { + "mimeType":"application/pdf", + "headers":[{"name":"Content-Disposition","value":"attachment; filename=\"report.pdf\""}], + "body":{"attachmentId":"att-123","size":5} + } + ] + } + }`)) + case strings.HasSuffix(r.URL.Path, "/messages"): + _, _ = w.Write([]byte(`{"messages":[{"id":"` + gmailID + `"}]}`)) + default: + http.NotFound(w, r) + } + }) + + importer := migr.NewGmailImporter(h.Pool). + WithHTTPClient(client). + WithStorage(h.AttachmentStorage, h.AttachmentsBucket) + job := &migr.Job{ + UserID: userID, + CursorJSON: map[string]any{}, + StatsJSON: map[string]any{}, + } + err = importer.ImportBatch(ctx, job, "token", false, func(status string, cursor, stats map[string]any, jobErr string) error { + if jobErr != "" { + t.Fatalf("import error: %s", jobErr) + } + if status != "completed" { + t.Fatalf("status = %q, want completed", status) + } + return nil + }) + integrationtest.FailIf(err, t, "import batch") + + var attCount int + var hasAttachments bool + err = h.Pool.QueryRow(ctx, ` + SELECT COUNT(*)::int, COALESCE(BOOL_OR(m.has_attachments), false) + FROM attachments a + JOIN messages m ON m.id = a.message_id + WHERE m.account_id = $1::uuid AND a.filename = 'report.pdf' + `, accountID).Scan(&attCount, &hasAttachments) + integrationtest.FailIf(err, t, "count attachments") + if attCount != 1 || !hasAttachments { + t.Fatalf("attachments = %d has_attachments = %v", attCount, hasAttachments) + } +} diff --git a/internal/integrationtest/migration/suite_test.go b/internal/integrationtest/migration/suite_test.go new file mode 100644 index 0000000..d1ef99e --- /dev/null +++ b/internal/integrationtest/migration/suite_test.go @@ -0,0 +1,14 @@ +//go:build integration + +package migration_test + +import ( + "os" + "testing" + + "github.com/ultisuite/ulti-backend/internal/integrationtest" +) + +func TestMain(m *testing.M) { + os.Exit(integrationtest.RunMain(m)) +} diff --git a/internal/integrationtest/oidc.go b/internal/integrationtest/oidc.go index b915103..f86f309 100644 --- a/internal/integrationtest/oidc.go +++ b/internal/integrationtest/oidc.go @@ -93,9 +93,11 @@ func (s *OIDCServer) IssueToken(claims *auth.Claims) (string, error) { Expiry: jwt.NewNumericDate(now.Add(time.Hour)), IssuedAt: jwt.NewNumericDate(now), }).Claims(map[string]any{ - "email": claims.Email, - "name": claims.Name, - "groups": claims.Groups, + "email": claims.Email, + "preferred_username": claims.PreferredUsername, + "upn": claims.UPN, + "name": claims.Name, + "groups": claims.Groups, }) return builder.Serialize() } diff --git a/internal/mail/hosted/dns_verify.go b/internal/mail/hosted/dns_verify.go new file mode 100644 index 0000000..496c412 --- /dev/null +++ b/internal/mail/hosted/dns_verify.go @@ -0,0 +1,166 @@ +package hosted + +import ( + "context" + "fmt" + "net" + "sort" + "strings" +) + +// DNSCheckReport summarizes live DNS checks for a hosted mail domain. +type DNSCheckReport struct { + Domain string `json:"domain"` + TXTVerified bool `json:"txt_verified"` + TXTRecords []string `json:"txt_records,omitempty"` + TXTExpected string `json:"txt_expected,omitempty"` + MXVerified bool `json:"mx_verified"` + MXRecords []string `json:"mx_records"` + ExpectedMX []string `json:"expected_mx"` + Warnings []string `json:"warnings,omitempty"` + Errors []string `json:"errors,omitempty"` +} + +func LookupDomainMX(ctx context.Context, domain string) ([]string, error) { + domain = strings.ToLower(strings.TrimSpace(domain)) + if domain == "" { + return nil, fmt.Errorf("domain required") + } + mxRecords, err := (&net.Resolver{}).LookupMX(ctx, domain) + if err != nil { + return nil, err + } + sort.Slice(mxRecords, func(i, j int) bool { + return mxRecords[i].Pref < mxRecords[j].Pref + }) + out := make([]string, 0, len(mxRecords)) + for _, mx := range mxRecords { + host := strings.TrimSuffix(strings.ToLower(mx.Host), ".") + if host != "" { + out = append(out, host) + } + } + return out, nil +} + +func LookupDomainTXT(ctx context.Context, name string) ([]string, error) { + name = strings.ToLower(strings.TrimSpace(name)) + if name == "" { + return nil, fmt.Errorf("txt name required") + } + records, err := (&net.Resolver{}).LookupTXT(ctx, name) + if err != nil { + return nil, err + } + out := make([]string, 0, len(records)) + for _, record := range records { + record = strings.TrimSpace(record) + if record != "" { + out = append(out, record) + } + } + return out, nil +} + +func MXMatchesExpected(mxHosts, expected []string) bool { + if len(mxHosts) == 0 || len(expected) == 0 { + return false + } + for _, mx := range mxHosts { + mx = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(mx)), ".") + for _, want := range expected { + want = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(want)), ".") + if want == "" { + continue + } + if mx == want || strings.HasSuffix(mx, "."+want) { + return true + } + } + } + return false +} + +func TXTContainsToken(records []string, token string) bool { + token = strings.TrimSpace(token) + if token == "" { + return false + } + for _, record := range records { + if strings.TrimSpace(record) == token { + return true + } + } + return false +} + +func (s *Service) CheckDomainDNS(ctx context.Context, domainID string, expectedMX []string) (DomainRow, DNSCheckReport, error) { + row, err := s.GetDomain(ctx, domainID) + if err != nil { + return DomainRow{}, DNSCheckReport{}, err + } + report := DNSCheckReport{ + Domain: row.Name, + ExpectedMX: append([]string(nil), expectedMX...), + TXTExpected: strings.TrimSpace(row.VerificationToken), + } + + txtName := "_ultisuite-verify." + row.Name + txtRecords, err := LookupDomainTXT(ctx, txtName) + if err != nil { + report.Errors = append(report.Errors, "txt lookup: "+err.Error()) + } else { + report.TXTRecords = txtRecords + report.TXTVerified = TXTContainsToken(txtRecords, row.VerificationToken) + if !report.TXTVerified && row.TXTVerifiedAt != nil { + report.Warnings = append(report.Warnings, "txt record not found but domain was previously verified") + report.TXTVerified = true + } + } + + mxRecords, err := LookupDomainMX(ctx, row.Name) + if err != nil { + report.Errors = append(report.Errors, "mx lookup: "+err.Error()) + } else { + report.MXRecords = mxRecords + report.MXVerified = MXMatchesExpected(mxRecords, expectedMX) + if !report.MXVerified && row.MXVerifiedAt != nil && len(expectedMX) == 0 { + report.MXVerified = len(mxRecords) > 0 + } + } + return row, report, nil +} + +func (s *Service) VerifyDomainTXTRecord(ctx context.Context, domainID string) (DomainRow, DNSCheckReport, error) { + row, report, err := s.CheckDomainDNS(ctx, domainID, nil) + if err != nil { + return DomainRow{}, DNSCheckReport{}, err + } + if !report.TXTVerified { + return row, report, fmt.Errorf("txt verification token not found at _ultisuite-verify.%s", row.Name) + } + updated, err := s.MarkDomainVerified(ctx, domainID) + if err != nil { + return row, report, err + } + return updated, report, nil +} + +func (s *Service) VerifyDomainMXRecord(ctx context.Context, domainID string, expectedMX []string) (DomainRow, DNSCheckReport, error) { + row, report, err := s.CheckDomainDNS(ctx, domainID, expectedMX) + if err != nil { + return DomainRow{}, DNSCheckReport{}, err + } + if len(expectedMX) == 0 { + report.Warnings = append(report.Warnings, "expected mx hosts not configured") + return row, report, fmt.Errorf("expected mx hosts not configured") + } + if !report.MXVerified { + return row, report, fmt.Errorf("mx records %v do not match expected %v", report.MXRecords, expectedMX) + } + updated, err := s.MarkDomainMXVerified(ctx, domainID) + if err != nil { + return row, report, err + } + return updated, report, nil +} diff --git a/internal/mail/hosted/dns_verify_test.go b/internal/mail/hosted/dns_verify_test.go new file mode 100644 index 0000000..f140bf8 --- /dev/null +++ b/internal/mail/hosted/dns_verify_test.go @@ -0,0 +1,24 @@ +package hosted + +import "testing" + +func TestMXMatchesExpected(t *testing.T) { + if !MXMatchesExpected([]string{"mail.acme.com."}, []string{"mail.acme.com"}) { + t.Fatal("exact mx match") + } + if !MXMatchesExpected([]string{"mx1.mail.ultisuite.fr"}, []string{"mail.ultisuite.fr"}) { + t.Fatal("suffix mx match") + } + if MXMatchesExpected([]string{"aspmx.l.google.com"}, []string{"mail.ultisuite.fr"}) { + t.Fatal("google mx should not match") + } +} + +func TestTXTContainsToken(t *testing.T) { + if !TXTContainsToken([]string{"abc123", "other"}, "abc123") { + t.Fatal("expected token match") + } + if TXTContainsToken([]string{"wrong"}, "abc123") { + t.Fatal("unexpected token match") + } +} diff --git a/internal/mail/hosted/service.go b/internal/mail/hosted/service.go new file mode 100644 index 0000000..cd77623 --- /dev/null +++ b/internal/mail/hosted/service.go @@ -0,0 +1,435 @@ +package hosted + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "strings" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/ultisuite/ulti-backend/internal/mail/credentials" + "github.com/ultisuite/ulti-backend/internal/mail/stalwart" +) + +var ( + ErrAddressTaken = errors.New("mail address already taken") + ErrInvalidLocalPart = errors.New("invalid local part") + ErrDomainNotActive = errors.New("mail domain not active") +) + +type Service struct { + db *pgxpool.Pool + stlw *stalwart.Client + creds *credentials.Manager + imapHost string + imapPort int + imapTLS bool + smtpHost string + smtpPort int + smtpTLS bool +} + +func NewService(db *pgxpool.Pool, stlw *stalwart.Client, creds *credentials.Manager) *Service { + s := &Service{db: db, stlw: stlw, creds: creds} + if stlw != nil { + s.imapHost, s.imapPort, s.imapTLS = stlw.IMAPEndpoint() + s.smtpHost, s.smtpPort, s.smtpTLS = stlw.SMTPEndpoint() + } + return s +} + +type DomainRow struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + VerificationToken string `json:"verification_token,omitempty"` + DKIMSelector string `json:"dkim_selector,omitempty"` + DKIMPublicKey string `json:"dkim_public_key,omitempty"` + StalwartDomainID string `json:"stalwart_domain_id,omitempty"` + IsPlatformDomain bool `json:"is_platform_domain"` + MXVerifiedAt *string `json:"mx_verified_at,omitempty"` + TXTVerifiedAt *string `json:"txt_verified_at,omitempty"` + CreatedAt string `json:"created_at"` +} + +type MailboxRow struct { + ID string `json:"id"` + DomainID string `json:"domain_id"` + LocalPart string `json:"local_part"` + Email string `json:"email"` + UserID string `json:"user_id,omitempty"` + MailAccountID string `json:"mail_account_id,omitempty"` + StalwartAccountID string `json:"stalwart_account_id,omitempty"` + QuotaBytes int64 `json:"quota_bytes"` + Status string `json:"status"` +} + +func normalizeLocalPart(v string) (string, error) { + v = strings.ToLower(strings.TrimSpace(v)) + if v == "" || len(v) > 64 { + return "", ErrInvalidLocalPart + } + for _, ch := range v { + if (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '.' || ch == '-' || ch == '_' || ch == '+' { + continue + } + return "", ErrInvalidLocalPart + } + return v, nil +} + +func (s *Service) IsAddressAvailable(ctx context.Context, domainName, localPart string) (bool, error) { + localPart, err := normalizeLocalPart(localPart) + if err != nil { + return false, err + } + domainName = strings.ToLower(strings.TrimSpace(domainName)) + var exists bool + err = s.db.QueryRow(ctx, ` + SELECT EXISTS( + SELECT 1 FROM mailboxes m + JOIN mail_domains d ON d.id = m.domain_id + WHERE d.name = $1 AND m.local_part = $2 + ) + `, domainName, localPart).Scan(&exists) + return !exists, err +} + +func (s *Service) EnsurePlatformDomain(ctx context.Context, name string) (DomainRow, error) { + name = strings.ToLower(strings.TrimSpace(name)) + if name == "" { + return DomainRow{}, fmt.Errorf("platform domain name required") + } + var row DomainRow + err := s.db.QueryRow(ctx, ` + SELECT id::text, name, status, verification_token, dkim_selector, dkim_public_key, + stalwart_domain_id, is_platform_domain, + mx_verified_at::text, txt_verified_at::text, created_at::text + FROM mail_domains WHERE name = $1 + `, name).Scan( + &row.ID, &row.Name, &row.Status, &row.VerificationToken, &row.DKIMSelector, &row.DKIMPublicKey, + &row.StalwartDomainID, &row.IsPlatformDomain, &row.MXVerifiedAt, &row.TXTVerifiedAt, &row.CreatedAt, + ) + if err == nil { + return row, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return DomainRow{}, err + } + token, err := randomToken(16) + if err != nil { + return DomainRow{}, err + } + stlwID := "" + if s.stlw != nil && s.stlw.Enabled() { + d, err := s.stlw.CreateDomain(ctx, name) + if err != nil { + return DomainRow{}, fmt.Errorf("stalwart create domain: %w", err) + } + stlwID = d.ID + } + status := "active" + if !strings.HasSuffix(name, ".local") { + status = "pending_verification" + } + err = s.db.QueryRow(ctx, ` + INSERT INTO mail_domains (name, status, verification_token, stalwart_domain_id, is_platform_domain, txt_verified_at) + VALUES ($1, $2, $3, $4, true, CASE WHEN $2 = 'active' THEN NOW() ELSE NULL END) + RETURNING id::text, name, status, verification_token, dkim_selector, dkim_public_key, + stalwart_domain_id, is_platform_domain, + mx_verified_at::text, txt_verified_at::text, created_at::text + `, name, status, token, stlwID).Scan( + &row.ID, &row.Name, &row.Status, &row.VerificationToken, &row.DKIMSelector, &row.DKIMPublicKey, + &row.StalwartDomainID, &row.IsPlatformDomain, &row.MXVerifiedAt, &row.TXTVerifiedAt, &row.CreatedAt, + ) + return row, err +} + +func (s *Service) CreateDomain(ctx context.Context, name string, platform bool) (DomainRow, error) { + name = strings.ToLower(strings.TrimSpace(name)) + if name == "" { + return DomainRow{}, fmt.Errorf("domain name required") + } + token, err := randomToken(16) + if err != nil { + return DomainRow{}, err + } + stlwID := "" + if s.stlw != nil && s.stlw.Enabled() { + d, err := s.stlw.CreateDomain(ctx, name) + if err != nil { + return DomainRow{}, fmt.Errorf("stalwart create domain: %w", err) + } + stlwID = d.ID + } + var row DomainRow + err = s.db.QueryRow(ctx, ` + INSERT INTO mail_domains (name, status, verification_token, stalwart_domain_id, is_platform_domain) + VALUES ($1, 'pending_verification', $2, $3, $4) + RETURNING id::text, name, status, verification_token, dkim_selector, dkim_public_key, + stalwart_domain_id, is_platform_domain, + mx_verified_at::text, txt_verified_at::text, created_at::text + `, name, token, stlwID, platform).Scan( + &row.ID, &row.Name, &row.Status, &row.VerificationToken, &row.DKIMSelector, &row.DKIMPublicKey, + &row.StalwartDomainID, &row.IsPlatformDomain, &row.MXVerifiedAt, &row.TXTVerifiedAt, &row.CreatedAt, + ) + return row, err +} + +func (s *Service) ListDomains(ctx context.Context) ([]DomainRow, error) { + rows, err := s.db.Query(ctx, ` + SELECT id::text, name, status, verification_token, dkim_selector, dkim_public_key, + stalwart_domain_id, is_platform_domain, + mx_verified_at::text, txt_verified_at::text, created_at::text + FROM mail_domains ORDER BY is_platform_domain DESC, name ASC + `) + if err != nil { + return nil, err + } + defer rows.Close() + var out []DomainRow + for rows.Next() { + var row DomainRow + if err := rows.Scan( + &row.ID, &row.Name, &row.Status, &row.VerificationToken, &row.DKIMSelector, &row.DKIMPublicKey, + &row.StalwartDomainID, &row.IsPlatformDomain, &row.MXVerifiedAt, &row.TXTVerifiedAt, &row.CreatedAt, + ); err != nil { + return nil, err + } + out = append(out, row) + } + return out, rows.Err() +} + +func (s *Service) GetDomain(ctx context.Context, domainID string) (DomainRow, error) { + var row DomainRow + err := s.db.QueryRow(ctx, ` + SELECT id::text, name, status, verification_token, dkim_selector, dkim_public_key, + stalwart_domain_id, is_platform_domain, + mx_verified_at::text, txt_verified_at::text, created_at::text + FROM mail_domains WHERE id = $1 + `, domainID).Scan( + &row.ID, &row.Name, &row.Status, &row.VerificationToken, &row.DKIMSelector, &row.DKIMPublicKey, + &row.StalwartDomainID, &row.IsPlatformDomain, &row.MXVerifiedAt, &row.TXTVerifiedAt, &row.CreatedAt, + ) + return row, err +} + +func (s *Service) MarkDomainVerified(ctx context.Context, domainID string) (DomainRow, error) { + var row DomainRow + err := s.db.QueryRow(ctx, ` + UPDATE mail_domains + SET status = 'active', txt_verified_at = COALESCE(txt_verified_at, NOW()), updated_at = NOW() + WHERE id = $1 + RETURNING id::text, name, status, verification_token, dkim_selector, dkim_public_key, + stalwart_domain_id, is_platform_domain, + mx_verified_at::text, txt_verified_at::text, created_at::text + `, domainID).Scan( + &row.ID, &row.Name, &row.Status, &row.VerificationToken, &row.DKIMSelector, &row.DKIMPublicKey, + &row.StalwartDomainID, &row.IsPlatformDomain, &row.MXVerifiedAt, &row.TXTVerifiedAt, &row.CreatedAt, + ) + return row, err +} + +func (s *Service) MarkDomainMXVerified(ctx context.Context, domainID string) (DomainRow, error) { + var row DomainRow + err := s.db.QueryRow(ctx, ` + UPDATE mail_domains + SET mx_verified_at = NOW(), status = 'active', updated_at = NOW() + WHERE id = $1 + RETURNING id::text, name, status, verification_token, dkim_selector, dkim_public_key, + stalwart_domain_id, is_platform_domain, + mx_verified_at::text, txt_verified_at::text, created_at::text + `, domainID).Scan( + &row.ID, &row.Name, &row.Status, &row.VerificationToken, &row.DKIMSelector, &row.DKIMPublicKey, + &row.StalwartDomainID, &row.IsPlatformDomain, &row.MXVerifiedAt, &row.TXTVerifiedAt, &row.CreatedAt, + ) + return row, err +} + +type ProvisionMailboxInput struct { + UserID string + Email string + DisplayName string + Password string + QuotaBytes int64 + DomainID string +} + +type ProvisionMailboxResult struct { + Mailbox MailboxRow + MailAccountID string +} + +func (s *Service) ProvisionMailbox(ctx context.Context, in ProvisionMailboxInput) (ProvisionMailboxResult, error) { + email := strings.ToLower(strings.TrimSpace(in.Email)) + at := strings.LastIndex(email, "@") + if at <= 0 { + return ProvisionMailboxResult{}, fmt.Errorf("invalid email") + } + localPart := email[:at] + domainName := email[at+1:] + localPart, err := normalizeLocalPart(localPart) + if err != nil { + return ProvisionMailboxResult{}, err + } + + var domain DomainRow + if strings.TrimSpace(in.DomainID) != "" { + domain, err = s.GetDomain(ctx, in.DomainID) + if err != nil { + return ProvisionMailboxResult{}, err + } + if !strings.EqualFold(domain.Name, domainName) { + return ProvisionMailboxResult{}, fmt.Errorf("email domain %q does not match project domain %q", domainName, domain.Name) + } + } else { + err = s.db.QueryRow(ctx, ` + SELECT id::text, name, status, verification_token, dkim_selector, dkim_public_key, + stalwart_domain_id, is_platform_domain, + mx_verified_at::text, txt_verified_at::text, created_at::text + FROM mail_domains WHERE name = $1 + `, domainName).Scan( + &domain.ID, &domain.Name, &domain.Status, &domain.VerificationToken, &domain.DKIMSelector, &domain.DKIMPublicKey, + &domain.StalwartDomainID, &domain.IsPlatformDomain, &domain.MXVerifiedAt, &domain.TXTVerifiedAt, &domain.CreatedAt, + ) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + domain, err = s.EnsurePlatformDomain(ctx, domainName) + if err != nil { + return ProvisionMailboxResult{}, err + } + } else { + return ProvisionMailboxResult{}, err + } + } + } + if domain.Status != "active" && !domain.IsPlatformDomain { + return ProvisionMailboxResult{}, ErrDomainNotActive + } + + available, err := s.IsAddressAvailable(ctx, domainName, localPart) + if err != nil { + return ProvisionMailboxResult{}, err + } + if !available { + return ProvisionMailboxResult{}, ErrAddressTaken + } + + quota := in.QuotaBytes + if quota <= 0 { + quota = 5 * 1024 * 1024 * 1024 + } + + stlwAccountID := "" + if s.stlw != nil { + acct, err := s.stlw.CreateAccount(ctx, domain.StalwartDomainID, localPart, in.Password, quota) + if err != nil && !errors.Is(err, stalwart.ErrDisabled) { + return ProvisionMailboxResult{}, fmt.Errorf("stalwart account: %w", err) + } + stlwAccountID = acct.ID + } + + tx, err := s.db.Begin(ctx) + if err != nil { + return ProvisionMailboxResult{}, err + } + defer tx.Rollback(ctx) + + var mailboxID string + err = tx.QueryRow(ctx, ` + INSERT INTO mailboxes (domain_id, local_part, user_id, stalwart_account_id, quota_bytes, status) + VALUES ($1, $2, NULLIF($3, '')::uuid, $4, $5, 'active') + RETURNING id::text + `, domain.ID, localPart, in.UserID, stlwAccountID, quota).Scan(&mailboxID) + if err != nil { + return ProvisionMailboxResult{}, err + } + + var mailAccountID string + if in.UserID != "" { + enc, err := s.encryptHostedCredential(email, in.Password) + if err != nil { + return ProvisionMailboxResult{}, err + } + err = tx.QueryRow(ctx, ` + INSERT INTO mail_accounts ( + user_id, name, email, provider, + imap_host, imap_port, imap_tls, + smtp_host, smtp_port, smtp_tls, + credentials, is_active + ) + VALUES ($1, $2, $3, 'hosted', $4, $5, $6, $7, $8, $9, $10, true) + RETURNING id::text + `, in.UserID, in.DisplayName, email, + s.imapHost, s.imapPort, s.imapTLS, + s.smtpHost, s.smtpPort, s.smtpTLS, + enc, + ).Scan(&mailAccountID) + if err != nil { + return ProvisionMailboxResult{}, err + } + _, err = tx.Exec(ctx, ` + UPDATE mailboxes SET user_id = $1::uuid, mail_account_id = $2::uuid, updated_at = NOW() + WHERE id = $3::uuid + `, in.UserID, mailAccountID, mailboxID) + if err != nil { + return ProvisionMailboxResult{}, err + } + } + + if err := tx.Commit(ctx); err != nil { + return ProvisionMailboxResult{}, err + } + + return ProvisionMailboxResult{ + Mailbox: MailboxRow{ + ID: mailboxID, + DomainID: domain.ID, + LocalPart: localPart, + Email: email, + UserID: in.UserID, + MailAccountID: mailAccountID, + StalwartAccountID: stlwAccountID, + QuotaBytes: quota, + Status: "active", + }, + MailAccountID: mailAccountID, + }, nil +} + +func (s *Service) encryptHostedCredential(email, password string) ([]byte, error) { + if s.creds == nil { + return nil, fmt.Errorf("credential manager not configured") + } + return s.creds.EncryptCredential(credentials.Credential{ + AuthType: credentials.AuthPassword, + Username: email, + Password: password, + }) +} + +func (s *Service) LinkMailboxToUser(ctx context.Context, mailboxID, userID string) error { + _, err := s.db.Exec(ctx, ` + UPDATE mailboxes SET user_id = $1::uuid, updated_at = NOW() + WHERE id = $2::uuid AND (user_id IS NULL OR user_id = $1::uuid) + `, userID, mailboxID) + return err +} + +func randomToken(n int) (string, error) { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +func NewInviteToken() (string, error) { + return uuid.NewString(), nil +} diff --git a/internal/mail/stalwart/client.go b/internal/mail/stalwart/client.go new file mode 100644 index 0000000..95f213e --- /dev/null +++ b/internal/mail/stalwart/client.go @@ -0,0 +1,233 @@ +package stalwart + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +var ( + ErrDisabled = errors.New("stalwart client disabled") + ErrNotFound = errors.New("stalwart resource not found") +) + +type Config struct { + Enabled bool + BaseURL string + APIKey string + IMAPHost string + IMAPPort int + IMAPTLS bool + SMTPHost string + SMTPPort int + SMTPTLS bool + HTTPClient *http.Client +} + +type Client struct { + cfg Config +} + +func NewClient(cfg Config) *Client { + if cfg.HTTPClient == nil { + cfg.HTTPClient = &http.Client{Timeout: 30 * time.Second} + } + if cfg.IMAPPort == 0 { + cfg.IMAPPort = 993 + } + if cfg.SMTPPort == 0 { + cfg.SMTPPort = 587 + } + return &Client{cfg: cfg} +} + +func (c *Client) Enabled() bool { + return c != nil && c.cfg.Enabled && strings.TrimSpace(c.cfg.BaseURL) != "" +} + +func (c *Client) IMAPEndpoint() (host string, port int, tls bool) { + return c.cfg.IMAPHost, c.cfg.IMAPPort, c.cfg.IMAPTLS +} + +func (c *Client) SMTPEndpoint() (host string, port int, tls bool) { + return c.cfg.SMTPHost, c.cfg.SMTPPort, c.cfg.SMTPTLS +} + +type Domain struct { + ID string + Name string +} + +type Account struct { + ID string + Email string +} + +type jmapRequest struct { + Using []string `json:"using"` + MethodCalls []any `json:"methodCalls"` +} + +type jmapResponse struct { + MethodResponses []json.RawMessage `json:"methodResponses"` +} + +func (c *Client) call(ctx context.Context, method string, args any, callID string) (json.RawMessage, error) { + if !c.Enabled() { + return nil, ErrDisabled + } + body, err := json.Marshal(jmapRequest{ + Using: []string{"urn:ietf:params:jmap:core", "urn:stalwart:jmap"}, + MethodCalls: []any{[]any{method, args, callID}}, + }) + if err != nil { + return nil, err + } + url := strings.TrimRight(c.cfg.BaseURL, "/") + "/api" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + if c.cfg.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey) + } + resp, err := c.cfg.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("stalwart api %s: %s", resp.Status, strings.TrimSpace(string(raw))) + } + var envelope jmapResponse + if err := json.Unmarshal(raw, &envelope); err != nil { + return nil, err + } + if len(envelope.MethodResponses) == 0 { + return nil, fmt.Errorf("stalwart: empty method response") + } + var parts []json.RawMessage + if err := json.Unmarshal(envelope.MethodResponses[0], &parts); err != nil { + return nil, err + } + if len(parts) < 2 { + return nil, fmt.Errorf("stalwart: malformed method response") + } + var status string + if err := json.Unmarshal(parts[0], &status); err != nil { + return nil, err + } + if status == "error" { + return nil, fmt.Errorf("stalwart error: %s", string(parts[1])) + } + return parts[1], nil +} + +func (c *Client) CreateDomain(ctx context.Context, name string) (Domain, error) { + name = strings.ToLower(strings.TrimSpace(name)) + if name == "" { + return Domain{}, fmt.Errorf("domain name required") + } + if !c.Enabled() { + return Domain{ID: "local-" + name, Name: name}, nil + } + raw, err := c.call(ctx, "x:Domain/set", map[string]any{ + "create": map[string]any{ + "d1": map[string]any{"name": name}, + }, + }, "c1") + if err != nil { + return Domain{}, err + } + var parsed struct { + Created map[string]struct { + ID string `json:"id"` + } `json:"created"` + } + if err := json.Unmarshal(raw, &parsed); err != nil { + return Domain{}, err + } + for _, v := range parsed.Created { + return Domain{ID: v.ID, Name: name}, nil + } + return Domain{}, fmt.Errorf("stalwart: domain not created") +} + +func (c *Client) CreateAccount(ctx context.Context, domainID, localPart, password string, quotaBytes int64) (Account, error) { + localPart = strings.ToLower(strings.TrimSpace(localPart)) + if localPart == "" { + return Account{}, fmt.Errorf("local part required") + } + if !c.Enabled() { + return Account{ID: "local-" + localPart, Email: localPart + "@local"}, nil + } + fields := map[string]any{ + "name": localPart, + "domainId": domainID, + } + if password != "" { + fields["credentials"] = map[string]any{"password": password} + } + if quotaBytes > 0 { + fields["quota"] = map[string]any{"maxDiskQuota": quotaBytes} + } + raw, err := c.call(ctx, "x:Account/set", map[string]any{ + "create": map[string]any{"a1": fields}, + }, "c1") + if err != nil { + return Account{}, err + } + var parsed struct { + Created map[string]struct { + ID string `json:"id"` + } `json:"created"` + } + if err := json.Unmarshal(raw, &parsed); err != nil { + return Account{}, err + } + for _, v := range parsed.Created { + return Account{ID: v.ID, Email: localPart}, nil + } + return Account{}, fmt.Errorf("stalwart: account not created") +} + +func (c *Client) SetAccountPassword(ctx context.Context, accountID, password string) error { + if password == "" { + return fmt.Errorf("password required") + } + if !c.Enabled() { + return nil + } + _, err := c.call(ctx, "x:Account/set", map[string]any{ + "update": map[string]any{ + accountID: map[string]any{ + "credentials": map[string]any{"password": password}, + }, + }, + }, "c1") + return err +} + +func (c *Client) DeleteAccount(ctx context.Context, accountID string) error { + if accountID == "" { + return nil + } + if !c.Enabled() { + return nil + } + _, err := c.call(ctx, "x:Account/set", map[string]any{ + "destroy": []string{accountID}, + }, "c1") + return err +} diff --git a/internal/mail/stalwart/client_test.go b/internal/mail/stalwart/client_test.go new file mode 100644 index 0000000..5bc333b --- /dev/null +++ b/internal/mail/stalwart/client_test.go @@ -0,0 +1,28 @@ +package stalwart + +import ( + "context" + "testing" +) + +func TestClientDisabledCreateDomain(t *testing.T) { + c := NewClient(Config{Enabled: false}) + d, err := c.CreateDomain(context.Background(), "example.com") + if err != nil { + t.Fatal(err) + } + if d.Name != "example.com" { + t.Fatalf("expected example.com, got %q", d.Name) + } +} + +func TestClientDisabledCreateAccount(t *testing.T) { + c := NewClient(Config{Enabled: false}) + a, err := c.CreateAccount(context.Background(), "local-example.com", "alice", "secret", 1024) + if err != nil { + t.Fatal(err) + } + if a.ID == "" { + t.Fatal("expected local account id") + } +} diff --git a/internal/migration/admin_jobs.go b/internal/migration/admin_jobs.go new file mode 100644 index 0000000..1e82dfc --- /dev/null +++ b/internal/migration/admin_jobs.go @@ -0,0 +1,161 @@ +package migration + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" +) + +// AdminJob is a migration job enriched with the invited user email for admin dashboards. +type AdminJob struct { + Job + UserEmail string `json:"user_email"` +} + +func (s *Service) ListProjectJobs(ctx context.Context, projectID string) ([]AdminJob, error) { + rows, err := s.db.Query(ctx, ` + SELECT j.id::text, j.project_id::text, j.user_id::text, j.service, j.status, + j.cursor_json, j.stats_json, j.error, j.started_at::text, j.completed_at::text, + COALESCE(i.email, '') + FROM migration_jobs j + LEFT JOIN migration_invites i + ON i.project_id = j.project_id AND i.user_id = j.user_id + WHERE j.project_id = $1::uuid + ORDER BY COALESCE(i.email, ''), j.service ASC + `, projectID) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []AdminJob + for rows.Next() { + var row AdminJob + var cursorRaw, statsRaw []byte + if err := rows.Scan( + &row.ID, &row.ProjectID, &row.UserID, &row.Service, &row.Status, + &cursorRaw, &statsRaw, &row.Error, &row.StartedAt, &row.CompletedAt, + &row.UserEmail, + ); err != nil { + return nil, err + } + _ = json.Unmarshal(cursorRaw, &row.CursorJSON) + _ = json.Unmarshal(statsRaw, &row.StatsJSON) + if row.CursorJSON == nil { + row.CursorJSON = map[string]any{} + } + if row.StatsJSON == nil { + row.StatsJSON = map[string]any{} + } + out = append(out, row) + } + return out, rows.Err() +} + +func (s *Service) RetryJob(ctx context.Context, projectID, jobID string) (Job, error) { + var row Job + var cursorRaw, statsRaw []byte + err := s.db.QueryRow(ctx, ` + UPDATE migration_jobs + SET status = 'pending', error = '', updated_at = NOW() + WHERE id = $1::uuid AND project_id = $2::uuid AND status = 'failed' + RETURNING id::text, project_id::text, user_id::text, service, status, + cursor_json, stats_json, error, started_at::text, completed_at::text + `, jobID, projectID).Scan( + &row.ID, &row.ProjectID, &row.UserID, &row.Service, &row.Status, + &cursorRaw, &statsRaw, &row.Error, &row.StartedAt, &row.CompletedAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return Job{}, fmt.Errorf("job not found or not retryable") + } + if err != nil { + return Job{}, err + } + _ = json.Unmarshal(cursorRaw, &row.CursorJSON) + _ = json.Unmarshal(statsRaw, &row.StatsJSON) + if row.CursorJSON == nil { + row.CursorJSON = map[string]any{} + } + if row.StatsJSON == nil { + row.StatsJSON = map[string]any{} + } + return row, nil +} + +func (s *Service) RetryFailedJobs(ctx context.Context, projectID string) (int64, error) { + tag, err := s.db.Exec(ctx, ` + UPDATE migration_jobs + SET status = 'pending', error = '', updated_at = NOW() + WHERE project_id = $1::uuid AND status = 'failed' + `, projectID) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} + +func (s *Service) ResetJobCursor(ctx context.Context, projectID, jobID string) (Job, error) { + tx, err := s.db.Begin(ctx) + if err != nil { + return Job{}, err + } + defer tx.Rollback(ctx) + + var status string + err = tx.QueryRow(ctx, ` + SELECT status FROM migration_jobs + WHERE id = $1::uuid AND project_id = $2::uuid + `, jobID, projectID).Scan(&status) + if errors.Is(err, pgx.ErrNoRows) { + return Job{}, fmt.Errorf("job not found") + } + if err != nil { + return Job{}, err + } + if status == "running" { + return Job{}, fmt.Errorf("job running; wait for completion before reset") + } + + if _, err := tx.Exec(ctx, ` + DELETE FROM migration_imported_items WHERE job_id = $1::uuid + `, jobID); err != nil { + return Job{}, err + } + + var row Job + var cursorRaw, statsRaw []byte + err = tx.QueryRow(ctx, ` + UPDATE migration_jobs + SET status = 'pending', + cursor_json = '{}'::jsonb, + stats_json = '{}'::jsonb, + error = '', + started_at = NULL, + completed_at = NULL, + updated_at = NOW() + WHERE id = $1::uuid AND project_id = $2::uuid + RETURNING id::text, project_id::text, user_id::text, service, status, + cursor_json, stats_json, error, started_at::text, completed_at::text + `, jobID, projectID).Scan( + &row.ID, &row.ProjectID, &row.UserID, &row.Service, &row.Status, + &cursorRaw, &statsRaw, &row.Error, &row.StartedAt, &row.CompletedAt, + ) + if err != nil { + return Job{}, err + } + if err := tx.Commit(ctx); err != nil { + return Job{}, err + } + _ = json.Unmarshal(cursorRaw, &row.CursorJSON) + _ = json.Unmarshal(statsRaw, &row.StatsJSON) + if row.CursorJSON == nil { + row.CursorJSON = map[string]any{} + } + if row.StatsJSON == nil { + row.StatsJSON = map[string]any{} + } + return row, nil +} diff --git a/internal/migration/auth_modes.go b/internal/migration/auth_modes.go new file mode 100644 index 0000000..adf5914 --- /dev/null +++ b/internal/migration/auth_modes.go @@ -0,0 +1,36 @@ +package migration + +import "strings" + +const AuthModeOAuth = "oauth" +const AuthModeGoogleDWD = "google_dwd" +const AuthModeMicrosoftApp = "microsoft_app" + +func NormalizeAuthMode(provider, authMode string) string { + authMode = strings.ToLower(strings.TrimSpace(authMode)) + if authMode == "" { + return AuthModeOAuth + } + provider = strings.ToLower(strings.TrimSpace(provider)) + switch authMode { + case AuthModeGoogleDWD: + if provider == "google" { + return AuthModeGoogleDWD + } + case AuthModeMicrosoftApp: + if provider == "microsoft" { + return AuthModeMicrosoftApp + } + } + return AuthModeOAuth +} + +func UsesUserOAuth(provider, authMode string) bool { + authMode = NormalizeAuthMode(provider, authMode) + switch authMode { + case AuthModeGoogleDWD, AuthModeMicrosoftApp: + return false + default: + return true + } +} diff --git a/internal/migration/batch_config.go b/internal/migration/batch_config.go new file mode 100644 index 0000000..4a79ff3 --- /dev/null +++ b/internal/migration/batch_config.go @@ -0,0 +1,46 @@ +package migration + +import "sync" + +const ( + defaultMailImportBatchSize = 25 + defaultDriveImportBatchSize = 10 +) + +// ImportBatchConfig controls how many items each migration importer processes per worker tick. +type ImportBatchConfig struct { + Mail int + Drive int +} + +var ( + importBatchMu sync.RWMutex + importBatchConfig = ImportBatchConfig{ + Mail: defaultMailImportBatchSize, + Drive: defaultDriveImportBatchSize, + } +) + +// ConfigureImportBatch sets package-wide batch sizes for migration importers. +func ConfigureImportBatch(cfg ImportBatchConfig) { + importBatchMu.Lock() + defer importBatchMu.Unlock() + if cfg.Mail > 0 { + importBatchConfig.Mail = cfg.Mail + } + if cfg.Drive > 0 { + importBatchConfig.Drive = cfg.Drive + } +} + +func mailImportBatchSize() int { + importBatchMu.RLock() + defer importBatchMu.RUnlock() + return importBatchConfig.Mail +} + +func driveImportBatchSize() int { + importBatchMu.RLock() + defer importBatchMu.RUnlock() + return importBatchConfig.Drive +} diff --git a/internal/migration/batch_config_test.go b/internal/migration/batch_config_test.go new file mode 100644 index 0000000..7a627f8 --- /dev/null +++ b/internal/migration/batch_config_test.go @@ -0,0 +1,25 @@ +package migration + +import "testing" + +func TestConfigureImportBatch(t *testing.T) { + t.Cleanup(func() { + ConfigureImportBatch(ImportBatchConfig{ + Mail: defaultMailImportBatchSize, + Drive: defaultDriveImportBatchSize, + }) + }) + + ConfigureImportBatch(ImportBatchConfig{Mail: 7, Drive: 3}) + if got := mailImportBatchSize(); got != 7 { + t.Fatalf("mail batch = %d", got) + } + if got := driveImportBatchSize(); got != 3 { + t.Fatalf("drive batch = %d", got) + } + + ConfigureImportBatch(ImportBatchConfig{Mail: 0, Drive: 0}) + if got := mailImportBatchSize(); got != 7 { + t.Fatalf("mail batch unchanged = %d", got) + } +} diff --git a/internal/migration/calendar_delta_test.go b/internal/migration/calendar_delta_test.go new file mode 100644 index 0000000..cc76b2c --- /dev/null +++ b/internal/migration/calendar_delta_test.go @@ -0,0 +1,70 @@ +package migration + +import ( + "context" + "net/http" + "strings" + "testing" +) + +func TestListGoogleCalendarCancelledEvent(t *testing.T) { + client := mockGoogleHTTPClient(t, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/calendar/v3/calendars/") { + _, _ = w.Write([]byte(`{ + "items":[{"id":"e1","status":"cancelled","summary":"gone"}], + "nextSyncToken":"sync-1" + }`)) + return + } + http.NotFound(w, r) + }) + + c := NewCalendarImporter(nil, nil).WithHTTPClient(client) + events, _, syncToken, err := c.listSourceEvents( + context.Background(), + "token", + "google", + sourceCalendar{ID: "primary"}, + "", + "sync-old", + ) + if err != nil { + t.Fatalf("list events: %v", err) + } + if len(events) != 1 || !events[0].Deleted { + t.Fatalf("events: %#v", events) + } + if syncToken != "sync-1" { + t.Fatalf("sync token = %q", syncToken) + } +} + +func TestListMicrosoftCalendarDeltaRemoved(t *testing.T) { + client := mockGraphHTTPClient(t, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/events/delta") { + _, _ = w.Write([]byte(`{ + "value":[{"id":"e1","@removed":{"reason":"deleted"}}], + "@odata.deltaLink":"https://graph.microsoft.com/delta/next" + }`)) + return + } + http.NotFound(w, r) + }) + + c := NewCalendarImporter(nil, nil).WithHTTPClient(client) + events, next, err := c.listMicrosoftCalendarDelta( + context.Background(), + "token", + "cal-1", + "https://graph.microsoft.com/v1.0/me/calendars/cal-1/events/delta", + ) + if err != nil { + t.Fatalf("list delta: %v", err) + } + if len(events) != 1 || !events[0].Deleted { + t.Fatalf("events: %#v", events) + } + if next == "" { + t.Fatal("expected next cursor") + } +} diff --git a/internal/migration/calendar_import.go b/internal/migration/calendar_import.go new file mode 100644 index 0000000..efeac08 --- /dev/null +++ b/internal/migration/calendar_import.go @@ -0,0 +1,542 @@ +package migration + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/ultisuite/ulti-backend/internal/nextcloud" +) + +const migrationCalendarID = "migration-import" + +type CalendarImporter struct { + db *pgxpool.Pool + nc *nextcloud.Client + client *http.Client + userUPN string +} + +func NewCalendarImporter(db *pgxpool.Pool, nc *nextcloud.Client) *CalendarImporter { + return &CalendarImporter{db: db, nc: nc, client: migrationHTTPClient()} +} + +func (c *CalendarImporter) WithUserPrincipal(upn string) *CalendarImporter { + c.userUPN = strings.TrimSpace(upn) + return c +} + +func (c *CalendarImporter) WithHTTPClient(client *http.Client) *CalendarImporter { + if client != nil { + c.client = client + } + return c +} + +func (c *CalendarImporter) ImportBatch(ctx context.Context, job *Job, accessToken, provider string, delta bool, update progressUpdater) error { + if c.nc == nil { + return fmt.Errorf("nextcloud required for calendar migration") + } + user, err := resolveMigrationUser(ctx, c.db, job.UserID) + if err != nil { + return err + } + ncUserID := nextcloud.UserIDFromClaims(user.Email, user.ExternalID) + if _, err := c.nc.EnsurePrincipal(ctx, user.Email, user.ExternalID, user.Name); err != nil { + return fmt.Errorf("nextcloud user: %w", err) + } + calPath, err := c.ensureMigrationCalendar(ctx, ncUserID) + if err != nil { + return err + } + items, err := LoadImportedItemStore(ctx, c.db, job.ID, job.CursorJSON) + if err != nil { + return err + } + + if delta && c.hasDeltaCursor(job, provider) { + return c.importDelta(ctx, job, accessToken, provider, ncUserID, calPath, items, update) + } + + return c.importFull(ctx, job, accessToken, provider, ncUserID, calPath, delta, items, update) +} + +func (c *CalendarImporter) hasDeltaCursor(job *Job, provider string) bool { + if provider == "google" { + return len(calendarSyncTokens(job.CursorJSON)) > 0 + } + return len(calendarDeltaLinks(job.CursorJSON)) > 0 +} + +func (c *CalendarImporter) importFull(ctx context.Context, job *Job, accessToken, provider, ncUserID, calPath string, captureDelta bool, items *ImportedItemStore, update progressUpdater) error { + imported, _ := job.StatsJSON["imported"].(float64) + batch := 0 + + calIndex := int(jsonNumber(job.CursorJSON["calendarIndex"])) + sourceCalendars, err := c.listSourceCalendars(ctx, accessToken, provider) + if err != nil { + return err + } + if len(sourceCalendars) == 0 { + job.StatsJSON["imported"] = imported + job.StatsJSON["phase"] = "imported" + return update("completed", job.CursorJSON, job.StatsJSON, "") + } + if calIndex >= len(sourceCalendars) { + job.StatsJSON["imported"] = imported + job.StatsJSON["phase"] = "imported" + return update("completed", job.CursorJSON, job.StatsJSON, "") + } + + sourceCal := sourceCalendars[calIndex] + pageToken, _ := job.CursorJSON["pageToken"].(string) + events, nextToken, syncToken, err := c.listSourceEvents(ctx, accessToken, provider, sourceCal, pageToken, "") + if err != nil { + return err + } + + listIndex := int(jsonNumber(job.CursorJSON["listIndex"])) + for i := listIndex; i < len(events) && batch < mailImportBatchSize(); i++ { + ev := events[i] + if alreadyImported(items, ev.SourceID) { + continue + } + if err := c.nc.CreateEvent(ctx, ncUserID, calPath, ev.ToNextcloudEvent(provider)); err != nil { + if markErr := items.MarkFailed(ctx, ev.SourceID, err.Error(), ""); markErr != nil { + return markErr + } + incJobStat(job.StatsJSON, "failed") + batch++ + continue + } + if err := items.MarkImported(ctx, ev.SourceID); err != nil { + return err + } + imported++ + batch++ + } + job.StatsJSON["imported"] = imported + + if listIndex+batch < len(events) { + job.CursorJSON["listIndex"] = float64(listIndex + batch) + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "listIndex") + + if nextToken != "" { + job.CursorJSON["pageToken"] = nextToken + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "pageToken") + + if captureDelta { + if provider == "google" && syncToken != "" { + setCalendarSyncToken(job.CursorJSON, sourceCal.ID, syncToken) + } + if provider != "google" { + if link, err := c.bootstrapCalendarDelta(ctx, accessToken, sourceCal.ID); err == nil && link != "" { + setCalendarDeltaLink(job.CursorJSON, sourceCal.ID, link) + } + } + } + + job.CursorJSON["calendarIndex"] = float64(calIndex + 1) + return update("pending", job.CursorJSON, job.StatsJSON, "") +} + +func (c *CalendarImporter) importDelta(ctx context.Context, job *Job, accessToken, provider, ncUserID, calPath string, items *ImportedItemStore, update progressUpdater) error { + calIndex := int(jsonNumber(job.CursorJSON["calendarIndex"])) + sourceCalendars, err := c.listSourceCalendars(ctx, accessToken, provider) + if err != nil { + return err + } + if calIndex >= len(sourceCalendars) { + job.StatsJSON["phase"] = "delta" + return update("completed", job.CursorJSON, job.StatsJSON, "") + } + + sourceCal := sourceCalendars[calIndex] + imported, _ := job.StatsJSON["delta_imported"].(float64) + updated, _ := job.StatsJSON["delta_updated"].(float64) + deleted, _ := job.StatsJSON["delta_deleted"].(float64) + + var events []sourceEvent + var nextCursor string + if provider == "google" { + syncToken := calendarSyncTokens(job.CursorJSON)[sourceCal.ID] + pageToken, _ := job.CursorJSON["pageToken"].(string) + var syncTokenOut string + events, nextCursor, syncTokenOut, err = c.listSourceEvents(ctx, accessToken, provider, sourceCal, pageToken, syncToken) + if syncTokenOut != "" { + setCalendarSyncToken(job.CursorJSON, sourceCal.ID, syncTokenOut) + } + } else { + deltaLink := calendarDeltaLinks(job.CursorJSON)[sourceCal.ID] + if deltaLink == "" { + deltaLink, _ = job.CursorJSON["pageToken"].(string) + } + events, nextCursor, err = c.listMicrosoftCalendarDelta(ctx, accessToken, sourceCal.ID, deltaLink) + if nextCursor != "" && strings.Contains(nextCursor, "delta") { + setCalendarDeltaLink(job.CursorJSON, sourceCal.ID, nextCursor) + } + } + if err != nil { + return err + } + + batch := 0 + listIndex := int(jsonNumber(job.CursorJSON["listIndex"])) + for i := listIndex; i < len(events) && batch < mailImportBatchSize(); i++ { + ev := events[i] + if ev.Deleted { + eventPath := migrationEventPath(calPath, provider, ev.SourceID) + if err := c.nc.DeleteEvent(ctx, ncUserID, eventPath); err != nil && !isDeleteNotFound(err) { + return err + } + if err := items.Unmark(ctx, ev.SourceID); err != nil { + return err + } + deleted++ + batch++ + continue + } + wasUpdate, err := c.upsertEvent(ctx, ncUserID, calPath, provider, ev, items) + if err != nil { + return err + } + if wasUpdate { + updated++ + } else if items.Has(ev.SourceID) { + imported++ + } else { + incJobStat(job.StatsJSON, "failed") + } + batch++ + } + job.StatsJSON["delta_imported"] = imported + job.StatsJSON["delta_updated"] = updated + job.StatsJSON["delta_deleted"] = deleted + + if listIndex+batch < len(events) { + job.CursorJSON["listIndex"] = float64(listIndex + batch) + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "listIndex") + + if nextCursor != "" { + job.CursorJSON["pageToken"] = nextCursor + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "pageToken") + job.CursorJSON["calendarIndex"] = float64(calIndex + 1) + return update("pending", job.CursorJSON, job.StatsJSON, "") +} + +func (c *CalendarImporter) bootstrapCalendarDelta(ctx context.Context, accessToken, calID string) (string, error) { + url := graphMicrosoftURL(c.userUPN, fmt.Sprintf("/calendars/%s/events/delta?$select=id,subject,body,start,end,isAllDay,location", url.PathEscape(calID))) + body, err := apiGet(ctx, c.client, url, accessToken) + if err != nil { + return "", err + } + var parsed struct { + NextLink string `json:"@odata.nextLink"` + DeltaLink string `json:"@odata.deltaLink"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return "", err + } + if parsed.DeltaLink != "" { + return parsed.DeltaLink, nil + } + return parsed.NextLink, nil +} + +func (c *CalendarImporter) listMicrosoftCalendarDelta(ctx context.Context, accessToken, calID, deltaLink string) ([]sourceEvent, string, error) { + if deltaLink == "" { + deltaLink = graphMicrosoftURL(c.userUPN, fmt.Sprintf("/calendars/%s/events/delta?$select=id,subject,body,start,end,isAllDay,location", url.PathEscape(calID))) + } + body, err := apiGet(ctx, c.client, deltaLink, accessToken) + if err != nil { + return nil, "", err + } + var parsed struct { + Value []graphCalendarEvent `json:"value"` + NextLink string `json:"@odata.nextLink"` + DeltaLink string `json:"@odata.deltaLink"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, "", err + } + out := make([]sourceEvent, 0, len(parsed.Value)) + for _, item := range parsed.Value { + out = append(out, item.toSourceEvent(calID)) + } + next := parsed.NextLink + if parsed.DeltaLink != "" && parsed.NextLink == "" { + next = parsed.DeltaLink + } + return out, next, nil +} + +func (c *CalendarImporter) ensureMigrationCalendar(ctx context.Context, ncUserID string) (string, error) { + path := fmt.Sprintf("/remote.php/dav/calendars/%s/%s/", ncUserID, migrationCalendarID) + if err := c.nc.CreateCalendar(ctx, ncUserID, migrationCalendarID, "Migration Import", "#1a73e8"); err != nil { + msg := strings.ToLower(err.Error()) + if !strings.Contains(msg, "405") && !strings.Contains(msg, "409") && !strings.Contains(msg, "423") { + return "", err + } + } + return path, nil +} + +type sourceCalendar struct { + ID string + Name string +} + +type sourceEvent struct { + SourceID string + Summary string + Description string + Location string + Start time.Time + End time.Time + AllDay bool + Deleted bool +} + +func (e sourceEvent) ToNextcloudEvent(provider string) *nextcloud.Event { + uid := sanitizeMigrationUID(provider, e.SourceID) + start := e.Start.UTC() + end := e.End.UTC() + if end.IsZero() || !end.After(start) { + end = start.Add(time.Hour) + } + ev := &nextcloud.Event{ + UID: uid, + Summary: e.Summary, + Description: e.Description, + Location: e.Location, + AllDay: e.AllDay, + } + if e.AllDay { + ev.Start = start.Format("20060102") + ev.End = end.Format("20060102") + } else { + ev.Start = start.Format("20060102T150405Z") + ev.End = end.Format("20060102T150405Z") + } + return ev +} + +func (c *CalendarImporter) listSourceCalendars(ctx context.Context, accessToken, provider string) ([]sourceCalendar, error) { + switch provider { + case "google": + body, err := apiGet(ctx, c.client, "https://www.googleapis.com/calendar/v3/users/me/calendarList?maxResults=100", accessToken) + if err != nil { + return nil, err + } + var parsed struct { + Items []struct { + ID string `json:"id"` + Summary string `json:"summary"` + } `json:"items"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, err + } + out := make([]sourceCalendar, 0, len(parsed.Items)) + for _, item := range parsed.Items { + out = append(out, sourceCalendar{ID: item.ID, Name: item.Summary}) + } + return out, nil + default: + body, err := apiGet(ctx, c.client, graphMicrosoftURL(c.userUPN, "/calendars?$top=100"), accessToken) + if err != nil { + return nil, err + } + var parsed struct { + Value []struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"value"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, err + } + out := make([]sourceCalendar, 0, len(parsed.Value)) + for _, item := range parsed.Value { + out = append(out, sourceCalendar{ID: item.ID, Name: item.Name}) + } + return out, nil + } +} + +func (c *CalendarImporter) listSourceEvents(ctx context.Context, accessToken, provider string, cal sourceCalendar, pageToken, syncToken string) ([]sourceEvent, string, string, error) { + switch provider { + case "google": + listURL := fmt.Sprintf( + "https://www.googleapis.com/calendar/v3/calendars/%s/events?maxResults=100&singleEvents=true&orderBy=startTime", + url.PathEscape(cal.ID), + ) + if syncToken != "" { + listURL += "&syncToken=" + url.QueryEscape(syncToken) + "&showDeleted=true" + } else if pageToken != "" { + listURL += "&pageToken=" + url.QueryEscape(pageToken) + } + body, err := apiGet(ctx, c.client, listURL, accessToken) + if err != nil { + return nil, "", "", err + } + var parsed struct { + Items []googleCalendarEvent `json:"items"` + NextPageToken string `json:"nextPageToken"` + NextSyncToken string `json:"nextSyncToken"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, "", "", err + } + out := make([]sourceEvent, 0, len(parsed.Items)) + for _, item := range parsed.Items { + out = append(out, item.toSourceEvent(cal.ID)) + } + return out, parsed.NextPageToken, parsed.NextSyncToken, nil + default: + listURL := graphMicrosoftURL(c.userUPN, fmt.Sprintf("/calendars/%s/events?$top=100&$select=id,subject,body,start,end,isAllDay,location", url.PathEscape(cal.ID))) + if pageToken != "" { + listURL = pageToken + } + body, err := apiGet(ctx, c.client, listURL, accessToken) + if err != nil { + return nil, "", "", err + } + var parsed struct { + Value []graphCalendarEvent `json:"value"` + NextLink string `json:"@odata.nextLink"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, "", "", err + } + out := make([]sourceEvent, 0, len(parsed.Value)) + for _, item := range parsed.Value { + out = append(out, item.toSourceEvent(cal.ID)) + } + return out, parsed.NextLink, "", nil + } +} + +type googleCalendarEvent struct { + ID string `json:"id"` + Status string `json:"status"` + Summary string `json:"summary"` + Description string `json:"description"` + Location string `json:"location"` + Start struct { + DateTime string `json:"dateTime"` + Date string `json:"date"` + } `json:"start"` + End struct { + DateTime string `json:"dateTime"` + Date string `json:"date"` + } `json:"end"` +} + +func (e googleCalendarEvent) toSourceEvent(calID string) sourceEvent { + allDay := e.Start.Date != "" + start := parseFlexibleTime(e.Start.DateTime, e.Start.Date) + end := parseFlexibleTime(e.End.DateTime, e.End.Date) + return sourceEvent{ + SourceID: calID + ":" + e.ID, + Summary: e.Summary, + Description: e.Description, + Location: e.Location, + Start: start, + End: end, + AllDay: allDay, + Deleted: e.Status == "cancelled", + } +} + +type graphCalendarEvent struct { + ID string `json:"id"` + Removed *struct { + Reason string `json:"reason"` + } `json:"@removed"` + Subject string `json:"subject"` + Body struct { + Content string `json:"content"` + } `json:"body"` + IsAllDay bool `json:"isAllDay"` + Location struct { + DisplayName string `json:"displayName"` + } `json:"location"` + Start graphDateTime `json:"start"` + End graphDateTime `json:"end"` +} + +type graphDateTime struct { + DateTime string `json:"dateTime"` + Date string `json:"date"` +} + +func (e graphCalendarEvent) toSourceEvent(calID string) sourceEvent { + allDay := e.IsAllDay || e.Start.Date != "" + start := parseFlexibleTime(e.Start.DateTime, e.Start.Date) + end := parseFlexibleTime(e.End.DateTime, e.End.Date) + return sourceEvent{ + SourceID: calID + ":" + e.ID, + Summary: e.Subject, + Description: e.Body.Content, + Location: e.Location.DisplayName, + Start: start, + End: end, + AllDay: allDay, + Deleted: e.Removed != nil, + } +} + +func parseFlexibleTime(dateTime, date string) time.Time { + if strings.TrimSpace(dateTime) != "" { + if t, err := time.Parse(time.RFC3339, dateTime); err == nil { + return t.UTC() + } + } + if strings.TrimSpace(date) != "" { + if t, err := time.Parse("2006-01-02", date); err == nil { + return t.UTC() + } + } + return time.Time{} +} + +func (c *CalendarImporter) upsertEvent( + ctx context.Context, + ncUserID, calPath, provider string, + ev sourceEvent, + items *ImportedItemStore, +) (updated bool, err error) { + ncEv := ev.ToNextcloudEvent(provider) + if alreadyImported(items, ev.SourceID) { + eventPath := migrationEventPath(calPath, provider, ev.SourceID) + if _, err := c.nc.UpdateEvent(ctx, ncUserID, eventPath, "", ncEv); err != nil { + return false, err + } + return true, nil + } + if err := c.nc.CreateEvent(ctx, ncUserID, calPath, ncEv); err != nil { + if markErr := items.MarkFailed(ctx, ev.SourceID, err.Error(), ""); markErr != nil { + return false, markErr + } + return false, nil + } + if err := items.MarkImported(ctx, ev.SourceID); err != nil { + return false, err + } + return false, nil +} diff --git a/internal/migration/claim_email_match.go b/internal/migration/claim_email_match.go new file mode 100644 index 0000000..1cbc901 --- /dev/null +++ b/internal/migration/claim_email_match.go @@ -0,0 +1,150 @@ +package migration + +import ( + "strings" + + "github.com/ultisuite/ulti-backend/internal/auth" +) + +// ClaimIdentity holds SSO identity fields checked against a migration invite. +type ClaimIdentity struct { + Email string + PreferredUsername string + UPN string +} + +func ClaimIdentityFromAuth(c *auth.Claims) ClaimIdentity { + if c == nil { + return ClaimIdentity{} + } + return ClaimIdentity{ + Email: c.Email, + PreferredUsername: c.PreferredUsername, + UPN: c.UPN, + } +} + +func normalizeInviteEmail(email string) string { + return strings.ToLower(strings.TrimSpace(email)) +} + +func isEmailAddress(s string) bool { + s = strings.TrimSpace(s) + at := strings.LastIndex(s, "@") + return at > 0 && at < len(s)-1 +} + +func identityCandidateEmails(id ClaimIdentity) []string { + seen := make(map[string]struct{}) + var out []string + for _, raw := range []string{id.Email, id.PreferredUsername, id.UPN} { + raw = strings.TrimSpace(raw) + if raw == "" || !isEmailAddress(raw) { + continue + } + norm := normalizeInviteEmail(raw) + if _, ok := seen[norm]; ok { + continue + } + seen[norm] = struct{}{} + out = append(out, norm) + } + return out +} + +func normalizeEmailLocalPart(local string) string { + local = strings.ToLower(strings.TrimSpace(local)) + if plus := strings.Index(local, "+"); plus >= 0 { + local = local[:plus] + } + return strings.ReplaceAll(local, ".", "") +} + +func emailLocalAndDomain(email string) (local, domain string, ok bool) { + email = normalizeInviteEmail(email) + at := strings.LastIndex(email, "@") + if at <= 0 || at == len(email)-1 { + return "", "", false + } + return email[:at], email[at+1:], true +} + +func inviteMatchTargets(inviteEmail string, alternateEmails []string) []string { + seen := make(map[string]struct{}) + var out []string + add := func(e string) { + e = normalizeInviteEmail(e) + if e == "" || !isEmailAddress(e) { + return + } + if _, ok := seen[e]; ok { + return + } + seen[e] = struct{}{} + out = append(out, e) + } + add(inviteEmail) + for _, alt := range alternateEmails { + add(alt) + } + return out +} + +func localPartAliasMatch(a, b string) bool { + aLocal, aDomain, okA := emailLocalAndDomain(a) + bLocal, bDomain, okB := emailLocalAndDomain(b) + if !okA || !okB { + return false + } + if !strings.EqualFold(aDomain, bDomain) { + return false + } + return normalizeEmailLocalPart(aLocal) == normalizeEmailLocalPart(bLocal) +} + +// projectDomainUPNMatch accepts claims when the invite is on the hosted project domain +// and a UPN-style identity (preferred_username / upn) shares the same mailbox local-part. +// Typical Microsoft case: invite alice@acme.com, SSO preferred_username alice@tenant.onmicrosoft.com. +func projectDomainUPNMatch(inviteEmail, projectDomain string, identity ClaimIdentity) bool { + if projectDomain == "" { + return false + } + projectDomain = strings.ToLower(strings.TrimSpace(projectDomain)) + invLocal, invDomain, ok := emailLocalAndDomain(inviteEmail) + if !ok || !strings.EqualFold(invDomain, projectDomain) { + return false + } + for _, raw := range []string{identity.PreferredUsername, identity.UPN} { + candLocal, _, ok := emailLocalAndDomain(raw) + if ok && strings.EqualFold(candLocal, invLocal) { + return true + } + } + return false +} + +// InviteEmailMatchesIdentity reports whether SSO identity may claim the invite. +// projectDomain is the hosted mail domain when the migration project is domain-bound. +func InviteEmailMatchesIdentity(inviteEmail string, alternateEmails []string, projectDomain string, identity ClaimIdentity) bool { + targets := inviteMatchTargets(inviteEmail, alternateEmails) + if len(targets) == 0 { + return false + } + candidates := identityCandidateEmails(identity) + if len(candidates) == 0 { + return false + } + + for _, target := range targets { + for _, candidate := range candidates { + if candidate == target { + return true + } + if localPartAliasMatch(target, candidate) { + return true + } + } + } + + return projectDomainUPNMatch(inviteEmail, projectDomain, identity) +} diff --git a/internal/migration/claim_email_match_test.go b/internal/migration/claim_email_match_test.go new file mode 100644 index 0000000..623ae8d --- /dev/null +++ b/internal/migration/claim_email_match_test.go @@ -0,0 +1,92 @@ +package migration + +import "testing" + +func TestInviteEmailMatchesIdentityExact(t *testing.T) { + id := ClaimIdentity{Email: "Alice@Acme.com"} + if !InviteEmailMatchesIdentity("alice@acme.com", nil, "", id) { + t.Fatal("expected case-insensitive exact match") + } +} + +func TestInviteEmailMatchesIdentityPreferredUsername(t *testing.T) { + id := ClaimIdentity{ + Email: "alice.smith@acme.com", + PreferredUsername: "alice@acme.com", + } + if !InviteEmailMatchesIdentity("alice@acme.com", nil, "", id) { + t.Fatal("expected preferred_username match") + } +} + +func TestInviteEmailMatchesIdentityUPN(t *testing.T) { + id := ClaimIdentity{ + Email: "alice.smith@acme.com", + UPN: "alice@acme.com", + } + if !InviteEmailMatchesIdentity("alice@acme.com", nil, "", id) { + t.Fatal("expected upn match") + } +} + +func TestInviteEmailMatchesIdentityAlternateEmail(t *testing.T) { + id := ClaimIdentity{Email: "alice.smith@acme.com"} + if !InviteEmailMatchesIdentity("alice@acme.com", []string{"alice.smith@acme.com"}, "", id) { + t.Fatal("expected alternate email match") + } +} + +func TestInviteEmailMatchesIdentityGmailDotAlias(t *testing.T) { + id := ClaimIdentity{Email: "alice.smith@acme.com"} + if !InviteEmailMatchesIdentity("alice.smith@acme.com", nil, "", id) { + t.Fatal("expected exact match baseline") + } + id = ClaimIdentity{Email: "a.l.i.c.e.smith@acme.com"} + if !InviteEmailMatchesIdentity("alice.smith@acme.com", nil, "", id) { + t.Fatal("expected dot-insensitive local-part match") + } +} + +func TestInviteEmailMatchesIdentityPlusTag(t *testing.T) { + id := ClaimIdentity{Email: "alice+tag@acme.com"} + if !InviteEmailMatchesIdentity("alice@acme.com", nil, "", id) { + t.Fatal("expected plus-tag stripped match") + } +} + +func TestInviteEmailMatchesIdentityProjectDomainUPN(t *testing.T) { + id := ClaimIdentity{ + Email: "alice.smith@acme.com", + PreferredUsername: "alice@contoso.onmicrosoft.com", + } + if !InviteEmailMatchesIdentity("alice@acme.com", nil, "acme.com", id) { + t.Fatal("expected project-domain UPN local-part match") + } +} + +func TestInviteEmailMatchesIdentityRejectsDifferentUserSameDomain(t *testing.T) { + id := ClaimIdentity{Email: "bob@acme.com"} + if InviteEmailMatchesIdentity("alice@acme.com", nil, "acme.com", id) { + t.Fatal("expected reject for different local-part on same domain") + } +} + +func TestInviteEmailMatchesIdentityRejectsUnrelatedDomain(t *testing.T) { + id := ClaimIdentity{Email: "alice@evil.com"} + if InviteEmailMatchesIdentity("alice@acme.com", nil, "", id) { + t.Fatal("expected reject for different domain without alias") + } +} + +func TestInviteEmailMatchesIdentityEmptyIdentity(t *testing.T) { + if InviteEmailMatchesIdentity("alice@acme.com", nil, "", ClaimIdentity{}) { + t.Fatal("expected reject for empty identity") + } +} + +func TestInviteEmailMatchesIdentityIgnoresNonEmailPreferredUsername(t *testing.T) { + id := ClaimIdentity{PreferredUsername: "alice"} + if InviteEmailMatchesIdentity("alice@acme.com", nil, "", id) { + t.Fatal("expected reject when preferred_username is not an email") + } +} diff --git a/internal/migration/contacts_delta_test.go b/internal/migration/contacts_delta_test.go new file mode 100644 index 0000000..a02dc3c --- /dev/null +++ b/internal/migration/contacts_delta_test.go @@ -0,0 +1,54 @@ +package migration + +import "testing" + +func TestGooglePersonDeletedMetadata(t *testing.T) { + person := googlePerson{ + ResourceName: "people/abc", + Metadata: &struct{ Deleted bool `json:"deleted"` }{Deleted: true}, + } + if person.Metadata == nil || !person.Metadata.Deleted { + t.Fatal("expected deleted metadata") + } +} + +func TestGraphContactRemoved(t *testing.T) { + removed := struct { + Reason string `json:"reason"` + }{Reason: "deleted"} + item := graphContact{ID: "c1", Removed: &removed} + if item.Removed == nil { + t.Fatal("expected removed marker") + } +} + +func TestGooglePersonToContact(t *testing.T) { + person := googlePerson{ + ResourceName: "people/abc", + Names: []struct{ DisplayName string `json:"displayName"` }{{DisplayName: "Alice"}}, + EmailAddresses: []struct { + Value string `json:"value"` + }{{Value: "Alice@Example.COM"}}, + } + contact := googlePersonToContact("people/abc", person) + if contact.FullName != "Alice" || contact.Email != "alice@example.com" { + t.Fatalf("contact: %#v", contact) + } +} + +func TestGraphContactToNC(t *testing.T) { + item := graphContact{ + ID: "c1", + GivenName: "Bob", + Surname: "Smith", + MobilePhone: "+33123456789", + CompanyName: "Acme", + EmailAddresses: []struct { + Address string `json:"address"` + }{{Address: "bob@example.com"}}, + } + contact := graphContactToNC("c1", item) + if contact.FullName != "Bob Smith" || contact.Org != "Acme" { + t.Fatalf("contact: %#v", contact) + } +} diff --git a/internal/migration/contacts_import.go b/internal/migration/contacts_import.go new file mode 100644 index 0000000..43af003 --- /dev/null +++ b/internal/migration/contacts_import.go @@ -0,0 +1,476 @@ +package migration + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/ultisuite/ulti-backend/internal/nextcloud" +) + +const migrationContactsBookID = "migration-import" + +type ContactsImporter struct { + db *pgxpool.Pool + nc *nextcloud.Client + client *http.Client + userUPN string +} + +func NewContactsImporter(db *pgxpool.Pool, nc *nextcloud.Client) *ContactsImporter { + return &ContactsImporter{db: db, nc: nc, client: migrationHTTPClient()} +} + +func (c *ContactsImporter) WithUserPrincipal(upn string) *ContactsImporter { + c.userUPN = strings.TrimSpace(upn) + return c +} + +func (c *ContactsImporter) WithHTTPClient(client *http.Client) *ContactsImporter { + if client != nil { + c.client = client + } + return c +} + +func (c *ContactsImporter) ImportBatch(ctx context.Context, job *Job, accessToken, provider string, delta bool, update progressUpdater) error { + if c.nc == nil { + return fmt.Errorf("nextcloud required for contacts migration") + } + user, err := resolveMigrationUser(ctx, c.db, job.UserID) + if err != nil { + return err + } + ncUserID := nextcloud.UserIDFromClaims(user.Email, user.ExternalID) + if _, err := c.nc.EnsurePrincipal(ctx, user.Email, user.ExternalID, user.Name); err != nil { + return fmt.Errorf("nextcloud user: %w", err) + } + bookPath := nextcloud.AddressBookPath(ncUserID, migrationContactsBookID) + items, err := LoadImportedItemStore(ctx, c.db, job.ID, job.CursorJSON) + if err != nil { + return err + } + + if delta { + if provider == "google" { + if syncToken, _ := job.CursorJSON["syncToken"].(string); strings.TrimSpace(syncToken) != "" { + return c.importGoogleDelta(ctx, job, accessToken, ncUserID, bookPath, items, update) + } + } else if deltaLink, _ := job.CursorJSON["deltaLink"].(string); strings.TrimSpace(deltaLink) != "" { + return c.importMicrosoftDelta(ctx, job, accessToken, ncUserID, bookPath, deltaLink, items, update) + } else { + return c.bootstrapMicrosoftDelta(ctx, job, accessToken, ncUserID, bookPath, update) + } + } + + switch provider { + case "google": + return c.importGoogleFull(ctx, job, accessToken, ncUserID, bookPath, delta, items, update) + default: + return c.importMicrosoftFull(ctx, job, accessToken, ncUserID, bookPath, delta, items, update) + } +} + +func (c *ContactsImporter) importGoogleFull(ctx context.Context, job *Job, accessToken, ncUserID, bookPath string, captureToken bool, items *ImportedItemStore, update progressUpdater) error { + imported, _ := job.StatsJSON["imported"].(float64) + batch := 0 + + pageToken, _ := job.CursorJSON["pageToken"].(string) + listURL := "https://people.googleapis.com/v1/people/me/connections?pageSize=100&personFields=names,emailAddresses,phoneNumbers,organizations,metadata" + if captureToken { + listURL += "&requestSyncToken=true" + } + if pageToken != "" { + listURL += "&pageToken=" + url.QueryEscape(pageToken) + } + body, err := apiGet(ctx, c.client, listURL, accessToken) + if err != nil { + return err + } + var listed struct { + Connections []googlePerson `json:"connections"` + NextPageToken string `json:"nextPageToken"` + NextSyncToken string `json:"nextSyncToken"` + } + if err := json.Unmarshal(body, &listed); err != nil { + return err + } + + listIndex := int(jsonNumber(job.CursorJSON["listIndex"])) + for i := listIndex; i < len(listed.Connections) && batch < mailImportBatchSize(); i++ { + person := listed.Connections[i] + sourceID := strings.TrimSpace(person.ResourceName) + if sourceID == "" { + sourceID = fmt.Sprintf("google-person-%d", i) + } + if alreadyImported(items, sourceID) { + continue + } + contact := googlePersonToContact(sourceID, person) + if contact.Email == "" && contact.FullName == "" { + continue + } + if _, err := c.nc.CreateContact(ctx, ncUserID, bookPath, contact); err != nil { + if markErr := items.MarkFailed(ctx, sourceID, err.Error(), ""); markErr != nil { + return markErr + } + incJobStat(job.StatsJSON, "failed") + batch++ + continue + } + if err := items.MarkImported(ctx, sourceID); err != nil { + return err + } + imported++ + batch++ + } + job.StatsJSON["imported"] = imported + + if listIndex+batch < len(listed.Connections) { + job.CursorJSON["listIndex"] = float64(listIndex + batch) + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "listIndex") + + if listed.NextPageToken != "" { + job.CursorJSON["pageToken"] = listed.NextPageToken + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "pageToken") + if listed.NextSyncToken != "" { + job.CursorJSON["syncToken"] = listed.NextSyncToken + } + job.StatsJSON["phase"] = "imported" + return update("completed", job.CursorJSON, job.StatsJSON, "") +} + +func (c *ContactsImporter) importGoogleDelta(ctx context.Context, job *Job, accessToken, ncUserID, bookPath string, items *ImportedItemStore, update progressUpdater) error { + syncToken, _ := job.CursorJSON["syncToken"].(string) + listURL := "https://people.googleapis.com/v1/people/me/connections?syncToken=" + url.QueryEscape(syncToken) + + "&personFields=names,emailAddresses,phoneNumbers,organizations,metadata&requestSyncToken=true" + body, err := apiGet(ctx, c.client, listURL, accessToken) + if err != nil { + return err + } + var listed struct { + Connections []googlePerson `json:"connections"` + NextSyncToken string `json:"nextSyncToken"` + } + if err := json.Unmarshal(body, &listed); err != nil { + return err + } + deltaCount, _ := job.StatsJSON["delta_imported"].(float64) + deltaUpdated, _ := job.StatsJSON["delta_updated"].(float64) + deleted, _ := job.StatsJSON["delta_deleted"].(float64) + for _, person := range listed.Connections { + sourceID := strings.TrimSpace(person.ResourceName) + if sourceID == "" { + continue + } + if person.Metadata != nil && person.Metadata.Deleted { + contactPath := migrationContactPath(bookPath, "google", sourceID) + if err := c.nc.DeleteContact(ctx, ncUserID, contactPath); err != nil && !isDeleteNotFound(err) { + return err + } + if err := items.Unmark(ctx, sourceID); err != nil { + return err + } + deleted++ + continue + } + contact := googlePersonToContact(sourceID, person) + if contact.Email == "" && contact.FullName == "" { + continue + } + updated, err := c.upsertContact(ctx, ncUserID, bookPath, "google", sourceID, contact, items) + if err != nil { + return err + } + if updated { + deltaUpdated++ + } else if items.Has(sourceID) { + deltaCount++ + } else { + incJobStat(job.StatsJSON, "failed") + } + } + if listed.NextSyncToken != "" { + job.CursorJSON["syncToken"] = listed.NextSyncToken + } + job.StatsJSON["delta_imported"] = deltaCount + job.StatsJSON["delta_updated"] = deltaUpdated + job.StatsJSON["delta_deleted"] = deleted + job.StatsJSON["phase"] = "delta" + return update("completed", job.CursorJSON, job.StatsJSON, "") +} + +func (c *ContactsImporter) importMicrosoftFull(ctx context.Context, job *Job, accessToken, ncUserID, bookPath string, captureDelta bool, items *ImportedItemStore, update progressUpdater) error { + imported, _ := job.StatsJSON["imported"].(float64) + batch := 0 + + nextLink, _ := job.CursorJSON["nextLink"].(string) + listURL := graphMicrosoftURL(c.userUPN, "/contacts?$top=100&$select=id,displayName,givenName,surname,emailAddresses,mobilePhone,businessPhones,companyName") + if nextLink != "" { + listURL = nextLink + } + body, err := apiGet(ctx, c.client, listURL, accessToken) + if err != nil { + return err + } + var listed struct { + Value []graphContact `json:"value"` + NextLink string `json:"@odata.nextLink"` + } + if err := json.Unmarshal(body, &listed); err != nil { + return err + } + + listIndex := int(jsonNumber(job.CursorJSON["listIndex"])) + for i := listIndex; i < len(listed.Value) && batch < mailImportBatchSize(); i++ { + item := listed.Value[i] + sourceID := strings.TrimSpace(item.ID) + if sourceID == "" { + sourceID = fmt.Sprintf("graph-contact-%d", i) + } + if alreadyImported(items, sourceID) { + continue + } + contact := graphContactToNC(sourceID, item) + if contact.Email == "" && contact.FullName == "" { + continue + } + if _, err := c.nc.CreateContact(ctx, ncUserID, bookPath, contact); err != nil { + if markErr := items.MarkFailed(ctx, sourceID, err.Error(), ""); markErr != nil { + return markErr + } + incJobStat(job.StatsJSON, "failed") + batch++ + continue + } + if err := items.MarkImported(ctx, sourceID); err != nil { + return err + } + imported++ + batch++ + } + job.StatsJSON["imported"] = imported + + if listIndex+batch < len(listed.Value) { + job.CursorJSON["listIndex"] = float64(listIndex + batch) + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "listIndex") + + if listed.NextLink != "" { + job.CursorJSON["nextLink"] = listed.NextLink + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "nextLink") + + if captureDelta { + return c.bootstrapMicrosoftDelta(ctx, job, accessToken, ncUserID, bookPath, update) + } + job.StatsJSON["phase"] = "imported" + return update("completed", job.CursorJSON, job.StatsJSON, "") +} + +func (c *ContactsImporter) bootstrapMicrosoftDelta(ctx context.Context, job *Job, accessToken, ncUserID, bookPath string, update progressUpdater) error { + body, err := apiGet(ctx, c.client, graphMicrosoftURL(c.userUPN, "/contacts/delta?$select=id,displayName,givenName,surname,emailAddresses,mobilePhone,businessPhones,companyName"), accessToken) + if err != nil { + return err + } + var listed struct { + Value []graphContact `json:"value"` + NextLink string `json:"@odata.nextLink"` + DeltaLink string `json:"@odata.deltaLink"` + } + if err := json.Unmarshal(body, &listed); err != nil { + return err + } + if listed.DeltaLink != "" { + job.CursorJSON["deltaLink"] = listed.DeltaLink + job.StatsJSON["phase"] = "delta_ready" + return update("completed", job.CursorJSON, job.StatsJSON, "") + } + if listed.NextLink != "" { + job.CursorJSON["nextLink"] = listed.NextLink + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + job.StatsJSON["phase"] = "imported" + return update("completed", job.CursorJSON, job.StatsJSON, "") +} + +func (c *ContactsImporter) importMicrosoftDelta(ctx context.Context, job *Job, accessToken, ncUserID, bookPath, deltaLink string, items *ImportedItemStore, update progressUpdater) error { + body, err := apiGet(ctx, c.client, deltaLink, accessToken) + if err != nil { + return err + } + var listed struct { + Value []graphContact `json:"value"` + NextLink string `json:"@odata.nextLink"` + DeltaLink string `json:"@odata.deltaLink"` + } + if err := json.Unmarshal(body, &listed); err != nil { + return err + } + deltaCount, _ := job.StatsJSON["delta_imported"].(float64) + deltaUpdated, _ := job.StatsJSON["delta_updated"].(float64) + deleted, _ := job.StatsJSON["delta_deleted"].(float64) + for _, item := range listed.Value { + sourceID := strings.TrimSpace(item.ID) + if sourceID == "" { + continue + } + if item.Removed != nil { + contactPath := migrationContactPath(bookPath, "microsoft", sourceID) + if err := c.nc.DeleteContact(ctx, ncUserID, contactPath); err != nil && !isDeleteNotFound(err) { + return err + } + if err := items.Unmark(ctx, sourceID); err != nil { + return err + } + deleted++ + continue + } + contact := graphContactToNC(sourceID, item) + if contact.Email == "" && contact.FullName == "" { + continue + } + updated, err := c.upsertContact(ctx, ncUserID, bookPath, "microsoft", sourceID, contact, items) + if err != nil { + return err + } + if updated { + deltaUpdated++ + } else if items.Has(sourceID) { + deltaCount++ + } else { + incJobStat(job.StatsJSON, "failed") + } + } + if listed.NextLink != "" { + job.CursorJSON["deltaLink"] = listed.NextLink + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + if listed.DeltaLink != "" { + job.CursorJSON["deltaLink"] = listed.DeltaLink + } + job.StatsJSON["delta_imported"] = deltaCount + job.StatsJSON["delta_updated"] = deltaUpdated + job.StatsJSON["delta_deleted"] = deleted + job.StatsJSON["phase"] = "delta" + return update("completed", job.CursorJSON, job.StatsJSON, "") +} + +type googlePerson struct { + ResourceName string `json:"resourceName"` + Metadata *struct { + Deleted bool `json:"deleted"` + } `json:"metadata"` + Names []struct { + DisplayName string `json:"displayName"` + } `json:"names"` + EmailAddresses []struct { + Value string `json:"value"` + } `json:"emailAddresses"` + PhoneNumbers []struct { + Value string `json:"value"` + } `json:"phoneNumbers"` + Organizations []struct { + Name string `json:"name"` + } `json:"organizations"` +} + +type graphContact struct { + ID string `json:"id"` + Removed *struct { + Reason string `json:"reason"` + } `json:"@removed"` + DisplayName string `json:"displayName"` + GivenName string `json:"givenName"` + Surname string `json:"surname"` + MobilePhone string `json:"mobilePhone"` + BusinessPhones []string `json:"businessPhones"` + CompanyName string `json:"companyName"` + EmailAddresses []struct { + Address string `json:"address"` + } `json:"emailAddresses"` +} + +func googlePersonToContact(sourceID string, p googlePerson) *nextcloud.Contact { + name := "" + if len(p.Names) > 0 { + name = strings.TrimSpace(p.Names[0].DisplayName) + } + email := "" + if len(p.EmailAddresses) > 0 { + email = strings.ToLower(strings.TrimSpace(p.EmailAddresses[0].Value)) + } + phone := "" + if len(p.PhoneNumbers) > 0 { + phone = strings.TrimSpace(p.PhoneNumbers[0].Value) + } + org := "" + if len(p.Organizations) > 0 { + org = strings.TrimSpace(p.Organizations[0].Name) + } + return &nextcloud.Contact{ + UID: sanitizeMigrationUID("google", sourceID), + FullName: name, + Email: email, + Phone: phone, + Org: org, + } +} + +func graphContactToNC(sourceID string, c graphContact) *nextcloud.Contact { + name := strings.TrimSpace(c.DisplayName) + if name == "" { + name = strings.TrimSpace(strings.TrimSpace(c.GivenName + " " + c.Surname)) + } + email := "" + if len(c.EmailAddresses) > 0 { + email = strings.ToLower(strings.TrimSpace(c.EmailAddresses[0].Address)) + } + phone := strings.TrimSpace(c.MobilePhone) + if phone == "" && len(c.BusinessPhones) > 0 { + phone = strings.TrimSpace(c.BusinessPhones[0]) + } + return &nextcloud.Contact{ + UID: sanitizeMigrationUID("microsoft", sourceID), + FullName: name, + Email: email, + Phone: phone, + Org: strings.TrimSpace(c.CompanyName), + } +} + +func (c *ContactsImporter) upsertContact( + ctx context.Context, + ncUserID, bookPath, provider, sourceID string, + contact *nextcloud.Contact, + items *ImportedItemStore, +) (updated bool, err error) { + if alreadyImported(items, sourceID) { + contactPath := migrationContactPath(bookPath, provider, sourceID) + if _, err := c.nc.UpdateContact(ctx, ncUserID, contactPath, "", contact); err != nil { + return false, err + } + return true, nil + } + if _, err := c.nc.CreateContact(ctx, ncUserID, bookPath, contact); err != nil { + if markErr := items.MarkFailed(ctx, sourceID, err.Error(), ""); markErr != nil { + return false, markErr + } + return false, nil + } + if err := items.MarkImported(ctx, sourceID); err != nil { + return false, err + } + return false, nil +} diff --git a/internal/migration/cutover_dns.go b/internal/migration/cutover_dns.go new file mode 100644 index 0000000..f55b475 --- /dev/null +++ b/internal/migration/cutover_dns.go @@ -0,0 +1,142 @@ +package migration + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/ultisuite/ulti-backend/internal/mail/hosted" +) + +// CutoverConfig controls DNS expectations during migration cutover. +type CutoverConfig struct { + ExpectedMXHosts []string + RequireMX bool +} + +// CutoverResult is returned when a migration project enters cutover. +type CutoverResult struct { + Project Project `json:"project"` + DNS hosted.DNSCheckReport `json:"dns"` +} + +var ErrCutoverMXNotReady = fmt.Errorf("migration cutover blocked: mx records not pointing to ultimail") + +func (s *Service) PreflightCutoverDNS(ctx context.Context, projectID string, cfg CutoverConfig) (hosted.DNSCheckReport, error) { + domainID, err := s.projectDomainID(ctx, projectID) + if err != nil { + return hosted.DNSCheckReport{}, err + } + if domainID == "" { + return hosted.DNSCheckReport{ + Warnings: []string{"project has no linked mail domain; mx/txt checks skipped"}, + }, nil + } + if s.hosted == nil { + return hosted.DNSCheckReport{}, fmt.Errorf("hosted mail service not configured") + } + _, report, err := s.hosted.CheckDomainDNS(ctx, domainID, cfg.ExpectedMXHosts) + return report, err +} + +func (s *Service) StartCutover(ctx context.Context, projectID string) (CutoverResult, error) { + return s.startCutover(ctx, projectID, s.cutover) +} + +func (s *Service) startCutover(ctx context.Context, projectID string, cfg CutoverConfig) (CutoverResult, error) { + domainID, err := s.projectDomainID(ctx, projectID) + if err != nil { + return CutoverResult{}, err + } + + var report hosted.DNSCheckReport + if domainID != "" { + if s.hosted == nil { + return CutoverResult{}, fmt.Errorf("hosted mail service not configured") + } + var checkErr error + _, report, checkErr = s.hosted.CheckDomainDNS(ctx, domainID, cfg.ExpectedMXHosts) + if checkErr != nil { + return CutoverResult{}, checkErr + } + if report.TXTVerified { + if _, _, err := s.hosted.VerifyDomainTXTRecord(ctx, domainID); err != nil { + report.Warnings = append(report.Warnings, "auto txt verify: "+err.Error()) + } + } + if report.MXVerified && len(cfg.ExpectedMXHosts) > 0 { + if _, _, err := s.hosted.VerifyDomainMXRecord(ctx, domainID, cfg.ExpectedMXHosts); err != nil { + report.Warnings = append(report.Warnings, "auto mx verify: "+err.Error()) + } else { + report.Warnings = append(report.Warnings, "mx verified and domain marked active") + } + } else if !report.MXVerified { + if cfg.RequireMX { + return CutoverResult{DNS: report}, ErrCutoverMXNotReady + } + report.Warnings = append(report.Warnings, "mx not pointing to ultimail yet; cutover flag set anyway") + } + } else { + report.Warnings = append(report.Warnings, "no domain_id on project; configure mail domain before mx cutover") + } + + rawDNS, _ := json.Marshal(report) + _, err = s.db.Exec(ctx, ` + UPDATE migration_projects + SET status = 'cutover', cutover_at = NOW(), delta_mode = true, + cutover_dns_json = $2, updated_at = NOW() + WHERE id = $1::uuid + `, projectID, rawDNS) + if err != nil { + return CutoverResult{}, err + } + _, _ = s.db.Exec(ctx, ` + UPDATE migration_jobs + SET status = 'pending', error = '', updated_at = NOW() + WHERE project_id = $1::uuid AND status = 'completed' + `, projectID) + + sc := newProjectScanner() + err = s.db.QueryRow(ctx, ` + SELECT `+projectSelectSQL("")+` + FROM migration_projects WHERE id = $1::uuid + `, projectID).Scan(sc.targets()...) + if err != nil { + return CutoverResult{}, err + } + return CutoverResult{Project: sc.result(), DNS: report}, nil +} + +func (s *Service) projectDomainID(ctx context.Context, projectID string) (string, error) { + var domainID string + err := s.db.QueryRow(ctx, ` + SELECT COALESCE(domain_id::text, '') FROM migration_projects WHERE id = $1::uuid + `, projectID).Scan(&domainID) + return domainID, err +} + +func ParseCutoverMXHosts(raw string, platformMailDomain, stalwartIMAPHost string) []string { + var out []string + for _, part := range strings.Split(raw, ",") { + part = strings.ToLower(strings.TrimSpace(part)) + if part != "" { + out = append(out, part) + } + } + if len(out) > 0 { + return out + } + platformMailDomain = strings.ToLower(strings.TrimSpace(platformMailDomain)) + if platformMailDomain != "" { + out = append(out, "mail."+platformMailDomain) + } + stalwartIMAPHost = strings.ToLower(strings.TrimSpace(stalwartIMAPHost)) + if stalwartIMAPHost != "" && !strings.Contains(stalwartIMAPHost, ".") { + return out + } + if stalwartIMAPHost != "" { + out = append(out, stalwartIMAPHost) + } + return out +} diff --git a/internal/migration/cutover_dns_test.go b/internal/migration/cutover_dns_test.go new file mode 100644 index 0000000..8e6834f --- /dev/null +++ b/internal/migration/cutover_dns_test.go @@ -0,0 +1,37 @@ +package migration + +import "testing" + +func TestParseCutoverMXHostsExplicit(t *testing.T) { + got := ParseCutoverMXHosts(" Mail.Acme.Com , mx2.acme.com ", "", "") + want := []string{"mail.acme.com", "mx2.acme.com"} + if len(got) != len(want) { + t.Fatalf("got %v want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got %v want %v", got, want) + } + } +} + +func TestParseCutoverMXHostsFallbackPlatform(t *testing.T) { + got := ParseCutoverMXHosts("", "Ultisuite.Fr", "") + if len(got) != 1 || got[0] != "mail.ultisuite.fr" { + t.Fatalf("got %v", got) + } +} + +func TestParseCutoverMXHostsFallbackStalwart(t *testing.T) { + got := ParseCutoverMXHosts("", "", "mail.hosted.example.com") + if len(got) != 1 || got[0] != "mail.hosted.example.com" { + t.Fatalf("got %v", got) + } +} + +func TestParseCutoverMXHostsSkipsBareStalwartHost(t *testing.T) { + got := ParseCutoverMXHosts("", "acme.fr", "stalwart") + if len(got) != 1 || got[0] != "mail.acme.fr" { + t.Fatalf("got %v", got) + } +} diff --git a/internal/migration/drive_delta.go b/internal/migration/drive_delta.go new file mode 100644 index 0000000..ae9dab7 --- /dev/null +++ b/internal/migration/drive_delta.go @@ -0,0 +1,425 @@ +package migration + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "path" + "strings" +) + +func (d *DriveImporter) hasDriveDeltaCursor(job *Job, provider string) bool { + if provider == "google" { + token, _ := job.CursorJSON["driveChangeToken"].(string) + return strings.TrimSpace(token) != "" + } + link, _ := job.CursorJSON["driveDeltaLink"].(string) + return strings.TrimSpace(link) != "" +} + +func (d *DriveImporter) bootstrapDriveDelta(ctx context.Context, accessToken, provider string, cursor map[string]any) error { + switch provider { + case "google": + body, err := apiGet(ctx, d.client, "https://www.googleapis.com/drive/v3/changes/startPageToken?spaces=drive", accessToken) + if err != nil { + return err + } + var parsed struct { + StartPageToken string `json:"startPageToken"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return err + } + if parsed.StartPageToken != "" { + cursor["driveChangeToken"] = parsed.StartPageToken + } + return nil + default: + link, err := d.walkMicrosoftDriveDelta(ctx, accessToken, graphMicrosoftURL(d.userUPN, "/drive/root/delta?$select=id,name,folder,file,size,parentReference,deleted")) + if err != nil { + return err + } + if link != "" { + cursor["driveDeltaLink"] = link + } + return nil + } +} + +func (d *DriveImporter) walkMicrosoftDriveDelta(ctx context.Context, accessToken, listURL string) (string, error) { + for listURL != "" { + body, err := apiGet(ctx, d.client, listURL, accessToken) + if err != nil { + return "", err + } + var parsed struct { + NextLink string `json:"@odata.nextLink"` + DeltaLink string `json:"@odata.deltaLink"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return "", err + } + if parsed.DeltaLink != "" { + return parsed.DeltaLink, nil + } + listURL = parsed.NextLink + } + return "", nil +} + +func (d *DriveImporter) importDriveDelta(ctx context.Context, job *Job, accessToken, provider, ncUserID, root string, items *ImportedItemStore, update progressUpdater) error { + switch provider { + case "google": + return d.importGoogleDriveDelta(ctx, job, accessToken, ncUserID, root, items, update) + default: + return d.importMicrosoftDriveDelta(ctx, job, accessToken, ncUserID, root, items, update) + } +} + +func (d *DriveImporter) importGoogleDriveDelta(ctx context.Context, job *Job, accessToken, ncUserID, root string, items *ImportedItemStore, update progressUpdater) error { + pageToken, _ := job.CursorJSON["driveChangeToken"].(string) + if pageToken == "" { + return fmt.Errorf("google drive delta token missing") + } + + listURL := "https://www.googleapis.com/drive/v3/changes?pageSize=100&spaces=drive&includeRemoved=true&fields=" + + url.QueryEscape("nextPageToken,newStartPageToken,changes(fileId,removed,file(id,name,mimeType,size,parents,trashed))") + + "&pageToken=" + url.QueryEscape(pageToken) + + body, err := apiGet(ctx, d.client, listURL, accessToken) + if err != nil { + return err + } + var parsed struct { + Changes []googleDriveChange `json:"changes"` + NextPageToken string `json:"nextPageToken"` + NewStartPageToken string `json:"newStartPageToken"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return err + } + + imported, _ := job.StatsJSON["delta_imported"].(float64) + deleted, _ := job.StatsJSON["delta_deleted"].(float64) + exported, _ := job.StatsJSON["exported"].(float64) + skipped, _ := job.StatsJSON["skipped"].(float64) + batch := 0 + listIndex := int(jsonNumber(job.CursorJSON["listIndex"])) + + for i := listIndex; i < len(parsed.Changes) && batch < driveImportBatchSize(); i++ { + change := parsed.Changes[i] + if change.Removed || change.File == nil || change.File.Trashed { + if err := d.deleteDriveItem(ctx, ncUserID, root, items, change.FileID); err != nil { + return err + } + deleted++ + batch++ + continue + } + item := googleFileToDriveItem(*change.File) + relPath := d.resolveDriveRelPath(items, item) + if err := d.uploadDriveItem(ctx, accessToken, ncUserID, root, relPath, item, items, &imported, &exported, &skipped, job.StatsJSON); err != nil { + return err + } + batch++ + } + + job.StatsJSON["delta_imported"] = imported + job.StatsJSON["delta_deleted"] = deleted + job.StatsJSON["exported"] = exported + job.StatsJSON["skipped"] = skipped + + if listIndex+batch < len(parsed.Changes) { + job.CursorJSON["listIndex"] = float64(listIndex + batch) + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "listIndex") + + if parsed.NextPageToken != "" { + job.CursorJSON["driveChangeToken"] = parsed.NextPageToken + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + if parsed.NewStartPageToken != "" { + job.CursorJSON["driveChangeToken"] = parsed.NewStartPageToken + } + job.StatsJSON["phase"] = "delta" + return update("completed", job.CursorJSON, job.StatsJSON, "") +} + +type googleDriveChange struct { + FileID string `json:"fileId"` + Removed bool `json:"removed"` + File *googleDriveFile `json:"file"` +} + +type googleDriveFile struct { + ID string `json:"id"` + Name string `json:"name"` + MimeType string `json:"mimeType"` + Size string `json:"size"` + Parents []string `json:"parents"` + Trashed bool `json:"trashed"` +} + +func googleFileToDriveItem(f googleDriveFile) driveItem { + size := int64(0) + if f.Size != "" { + fmt.Sscan(f.Size, &size) + } + item := driveItem{ + ID: f.ID, + Name: f.Name, + IsFolder: f.MimeType == "application/vnd.google-apps.folder", + Size: size, + MimeType: f.MimeType, + } + if len(f.Parents) > 0 { + item.ParentID = f.Parents[0] + } + if item.IsFolder { + return item + } + if exportMime, ext, ok := googleWorkspaceExport(f.MimeType); ok { + item.Export = true + item.ExportMime = exportMime + item.ExportExt = ext + item.Name = driveExportFileName(f.Name, ext) + } else { + item.Download = "https://www.googleapis.com/drive/v3/files/" + url.PathEscape(f.ID) + "?alt=media" + } + return item +} + +func (d *DriveImporter) importMicrosoftDriveDelta(ctx context.Context, job *Job, accessToken, ncUserID, root string, items *ImportedItemStore, update progressUpdater) error { + deltaLink, _ := job.CursorJSON["driveDeltaLink"].(string) + if deltaLink == "" { + return fmt.Errorf("microsoft drive delta link missing") + } + + body, err := apiGet(ctx, d.client, deltaLink, accessToken) + if err != nil { + return err + } + var parsed struct { + Value []graphDriveItem `json:"value"` + NextLink string `json:"@odata.nextLink"` + DeltaLink string `json:"@odata.deltaLink"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return err + } + + imported, _ := job.StatsJSON["delta_imported"].(float64) + deleted, _ := job.StatsJSON["delta_deleted"].(float64) + skipped, _ := job.StatsJSON["skipped"].(float64) + batch := 0 + listIndex := int(jsonNumber(job.CursorJSON["listIndex"])) + + for i := listIndex; i < len(parsed.Value) && batch < driveImportBatchSize(); i++ { + item := parsed.Value[i] + sourceID := strings.TrimSpace(item.ID) + if sourceID == "" { + continue + } + if item.Removed != nil || item.Deleted != nil { + if err := d.deleteDriveItem(ctx, ncUserID, root, items, sourceID); err != nil { + return err + } + deleted++ + batch++ + continue + } + driveItem := graphDriveToItem(d.userUPN, item) + relPath := d.resolveDriveRelPath(items, driveItem) + if err := d.uploadDriveItem(ctx, accessToken, ncUserID, root, relPath, driveItem, items, &imported, nil, &skipped, job.StatsJSON); err != nil { + return err + } + batch++ + } + + job.StatsJSON["delta_imported"] = imported + job.StatsJSON["delta_deleted"] = deleted + job.StatsJSON["skipped"] = skipped + + if listIndex+batch < len(parsed.Value) { + job.CursorJSON["listIndex"] = float64(listIndex + batch) + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "listIndex") + + if parsed.NextLink != "" { + job.CursorJSON["driveDeltaLink"] = parsed.NextLink + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + if parsed.DeltaLink != "" { + job.CursorJSON["driveDeltaLink"] = parsed.DeltaLink + } + job.StatsJSON["phase"] = "delta" + return update("completed", job.CursorJSON, job.StatsJSON, "") +} + +type graphDriveItem struct { + ID string `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + Folder *struct{ ChildCount int `json:"childCount"` } `json:"folder"` + File *struct{ MimeType string `json:"mimeType"` } `json:"file"` + ParentReference *struct { + ID string `json:"id"` + } `json:"parentReference"` + Removed *struct { + Reason string `json:"reason"` + } `json:"@removed"` + Deleted *struct{} `json:"deleted"` +} + +func graphDriveToItem(userUPN string, item graphDriveItem) driveItem { + out := driveItem{ + ID: item.ID, + Name: item.Name, + Size: item.Size, + } + if item.ParentReference != nil { + out.ParentID = item.ParentReference.ID + } + if item.Folder != nil { + out.IsFolder = true + return out + } + mime := "" + if item.File != nil { + mime = item.File.MimeType + } + out.MimeType = mime + out.Download = graphMicrosoftURL(userUPN, "/drive/items/"+url.PathEscape(item.ID)+"/content") + return out +} + +func (d *DriveImporter) resolveDriveRelPath(items *ImportedItemStore, item driveItem) string { + if stored := items.Path(item.ID); stored != "" { + return stored + } + parentRel := "" + if item.ParentID != "" { + parentRel = items.Path(item.ParentID) + } + return path.Join(parentRel, sanitizeDrivePath(item.Name)) +} + +func (d *DriveImporter) uploadDriveItem(ctx context.Context, accessToken, ncUserID, root, relPath string, item driveItem, items *ImportedItemStore, imported, exported, skipped *float64, stats map[string]any) error { + targetPath := path.Join(root, relPath) + if item.IsFolder { + if err := d.nc.CreateFolder(ctx, ncUserID, targetPath); err != nil { + if markErr := items.MarkFailed(ctx, item.ID, err.Error(), relPath); markErr != nil { + return markErr + } + incJobStat(stats, "failed") + return nil + } + if err := items.MarkPath(ctx, item.ID, relPath); err != nil { + return err + } + if imported != nil { + *imported++ + } + return nil + } + if item.Export { + content, contentType, fileName, err := d.downloadGoogleExport(ctx, accessToken, item) + if err != nil { + if skipped != nil { + *skipped++ + } + return items.MarkSkipped(ctx, item.ID, "export: "+err.Error(), relPath) + } + targetPath = path.Join(path.Dir(targetPath), fileName) + relPath = path.Join(path.Dir(relPath), fileName) + if err := d.nc.Upload(ctx, ncUserID, targetPath, content, contentType); err != nil { + if markErr := items.MarkFailed(ctx, item.ID, err.Error(), relPath); markErr != nil { + return markErr + } + incJobStat(stats, "failed") + return nil + } + if exported != nil { + *exported++ + } + if pdfMime, pdfExt, ok := googleSlidesPDFExport(item.MimeType); ok { + pdfItem := item + pdfItem.ExportMime = pdfMime + pdfItem.ExportExt = pdfExt + pdfContent, pdfType, pdfName, err := d.downloadGoogleExport(ctx, accessToken, pdfItem) + if err == nil { + pdfRel := path.Join(path.Dir(relPath), pdfName) + pdfTarget := path.Join(root, pdfRel) + if err := d.nc.Upload(ctx, ncUserID, pdfTarget, pdfContent, pdfType); err == nil { + if err := items.MarkPath(ctx, item.ID+"_pdf", pdfRel); err != nil { + return err + } + } + } + } + } else { + if item.Size > maxDriveFileBytes { + if skipped != nil { + *skipped++ + } + reason := fmt.Sprintf("file exceeds %d byte limit", maxDriveFileBytes) + return items.MarkSkipped(ctx, item.ID, reason, relPath) + } + content, contentType, err := d.downloadDriveFile(ctx, accessToken, item) + if err != nil { + if markErr := items.MarkFailed(ctx, item.ID, err.Error(), relPath); markErr != nil { + return markErr + } + incJobStat(stats, "failed") + return nil + } + if err := d.nc.Upload(ctx, ncUserID, targetPath, content, contentType); err != nil { + if markErr := items.MarkFailed(ctx, item.ID, err.Error(), relPath); markErr != nil { + return markErr + } + incJobStat(stats, "failed") + return nil + } + } + if err := items.MarkImported(ctx, item.ID); err != nil { + return err + } + if err := items.MarkPath(ctx, item.ID, relPath); err != nil { + return err + } + if imported != nil { + *imported++ + } + return nil +} + +func (d *DriveImporter) deleteDriveItem(ctx context.Context, ncUserID, root string, items *ImportedItemStore, fileID string) error { + if fileID == "" { + return nil + } + for _, suffix := range []string{"", "_pdf"} { + rel := items.Path(fileID + suffix) + if rel == "" { + continue + } + target := path.Join(root, rel) + if err := d.nc.Delete(ctx, ncUserID, target); err != nil && !isDeleteNotFound(err) { + return err + } + } + if err := items.Unmark(ctx, fileID); err != nil { + return err + } + return items.Unmark(ctx, fileID+"_pdf") +} + +func isDeleteNotFound(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "404") || strings.Contains(msg, "not found") +} diff --git a/internal/migration/drive_delta_test.go b/internal/migration/drive_delta_test.go new file mode 100644 index 0000000..cbc98bc --- /dev/null +++ b/internal/migration/drive_delta_test.go @@ -0,0 +1,110 @@ +package migration + +import ( + "context" + "net/http" + "strings" + "testing" +) + +func TestBootstrapDriveDeltaGoogle(t *testing.T) { + client := mockGoogleHTTPClient(t, func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/drive/v3/changes/startPageToken") { + _, _ = w.Write([]byte(`{"startPageToken":"start-123"}`)) + return + } + http.NotFound(w, r) + }) + + d := NewDriveImporter(nil, nil).WithHTTPClient(client) + cursor := map[string]any{} + if err := d.bootstrapDriveDelta(context.Background(), "token", "google", cursor); err != nil { + t.Fatalf("bootstrap: %v", err) + } + if cursor["driveChangeToken"] != "start-123" { + t.Fatalf("token = %v", cursor["driveChangeToken"]) + } +} + +func TestHasDriveDeltaCursor(t *testing.T) { + d := &DriveImporter{} + job := &Job{CursorJSON: map[string]any{}} + if d.hasDriveDeltaCursor(job, "google") { + t.Fatal("expected no google token") + } + job.CursorJSON["driveChangeToken"] = "tok" + if !d.hasDriveDeltaCursor(job, "google") { + t.Fatal("expected google token") + } + job.CursorJSON = map[string]any{"driveDeltaLink": "https://graph.microsoft.com/delta"} + if !d.hasDriveDeltaCursor(job, "microsoft") { + t.Fatal("expected microsoft delta link") + } +} + +func TestGoogleFileToDriveItem(t *testing.T) { + folder := googleFileToDriveItem(googleDriveFile{ + ID: "f1", Name: "Docs", MimeType: "application/vnd.google-apps.folder", Parents: []string{"root"}, + }) + if !folder.IsFolder || folder.ParentID != "root" { + t.Fatalf("folder: %#v", folder) + } + + slides := googleFileToDriveItem(googleDriveFile{ + ID: "s1", Name: "Deck", MimeType: "application/vnd.google-apps.presentation", + }) + if !slides.Export || slides.ExportExt != ".pptx" { + t.Fatalf("slides export: %#v", slides) + } + + binary := googleFileToDriveItem(googleDriveFile{ + ID: "b1", Name: "photo.png", MimeType: "image/png", Size: "1024", + }) + if binary.Export || binary.Download == "" || binary.Size != 1024 { + t.Fatalf("binary file: %#v", binary) + } +} + +func TestGraphDriveToItem(t *testing.T) { + item := graphDriveToItem("", graphDriveItem{ + ID: "item-1", + Name: "report.pdf", + Size: 4096, + File: &struct{ MimeType string `json:"mimeType"` }{MimeType: "application/pdf"}, + ParentReference: &struct { + ID string `json:"id"` + }{ID: "parent-1"}, + }) + if item.IsFolder || item.ParentID != "parent-1" || item.Download == "" { + t.Fatalf("file item: %#v", item) + } + + folder := graphDriveToItem("", graphDriveItem{ + ID: "dir-1", Name: "Shared", Folder: &struct{ ChildCount int `json:"childCount"` }{ChildCount: 2}, + }) + if !folder.IsFolder { + t.Fatalf("folder item: %#v", folder) + } +} + +func TestResolveDriveRelPath(t *testing.T) { + d := &DriveImporter{} + store := NewImportedItemStoreMemory() + ctx := context.Background() + if err := store.MarkPath(ctx, "parent", "Projects"); err != nil { + t.Fatal(err) + } + if err := store.MarkPath(ctx, "file-1", "Projects/old-name.docx"); err != nil { + t.Fatal(err) + } + + if got := d.resolveDriveRelPath(store, driveItem{ID: "file-1", Name: "ignored.docx"}); got != "Projects/old-name.docx" { + t.Fatalf("stored path wins: %q", got) + } + if got := d.resolveDriveRelPath(store, driveItem{ID: "new", Name: "readme.txt", ParentID: "parent"}); got != "Projects/readme.txt" { + t.Fatalf("parent path join: %q", got) + } + if got := d.resolveDriveRelPath(store, driveItem{ID: "orphan", Name: "solo.txt"}); got != "solo.txt" { + t.Fatalf("root file: %q", got) + } +} diff --git a/internal/migration/drive_helpers.go b/internal/migration/drive_helpers.go new file mode 100644 index 0000000..b2b7a9a --- /dev/null +++ b/internal/migration/drive_helpers.go @@ -0,0 +1,87 @@ +package migration + +import "strings" + +// googleWorkspaceExport maps Google native mime types to export targets. +func googleWorkspaceExport(mimeType string) (exportMime, ext string, ok bool) { + switch mimeType { + case "application/vnd.google-apps.document": + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".docx", true + case "application/vnd.google-apps.spreadsheet": + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ".xlsx", true + case "application/vnd.google-apps.presentation": + return "application/vnd.openxmlformats-officedocument.presentationml.presentation", ".pptx", true + case "application/vnd.google-apps.drawing": + return "application/pdf", ".pdf", true + case "application/vnd.google-apps.script": + return "application/vnd.google-apps.script+json", ".json", true + case "application/vnd.google-apps.site": + return "text/plain", ".txt", true + default: + return "", "", false + } +} + +// googleSlidesPDFExport returns PDF export for Google Slides (companion copy). +func googleSlidesPDFExport(mimeType string) (exportMime, ext string, ok bool) { + if mimeType == "application/vnd.google-apps.presentation" { + return "application/pdf", ".pdf", true + } + return "", "", false +} + +func driveExportFileName(name, ext string) string { + name = strings.TrimSpace(name) + if name == "" { + name = "untitled" + } + if ext != "" && !strings.HasSuffix(strings.ToLower(name), strings.ToLower(ext)) { + return name + ext + } + return name +} + +type driveFolderRef struct { + ID string + Path string +} + +func readDriveFolderQueue(cursor map[string]any, provider string) []driveFolderRef { + raw, _ := cursor["folderQueue"].([]any) + out := make([]driveFolderRef, 0, len(raw)) + for _, item := range raw { + m, _ := item.(map[string]any) + if m == nil { + continue + } + id, _ := m["id"].(string) + p, _ := m["path"].(string) + if id != "" { + out = append(out, driveFolderRef{ID: id, Path: p}) + } + } + if len(out) == 0 { + if provider == "google" { + return []driveFolderRef{{ID: "root", Path: ""}} + } + return []driveFolderRef{{ID: "root", Path: ""}} + } + return out +} + +func writeDriveFolderQueue(cursor map[string]any, queue []driveFolderRef) { + raw := make([]any, 0, len(queue)) + for _, f := range queue { + raw = append(raw, map[string]any{"id": f.ID, "path": f.Path}) + } + cursor["folderQueue"] = raw +} + +func enqueueDriveFolder(queue []driveFolderRef, folder driveFolderRef) []driveFolderRef { + for _, existing := range queue { + if existing.ID == folder.ID { + return queue + } + } + return append(queue, folder) +} diff --git a/internal/migration/drive_helpers_test.go b/internal/migration/drive_helpers_test.go new file mode 100644 index 0000000..8870526 --- /dev/null +++ b/internal/migration/drive_helpers_test.go @@ -0,0 +1,95 @@ +package migration + +import ( + "context" + "testing" +) + +func TestGoogleWorkspaceExport(t *testing.T) { + mime, ext, ok := googleWorkspaceExport("application/vnd.google-apps.document") + if !ok || ext != ".docx" || mime == "" { + t.Fatalf("document export: mime=%q ext=%q ok=%v", mime, ext, ok) + } + _, _, ok = googleWorkspaceExport("application/vnd.google-apps.folder") + if ok { + t.Fatal("folder should not export") + } +} + +func TestDriveExportFileName(t *testing.T) { + got := driveExportFileName("Report", ".docx") + if got != "Report.docx" { + t.Fatalf("got %q", got) + } + got = driveExportFileName("Report.docx", ".docx") + if got != "Report.docx" { + t.Fatalf("got %q", got) + } +} + +func TestDriveFolderQueue(t *testing.T) { + cursor := map[string]any{} + queue := readDriveFolderQueue(cursor, "google") + if len(queue) != 1 || queue[0].ID != "root" { + t.Fatalf("default queue: %#v", queue) + } + queue = enqueueDriveFolder(queue, driveFolderRef{ID: "abc", Path: "Docs"}) + writeDriveFolderQueue(cursor, queue) + if len(readDriveFolderQueue(cursor, "google")) != 2 { + t.Fatal("expected 2 folders in queue") + } +} + +func TestGoogleSlidesPDFExport(t *testing.T) { + mime, ext, ok := googleSlidesPDFExport("application/vnd.google-apps.presentation") + if !ok || ext != ".pdf" || mime != "application/pdf" { + t.Fatalf("slides pdf export: mime=%q ext=%q ok=%v", mime, ext, ok) + } + _, _, ok = googleSlidesPDFExport("application/vnd.google-apps.document") + if ok { + t.Fatal("document should not have slides pdf export") + } +} + +func TestImportedPathHelpers(t *testing.T) { + store := NewImportedItemStoreMemory() + ctx := context.Background() + if err := store.MarkPath(ctx, "file-1", "Docs/report.docx"); err != nil { + t.Fatal(err) + } + if got := store.Path("file-1"); got != "Docs/report.docx" { + t.Fatalf("imported path: got %q", got) + } + if err := store.Unmark(ctx, "file-1"); err != nil { + t.Fatal(err) + } + if got := store.Path("file-1"); got != "" { + t.Fatalf("expected empty after unmark, got %q", got) + } +} + +func TestGoogleCalendarCancelledEvent(t *testing.T) { + ev := googleCalendarEvent{ID: "evt1", Status: "cancelled"}.toSourceEvent("primary") + if !ev.Deleted { + t.Fatal("cancelled event should be deleted") + } +} + +func TestGraphCalendarRemovedEvent(t *testing.T) { + removed := struct { + Reason string `json:"reason"` + }{Reason: "deleted"} + ev := graphCalendarEvent{ID: "evt1", Removed: &removed}.toSourceEvent("cal1") + if !ev.Deleted { + t.Fatal("removed event should be deleted") + } +} + +func TestCalendarSyncTokenHelpers(t *testing.T) { + cursor := map[string]any{} + setCalendarSyncToken(cursor, "primary", "token-1") + tokens := calendarSyncTokens(cursor) + if tokens["primary"] != "token-1" { + t.Fatalf("got %#v", tokens) + } +} diff --git a/internal/migration/drive_import.go b/internal/migration/drive_import.go new file mode 100644 index 0000000..aed8f52 --- /dev/null +++ b/internal/migration/drive_import.go @@ -0,0 +1,405 @@ +package migration + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/ultisuite/ulti-backend/internal/nextcloud" +) + +const maxDriveFileBytes = 25 * 1024 * 1024 + +type DriveImporter struct { + db *pgxpool.Pool + nc *nextcloud.Client + client *http.Client + userUPN string +} + +func NewDriveImporter(db *pgxpool.Pool, nc *nextcloud.Client) *DriveImporter { + return &DriveImporter{db: db, nc: nc, client: migrationHTTPClient()} +} + +func (d *DriveImporter) WithUserPrincipal(upn string) *DriveImporter { + d.userUPN = strings.TrimSpace(upn) + return d +} + +func (d *DriveImporter) WithHTTPClient(c *http.Client) *DriveImporter { + if c != nil { + d.client = c + } + return d +} + +func (d *DriveImporter) ImportBatch(ctx context.Context, job *Job, accessToken, provider string, delta bool, update progressUpdater) error { + if d.nc == nil { + return fmt.Errorf("nextcloud required for drive migration") + } + user, err := resolveMigrationUser(ctx, d.db, job.UserID) + if err != nil { + return err + } + ncUserID := nextcloud.UserIDFromClaims(user.Email, user.ExternalID) + if _, err := d.nc.EnsurePrincipal(ctx, user.Email, user.ExternalID, user.Name); err != nil { + return fmt.Errorf("nextcloud user: %w", err) + } + root := fmt.Sprintf("/Migration/%s", provider) + _ = d.nc.CreateFolder(ctx, ncUserID, root) + store, err := LoadImportedItemStore(ctx, d.db, job.ID, job.CursorJSON) + if err != nil { + return err + } + + if delta && d.hasDriveDeltaCursor(job, provider) { + return d.importDriveDelta(ctx, job, accessToken, provider, ncUserID, root, store, update) + } + + imported, _ := job.StatsJSON["imported"].(float64) + skipped, _ := job.StatsJSON["skipped"].(float64) + exported, _ := job.StatsJSON["exported"].(float64) + batch := 0 + + queue := readDriveFolderQueue(job.CursorJSON, provider) + folderIndex := int(jsonNumber(job.CursorJSON["folderIndex"])) + if folderIndex >= len(queue) { + if delta && !d.hasDriveDeltaCursor(job, provider) { + if err := d.bootstrapDriveDelta(ctx, accessToken, provider, job.CursorJSON); err != nil { + return err + } + } + job.StatsJSON["imported"] = imported + job.StatsJSON["skipped"] = skipped + job.StatsJSON["exported"] = exported + if delta && d.hasDriveDeltaCursor(job, provider) { + job.StatsJSON["phase"] = "delta_ready" + } else { + job.StatsJSON["phase"] = "imported" + } + return update("completed", job.CursorJSON, job.StatsJSON, "") + } + + current := queue[folderIndex] + folderItems, nextCursor, subfolders, err := d.listDriveFolderItems(ctx, accessToken, provider, current, job.CursorJSON) + if err != nil { + return err + } + + listIndex := int(jsonNumber(job.CursorJSON["listIndex"])) + for i := listIndex; i < len(folderItems) && batch < driveImportBatchSize(); i++ { + item := folderItems[i] + if alreadyImported(store, item.ID) { + continue + } + relPath := path.Join(current.Path, sanitizeDrivePath(item.Name)) + targetPath := path.Join(root, relPath) + if item.IsFolder { + if err := d.nc.CreateFolder(ctx, ncUserID, targetPath); err != nil { + if markErr := store.MarkFailed(ctx, item.ID, err.Error(), relPath); markErr != nil { + return markErr + } + incJobStat(job.StatsJSON, "failed") + batch++ + continue + } + if err := store.MarkPath(ctx, item.ID, relPath); err != nil { + return err + } + queue = enqueueDriveFolder(queue, driveFolderRef{ID: item.ID, Path: relPath}) + } else { + if item.Export { + content, contentType, fileName, err := d.downloadGoogleExport(ctx, accessToken, item) + if err != nil { + skipped++ + if err := store.MarkSkipped(ctx, item.ID, "export: "+err.Error(), relPath); err != nil { + return err + } + batch++ + continue + } + targetPath = path.Join(path.Dir(targetPath), fileName) + relPath = path.Join(path.Dir(relPath), fileName) + if err := d.nc.Upload(ctx, ncUserID, targetPath, content, contentType); err != nil { + if markErr := store.MarkFailed(ctx, item.ID, err.Error(), relPath); markErr != nil { + return markErr + } + incJobStat(job.StatsJSON, "failed") + batch++ + continue + } + exported++ + if pdfMime, pdfExt, ok := googleSlidesPDFExport(item.MimeType); ok { + pdfItem := item + pdfItem.ExportMime = pdfMime + pdfItem.ExportExt = pdfExt + if pdfContent, pdfType, pdfName, err := d.downloadGoogleExport(ctx, accessToken, pdfItem); err == nil { + pdfRel := path.Join(path.Dir(relPath), pdfName) + pdfTarget := path.Join(root, pdfRel) + if err := d.nc.Upload(ctx, ncUserID, pdfTarget, pdfContent, pdfType); err == nil { + if err := store.MarkPath(ctx, item.ID+"_pdf", pdfRel); err != nil { + return err + } + } + } + } + } else { + if item.Size > maxDriveFileBytes { + skipped++ + reason := fmt.Sprintf("file exceeds %d byte limit", maxDriveFileBytes) + if err := store.MarkSkipped(ctx, item.ID, reason, relPath); err != nil { + return err + } + batch++ + continue + } + content, contentType, err := d.downloadDriveFile(ctx, accessToken, item) + if err != nil { + if markErr := store.MarkFailed(ctx, item.ID, err.Error(), relPath); markErr != nil { + return markErr + } + incJobStat(job.StatsJSON, "failed") + batch++ + continue + } + if err := d.nc.Upload(ctx, ncUserID, targetPath, content, contentType); err != nil { + if markErr := store.MarkFailed(ctx, item.ID, err.Error(), relPath); markErr != nil { + return markErr + } + incJobStat(job.StatsJSON, "failed") + batch++ + continue + } + } + } + if err := store.MarkImported(ctx, item.ID); err != nil { + return err + } + if !item.IsFolder { + if err := store.MarkPath(ctx, item.ID, relPath); err != nil { + return err + } + } + imported++ + batch++ + } + + for _, sub := range subfolders { + relPath := path.Join(current.Path, sanitizeDrivePath(sub.Name)) + queue = enqueueDriveFolder(queue, driveFolderRef{ID: sub.ID, Path: relPath}) + } + writeDriveFolderQueue(job.CursorJSON, queue) + + job.StatsJSON["imported"] = imported + job.StatsJSON["skipped"] = skipped + job.StatsJSON["exported"] = exported + + if listIndex+batch < len(folderItems) { + job.CursorJSON["listIndex"] = float64(listIndex + batch) + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "listIndex") + + if nextCursor != "" { + if provider == "google" { + job.CursorJSON["pageToken"] = nextCursor + } else { + job.CursorJSON["nextLink"] = nextCursor + } + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "pageToken") + delete(job.CursorJSON, "nextLink") + + job.CursorJSON["folderIndex"] = float64(folderIndex + 1) + delete(job.CursorJSON, "listIndex") + return update("pending", job.CursorJSON, job.StatsJSON, "") +} + +type driveItem struct { + ID string + Name string + ParentID string + IsFolder bool + Size int64 + MimeType string + Download string + Export bool + ExportMime string + ExportExt string +} + +type driveSubfolder struct { + ID string + Name string +} + +func (d *DriveImporter) listDriveFolderItems(ctx context.Context, accessToken, provider string, folder driveFolderRef, cursor map[string]any) ([]driveItem, string, []driveSubfolder, error) { + switch provider { + case "google": + pageToken, _ := cursor["pageToken"].(string) + q := url.QueryEscape("'" + folder.ID + "' in parents and trashed=false") + listURL := "https://www.googleapis.com/drive/v3/files?pageSize=100&fields=nextPageToken,files(id,name,mimeType,size)&q=" + q + if pageToken != "" { + listURL += "&pageToken=" + url.QueryEscape(pageToken) + } + body, err := apiGet(ctx, d.client, listURL, accessToken) + if err != nil { + return nil, "", nil, err + } + var parsed struct { + Files []struct { + ID string `json:"id"` + Name string `json:"name"` + MimeType string `json:"mimeType"` + Size string `json:"size"` + } `json:"files"` + NextPageToken string `json:"nextPageToken"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, "", nil, err + } + out := make([]driveItem, 0, len(parsed.Files)) + for _, f := range parsed.Files { + size := int64(0) + if f.Size != "" { + fmt.Sscan(f.Size, &size) + } + item := driveItem{ + ID: f.ID, + Name: f.Name, + IsFolder: f.MimeType == "application/vnd.google-apps.folder", + Size: size, + MimeType: f.MimeType, + } + if item.IsFolder { + out = append(out, item) + continue + } + if exportMime, ext, ok := googleWorkspaceExport(f.MimeType); ok { + item.Export = true + item.ExportMime = exportMime + item.ExportExt = ext + item.Name = driveExportFileName(f.Name, ext) + } else { + item.Download = "https://www.googleapis.com/drive/v3/files/" + url.PathEscape(f.ID) + "?alt=media" + } + out = append(out, item) + } + return out, parsed.NextPageToken, nil, nil + default: + nextLink, _ := cursor["nextLink"].(string) + var listURL string + if folder.ID == "root" { + listURL = graphMicrosoftURL(d.userUPN, "/drive/root/children?$top=100&$select=id,name,folder,file,size") + } else { + listURL = graphMicrosoftURL(d.userUPN, "/drive/items/"+url.PathEscape(folder.ID)+"/children?$top=100&$select=id,name,folder,file,size") + } + if nextLink != "" { + listURL = nextLink + } + body, err := apiGet(ctx, d.client, listURL, accessToken) + if err != nil { + return nil, "", nil, err + } + var parsed struct { + Value []struct { + ID string `json:"id"` + Name string `json:"name"` + Folder *struct { + ChildCount int `json:"childCount"` + } `json:"folder"` + File *struct { + MimeType string `json:"mimeType"` + } `json:"file"` + Size int64 `json:"size"` + } `json:"value"` + NextLink string `json:"@odata.nextLink"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, "", nil, err + } + out := make([]driveItem, 0, len(parsed.Value)) + var subs []driveSubfolder + for _, f := range parsed.Value { + if f.Folder != nil { + out = append(out, driveItem{ID: f.ID, Name: f.Name, IsFolder: true}) + if f.Folder.ChildCount > 0 { + subs = append(subs, driveSubfolder{ID: f.ID, Name: f.Name}) + } + continue + } + mime := "" + if f.File != nil { + mime = f.File.MimeType + } + out = append(out, driveItem{ + ID: f.ID, + Name: f.Name, + Size: f.Size, + MimeType: mime, + Download: graphMicrosoftURL(d.userUPN, "/drive/items/"+url.PathEscape(f.ID)+"/content"), + }) + } + return out, parsed.NextLink, subs, nil + } +} + +func (d *DriveImporter) downloadDriveFile(ctx context.Context, accessToken string, item driveItem) (io.ReadCloser, string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, item.Download, nil) + if err != nil { + return nil, "", err + } + req.Header.Set("Authorization", "Bearer "+accessToken) + resp, err := migrationDo(ctx, d.client, req) + if err != nil { + return nil, "", fmt.Errorf("download %s: %w", item.Name, err) + } + contentType := resp.Header.Get("Content-Type") + if contentType == "" { + contentType = item.MimeType + } + if contentType == "" { + contentType = "application/octet-stream" + } + return resp.Body, contentType, nil +} + +func (d *DriveImporter) downloadGoogleExport(ctx context.Context, accessToken string, item driveItem) (io.ReadCloser, string, string, error) { + exportURL := fmt.Sprintf( + "https://www.googleapis.com/drive/v3/files/%s/export?mimeType=%s", + url.PathEscape(item.ID), + url.QueryEscape(item.ExportMime), + ) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, exportURL, nil) + if err != nil { + return nil, "", "", err + } + req.Header.Set("Authorization", "Bearer "+accessToken) + resp, err := migrationDo(ctx, d.client, req) + if err != nil { + return nil, "", "", fmt.Errorf("export %s: %w", item.Name, err) + } + contentType := resp.Header.Get("Content-Type") + if contentType == "" { + contentType = item.ExportMime + } + return resp.Body, contentType, driveExportFileName(item.Name, item.ExportExt), nil +} + +func sanitizeDrivePath(name string) string { + name = strings.TrimSpace(name) + name = strings.ReplaceAll(name, "/", "-") + name = strings.ReplaceAll(name, "\\", "-") + if name == "" { + return "untitled" + } + return name +} diff --git a/internal/migration/gmail_attachments.go b/internal/migration/gmail_attachments.go new file mode 100644 index 0000000..77cd404 --- /dev/null +++ b/internal/migration/gmail_attachments.go @@ -0,0 +1,285 @@ +package migration + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "mime" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/ultisuite/ulti-backend/internal/mail/limits" + "github.com/ultisuite/ulti-backend/internal/mail/storage" +) + +type gmailAttachmentPart struct { + Filename string + MimeType string + ContentID string + IsInline bool + Size int + BodyData string + AttachmentID string +} + +func extractGmailAttachmentParts(payload gmailPayload) []gmailAttachmentPart { + var out []gmailAttachmentPart + var walk func(gmailPayload) + walk = func(node gmailPayload) { + headers := gmailHeaderMap(node.Headers) + mimeType := strings.TrimSpace(node.MimeType) + if strings.HasPrefix(strings.ToLower(mimeType), "multipart/") { + for _, part := range node.Parts { + walk(part) + } + return + } + if isGmailAttachmentPart(mimeType, headers, node.Body) { + out = append(out, buildGmailAttachmentPart(mimeType, headers, node.Body)) + } + } + walk(payload) + if len(out) > limits.MaxAttachmentsPerMessage { + return out[:limits.MaxAttachmentsPerMessage] + } + return out +} + +func gmailHeaderMap(headers []gmailHeader) map[string]string { + out := make(map[string]string, len(headers)) + for _, h := range headers { + key := strings.ToLower(strings.TrimSpace(h.Name)) + if key != "" && out[key] == "" { + out[key] = h.Value + } + } + return out +} + +func isGmailAttachmentPart(mimeType string, headers map[string]string, body gmailBody) bool { + if body.AttachmentID != "" { + return true + } + disposition := strings.ToLower(headers["content-disposition"]) + if strings.HasPrefix(disposition, "attachment") { + return true + } + contentID := normalizeGmailContentID(headers["content-id"]) + filename := gmailPartFilename(headers) + if strings.HasPrefix(disposition, "inline") && (filename != "" || contentID != "") { + return true + } + if contentID != "" && body.Data != "" { + return true + } + lower := strings.ToLower(strings.TrimSpace(mimeType)) + if lower == "text/plain" || lower == "text/html" || lower == "" { + return false + } + return filename != "" && body.Data != "" +} + +func buildGmailAttachmentPart(mimeType string, headers map[string]string, body gmailBody) gmailAttachmentPart { + disposition := strings.ToLower(headers["content-disposition"]) + contentID := normalizeGmailContentID(headers["content-id"]) + filename := gmailPartFilename(headers) + isInline := strings.HasPrefix(disposition, "inline") || (contentID != "" && !strings.HasPrefix(disposition, "attachment")) + if filename == "" { + filename = inlineGmailAttachmentFilename(contentID, mimeType) + } + size := body.Size + if size <= 0 && body.Data != "" { + size = len(body.Data) + } + return gmailAttachmentPart{ + Filename: filename, + MimeType: normalizeAttachmentMimeType(mimeType), + ContentID: contentID, + IsInline: isInline, + Size: size, + BodyData: body.Data, + AttachmentID: body.AttachmentID, + } +} + +func gmailPartFilename(headers map[string]string) string { + if name := strings.TrimSpace(headers["filename"]); name != "" { + return name + } + disposition := headers["content-disposition"] + if disposition == "" { + return "" + } + _, params, err := mime.ParseMediaType(disposition) + if err != nil { + return "" + } + return strings.TrimSpace(params["filename"]) +} + +func normalizeGmailContentID(raw string) string { + return strings.Trim(raw, "<> \t") +} + +func inlineGmailAttachmentFilename(contentID, mimeType string) string { + base := "inline" + if contentID != "" { + base = strings.Map(func(r rune) rune { + switch r { + case '<', '>', '/', '\\', ':', '"', '\'', '?', '*': + return '_' + default: + return r + } + }, contentID) + } + ext := extensionFromMimeType(mimeType) + if ext != "" && !strings.HasSuffix(strings.ToLower(base), ext) { + return base + ext + } + return base +} + +func extensionFromMimeType(mimeType string) string { + switch strings.ToLower(strings.TrimSpace(strings.Split(mimeType, ";")[0])) { + case "image/jpeg", "image/jpg": + return ".jpg" + case "image/png": + return ".png" + case "image/gif": + return ".gif" + case "image/webp": + return ".webp" + case "application/pdf": + return ".pdf" + default: + return "" + } +} + +func normalizeAttachmentMimeType(mimeType string) string { + mimeType = strings.TrimSpace(mimeType) + if mimeType == "" { + return "application/octet-stream" + } + mediaType, _, err := mime.ParseMediaType(mimeType) + if err != nil || mediaType == "" { + return mimeType + } + return mediaType +} + +func (g *GmailImporter) storeGmailAttachments( + ctx context.Context, + userID, messageID, gmailID, accessToken string, + payload gmailPayload, + messageExisted bool, +) error { + if g.storage == nil { + return nil + } + parts := extractGmailAttachmentParts(payload) + if len(parts) == 0 { + return nil + } + + var existingCount int + var existingTotal int64 + if err := g.db.QueryRow(ctx, ` + SELECT COUNT(*)::int, COALESCE(SUM(size), 0)::bigint + FROM attachments WHERE message_id = $1::uuid + `, messageID).Scan(&existingCount, &existingTotal); err != nil { + return err + } + + stored := 0 + for _, part := range parts { + if messageExisted && gmailAttachmentExists(ctx, g.db, messageID, part) { + continue + } + data, err := g.loadGmailAttachmentData(ctx, accessToken, gmailID, part) + if err != nil { + return err + } + if len(data) == 0 { + continue + } + if err := limits.ValidateAttachmentSize(int64(len(data))); err != nil { + continue + } + if err := limits.ValidateAttachmentQuota(existingCount, existingTotal, int64(len(data))); err != nil { + break + } + + objectKey := storage.MessageObjectKey(userID, messageID, part.Filename) + if err := g.storage.Put(ctx, objectKey, bytes.NewReader(data), int64(len(data)), part.MimeType); err != nil { + return err + } + bucket := g.attachBucket + if bucket == "" { + bucket = "mail-attachments" + } + _, err = g.db.Exec(ctx, ` + INSERT INTO attachments (message_id, filename, content_type, size, s3_bucket, s3_key, content_id, is_inline, virus_scan_status) + VALUES ($1::uuid, $2, $3, $4, $5, $6, $7, $8, 'skipped') + `, messageID, part.Filename, part.MimeType, len(data), bucket, objectKey, part.ContentID, part.IsInline) + if err != nil { + _ = g.storage.Delete(ctx, objectKey) + return err + } + existingCount++ + existingTotal += int64(len(data)) + stored++ + } + + if stored > 0 { + _, err := g.db.Exec(ctx, `UPDATE messages SET has_attachments = true, updated_at = NOW() WHERE id = $1::uuid`, messageID) + return err + } + return nil +} + +func (g *GmailImporter) loadGmailAttachmentData(ctx context.Context, accessToken, gmailID string, part gmailAttachmentPart) ([]byte, error) { + if part.AttachmentID != "" { + url := fmt.Sprintf( + "https://gmail.googleapis.com/gmail/v1/users/me/messages/%s/attachments/%s", + gmailID, + part.AttachmentID, + ) + raw, err := g.apiGet(ctx, url, accessToken) + if err != nil { + return nil, err + } + var parsed struct { + Data string `json:"data"` + } + if err := json.Unmarshal(raw, &parsed); err != nil { + return nil, err + } + if parsed.Data == "" { + return nil, nil + } + return []byte(decodeGmailBody(parsed.Data)), nil + } + if part.BodyData == "" { + return nil, nil + } + return []byte(decodeGmailBody(part.BodyData)), nil +} + +func gmailAttachmentExists(ctx context.Context, db *pgxpool.Pool, messageID string, part gmailAttachmentPart) bool { + var count int + if part.ContentID != "" { + _ = db.QueryRow(ctx, ` + SELECT COUNT(*) FROM attachments + WHERE message_id = $1::uuid AND (content_id = $2 OR filename = $3) + `, messageID, part.ContentID, part.Filename).Scan(&count) + return count > 0 + } + _ = db.QueryRow(ctx, ` + SELECT COUNT(*) FROM attachments WHERE message_id = $1::uuid AND filename = $2 + `, messageID, part.Filename).Scan(&count) + return count > 0 +} diff --git a/internal/migration/gmail_attachments_test.go b/internal/migration/gmail_attachments_test.go new file mode 100644 index 0000000..fb712b2 --- /dev/null +++ b/internal/migration/gmail_attachments_test.go @@ -0,0 +1,55 @@ +package migration + +import "testing" + +func TestExtractGmailAttachmentPartsInlineData(t *testing.T) { + payload := gmailPayload{ + MimeType: "multipart/mixed", + Parts: []gmailPayload{ + { + MimeType: "text/plain", + Body: gmailBody{Data: "dGV4dA=="}, + }, + { + MimeType: "image/png", + Headers: []gmailHeader{ + {Name: "Content-Disposition", Value: `attachment; filename="logo.png"`}, + }, + Body: gmailBody{Data: "aW1n", Size: 3}, + }, + }, + } + parts := extractGmailAttachmentParts(payload) + if len(parts) != 1 { + t.Fatalf("expected 1 attachment, got %d", len(parts)) + } + if parts[0].Filename != "logo.png" || parts[0].MimeType != "image/png" { + t.Fatalf("unexpected part: %#v", parts[0]) + } +} + +func TestExtractGmailAttachmentPartsAttachmentID(t *testing.T) { + payload := gmailPayload{ + MimeType: "multipart/mixed", + Parts: []gmailPayload{ + { + MimeType: "application/pdf", + Headers: []gmailHeader{ + {Name: "Content-Disposition", Value: `attachment; filename="report.pdf"`}, + }, + Body: gmailBody{AttachmentID: "ANGjdJ_test", Size: 4096}, + }, + }, + } + parts := extractGmailAttachmentParts(payload) + if len(parts) != 1 || parts[0].AttachmentID != "ANGjdJ_test" { + t.Fatalf("unexpected parts: %#v", parts) + } +} + +func TestDecodeGmailBodyBytes(t *testing.T) { + got := decodeGmailBody("aGVsbG8=") + if got != "hello" { + t.Fatalf("decode = %q", got) + } +} diff --git a/internal/migration/gmail_delta_test.go b/internal/migration/gmail_delta_test.go new file mode 100644 index 0000000..cb20c1d --- /dev/null +++ b/internal/migration/gmail_delta_test.go @@ -0,0 +1,30 @@ +package migration + +import "testing" + +func TestIsGmailHistoryNotFound(t *testing.T) { + if !isGmailHistoryNotFound(fmtError("gmail api 404 Not Found: historyId")) { + t.Fatal("expected history 404") + } + if isGmailHistoryNotFound(fmtError("gmail api 403 Forbidden")) { + t.Fatal("403 is not history not found") + } +} + +func fmtError(msg string) error { + return &simpleError{msg: msg} +} + +type simpleError struct{ msg string } + +func (e *simpleError) Error() string { return e.msg } + +func TestGraphMessageRemoved(t *testing.T) { + removed := struct { + Reason string `json:"reason"` + }{Reason: "deleted"} + msg := graphMessage{ID: "msg-1", Removed: &removed} + if msg.Removed == nil { + t.Fatal("expected removed marker") + } +} diff --git a/internal/migration/gmail_import.go b/internal/migration/gmail_import.go new file mode 100644 index 0000000..d880619 --- /dev/null +++ b/internal/migration/gmail_import.go @@ -0,0 +1,727 @@ +package migration + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "hash/fnv" + "net/http" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/ultisuite/ulti-backend/internal/mail/sanitize" + "github.com/ultisuite/ulti-backend/internal/mail/storage" + "github.com/ultisuite/ulti-backend/internal/mail/threading" +) + +type GmailImporter struct { + db *pgxpool.Pool + client *http.Client + storage *storage.Client + attachBucket string +} + +func NewGmailImporter(db *pgxpool.Pool) *GmailImporter { + return &GmailImporter{ + db: db, + client: &http.Client{Timeout: 90 * time.Second}, + } +} + +func (g *GmailImporter) WithHTTPClient(c *http.Client) *GmailImporter { + if c != nil { + g.client = c + } + return g +} + +func (g *GmailImporter) WithStorage(storage *storage.Client, bucket string) *GmailImporter { + g.storage = storage + g.attachBucket = strings.TrimSpace(bucket) + return g +} + +type gmailMessage struct { + ID string `json:"id"` + ThreadID string `json:"threadId"` + LabelIDs []string `json:"labelIds"` + Snippet string `json:"snippet"` + InternalDate string `json:"internalDate"` + Payload gmailPayload `json:"payload"` +} + +type gmailPayload struct { + MimeType string `json:"mimeType"` + Headers []gmailHeader `json:"headers"` + Body gmailBody `json:"body"` + Parts []gmailPayload `json:"parts"` +} + +type gmailHeader struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type gmailBody struct { + Size int `json:"size"` + Data string `json:"data"` + AttachmentID string `json:"attachmentId"` +} + +func (g *GmailImporter) ImportBatch( + ctx context.Context, + job *Job, + accessToken string, + delta bool, + update func(status string, cursor, stats map[string]any, jobErr string) error, +) error { + accountID, err := g.resolveMailAccountID(ctx, job.UserID) + if err != nil { + return err + } + if err := ensureDefaultMailFolders(ctx, g.db, accountID); err != nil { + return err + } + items, err := LoadImportedItemStore(ctx, g.db, job.ID, job.CursorJSON) + if err != nil { + return err + } + + if delta { + historyID, _ := job.CursorJSON["historyId"].(string) + if historyID != "" { + more, err := g.importHistory(ctx, job, accessToken, accountID, historyID, items) + if err != nil { + if isGmailHistoryNotFound(err) { + if newID, fetchErr := g.fetchHistoryID(ctx, accessToken); fetchErr == nil && newID != "" { + job.CursorJSON["historyId"] = newID + delete(job.CursorJSON, "historyPageToken") + job.StatsJSON["history_reset"] = float64(1) + job.StatsJSON["phase"] = "delta" + return update("completed", job.CursorJSON, job.StatsJSON, "") + } + } + return err + } + if more { + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + return update("completed", job.CursorJSON, job.StatsJSON, "") + } + } + + pageToken, _ := job.CursorJSON["pageToken"].(string) + listIndex := int(jsonNumber(job.CursorJSON["listIndex"])) + + listURL := "https://gmail.googleapis.com/gmail/v1/users/me/messages?maxResults=100" + if pageToken != "" { + listURL += "&pageToken=" + pageToken + } + body, err := g.apiGet(ctx, listURL, accessToken) + if err != nil { + return err + } + var listed struct { + Messages []struct{ ID string `json:"id"` } `json:"messages"` + NextPageToken string `json:"nextPageToken"` + ResultSizeEstimate int `json:"resultSizeEstimate"` + } + if err := json.Unmarshal(body, &listed); err != nil { + return err + } + if listed.ResultSizeEstimate > 0 { + job.StatsJSON["estimated_total"] = float64(listed.ResultSizeEstimate) + } + + imported, _ := job.StatsJSON["imported"].(float64) + batch := 0 + for i := listIndex; i < len(listed.Messages) && batch < mailImportBatchSize(); i++ { + msgID := listed.Messages[i].ID + if alreadyImported(items, msgID) { + listIndex = i + 1 + continue + } + created, err := g.importOne(ctx, accessToken, job.UserID, accountID, msgID) + if err != nil { + if markErr := items.MarkFailed(ctx, msgID, err.Error(), ""); markErr != nil { + return markErr + } + incJobStat(job.StatsJSON, "failed") + batch++ + listIndex = i + 1 + continue + } + if err := items.MarkImported(ctx, msgID); err != nil { + return err + } + if created { + imported++ + } + batch++ + listIndex = i + 1 + } + job.StatsJSON["imported"] = imported + job.CursorJSON["listIndex"] = float64(listIndex) + + if listIndex < len(listed.Messages) { + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + + // page complete + delete(job.CursorJSON, "listIndex") + if listed.NextPageToken != "" { + job.CursorJSON["pageToken"] = listed.NextPageToken + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + + delete(job.CursorJSON, "pageToken") + if hid, err := g.fetchHistoryID(ctx, accessToken); err == nil && hid != "" { + job.CursorJSON["historyId"] = hid + } + job.StatsJSON["phase"] = "imported" + return update("completed", job.CursorJSON, job.StatsJSON, "") +} + +func jsonNumber(v any) float64 { + if v == nil { + return 0 + } + if f, ok := v.(float64); ok { + return f + } + return 0 +} + +func (g *GmailImporter) importOne(ctx context.Context, accessToken, userID, accountID, gmailID string) (bool, error) { + raw, err := g.apiGet(ctx, "https://gmail.googleapis.com/gmail/v1/users/me/messages/"+gmailID+"?format=full", accessToken) + if err != nil { + return false, err + } + var msg gmailMessage + if err := json.Unmarshal(raw, &msg); err != nil { + return false, err + } + + remoteName, folderType := primaryGmailFolder(msg.LabelIDs) + folderID, err := ensureMailFolder(ctx, g.db, accountID, displayFolderName(remoteName, folderType), remoteName, folderType) + if err != nil { + return false, err + } + + headers := indexHeaders(msg.Payload) + subject := headers["subject"] + fromJSON := parseAddressListJSON(headers["from"]) + toJSON := parseAddressListJSON(headers["to"]) + ccJSON := parseAddressListJSON(headers["cc"]) + replyToJSON := parseAddressListJSON(headers["reply-to"]) + rfcID := threading.NormalizeMessageID(headers["message-id"]) + if rfcID == "" { + rfcID = threading.NormalizeMessageID("") + } + inReplyTo := threading.NormalizeMessageID(headers["in-reply-to"]) + references := parseReferences(headers["references"]) + bodyText, bodyHTML := extractGmailBodies(msg.Payload) + snippet := strings.TrimSpace(msg.Snippet) + if snippet == "" { + snippet = truncateRunes(bodyText, 200) + } + + date := parseMailDate(headers["date"]) + if msg.InternalDate != "" { + if ms, err := parseInternalDate(msg.InternalDate); err == nil { + date = ms + } + } + + flags := gmailFlags(msg.LabelIDs) + labels := gmailUserLabels(msg.LabelIDs) + + uid := gmailUID(gmailID) + var messageID string + var existed bool + _ = g.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM messages WHERE folder_id = $1 AND uid = $2)`, folderID, uid).Scan(&existed) + + err = g.db.QueryRow(ctx, ` + INSERT INTO messages ( + account_id, folder_id, uid, message_id, subject, + from_addr, to_addrs, cc_addrs, reply_to, + date, snippet, body_text, body_html, flags, labels, + in_reply_to, references_header + ) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17) + ON CONFLICT (folder_id, uid) DO UPDATE SET + message_id = EXCLUDED.message_id, + subject = EXCLUDED.subject, + from_addr = EXCLUDED.from_addr, + to_addrs = EXCLUDED.to_addrs, + cc_addrs = EXCLUDED.cc_addrs, + reply_to = EXCLUDED.reply_to, + date = EXCLUDED.date, + snippet = EXCLUDED.snippet, + body_text = EXCLUDED.body_text, + body_html = EXCLUDED.body_html, + flags = EXCLUDED.flags, + labels = EXCLUDED.labels, + in_reply_to = EXCLUDED.in_reply_to, + references_header = EXCLUDED.references_header, + updated_at = NOW() + RETURNING id + `, accountID, folderID, uid, rfcID, subject, + fromJSON, toJSON, ccJSON, replyToJSON, + date, snippet, bodyText, sanitize.SanitizeHTML(bodyHTML), flags, labels, + inReplyTo, references, + ).Scan(&messageID) + if err != nil { + return false, err + } + + if err := threading.ApplyMessageThread(ctx, g.db, accountID, messageID, rfcID, inReplyTo, references); err != nil { + return false, err + } + if err := g.storeGmailAttachments(ctx, userID, messageID, gmailID, accessToken, msg.Payload, existed); err != nil { + return false, err + } + return !existed, nil +} + +func (g *GmailImporter) importHistory(ctx context.Context, job *Job, accessToken, accountID, historyID string, items *ImportedItemStore) (more bool, err error) { + pageToken, _ := job.CursorJSON["historyPageToken"].(string) + listURL := fmt.Sprintf( + "https://gmail.googleapis.com/gmail/v1/users/me/history?startHistoryId=%s&maxResults=100&historyTypes=messageAdded&historyTypes=messageDeleted&historyTypes=labelAdded&historyTypes=labelRemoved", + historyID, + ) + if pageToken != "" { + listURL += "&pageToken=" + pageToken + } + body, err := g.apiGet(ctx, listURL, accessToken) + if err != nil { + return false, err + } + var parsed struct { + History []struct { + MessagesAdded []struct { + Message struct{ ID string `json:"id"` } `json:"message"` + } `json:"messagesAdded"` + MessagesDeleted []struct { + Message struct{ ID string `json:"id"` } `json:"message"` + } `json:"messagesDeleted"` + LabelsAdded []struct { + Message struct{ ID string `json:"id"` } `json:"message"` + } `json:"labelsAdded"` + LabelsRemoved []struct { + Message struct{ ID string `json:"id"` } `json:"message"` + } `json:"labelsRemoved"` + } `json:"history"` + NextPageToken string `json:"nextPageToken"` + HistoryID string `json:"historyId"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return false, err + } + + deltaCount, _ := job.StatsJSON["delta_imported"].(float64) + deleted, _ := job.StatsJSON["delta_deleted"].(float64) + batch := 0 + listIndex := int(jsonNumber(job.CursorJSON["historyListIndex"])) + + for hi := listIndex; hi < len(parsed.History) && batch < mailImportBatchSize(); hi++ { + h := parsed.History[hi] + for _, added := range h.MessagesAdded { + if batch >= mailImportBatchSize() { + job.CursorJSON["historyListIndex"] = float64(hi) + return true, nil + } + msgID := added.Message.ID + if alreadyImported(items, msgID) { + batch++ + continue + } + ok, err := g.importOne(ctx, accessToken, job.UserID, accountID, msgID) + if err != nil { + if markErr := items.MarkFailed(ctx, msgID, err.Error(), ""); markErr != nil { + return false, markErr + } + incJobStat(job.StatsJSON, "failed") + batch++ + continue + } + if err := items.MarkImported(ctx, msgID); err != nil { + return false, err + } + if ok { + deltaCount++ + } + batch++ + } + for _, removed := range h.MessagesDeleted { + if batch >= mailImportBatchSize() { + job.CursorJSON["historyListIndex"] = float64(hi) + return true, nil + } + if err := g.deleteByGmailID(ctx, accountID, removed.Message.ID); err != nil { + return false, err + } + deleted++ + batch++ + } + for _, labeled := range h.LabelsAdded { + if batch >= mailImportBatchSize() { + job.CursorJSON["historyListIndex"] = float64(hi) + return true, nil + } + if _, err := g.importOne(ctx, accessToken, job.UserID, accountID, labeled.Message.ID); err != nil { + return false, err + } + deltaCount++ + batch++ + } + for _, labeled := range h.LabelsRemoved { + if batch >= mailImportBatchSize() { + job.CursorJSON["historyListIndex"] = float64(hi) + return true, nil + } + if _, err := g.importOne(ctx, accessToken, job.UserID, accountID, labeled.Message.ID); err != nil { + return false, err + } + deltaCount++ + batch++ + } + } + delete(job.CursorJSON, "historyListIndex") + + job.StatsJSON["delta_imported"] = deltaCount + job.StatsJSON["delta_deleted"] = deleted + if parsed.NextPageToken != "" { + job.CursorJSON["historyPageToken"] = parsed.NextPageToken + if parsed.HistoryID != "" { + job.CursorJSON["historyId"] = parsed.HistoryID + } + job.StatsJSON["phase"] = "delta" + return true, nil + } + delete(job.CursorJSON, "historyPageToken") + if parsed.HistoryID != "" { + job.CursorJSON["historyId"] = parsed.HistoryID + } + job.StatsJSON["phase"] = "delta" + return false, nil +} + +func (g *GmailImporter) deleteByGmailID(ctx context.Context, accountID, gmailID string) error { + if strings.TrimSpace(gmailID) == "" { + return nil + } + uid := gmailUID(gmailID) + _, err := g.db.Exec(ctx, `DELETE FROM messages WHERE account_id = $1::uuid AND uid = $2`, accountID, uid) + return err +} + +func isGmailHistoryNotFound(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "404") && strings.Contains(msg, "history") +} + +func (g *GmailImporter) fetchHistoryID(ctx context.Context, accessToken string) (string, error) { + body, err := g.apiGet(ctx, "https://gmail.googleapis.com/gmail/v1/users/me/profile", accessToken) + if err != nil { + return "", err + } + var parsed struct { + HistoryID string `json:"historyId"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return "", err + } + return parsed.HistoryID, nil +} + +func (g *GmailImporter) apiGet(ctx context.Context, url, accessToken string) ([]byte, error) { + raw, err := apiGet(ctx, g.client, url, accessToken) + if err != nil { + return nil, fmt.Errorf("gmail api: %w", err) + } + return raw, nil +} + +func (g *GmailImporter) resolveMailAccountID(ctx context.Context, userID string) (string, error) { + var accountID string + err := g.db.QueryRow(ctx, ` + SELECT COALESCE( + (SELECT mail_account_id::text FROM mailboxes WHERE user_id = $1::uuid AND mail_account_id IS NOT NULL LIMIT 1), + (SELECT id::text FROM mail_accounts WHERE user_id = $1::uuid AND is_active ORDER BY created_at LIMIT 1) + ) + `, userID).Scan(&accountID) + if err != nil || accountID == "" { + return "", fmt.Errorf("no mail account for migration user") + } + return accountID, nil +} + +func ensureDefaultMailFolders(ctx context.Context, db *pgxpool.Pool, accountID string) error { + defaults := []struct{ name, remote, ftype string }{ + {"Boîte de réception", "INBOX", "inbox"}, + {"Envoyés", "SENT", "sent"}, + {"Brouillons", "DRAFT", "drafts"}, + {"Corbeille", "TRASH", "trash"}, + {"Spam", "SPAM", "spam"}, + {"Archives", "ARCHIVE", "archive"}, + } + for _, d := range defaults { + if _, err := ensureMailFolder(ctx, db, accountID, d.name, d.remote, d.ftype); err != nil { + return err + } + } + return nil +} + +func ensureMailFolder(ctx context.Context, db *pgxpool.Pool, accountID, name, remoteName, folderType string) (string, error) { + var folderID string + err := db.QueryRow(ctx, ` + INSERT INTO mail_folders (account_id, name, remote_name, folder_type) + VALUES ($1, $2, $3, $4) + ON CONFLICT (account_id, remote_name) DO UPDATE + SET name = EXCLUDED.name, folder_type = EXCLUDED.folder_type, updated_at = NOW() + RETURNING id::text + `, accountID, name, remoteName, folderType).Scan(&folderID) + return folderID, err +} + +func displayFolderName(remote, folderType string) string { + switch folderType { + case "inbox": + return "Boîte de réception" + case "sent": + return "Envoyés" + case "drafts": + return "Brouillons" + case "trash": + return "Corbeille" + case "spam": + return "Spam" + case "archive": + return "Archives" + default: + return remote + } +} + +func primaryGmailFolder(labelIDs []string) (remoteName, folderType string) { + priority := []struct{ label, remote, ftype string }{ + {"INBOX", "INBOX", "inbox"}, + {"SENT", "SENT", "sent"}, + {"DRAFT", "DRAFT", "drafts"}, + {"TRASH", "TRASH", "trash"}, + {"SPAM", "SPAM", "spam"}, + } + set := make(map[string]struct{}, len(labelIDs)) + for _, l := range labelIDs { + set[l] = struct{}{} + } + for _, p := range priority { + if _, ok := set[p.label]; ok { + return p.remote, p.ftype + } + } + return "ARCHIVE", "archive" +} + +func gmailUserLabels(labelIDs []string) []string { + system := map[string]struct{}{ + "INBOX": {}, "SENT": {}, "DRAFT": {}, "TRASH": {}, "SPAM": {}, + "STARRED": {}, "IMPORTANT": {}, "UNREAD": {}, "CATEGORY_PERSONAL": {}, + "CATEGORY_SOCIAL": {}, "CATEGORY_PROMOTIONS": {}, "CATEGORY_UPDATES": {}, + "CATEGORY_FORUMS": {}, + } + out := make([]string, 0, len(labelIDs)) + for _, l := range labelIDs { + if _, skip := system[l]; skip { + continue + } + out = append(out, strings.ToLower(l)) + } + return out +} + +func gmailFlags(labelIDs []string) []string { + flags := []string{} + unread := false + for _, l := range labelIDs { + switch l { + case "UNREAD": + unread = true + case "STARRED": + flags = append(flags, "\\Flagged") + case "IMPORTANT": + flags = append(flags, "important") + } + } + if !unread { + flags = append(flags, "\\Seen") + } + return flags +} + +func gmailUID(gmailID string) int64 { + h := fnv.New64a() + _, _ = h.Write([]byte(gmailID)) + v := int64(h.Sum64() & 0x7fffffffffffffff) + if v == 0 { + return 1 + } + return v +} + +func indexHeaders(p gmailPayload) map[string]string { + out := map[string]string{} + var walk func(gmailPayload) + walk = func(node gmailPayload) { + for _, h := range node.Headers { + key := strings.ToLower(strings.TrimSpace(h.Name)) + if key != "" && out[key] == "" { + out[key] = h.Value + } + } + for _, part := range node.Parts { + walk(part) + } + } + walk(p) + return out +} + +func extractGmailBodies(p gmailPayload) (text, html string) { + var walk func(gmailPayload) + walk = func(node gmailPayload) { + if text == "" && node.MimeType == "text/plain" && node.Body.Data != "" { + text = decodeGmailBody(node.Body.Data) + } + if html == "" && node.MimeType == "text/html" && node.Body.Data != "" { + html = decodeGmailBody(node.Body.Data) + } + for _, part := range node.Parts { + walk(part) + } + } + walk(p) + return text, html +} + +func decodeGmailBody(data string) string { + data = strings.ReplaceAll(data, "-", "+") + data = strings.ReplaceAll(data, "_", "/") + raw, err := base64.StdEncoding.DecodeString(data) + if err != nil { + return "" + } + return string(raw) +} + +func parseAddressListJSON(raw string) []byte { + raw = strings.TrimSpace(raw) + if raw == "" { + return []byte("[]") + } + parts := splitAddresses(raw) + type addr struct { + Name string `json:"name,omitempty"` + Email string `json:"email"` + } + out := make([]addr, 0, len(parts)) + for _, p := range parts { + name, email := parseSingleAddress(p) + out = append(out, addr{Name: name, Email: email}) + } + b, _ := json.Marshal(out) + return b +} + +func splitAddresses(raw string) []string { + return strings.Split(raw, ",") +} + +func parseSingleAddress(raw string) (name, email string) { + raw = strings.TrimSpace(raw) + if i := strings.Index(raw, "<"); i >= 0 && strings.HasSuffix(raw, ">") { + name = strings.Trim(strings.TrimSpace(raw[:i]), `"`) + email = strings.Trim(raw[i+1:len(raw)-1], " <>") + return name, strings.ToLower(email) + } + return "", strings.ToLower(raw) +} + +func parseReferences(raw string) []string { + raw = strings.TrimSpace(raw) + if raw == "" { + return []string{} + } + var out []string + for _, part := range strings.Fields(raw) { + if id := threading.NormalizeMessageID(part); id != "" { + out = append(out, id) + } + } + return out +} + +func parseMailDate(raw string) time.Time { + raw = strings.TrimSpace(raw) + if raw == "" { + return time.Now().UTC() + } + layouts := []string{time.RFC1123Z, time.RFC1123, time.RFC3339} + for _, layout := range layouts { + if t, err := time.Parse(layout, raw); err == nil { + return t.UTC() + } + } + return time.Now().UTC() +} + +func parseInternalDate(raw string) (time.Time, error) { + var ms int64 + if _, err := fmt.Sscan(raw, &ms); err != nil { + return time.Time{}, err + } + return time.UnixMilli(ms).UTC(), nil +} + +func truncateRunes(s string, n int) string { + r := []rune(strings.TrimSpace(s)) + if len(r) <= n { + return string(r) + } + return string(r[:n]) +} + +func LinkHostedMailboxByEmail(ctx context.Context, db *pgxpool.Pool, userID, email string) error { + email = strings.ToLower(strings.TrimSpace(email)) + if email == "" { + return nil + } + _, err := db.Exec(ctx, ` + UPDATE mailboxes SET user_id = $1::uuid, updated_at = NOW() + WHERE user_id IS NULL AND lower(local_part || '@' || (SELECT name FROM mail_domains d WHERE d.id = mailboxes.domain_id)) = $2 + `, userID, email) + if err != nil { + return err + } + _, err = db.Exec(ctx, ` + UPDATE mail_accounts ma SET user_id = $1::uuid, updated_at = NOW() + FROM mailboxes mb + JOIN mail_domains md ON md.id = mb.domain_id + WHERE mb.mail_account_id = ma.id + AND ma.user_id IS NULL + AND mb.user_id = $1::uuid + AND lower(mb.local_part || '@' || md.name) = $2 + `, userID, email) + return err +} + +var _ = pgx.ErrNoRows diff --git a/internal/migration/gmail_import_test.go b/internal/migration/gmail_import_test.go new file mode 100644 index 0000000..a20311d --- /dev/null +++ b/internal/migration/gmail_import_test.go @@ -0,0 +1,57 @@ +package migration + +import ( + "encoding/json" + "testing" +) + +func TestGmailUIDStable(t *testing.T) { + a := gmailUID("18c4f2a1b2d3e4f5") + b := gmailUID("18c4f2a1b2d3e4f5") + if a != b { + t.Fatalf("expected stable uid, got %d vs %d", a, b) + } + if a <= 0 { + t.Fatalf("expected positive uid, got %d", a) + } +} + +func TestPrimaryGmailFolder(t *testing.T) { + remote, folderType := primaryGmailFolder([]string{"INBOX", "UNREAD"}) + if remote != "INBOX" || folderType != "inbox" { + t.Fatalf("got %q / %q", remote, folderType) + } + + remote, folderType = primaryGmailFolder([]string{"SENT"}) + if remote != "SENT" || folderType != "sent" { + t.Fatalf("got %q / %q", remote, folderType) + } + + remote, folderType = primaryGmailFolder([]string{"Label_123", "INBOX"}) + if remote != "INBOX" || folderType != "inbox" { + t.Fatalf("inbox wins over label: got %q / %q", remote, folderType) + } +} + +func TestParseAddressListJSON(t *testing.T) { + raw := parseAddressListJSON(`Alice , bob@example.com`) + var addrs []map[string]string + if err := json.Unmarshal(raw, &addrs); err != nil { + t.Fatal(err) + } + if len(addrs) != 2 { + t.Fatalf("expected 2 addresses, got %d", len(addrs)) + } + if addrs[0]["email"] != "alice@example.com" { + t.Fatalf("unexpected first email: %v", addrs[0]) + } +} + +func TestDisplayFolderName(t *testing.T) { + if got := displayFolderName("Label_42", "custom"); got != "Label_42" { + t.Fatalf("custom label: %q", got) + } + if got := displayFolderName("INBOX", "inbox"); got != "Boîte de réception" { + t.Fatalf("inbox display: %q", got) + } +} diff --git a/internal/migration/google_dwd.go b/internal/migration/google_dwd.go new file mode 100644 index 0000000..5cd0243 --- /dev/null +++ b/internal/migration/google_dwd.go @@ -0,0 +1,77 @@ +package migration + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" +) + +var googleDWDScopes = []string{ + "https://www.googleapis.com/auth/gmail.readonly", + "https://www.googleapis.com/auth/drive.readonly", + "https://www.googleapis.com/auth/calendar.readonly", + "https://www.googleapis.com/auth/contacts.readonly", +} + +// GoogleDWD mints access tokens via a service account with domain-wide delegation. +type GoogleDWD struct { + jwtConfig *jwt.Config +} + +func NewGoogleDWD(jsonKey string) (*GoogleDWD, error) { + jsonKey = strings.TrimSpace(jsonKey) + if jsonKey == "" { + return nil, nil + } + conf, err := google.JWTConfigFromJSON([]byte(jsonKey), googleDWDScopes...) + if err != nil { + return nil, fmt.Errorf("parse google service account: %w", err) + } + return &GoogleDWD{jwtConfig: conf}, nil +} + +func (g *GoogleDWD) Enabled() bool { + return g != nil && g.jwtConfig != nil +} + +func (g *GoogleDWD) AccessToken(ctx context.Context, subjectEmail string) (string, error) { + if !g.Enabled() { + return "", fmt.Errorf("google domain-wide delegation not configured") + } + subjectEmail = strings.ToLower(strings.TrimSpace(subjectEmail)) + if subjectEmail == "" { + return "", fmt.Errorf("subject email required for domain-wide delegation") + } + conf := *g.jwtConfig + conf.Subject = subjectEmail + token, err := conf.TokenSource(ctx).Token() + if err != nil { + return "", fmt.Errorf("google dwd token: %w", err) + } + if token.AccessToken == "" { + return "", fmt.Errorf("google dwd token empty") + } + return token.AccessToken, nil +} + +func validateServiceAccountJSON(raw string) error { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil + } + var probe struct { + ClientEmail string `json:"client_email"` + PrivateKey string `json:"private_key"` + } + if err := json.Unmarshal([]byte(raw), &probe); err != nil { + return fmt.Errorf("invalid service account json: %w", err) + } + if probe.ClientEmail == "" || probe.PrivateKey == "" { + return fmt.Errorf("service account json missing client_email or private_key") + } + return nil +} diff --git a/internal/migration/google_dwd_test.go b/internal/migration/google_dwd_test.go new file mode 100644 index 0000000..38bd82e --- /dev/null +++ b/internal/migration/google_dwd_test.go @@ -0,0 +1,37 @@ +package migration + +import "testing" + +func TestNormalizeAuthMode(t *testing.T) { + if got := NormalizeAuthMode("google", "google_dwd"); got != AuthModeGoogleDWD { + t.Fatalf("got %q", got) + } + if got := NormalizeAuthMode("microsoft", "google_dwd"); got != AuthModeOAuth { + t.Fatalf("microsoft ignores dwd: got %q", got) + } + if got := NormalizeAuthMode("microsoft", "microsoft_app"); got != AuthModeMicrosoftApp { + t.Fatalf("microsoft app: got %q", got) + } + if got := NormalizeAuthMode("google", ""); got != AuthModeOAuth { + t.Fatalf("default oauth: got %q", got) + } +} + +func TestValidateServiceAccountJSON(t *testing.T) { + if err := validateServiceAccountJSON(""); err != nil { + t.Fatalf("empty ok: %v", err) + } + if err := validateServiceAccountJSON(`{"client_email":"a@b.iam.gserviceaccount.com","private_key":"-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----\n"}`); err != nil { + t.Fatalf("valid json: %v", err) + } + if err := validateServiceAccountJSON(`{`); err == nil { + t.Fatal("expected invalid json error") + } +} + +func TestNewGoogleDWDEmpty(t *testing.T) { + dwd, err := NewGoogleDWD("") + if err != nil || dwd != nil { + t.Fatalf("empty config: dwd=%v err=%v", dwd, err) + } +} diff --git a/internal/migration/graph_import.go b/internal/migration/graph_import.go new file mode 100644 index 0000000..60b5143 --- /dev/null +++ b/internal/migration/graph_import.go @@ -0,0 +1,531 @@ +package migration + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/ultisuite/ulti-backend/internal/mail/sanitize" + "github.com/ultisuite/ulti-backend/internal/mail/threading" +) + +const graphMessageSelect = "id,subject,bodyPreview,body,from,toRecipients,ccRecipients,replyTo," + + "receivedDateTime,sentDateTime,parentFolderId,isRead,flag,internetMessageId,internetMessageHeaders" + +type GraphImporter struct { + db *pgxpool.Pool + client *http.Client + baseURL string + userUPN string + folders map[string]graphFolderMeta +} + +type graphFolderMeta struct { + RemoteName string + FolderType string +} + +func NewGraphImporter(db *pgxpool.Pool) *GraphImporter { + return &GraphImporter{ + db: db, + client: &http.Client{Timeout: 90 * time.Second}, + folders: map[string]graphFolderMeta{}, + } +} + +func (g *GraphImporter) WithHTTPClient(c *http.Client) *GraphImporter { + if c != nil { + g.client = c + } + return g +} + +func (g *GraphImporter) WithUserPrincipal(upn string) *GraphImporter { + g.userUPN = strings.TrimSpace(upn) + return g +} + +func (g *GraphImporter) userBase() string { + return graphUserBase(g.userUPN) +} + +func (g *GraphImporter) WithBaseURL(baseURL string) *GraphImporter { + g.baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + return g +} + +func (g *GraphImporter) graphURL(path string) string { + if g.baseURL != "" { + return g.baseURL + path + } + return "https://graph.microsoft.com" + path +} + +type graphMessage struct { + ID string `json:"id"` + Subject string `json:"subject"` + BodyPreview string `json:"bodyPreview"` + Body graphBody `json:"body"` + From graphRecipient `json:"from"` + ToRecipients []graphRecipient `json:"toRecipients"` + CcRecipients []graphRecipient `json:"ccRecipients"` + ReplyTo []graphRecipient `json:"replyTo"` + ReceivedDateTime string `json:"receivedDateTime"` + SentDateTime string `json:"sentDateTime"` + ParentFolderID string `json:"parentFolderId"` + IsRead bool `json:"isRead"` + Flag graphFlag `json:"flag"` + InternetMessageID string `json:"internetMessageId"` + InternetMessageHeaders []graphHeader `json:"internetMessageHeaders"` + Removed *struct { + Reason string `json:"reason"` + } `json:"@removed"` +} + +type graphBody struct { + ContentType string `json:"contentType"` + Content string `json:"content"` +} + +type graphRecipient struct { + EmailAddress graphEmailAddress `json:"emailAddress"` +} + +type graphEmailAddress struct { + Name string `json:"name"` + Address string `json:"address"` +} + +type graphFlag struct { + FlagStatus string `json:"flagStatus"` +} + +type graphHeader struct { + Name string `json:"name"` + Value string `json:"value"` +} + +func (g *GraphImporter) ImportBatch( + ctx context.Context, + job *Job, + accessToken string, + delta bool, + update func(status string, cursor, stats map[string]any, jobErr string) error, +) error { + accountID, err := g.resolveMailAccountID(ctx, job.UserID) + if err != nil { + return err + } + if err := ensureDefaultMailFolders(ctx, g.db, accountID); err != nil { + return err + } + if err := g.ensureGraphFolders(ctx, accessToken); err != nil { + return err + } + items, err := LoadImportedItemStore(ctx, g.db, job.ID, job.CursorJSON) + if err != nil { + return err + } + + if delta { + deltaLink, _ := job.CursorJSON["deltaLink"].(string) + if deltaLink != "" { + more, err := g.importDeltaPage(ctx, job, accessToken, accountID, deltaLink, items) + if err != nil { + return err + } + if more { + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + return update("completed", job.CursorJSON, job.StatsJSON, "") + } + } + + nextLink, _ := job.CursorJSON["nextLink"].(string) + var listURL string + if nextLink != "" { + listURL = nextLink + } else { + listURL = g.graphURL(g.userBase()+"/messages?$top=100&$orderby="+url.QueryEscape("receivedDateTime desc")+"&$select="+graphMessageSelect) + } + + body, err := g.apiGet(ctx, listURL, accessToken) + if err != nil { + return err + } + var listed struct { + Value []graphMessage `json:"value"` + NextLink string `json:"@odata.nextLink"` + DeltaLink string `json:"@odata.deltaLink"` + } + if err := json.Unmarshal(body, &listed); err != nil { + return err + } + + imported, _ := job.StatsJSON["imported"].(float64) + batch := 0 + listIndex := int(jsonNumber(job.CursorJSON["listIndex"])) + for i := listIndex; i < len(listed.Value) && batch < mailImportBatchSize(); i++ { + msg := listed.Value[i] + if alreadyImported(items, msg.ID) { + listIndex = i + 1 + continue + } + created, err := g.importOne(ctx, accountID, msg) + if err != nil { + if markErr := items.MarkFailed(ctx, msg.ID, err.Error(), ""); markErr != nil { + return markErr + } + incJobStat(job.StatsJSON, "failed") + batch++ + listIndex = i + 1 + continue + } + if err := items.MarkImported(ctx, msg.ID); err != nil { + return err + } + if created { + imported++ + } + batch++ + listIndex = i + 1 + } + job.StatsJSON["imported"] = imported + job.CursorJSON["listIndex"] = float64(listIndex) + + if listIndex < len(listed.Value) { + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + + delete(job.CursorJSON, "listIndex") + if listed.NextLink != "" { + job.CursorJSON["nextLink"] = listed.NextLink + return update("pending", job.CursorJSON, job.StatsJSON, "") + } + delete(job.CursorJSON, "nextLink") + + if delta { + if listed.DeltaLink != "" { + job.CursorJSON["deltaLink"] = listed.DeltaLink + } else if link, err := g.initDeltaLink(ctx, accessToken); err == nil && link != "" { + job.CursorJSON["deltaLink"] = link + } + } + job.StatsJSON["phase"] = "imported" + return update("completed", job.CursorJSON, job.StatsJSON, "") +} + +func (g *GraphImporter) importDeltaPage(ctx context.Context, job *Job, accessToken, accountID, deltaLink string, items *ImportedItemStore) (more bool, err error) { + body, err := g.apiGet(ctx, deltaLink, accessToken) + if err != nil { + return false, err + } + var parsed struct { + Value []graphMessage `json:"value"` + NextLink string `json:"@odata.nextLink"` + DeltaLink string `json:"@odata.deltaLink"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return false, err + } + deltaCount, _ := job.StatsJSON["delta_imported"].(float64) + deleted, _ := job.StatsJSON["delta_deleted"].(float64) + for _, msg := range parsed.Value { + if msg.Removed != nil { + if err := g.deleteByGraphID(ctx, accountID, msg.ID); err != nil { + return false, err + } + deleted++ + continue + } + if alreadyImported(items, msg.ID) { + continue + } + ok, err := g.importOne(ctx, accountID, msg) + if err != nil { + if markErr := items.MarkFailed(ctx, msg.ID, err.Error(), ""); markErr != nil { + return false, markErr + } + incJobStat(job.StatsJSON, "failed") + continue + } + if err := items.MarkImported(ctx, msg.ID); err != nil { + return false, err + } + if ok { + deltaCount++ + } + } + job.StatsJSON["delta_imported"] = deltaCount + job.StatsJSON["delta_deleted"] = deleted + if parsed.NextLink != "" { + job.CursorJSON["deltaLink"] = parsed.NextLink + job.StatsJSON["phase"] = "delta" + return true, nil + } + if parsed.DeltaLink != "" { + job.CursorJSON["deltaLink"] = parsed.DeltaLink + } + job.StatsJSON["phase"] = "delta" + return false, nil +} + +func (g *GraphImporter) initDeltaLink(ctx context.Context, accessToken string) (string, error) { + body, err := g.apiGet(ctx, g.graphURL(g.userBase()+"/messages/delta?$select=id"), accessToken) + if err != nil { + return "", err + } + var parsed struct { + DeltaLink string `json:"@odata.deltaLink"` + NextLink string `json:"@odata.nextLink"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return "", err + } + if parsed.DeltaLink != "" { + return parsed.DeltaLink, nil + } + return parsed.NextLink, nil +} + +func (g *GraphImporter) importOne(ctx context.Context, accountID string, msg graphMessage) (bool, error) { + meta := g.folders[msg.ParentFolderID] + if meta.RemoteName == "" { + meta = graphFolderMeta{RemoteName: "ARCHIVE", FolderType: "archive"} + } + folderID, err := ensureMailFolder(ctx, g.db, accountID, displayFolderName(meta.RemoteName, meta.FolderType), meta.RemoteName, meta.FolderType) + if err != nil { + return false, err + } + + headers := indexGraphHeaders(msg.InternetMessageHeaders) + rfcID := threading.NormalizeMessageID(msg.InternetMessageID) + if rfcID == "" { + rfcID = threading.NormalizeMessageID(headers["message-id"]) + } + if rfcID == "" { + rfcID = threading.NormalizeMessageID("") + } + inReplyTo := threading.NormalizeMessageID(headers["in-reply-to"]) + references := parseReferences(headers["references"]) + + bodyText, bodyHTML := extractGraphBody(msg.Body) + snippet := strings.TrimSpace(msg.BodyPreview) + if snippet == "" { + snippet = truncateRunes(bodyText, 200) + } + + date := parseGraphTime(msg.ReceivedDateTime) + if date.IsZero() { + date = parseGraphTime(msg.SentDateTime) + } + if date.IsZero() { + date = time.Now().UTC() + } + + fromJSON := graphRecipientJSON(msg.From) + toJSON := graphRecipientsJSON(msg.ToRecipients) + ccJSON := graphRecipientsJSON(msg.CcRecipients) + replyToJSON := graphRecipientsJSON(msg.ReplyTo) + flags := graphFlags(msg.IsRead, msg.Flag.FlagStatus) + + uid := remoteMessageUID(msg.ID) + var messageID string + var existed bool + _ = g.db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM messages WHERE folder_id = $1 AND uid = $2)`, folderID, uid).Scan(&existed) + + err = g.db.QueryRow(ctx, ` + INSERT INTO messages ( + account_id, folder_id, uid, message_id, subject, + from_addr, to_addrs, cc_addrs, reply_to, + date, snippet, body_text, body_html, flags, labels, + in_reply_to, references_header + ) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17) + ON CONFLICT (folder_id, uid) DO UPDATE SET + message_id = EXCLUDED.message_id, + subject = EXCLUDED.subject, + from_addr = EXCLUDED.from_addr, + to_addrs = EXCLUDED.to_addrs, + cc_addrs = EXCLUDED.cc_addrs, + reply_to = EXCLUDED.reply_to, + date = EXCLUDED.date, + snippet = EXCLUDED.snippet, + body_text = EXCLUDED.body_text, + body_html = EXCLUDED.body_html, + flags = EXCLUDED.flags, + in_reply_to = EXCLUDED.in_reply_to, + references_header = EXCLUDED.references_header, + updated_at = NOW() + RETURNING id + `, accountID, folderID, uid, rfcID, msg.Subject, + fromJSON, toJSON, ccJSON, replyToJSON, + date, snippet, bodyText, sanitize.SanitizeHTML(bodyHTML), flags, []string{}, + inReplyTo, references, + ).Scan(&messageID) + if err != nil { + return false, err + } + if err := threading.ApplyMessageThread(ctx, g.db, accountID, messageID, rfcID, inReplyTo, references); err != nil { + return false, err + } + return !existed, nil +} + +func (g *GraphImporter) deleteByGraphID(ctx context.Context, accountID, graphID string) error { + if strings.TrimSpace(graphID) == "" { + return nil + } + uid := remoteMessageUID(graphID) + _, err := g.db.Exec(ctx, `DELETE FROM messages WHERE account_id = $1::uuid AND uid = $2`, accountID, uid) + return err +} + +func (g *GraphImporter) ensureGraphFolders(ctx context.Context, accessToken string) error { + if len(g.folders) > 0 { + return nil + } + body, err := g.apiGet(ctx, g.graphURL(g.userBase()+"/mailFolders?$top=100&$select=id,displayName,wellKnownName"), accessToken) + if err != nil { + return err + } + var parsed struct { + Value []struct { + ID string `json:"id"` + DisplayName string `json:"displayName"` + WellKnownName string `json:"wellKnownName"` + } `json:"value"` + } + if err := json.Unmarshal(body, &parsed); err != nil { + return err + } + for _, f := range parsed.Value { + remote, ftype := graphWellKnownFolder(f.WellKnownName, f.DisplayName) + g.folders[f.ID] = graphFolderMeta{RemoteName: remote, FolderType: ftype} + } + return nil +} + +func graphWellKnownFolder(wellKnown, displayName string) (remoteName, folderType string) { + switch strings.ToLower(strings.TrimSpace(wellKnown)) { + case "inbox": + return "INBOX", "inbox" + case "sentitems": + return "SENT", "sent" + case "drafts": + return "DRAFT", "drafts" + case "deleteditems": + return "TRASH", "trash" + case "junkemail": + return "SPAM", "spam" + case "archive": + return "ARCHIVE", "archive" + default: + name := strings.TrimSpace(displayName) + if name == "" { + name = "CUSTOM" + } + return strings.ToUpper(strings.ReplaceAll(name, " ", "_")), "custom" + } +} + +func graphFlags(isRead bool, flagStatus string) []string { + flags := []string{} + if isRead { + flags = append(flags, "\\Seen") + } + if strings.EqualFold(flagStatus, "flagged") { + flags = append(flags, "\\Flagged") + } + return flags +} + +func extractGraphBody(body graphBody) (text, html string) { + content := body.Content + switch strings.ToLower(body.ContentType) { + case "html": + html = content + case "text": + text = content + default: + text = content + } + return text, html +} + +func graphRecipientJSON(r graphRecipient) []byte { + if strings.TrimSpace(r.EmailAddress.Address) == "" { + return []byte("[]") + } + type addr struct { + Name string `json:"name,omitempty"` + Email string `json:"email"` + } + b, _ := json.Marshal([]addr{{Name: r.EmailAddress.Name, Email: strings.ToLower(r.EmailAddress.Address)}}) + return b +} + +func graphRecipientsJSON(recipients []graphRecipient) []byte { + type addr struct { + Name string `json:"name,omitempty"` + Email string `json:"email"` + } + out := make([]addr, 0, len(recipients)) + for _, r := range recipients { + email := strings.ToLower(strings.TrimSpace(r.EmailAddress.Address)) + if email == "" { + continue + } + out = append(out, addr{Name: r.EmailAddress.Name, Email: email}) + } + b, _ := json.Marshal(out) + return b +} + +func indexGraphHeaders(headers []graphHeader) map[string]string { + out := map[string]string{} + for _, h := range headers { + key := strings.ToLower(strings.TrimSpace(h.Name)) + if key != "" && out[key] == "" { + out[key] = h.Value + } + } + return out +} + +func parseGraphTime(raw string) time.Time { + raw = strings.TrimSpace(raw) + if raw == "" { + return time.Time{} + } + if t, err := time.Parse(time.RFC3339Nano, raw); err == nil { + return t.UTC() + } + if t, err := time.Parse(time.RFC3339, raw); err == nil { + return t.UTC() + } + return time.Time{} +} + +func (g *GraphImporter) apiGet(ctx context.Context, url, accessToken string) ([]byte, error) { + raw, err := apiGet(ctx, g.client, url, accessToken) + if err != nil { + return nil, fmt.Errorf("graph api: %w", err) + } + return raw, nil +} + +func (g *GraphImporter) resolveMailAccountID(ctx context.Context, userID string) (string, error) { + importer := NewGmailImporter(g.db) + return importer.resolveMailAccountID(ctx, userID) +} + +func remoteMessageUID(remoteID string) int64 { + return gmailUID(remoteID) +} diff --git a/internal/migration/graph_import_test.go b/internal/migration/graph_import_test.go new file mode 100644 index 0000000..ac9ea3b --- /dev/null +++ b/internal/migration/graph_import_test.go @@ -0,0 +1,49 @@ +package migration + +import "testing" + +func TestGraphWellKnownFolder(t *testing.T) { + remote, ftype := graphWellKnownFolder("inbox", "Inbox") + if remote != "INBOX" || ftype != "inbox" { + t.Fatalf("got %q / %q", remote, ftype) + } + + remote, ftype = graphWellKnownFolder("", "Projects") + if remote != "PROJECTS" || ftype != "custom" { + t.Fatalf("custom folder: got %q / %q", remote, ftype) + } +} + +func TestGraphFlags(t *testing.T) { + flags := graphFlags(true, "notFlagged") + if len(flags) != 1 || flags[0] != "\\Seen" { + t.Fatalf("read flags: %v", flags) + } + flags = graphFlags(false, "flagged") + if len(flags) != 1 || flags[0] != "\\Flagged" { + t.Fatalf("flagged: %v", flags) + } +} + +func TestGraphRecipientsJSON(t *testing.T) { + raw := graphRecipientsJSON([]graphRecipient{ + {EmailAddress: graphEmailAddress{Name: "Bob", Address: "bob@example.com"}}, + }) + if string(raw) != `[{"name":"Bob","email":"bob@example.com"}]` { + t.Fatalf("unexpected json: %s", raw) + } +} + +func TestParseGraphTime(t *testing.T) { + tm := parseGraphTime("2024-05-01T12:34:56Z") + if tm.IsZero() { + t.Fatal("expected parsed time") + } +} + +func TestRemoteMessageUIDMatchesGmailUID(t *testing.T) { + id := "abc123" + if remoteMessageUID(id) != gmailUID(id) { + t.Fatal("uid helpers diverged") + } +} diff --git a/internal/migration/graph_user.go b/internal/migration/graph_user.go new file mode 100644 index 0000000..03e1d14 --- /dev/null +++ b/internal/migration/graph_user.go @@ -0,0 +1,22 @@ +package migration + +import ( + "net/url" + "strings" +) + +func graphUserBase(userUPN string) string { + userUPN = strings.TrimSpace(userUPN) + if userUPN == "" { + return "/v1.0/me" + } + return "/v1.0/users/" + url.PathEscape(userUPN) +} + +func graphMicrosoftURL(userUPN, suffix string) string { + suffix = strings.TrimSpace(suffix) + if suffix != "" && !strings.HasPrefix(suffix, "/") { + suffix = "/" + suffix + } + return "https://graph.microsoft.com" + graphUserBase(userUPN) + suffix +} diff --git a/internal/migration/http_retry.go b/internal/migration/http_retry.go new file mode 100644 index 0000000..4729381 --- /dev/null +++ b/internal/migration/http_retry.go @@ -0,0 +1,186 @@ +package migration + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/ultisuite/ulti-backend/internal/observability" +) + +const ( + defaultRateLimitMaxRetries = 6 + defaultRateLimitBaseDelay = 2 * time.Second + defaultRateLimitMaxDelay = 2 * time.Minute +) + +// RateLimitConfig controls exponential backoff for migration provider API calls. +type RateLimitConfig struct { + MaxRetries int + BaseDelay time.Duration + MaxDelay time.Duration +} + +// RateLimitError is returned when a provider keeps responding with HTTP 429 +// after all configured retries are exhausted. +type RateLimitError struct { + Cause error + RetryAfter time.Duration +} + +func (e *RateLimitError) Error() string { + if e == nil || e.Cause == nil { + return "migration api rate limited" + } + return e.Cause.Error() +} + +func (e *RateLimitError) Unwrap() error { + if e == nil { + return nil + } + return e.Cause +} + +// IsRateLimitError reports whether err is a terminal migration rate-limit error. +func IsRateLimitError(err error) bool { + var rl *RateLimitError + return errors.As(err, &rl) +} + +var ( + rateLimitMu sync.RWMutex + rateLimitConfig = RateLimitConfig{ + MaxRetries: defaultRateLimitMaxRetries, + BaseDelay: defaultRateLimitBaseDelay, + MaxDelay: defaultRateLimitMaxDelay, + } +) + +// ConfigureRateLimit sets package-wide retry settings for migration HTTP calls. +func ConfigureRateLimit(cfg RateLimitConfig) { + rateLimitMu.Lock() + defer rateLimitMu.Unlock() + if cfg.MaxRetries > 0 { + rateLimitConfig.MaxRetries = cfg.MaxRetries + } + if cfg.BaseDelay > 0 { + rateLimitConfig.BaseDelay = cfg.BaseDelay + } + if cfg.MaxDelay > 0 { + rateLimitConfig.MaxDelay = cfg.MaxDelay + } +} + +func currentRateLimitConfig() RateLimitConfig { + rateLimitMu.RLock() + defer rateLimitMu.RUnlock() + return rateLimitConfig +} + +func parseRetryAfter(v string) time.Duration { + v = strings.TrimSpace(v) + if v == "" { + return 0 + } + if secs, err := strconv.Atoi(v); err == nil && secs >= 0 { + return time.Duration(secs) * time.Second + } + if t, err := http.ParseTime(v); err == nil { + d := time.Until(t) + if d < 0 { + return 0 + } + return d + } + return 0 +} + +func rateLimitDelay(attempt int, retryAfter time.Duration) time.Duration { + cfg := currentRateLimitConfig() + delay := cfg.BaseDelay + for i := 1; i < attempt; i++ { + if delay >= cfg.MaxDelay { + delay = cfg.MaxDelay + break + } + delay *= 2 + } + if retryAfter > delay { + delay = retryAfter + } + if delay > cfg.MaxDelay { + delay = cfg.MaxDelay + } + return delay +} + +func migrationDo(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) { + if client == nil { + client = migrationHTTPClient() + } + cfg := currentRateLimitConfig() + var lastErr error + var lastRetryAfter time.Duration + + for attempt := 0; attempt <= cfg.MaxRetries; attempt++ { + if attempt > 0 { + observability.IncMigrationRateLimitRetry() + delay := rateLimitDelay(attempt, lastRetryAfter) + slog.Default().Warn( + "migration api rate limited, backing off", + "component", "migration-http", + "attempt", attempt, + "max_retries", cfg.MaxRetries, + "delay", delay.String(), + "method", req.Method, + "host", req.URL.Host, + "path", req.URL.Path, + ) + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + + cloned := req.Clone(ctx) + resp, err := client.Do(cloned) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusTooManyRequests { + lastRetryAfter = parseRetryAfter(resp.Header.Get("Retry-After")) + raw, _ := io.ReadAll(resp.Body) + resp.Body.Close() + lastErr = fmt.Errorf("api rate limited (429): %s", strings.TrimSpace(string(raw))) + if attempt >= cfg.MaxRetries { + return nil, &RateLimitError{Cause: lastErr, RetryAfter: lastRetryAfter} + } + continue + } + + if resp.StatusCode >= 400 { + raw, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return nil, fmt.Errorf("api %s: %s", resp.Status, strings.TrimSpace(string(raw))) + } + + return resp, nil + } + + if lastErr == nil { + lastErr = errors.New("migration api rate limited") + } + return nil, &RateLimitError{Cause: lastErr, RetryAfter: lastRetryAfter} +} diff --git a/internal/migration/http_retry_test.go b/internal/migration/http_retry_test.go new file mode 100644 index 0000000..402de9f --- /dev/null +++ b/internal/migration/http_retry_test.go @@ -0,0 +1,103 @@ +package migration + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestParseRetryAfter(t *testing.T) { + if got := parseRetryAfter("30"); got != 30*time.Second { + t.Fatalf("seconds = %v", got) + } + if got := parseRetryAfter(""); got != 0 { + t.Fatalf("empty = %v", got) + } + future := time.Now().Add(45 * time.Second).UTC().Format(http.TimeFormat) + if got := parseRetryAfter(future); got < 40*time.Second || got > 50*time.Second { + t.Fatalf("http date = %v", got) + } +} + +func TestRateLimitDelayUsesRetryAfter(t *testing.T) { + ConfigureRateLimit(RateLimitConfig{ + MaxRetries: 3, + BaseDelay: 100 * time.Millisecond, + MaxDelay: time.Second, + }) + delay := rateLimitDelay(1, 500*time.Millisecond) + if delay != 500*time.Millisecond { + t.Fatalf("delay = %v", delay) + } + delay = rateLimitDelay(3, 0) + if delay != 400*time.Millisecond { + t.Fatalf("exponential delay = %v", delay) + } +} + +func TestAPIGETRetries429ThenSucceeds(t *testing.T) { + ConfigureRateLimit(RateLimitConfig{ + MaxRetries: 5, + BaseDelay: 5 * time.Millisecond, + MaxDelay: 50 * time.Millisecond, + }) + + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls < 3 { + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"error":"rateLimitExceeded"}`)) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(srv.Close) + + body, err := apiGet(context.Background(), srv.Client(), srv.URL, "token") + if err != nil { + t.Fatalf("apiGet: %v", err) + } + if string(body) != `{"ok":true}` { + t.Fatalf("body = %q", body) + } + if calls != 3 { + t.Fatalf("calls = %d", calls) + } +} + +func TestAPIGETReturnsRateLimitErrorAfterMaxRetries(t *testing.T) { + ConfigureRateLimit(RateLimitConfig{ + MaxRetries: 2, + BaseDelay: time.Millisecond, + MaxDelay: 10 * time.Millisecond, + }) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"error":"quota"}`)) + })) + t.Cleanup(srv.Close) + + _, err := apiGet(context.Background(), srv.Client(), srv.URL, "token") + if !IsRateLimitError(err) { + t.Fatalf("expected RateLimitError, got %v", err) + } +} + +func TestWorkerRateLimitErrorIsPending(t *testing.T) { + if !IsRateLimitError(&RateLimitError{Cause: errTestRateLimit}) { + t.Fatal("expected typed rate limit error") + } +} + +var errTestRateLimit = &RateLimitError{Cause: errTestRateLimitCause{}} + +type errTestRateLimitCause struct{} + +func (errTestRateLimitCause) Error() string { return "api rate limited (429): quota" } diff --git a/internal/migration/httpmock_test.go b/internal/migration/httpmock_test.go new file mode 100644 index 0000000..c719279 --- /dev/null +++ b/internal/migration/httpmock_test.go @@ -0,0 +1,59 @@ +package migration + +import ( + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +type hostRewriteTransport struct { + mockBase string + match func(host string) bool + base http.RoundTripper +} + +func (rt *hostRewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if rt.match(req.URL.Host) { + mockURL, err := url.Parse(rt.mockBase) + if err != nil { + return nil, err + } + req.URL.Scheme = mockURL.Scheme + req.URL.Host = mockURL.Host + } + base := rt.base + if base == nil { + base = http.DefaultTransport + } + return base.RoundTrip(req) +} + +func mockGoogleHTTPClient(t *testing.T, handler http.HandlerFunc) *http.Client { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return &http.Client{ + Transport: &hostRewriteTransport{ + mockBase: srv.URL, + match: func(host string) bool { + return strings.Contains(host, "googleapis.com") + }, + }, + } +} + +func mockGraphHTTPClient(t *testing.T, handler http.HandlerFunc) *http.Client { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return &http.Client{ + Transport: &hostRewriteTransport{ + mockBase: srv.URL, + match: func(host string) bool { + return strings.Contains(host, "graph.microsoft.com") + }, + }, + } +} diff --git a/internal/migration/import_helpers.go b/internal/migration/import_helpers.go new file mode 100644 index 0000000..127a9fd --- /dev/null +++ b/internal/migration/import_helpers.go @@ -0,0 +1,142 @@ +package migration + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/ultisuite/ulti-backend/internal/mail/credentials" +) + +type progressUpdater func(status string, cursor, stats map[string]any, jobErr string) error + +type migrationUser struct { + Email string + ExternalID string + Name string +} + +func resolveMigrationUser(ctx context.Context, db *pgxpool.Pool, userID string) (migrationUser, error) { + var u migrationUser + err := db.QueryRow(ctx, ` + SELECT COALESCE(email, ''), COALESCE(external_id, ''), COALESCE(name, '') + FROM users WHERE id = $1::uuid + `, userID).Scan(&u.Email, &u.ExternalID, &u.Name) + if err != nil { + return migrationUser{}, fmt.Errorf("migration user not found") + } + if u.Email == "" { + return migrationUser{}, fmt.Errorf("migration user email missing") + } + return u, nil +} + +func migrationHTTPClient() *http.Client { + return &http.Client{Timeout: 90 * time.Second} +} + +func apiGet(ctx context.Context, client *http.Client, url, accessToken string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+accessToken) + resp, err := migrationDo(ctx, client, req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(resp.Body) +} + +func alreadyImported(store *ImportedItemStore, id string) bool { + if store == nil { + return false + } + return store.Has(id) +} + +func calendarSyncTokens(cursor map[string]any) map[string]string { + raw, _ := cursor["calendarSyncTokens"].(map[string]any) + out := make(map[string]string, len(raw)) + for k, v := range raw { + if s, ok := v.(string); ok && s != "" { + out[k] = s + } + } + return out +} + +func setCalendarSyncToken(cursor map[string]any, calID, token string) { + if calID == "" || token == "" { + return + } + raw, _ := cursor["calendarSyncTokens"].(map[string]any) + if raw == nil { + raw = map[string]any{} + cursor["calendarSyncTokens"] = raw + } + raw[calID] = token +} + +func calendarDeltaLinks(cursor map[string]any) map[string]string { + raw, _ := cursor["calendarDeltaLinks"].(map[string]any) + out := make(map[string]string, len(raw)) + for k, v := range raw { + if s, ok := v.(string); ok && s != "" { + out[k] = s + } + } + return out +} + +func setCalendarDeltaLink(cursor map[string]any, calID, link string) { + if calID == "" || link == "" { + return + } + raw, _ := cursor["calendarDeltaLinks"].(map[string]any) + if raw == nil { + raw = map[string]any{} + cursor["calendarDeltaLinks"] = raw + } + raw[calID] = link +} + +func migrationContactPath(bookPath, provider, sourceID string) string { + uid := sanitizeMigrationUID(provider, sourceID) + return bookPath + uid + ".vcf" +} + +func migrationEventPath(calPath, provider, sourceID string) string { + uid := sanitizeMigrationUID(provider, sourceID) + return calPath + uid + ".ics" +} + +func sanitizeMigrationUID(provider, sourceID string) string { + sourceID = strings.TrimSpace(sourceID) + sourceID = strings.ReplaceAll(sourceID, "/", "-") + return provider + "-" + sourceID + "@ultimail.migrated" +} + +func applyOAuthToken(cred credentials.Credential, token *oauthToken) credentials.Credential { + cred.AuthType = credentials.AuthOAuth2 + cred.AccessToken = token.AccessToken + if token.RefreshToken != "" { + cred.RefreshToken = token.RefreshToken + } + if !token.Expiry.IsZero() { + cred.Expiry = token.Expiry.UTC() + } + return cred +} + +type oauthToken struct { + AccessToken string + RefreshToken string + Expiry time.Time +} diff --git a/internal/migration/import_helpers_test.go b/internal/migration/import_helpers_test.go new file mode 100644 index 0000000..44485fc --- /dev/null +++ b/internal/migration/import_helpers_test.go @@ -0,0 +1,35 @@ +package migration + +import ( + "errors" + "fmt" + "testing" +) + +func TestMigrationContactAndEventPaths(t *testing.T) { + book := "/remote.php/dav/addressbooks/user%40test.local/migration-import/" + contact := migrationContactPath(book, "google", "people/abc") + if contact != book+"google-people-abc@ultimail.migrated.vcf" { + t.Fatalf("contact path: %q", contact) + } + cal := "/remote.php/dav/calendars/user%40test.local/migration-import/" + event := migrationEventPath(cal, "microsoft", "cal1:evt1") + if event != cal+"microsoft-cal1:evt1@ultimail.migrated.ics" { + t.Fatalf("event path: %q", event) + } +} + +func TestIsDeleteNotFound(t *testing.T) { + if isDeleteNotFound(nil) { + t.Fatal("nil is not not-found") + } + if !isDeleteNotFound(errors.New("delete failed: 404")) { + t.Fatal("expected 404 as not-found") + } + if !isDeleteNotFound(fmt.Errorf("not found")) { + t.Fatal("expected not found message") + } + if isDeleteNotFound(errors.New("permission denied")) { + t.Fatal("unexpected not-found") + } +} diff --git a/internal/migration/imported_items.go b/internal/migration/imported_items.go new file mode 100644 index 0000000..bf1dd5a --- /dev/null +++ b/internal/migration/imported_items.go @@ -0,0 +1,206 @@ +package migration + +import ( + "context" + "fmt" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// ImportedItemStore tracks imported source IDs and optional relative paths for a migration job. +// Data lives in migration_imported_items instead of unbounded cursor_json maps. +type ImportedItemStore struct { + db *pgxpool.Pool + jobID string + done map[string]struct{} // imported or skipped — resume skips these + paths map[string]string +} + +func NewImportedItemStoreMemory() *ImportedItemStore { + return &ImportedItemStore{ + done: map[string]struct{}{}, + paths: map[string]string{}, + } +} + +func LoadImportedItemStore(ctx context.Context, db *pgxpool.Pool, jobID string, cursor map[string]any) (*ImportedItemStore, error) { + store := &ImportedItemStore{ + db: db, + jobID: jobID, + done: map[string]struct{}{}, + paths: map[string]string{}, + } + if db != nil && jobID != "" { + rows, err := db.Query(ctx, ` + SELECT source_id, rel_path, status + FROM migration_imported_items + WHERE job_id = $1::uuid + `, jobID) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var sourceID, relPath, status string + if err := rows.Scan(&sourceID, &relPath, &status); err != nil { + return nil, err + } + if isImportedItemDone(status) { + store.done[sourceID] = struct{}{} + } + if relPath != "" { + store.paths[sourceID] = relPath + } + } + if err := rows.Err(); err != nil { + return nil, err + } + } + if err := store.migrateLegacyCursor(ctx, cursor); err != nil { + return nil, err + } + stripImportedCursorKeys(cursor) + return store, nil +} + +func isImportedItemDone(status string) bool { + switch status { + case "", ItemStatusImported, ItemStatusSkipped: + return true + default: + return false + } +} + +func stripImportedCursorKeys(cursor map[string]any) { + if cursor == nil { + return + } + delete(cursor, "imported_ids") + delete(cursor, "imported_paths") +} + +func (s *ImportedItemStore) Has(id string) bool { + _, ok := s.done[id] + return ok +} + +func (s *ImportedItemStore) Path(id string) string { + return s.paths[id] +} + +func (s *ImportedItemStore) MarkImported(ctx context.Context, id string) error { + return s.upsertItem(ctx, id, "", ItemStatusImported, "") +} + +func (s *ImportedItemStore) MarkPath(ctx context.Context, id, relPath string) error { + return s.upsertItem(ctx, id, relPath, ItemStatusImported, "") +} + +func (s *ImportedItemStore) MarkSkipped(ctx context.Context, id, reason, relPath string) error { + return s.upsertItem(ctx, id, relPath, ItemStatusSkipped, reason) +} + +func (s *ImportedItemStore) MarkFailed(ctx context.Context, id, reason, relPath string) error { + delete(s.done, id) + delete(s.paths, id) + if s.db == nil || s.jobID == "" || id == "" { + return nil + } + _, err := s.db.Exec(ctx, ` + INSERT INTO migration_imported_items (job_id, source_id, rel_path, status, reason) + VALUES ($1::uuid, $2, $3, $4, $5) + ON CONFLICT (job_id, source_id) DO UPDATE + SET rel_path = EXCLUDED.rel_path, + status = EXCLUDED.status, + reason = EXCLUDED.reason, + imported_at = NOW() + `, s.jobID, id, relPath, ItemStatusFailed, truncateReason(reason)) + return err +} + +func (s *ImportedItemStore) upsertItem(ctx context.Context, id, relPath, status, reason string) error { + if id == "" { + return nil + } + if isImportedItemDone(status) { + s.done[id] = struct{}{} + } else { + delete(s.done, id) + } + if relPath != "" { + s.paths[id] = relPath + } + if s.db == nil || s.jobID == "" { + return nil + } + _, err := s.db.Exec(ctx, ` + INSERT INTO migration_imported_items (job_id, source_id, rel_path, status, reason) + VALUES ($1::uuid, $2, $3, $4, $5) + ON CONFLICT (job_id, source_id) DO UPDATE + SET rel_path = EXCLUDED.rel_path, + status = EXCLUDED.status, + reason = EXCLUDED.reason, + imported_at = NOW() + `, s.jobID, id, relPath, status, truncateReason(reason)) + return err +} + +func truncateReason(reason string) string { + reason = strings.TrimSpace(reason) + const maxLen = 2000 + if len(reason) <= maxLen { + return reason + } + return reason[:maxLen] +} + +func (s *ImportedItemStore) Unmark(ctx context.Context, id string) error { + if id == "" { + return nil + } + delete(s.done, id) + delete(s.paths, id) + if s.db == nil || s.jobID == "" { + return nil + } + _, err := s.db.Exec(ctx, ` + DELETE FROM migration_imported_items + WHERE job_id = $1::uuid AND source_id = $2 + `, s.jobID, id) + return err +} + +func (s *ImportedItemStore) migrateLegacyCursor(ctx context.Context, cursor map[string]any) error { + if cursor == nil { + return nil + } + rawIDs, _ := cursor["imported_ids"].(map[string]any) + rawPaths, _ := cursor["imported_paths"].(map[string]any) + if len(rawIDs) == 0 && len(rawPaths) == 0 { + return nil + } + + seen := map[string]struct{}{} + for id := range rawIDs { + seen[id] = struct{}{} + } + for id := range rawPaths { + seen[id] = struct{}{} + } + + for id := range seen { + relPath, _ := rawPaths[id].(string) + if relPath != "" { + if err := s.MarkPath(ctx, id, relPath); err != nil { + return fmt.Errorf("migrate imported path %q: %w", id, err) + } + continue + } + if err := s.MarkImported(ctx, id); err != nil { + return fmt.Errorf("migrate imported id %q: %w", id, err) + } + } + return nil +} diff --git a/internal/migration/imported_items_test.go b/internal/migration/imported_items_test.go new file mode 100644 index 0000000..4a478c8 --- /dev/null +++ b/internal/migration/imported_items_test.go @@ -0,0 +1,109 @@ +package migration + +import ( + "testing" +) + +func TestImportedItemStoreStatusMemory(t *testing.T) { + ctx := t.Context() + store := NewImportedItemStoreMemory() + + if err := store.MarkImported(ctx, "ok-1"); err != nil { + t.Fatal(err) + } + if !store.Has("ok-1") { + t.Fatal("expected imported id in done set") + } + + if err := store.MarkSkipped(ctx, "skip-1", "too large", "big.bin"); err != nil { + t.Fatal(err) + } + if !store.Has("skip-1") { + t.Fatal("expected skipped id in done set") + } + if got := store.Path("skip-1"); got != "big.bin" { + t.Fatalf("path = %q", got) + } + + if err := store.MarkFailed(ctx, "fail-1", "upload error", ""); err != nil { + t.Fatal(err) + } + if store.Has("fail-1") { + t.Fatal("failed item should not be in done set") + } + + // retry success clears failure + if err := store.MarkImported(ctx, "fail-1"); err != nil { + t.Fatal(err) + } + if !store.Has("fail-1") { + t.Fatal("expected retried id in done set") + } +} + +func TestImportedItemStoreMemory(t *testing.T) { + ctx := t.Context() + store := NewImportedItemStoreMemory() + + if err := store.MarkPath(ctx, "id-1", "Docs/a.docx"); err != nil { + t.Fatal(err) + } + if !store.Has("id-1") { + t.Fatal("expected imported id") + } + if got := store.Path("id-1"); got != "Docs/a.docx" { + t.Fatalf("path = %q", got) + } + if err := store.Unmark(ctx, "id-1"); err != nil { + t.Fatal(err) + } + if store.Has("id-1") { + t.Fatal("expected id removed") + } +} + +func TestImportedItemStoreMigratesLegacyCursor(t *testing.T) { + ctx := t.Context() + cursor := map[string]any{ + "imported_ids": map[string]any{ + "file-1": true, + }, + "imported_paths": map[string]any{ + "file-1": "Docs/report.docx", + }, + } + store, err := LoadImportedItemStore(ctx, nil, "", cursor) + if err != nil { + t.Fatal(err) + } + if !store.Has("file-1") { + t.Fatal("expected migrated id") + } + if got := store.Path("file-1"); got != "Docs/report.docx" { + t.Fatalf("path = %q", got) + } + if _, ok := cursor["imported_ids"]; ok { + t.Fatal("expected imported_ids stripped from cursor") + } + if _, ok := cursor["imported_paths"]; ok { + t.Fatal("expected imported_paths stripped from cursor") + } +} + +func TestNormalizeAuditStatusFilter(t *testing.T) { + if got := normalizeAuditStatusFilter("failed"); got != ItemStatusFailed { + t.Fatalf("got %q", got) + } + if got := normalizeAuditStatusFilter("bogus"); got != "" { + t.Fatalf("got %q", got) + } +} + +func TestIncJobStat(t *testing.T) { + stats := map[string]any{} + incJobStat(stats, "failed") + incJobStat(stats, "failed") + if stats["failed"] != float64(2) { + t.Fatalf("failed = %v", stats["failed"]) + } +} diff --git a/internal/migration/job_audit.go b/internal/migration/job_audit.go new file mode 100644 index 0000000..ba807f7 --- /dev/null +++ b/internal/migration/job_audit.go @@ -0,0 +1,158 @@ +package migration + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "github.com/ultisuite/ulti-backend/internal/api/query" +) + +const ( + ItemStatusImported = "imported" + ItemStatusFailed = "failed" + ItemStatusSkipped = "skipped" +) + +// JobAuditItem is one row in a migration job item audit report. +type JobAuditItem struct { + SourceID string `json:"source_id"` + RelPath string `json:"rel_path,omitempty"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + ImportedAt string `json:"imported_at"` +} + +// JobAuditSummary counts items by status for a migration job. +type JobAuditSummary struct { + Service string `json:"service"` + Imported int64 `json:"imported"` + Failed int64 `json:"failed"` + Skipped int64 `json:"skipped"` + Total int64 `json:"total"` + ByStatus map[string]int64 `json:"by_status,omitempty"` +} + +func (s *Service) verifyJobInProject(ctx context.Context, projectID, jobID string) (service string, err error) { + err = s.db.QueryRow(ctx, ` + SELECT service FROM migration_jobs + WHERE id = $1::uuid AND project_id = $2::uuid + `, jobID, projectID).Scan(&service) + if errors.Is(err, pgx.ErrNoRows) { + return "", fmt.Errorf("job not found") + } + return service, err +} + +func (s *Service) JobAuditSummary(ctx context.Context, projectID, jobID string) (JobAuditSummary, error) { + service, err := s.verifyJobInProject(ctx, projectID, jobID) + if err != nil { + return JobAuditSummary{}, err + } + + rows, err := s.db.Query(ctx, ` + SELECT status, COUNT(*) + FROM migration_imported_items + WHERE job_id = $1::uuid + GROUP BY status + `, jobID) + if err != nil { + return JobAuditSummary{}, err + } + defer rows.Close() + + summary := JobAuditSummary{Service: service, ByStatus: map[string]int64{}} + for rows.Next() { + var status string + var count int64 + if err := rows.Scan(&status, &count); err != nil { + return JobAuditSummary{}, err + } + summary.ByStatus[status] = count + summary.Total += count + switch status { + case ItemStatusImported: + summary.Imported = count + case ItemStatusFailed: + summary.Failed = count + case ItemStatusSkipped: + summary.Skipped = count + } + } + return summary, rows.Err() +} + +func (s *Service) ListJobAudit( + ctx context.Context, + projectID, jobID, statusFilter string, + params query.ListParams, +) ([]JobAuditItem, query.PaginationMeta, error) { + if _, err := s.verifyJobInProject(ctx, projectID, jobID); err != nil { + return nil, query.PaginationMeta{}, err + } + + statusFilter = normalizeAuditStatusFilter(statusFilter) + + var total int64 + countSQL := `SELECT COUNT(*) FROM migration_imported_items WHERE job_id = $1::uuid` + countArgs := []any{jobID} + if statusFilter != "" { + countSQL += ` AND status = $2` + countArgs = append(countArgs, statusFilter) + } + if err := s.db.QueryRow(ctx, countSQL, countArgs...).Scan(&total); err != nil { + return nil, query.PaginationMeta{}, err + } + + listSQL := ` + SELECT source_id, rel_path, status, reason, imported_at::text + FROM migration_imported_items + WHERE job_id = $1::uuid + ` + listArgs := []any{jobID} + if statusFilter != "" { + listSQL += ` AND status = $2` + listArgs = append(listArgs, statusFilter) + } + listSQL += ` ORDER BY imported_at DESC, source_id ASC LIMIT $` + fmt.Sprint(len(listArgs)+1) + + ` OFFSET $` + fmt.Sprint(len(listArgs)+2) + listArgs = append(listArgs, params.Limit(), params.Offset()) + + rows, err := s.db.Query(ctx, listSQL, listArgs...) + if err != nil { + return nil, query.PaginationMeta{}, err + } + defer rows.Close() + + out := make([]JobAuditItem, 0, params.Limit()) + for rows.Next() { + var item JobAuditItem + if err := rows.Scan(&item.SourceID, &item.RelPath, &item.Status, &item.Reason, &item.ImportedAt); err != nil { + return nil, query.PaginationMeta{}, err + } + out = append(out, item) + } + if err := rows.Err(); err != nil { + return nil, query.PaginationMeta{}, err + } + return out, params.Meta(&total), nil +} + +func normalizeAuditStatusFilter(raw string) string { + switch raw { + case ItemStatusImported, ItemStatusFailed, ItemStatusSkipped: + return raw + default: + return "" + } +} + +func incJobStat(stats map[string]any, key string) { + if stats == nil { + return + } + v, _ := stats[key].(float64) + stats[key] = v + 1 +} diff --git a/internal/migration/microsoft_admin_consent.go b/internal/migration/microsoft_admin_consent.go new file mode 100644 index 0000000..7ab4096 --- /dev/null +++ b/internal/migration/microsoft_admin_consent.go @@ -0,0 +1,143 @@ +package migration + +import ( + "context" + "fmt" + "strings" +) + +const adminConsentStatePrefix = "project:" + +// MicrosoftAdminConsentRecord stores the outcome of a Microsoft admin consent redirect. +type MicrosoftAdminConsentRecord struct { + TenantID string + ClientID string + ProjectID string + Granted bool + ErrorCode string + ErrorDescription string +} + +// MicrosoftAdminConsent is a persisted tenant-level admin consent row. +type MicrosoftAdminConsent struct { + TenantID string `json:"tenant_id"` + ClientID string `json:"client_id"` + ProjectID string `json:"project_id,omitempty"` + Granted bool `json:"granted"` + ErrorCode string `json:"error_code,omitempty"` + ErrorDescription string `json:"error_description,omitempty"` + ConsentedAt string `json:"consented_at"` + UpdatedAt string `json:"updated_at"` +} + +func EncodeAdminConsentState(projectID string) string { + projectID = strings.TrimSpace(projectID) + if projectID == "" { + return "" + } + return adminConsentStatePrefix + projectID +} + +func ParseAdminConsentProjectID(state string) string { + state = strings.TrimSpace(state) + if !strings.HasPrefix(state, adminConsentStatePrefix) { + return "" + } + return strings.TrimSpace(strings.TrimPrefix(state, adminConsentStatePrefix)) +} + +func (s *Service) RecordMicrosoftAdminConsent(ctx context.Context, in MicrosoftAdminConsentRecord) error { + if s.db == nil { + return fmt.Errorf("database not configured") + } + tenantID := strings.TrimSpace(in.TenantID) + clientID := strings.TrimSpace(in.ClientID) + projectID := strings.TrimSpace(in.ProjectID) + if tenantID == "" { + return fmt.Errorf("tenant id required") + } + if clientID == "" { + return fmt.Errorf("client id required") + } + + tx, err := s.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + + _, err = tx.Exec(ctx, ` + INSERT INTO migration_microsoft_admin_consents ( + tenant_id, client_id, project_id, granted, error_code, error_description + ) VALUES ($1, $2, NULLIF($3, '')::uuid, $4, $5, $6) + ON CONFLICT (tenant_id, client_id) DO UPDATE SET + project_id = COALESCE(EXCLUDED.project_id, migration_microsoft_admin_consents.project_id), + granted = EXCLUDED.granted, + error_code = EXCLUDED.error_code, + error_description = EXCLUDED.error_description, + consented_at = NOW(), + updated_at = NOW() + `, tenantID, clientID, projectID, in.Granted, in.ErrorCode, in.ErrorDescription) + if err != nil { + return err + } + + if projectID != "" { + if in.Granted { + _, err = tx.Exec(ctx, ` + UPDATE migration_projects SET + microsoft_tenant_id = $2, + microsoft_admin_consent_at = NOW(), + microsoft_admin_consent_error = '', + updated_at = NOW() + WHERE id = $1::uuid + `, projectID, tenantID) + } else { + errMsg := strings.TrimSpace(in.ErrorDescription) + if errMsg == "" { + errMsg = strings.TrimSpace(in.ErrorCode) + } + _, err = tx.Exec(ctx, ` + UPDATE migration_projects SET + microsoft_tenant_id = CASE + WHEN microsoft_tenant_id = '' THEN $2 + ELSE microsoft_tenant_id + END, + microsoft_admin_consent_error = $3, + updated_at = NOW() + WHERE id = $1::uuid + `, projectID, tenantID, errMsg) + } + if err != nil { + return err + } + } + + return tx.Commit(ctx) +} + +func (s *Service) ListMicrosoftAdminConsents(ctx context.Context) ([]MicrosoftAdminConsent, error) { + rows, err := s.db.Query(ctx, ` + SELECT tenant_id, client_id, COALESCE(project_id::text, ''), granted, + error_code, error_description, consented_at::text, updated_at::text + FROM migration_microsoft_admin_consents + ORDER BY updated_at DESC + `) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []MicrosoftAdminConsent + for rows.Next() { + var row MicrosoftAdminConsent + if err := rows.Scan( + &row.TenantID, &row.ClientID, &row.ProjectID, &row.Granted, + &row.ErrorCode, &row.ErrorDescription, &row.ConsentedAt, &row.UpdatedAt, + ); err != nil { + return nil, err + } + out = append(out, row) + } + return out, rows.Err() +} diff --git a/internal/migration/microsoft_admin_consent_test.go b/internal/migration/microsoft_admin_consent_test.go new file mode 100644 index 0000000..732b7ee --- /dev/null +++ b/internal/migration/microsoft_admin_consent_test.go @@ -0,0 +1,17 @@ +package migration + +import "testing" + +func TestAdminConsentStateRoundTrip(t *testing.T) { + const projectID = "550e8400-e29b-41d4-a716-446655440000" + state := EncodeAdminConsentState(projectID) + if state != adminConsentStatePrefix+projectID { + t.Fatalf("encode: %q", state) + } + if got := ParseAdminConsentProjectID(state); got != projectID { + t.Fatalf("parse: got %q", got) + } + if ParseAdminConsentProjectID("other") != "" { + t.Fatal("expected empty for unrelated state") + } +} diff --git a/internal/migration/microsoft_app.go b/internal/migration/microsoft_app.go new file mode 100644 index 0000000..7a1413d --- /dev/null +++ b/internal/migration/microsoft_app.go @@ -0,0 +1,103 @@ +package migration + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" +) + +// MicrosoftApp mints Graph access tokens via the client credentials flow. +type MicrosoftApp struct { + clientID string + clientSecret string + defaultTenant string + tokenURL string + client *http.Client +} + +type MicrosoftAppConfig struct { + ClientID string + ClientSecret string + DefaultTenant string + TokenURL string + HTTPClient *http.Client +} + +func NewMicrosoftApp(cfg MicrosoftAppConfig) (*MicrosoftApp, error) { + clientID := strings.TrimSpace(cfg.ClientID) + clientSecret := strings.TrimSpace(cfg.ClientSecret) + if clientID == "" || clientSecret == "" { + return nil, nil + } + client := cfg.HTTPClient + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + return &MicrosoftApp{ + clientID: clientID, + clientSecret: clientSecret, + defaultTenant: strings.TrimSpace(cfg.DefaultTenant), + tokenURL: strings.TrimSpace(cfg.TokenURL), + client: client, + }, nil +} + +func (m *MicrosoftApp) Enabled() bool { + return m != nil && m.clientID != "" && m.clientSecret != "" +} + +func (m *MicrosoftApp) AccessToken(ctx context.Context, tenantID string) (string, error) { + if !m.Enabled() { + return "", fmt.Errorf("microsoft app-only auth not configured") + } + tenantID = strings.TrimSpace(tenantID) + if tenantID == "" { + tenantID = m.defaultTenant + } + if tenantID == "" { + return "", fmt.Errorf("microsoft tenant id required for app-only auth") + } + tokenURL := m.tokenURL + if tokenURL == "" { + tokenURL = fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/v2.0/token", url.PathEscape(tenantID)) + } + cc := clientcredentials.Config{ + ClientID: m.clientID, + ClientSecret: m.clientSecret, + TokenURL: tokenURL, + Scopes: []string{"https://graph.microsoft.com/.default"}, + } + if m.client != nil { + ctx = context.WithValue(ctx, oauth2.HTTPClient, m.client) + } + token, err := cc.Token(ctx) + if err != nil { + return "", fmt.Errorf("microsoft app token: %w", err) + } + if token.AccessToken == "" { + return "", fmt.Errorf("microsoft app token empty") + } + return token.AccessToken, nil +} + +// WithClient overrides the HTTP client (tests). +func (m *MicrosoftApp) WithClient(c *http.Client) *MicrosoftApp { + if m != nil && c != nil { + m.client = c + } + return m +} + +func microsoftAppTokenURL(tenantID string) string { + tenantID = strings.TrimSpace(tenantID) + if tenantID == "" { + tenantID = "common" + } + return fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/v2.0/token", url.PathEscape(tenantID)) +} diff --git a/internal/migration/microsoft_app_test.go b/internal/migration/microsoft_app_test.go new file mode 100644 index 0000000..f13eed9 --- /dev/null +++ b/internal/migration/microsoft_app_test.go @@ -0,0 +1,141 @@ +package migration + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestNormalizeAuthModeMicrosoftApp(t *testing.T) { + if got := NormalizeAuthMode("microsoft", "microsoft_app"); got != AuthModeMicrosoftApp { + t.Fatalf("got %q", got) + } + if got := NormalizeAuthMode("google", "microsoft_app"); got != AuthModeOAuth { + t.Fatalf("google ignores ms app: got %q", got) + } +} + +func TestUsesUserOAuth(t *testing.T) { + if UsesUserOAuth("google", AuthModeGoogleDWD) { + t.Fatal("google dwd should skip user oauth") + } + if UsesUserOAuth("microsoft", AuthModeMicrosoftApp) { + t.Fatal("microsoft app should skip user oauth") + } + if !UsesUserOAuth("microsoft", "oauth") { + t.Fatal("microsoft oauth needs user oauth") + } +} + +func TestGraphUserBase(t *testing.T) { + if got := graphUserBase(""); got != "/v1.0/me" { + t.Fatalf("empty upn: %q", got) + } + if got := graphUserBase("alice@contoso.com"); got != "/v1.0/users/alice@contoso.com" { + t.Fatalf("encoded upn: %q", got) + } +} + +func TestNewMicrosoftAppEmpty(t *testing.T) { + app, err := NewMicrosoftApp(MicrosoftAppConfig{}) + if err != nil || app != nil { + t.Fatalf("empty config: app=%v err=%v", app, err) + } +} + +func TestMicrosoftAppAccessToken(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Fatalf("method %s", r.Method) + } + if err := r.ParseForm(); err != nil { + t.Fatal(err) + } + if r.Form.Get("grant_type") != "client_credentials" { + t.Fatalf("grant_type=%q", r.Form.Get("grant_type")) + } + auth := r.Header.Get("Authorization") + if !strings.HasPrefix(auth, "Basic ") { + t.Fatalf("expected basic auth, got %q", auth) + } + if !strings.Contains(r.Form.Get("scope"), "graph.microsoft.com/.default") { + t.Fatalf("scope=%q", r.Form.Get("scope")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"access_token":"app-token","token_type":"Bearer","expires_in":3600}`) + })) + t.Cleanup(srv.Close) + + app, err := NewMicrosoftApp(MicrosoftAppConfig{ + ClientID: "client-id", + ClientSecret: "client-secret", + TokenURL: srv.URL, + }) + if err != nil { + t.Fatal(err) + } + token, err := app.AccessToken(t.Context(), "tenant-123") + if err != nil { + t.Fatal(err) + } + if token != "app-token" { + t.Fatalf("token=%q", token) + } +} + +func TestMicrosoftAppRequiresTenant(t *testing.T) { + app, err := NewMicrosoftApp(MicrosoftAppConfig{ + ClientID: "client-id", + ClientSecret: "client-secret", + TokenURL: "http://example.invalid/token", + }) + if err != nil { + t.Fatal(err) + } + if _, err := app.AccessToken(t.Context(), ""); err == nil { + t.Fatal("expected tenant required error") + } +} + +func TestWorkerAuthPathSelection(t *testing.T) { + cases := []struct { + provider string + authMode string + userOAuth bool + }{ + {"google", AuthModeOAuth, true}, + {"google", AuthModeGoogleDWD, false}, + {"microsoft", AuthModeOAuth, true}, + {"microsoft", AuthModeMicrosoftApp, false}, + {"microsoft", "google_dwd", true}, + } + for _, tc := range cases { + got := UsesUserOAuth(tc.provider, tc.authMode) + if got != tc.userOAuth { + t.Fatalf("%s/%s: got userOAuth=%v want %v", tc.provider, tc.authMode, got, tc.userOAuth) + } + } +} + +func TestMicrosoftAppTokenError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _ = json.NewEncoder(w).Encode(map[string]string{"error": "invalid_client"}) + })) + t.Cleanup(srv.Close) + + app, err := NewMicrosoftApp(MicrosoftAppConfig{ + ClientID: "bad", + ClientSecret: "bad", + TokenURL: srv.URL, + }) + if err != nil { + t.Fatal(err) + } + if _, err := app.AccessToken(t.Context(), "tenant"); err == nil { + t.Fatal("expected token error") + } +} diff --git a/internal/migration/oauth.go b/internal/migration/oauth.go new file mode 100644 index 0000000..e73b757 --- /dev/null +++ b/internal/migration/oauth.go @@ -0,0 +1,228 @@ +package migration + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/redis/go-redis/v9" + "golang.org/x/oauth2" +) + +const pendingKeyPrefix = "migration_oauth_pending:" +const pendingTTL = 15 * time.Minute + +var ErrUnknownState = errors.New("migration oauth state expired or unknown") +var ErrProviderDisabled = errors.New("migration oauth provider not configured") + +type Provider string + +const ( + ProviderGoogle Provider = "google" + ProviderMicrosoft Provider = "microsoft" +) + +type PendingOAuth struct { + UserID string `json:"user_id"` + ProjectID string `json:"project_id"` + Provider string `json:"provider"` + InviteToken string `json:"invite_token,omitempty"` + PKCEVerifier string `json:"pkce_verifier"` +} + +type OAuthConfig struct { + GoogleClientID string + GoogleClientSecret string + MicrosoftClientID string + MicrosoftSecret string + MicrosoftTenant string + RedirectURL string +} + +type OAuthService struct { + cfg OAuthConfig + rdb *redis.Client +} + +func NewOAuthService(cfg OAuthConfig, rdb *redis.Client) *OAuthService { + return &OAuthService{cfg: cfg, rdb: rdb} +} + +func (s *OAuthService) EnabledProviders() []string { + var out []string + if s.providerConfig(ProviderGoogle) != nil { + out = append(out, string(ProviderGoogle)) + } + if s.providerConfig(ProviderMicrosoft) != nil { + out = append(out, string(ProviderMicrosoft)) + } + return out +} + +func (s *OAuthService) Start(ctx context.Context, pending PendingOAuth, provider Provider) (authURL, state string, err error) { + oauthCfg := s.providerConfig(provider) + if oauthCfg == nil { + return "", "", ErrProviderDisabled + } + verifier, challenge, err := newPKCE() + if err != nil { + return "", "", err + } + state, err = randomState() + if err != nil { + return "", "", err + } + pending.Provider = string(provider) + pending.PKCEVerifier = verifier + if err := s.savePending(ctx, state, pending); err != nil { + return "", "", err + } + authURL = oauthCfg.AuthCodeURL(state, + oauth2.AccessTypeOffline, + oauth2.SetAuthURLParam("code_challenge", challenge), + oauth2.SetAuthURLParam("code_challenge_method", "S256"), + oauth2.SetAuthURLParam("prompt", "consent"), + ) + return authURL, state, nil +} + +func (s *OAuthService) Exchange(ctx context.Context, state, code string) (PendingOAuth, *oauth2.Token, []string, error) { + pending, err := s.loadPending(ctx, state) + if err != nil { + return PendingOAuth{}, nil, nil, err + } + oauthCfg := s.providerConfig(Provider(pending.Provider)) + if oauthCfg == nil { + return PendingOAuth{}, nil, nil, ErrProviderDisabled + } + token, err := oauthCfg.Exchange(ctx, code, oauth2.SetAuthURLParam("code_verifier", pending.PKCEVerifier)) + if err != nil { + return PendingOAuth{}, nil, nil, fmt.Errorf("token exchange: %w", err) + } + _ = s.rdb.Del(ctx, pendingKeyPrefix+state).Err() + return pending, token, oauthCfg.Scopes, nil +} + +func (s *OAuthService) Refresh(ctx context.Context, provider Provider, refreshToken string) (*oauth2.Token, error) { + oauthCfg := s.providerConfig(provider) + if oauthCfg == nil { + return nil, ErrProviderDisabled + } + if strings.TrimSpace(refreshToken) == "" { + return nil, fmt.Errorf("refresh token required") + } + token, err := oauthCfg.TokenSource(ctx, &oauth2.Token{RefreshToken: refreshToken}).Token() + if err != nil { + return nil, fmt.Errorf("token refresh: %w", err) + } + return token, nil +} + +// AdminConsentURL returns the Microsoft tenant admin consent URL for migration scopes. +func (s *OAuthService) AdminConsentURL(tenant, state string) (string, error) { + if s.cfg.MicrosoftClientID == "" || s.cfg.RedirectURL == "" { + return "", ErrProviderDisabled + } + tenant = strings.TrimSpace(tenant) + if tenant == "" { + tenant = s.cfg.MicrosoftTenant + } + if tenant == "" { + tenant = "common" + } + values := url.Values{} + values.Set("client_id", s.cfg.MicrosoftClientID) + values.Set("redirect_uri", s.cfg.RedirectURL) + if state = strings.TrimSpace(state); state != "" { + values.Set("state", state) + } + return fmt.Sprintf("https://login.microsoftonline.com/%s/adminconsent?%s", url.PathEscape(tenant), values.Encode()), nil +} + +func (s *OAuthService) MicrosoftClientID() string { + return s.cfg.MicrosoftClientID +} + +func (s *OAuthService) providerConfig(provider Provider) *oauth2.Config { + switch provider { + case ProviderGoogle: + if s.cfg.GoogleClientID == "" || s.cfg.GoogleClientSecret == "" || s.cfg.RedirectURL == "" { + return nil + } + return &oauth2.Config{ + ClientID: s.cfg.GoogleClientID, + ClientSecret: s.cfg.GoogleClientSecret, + RedirectURL: s.cfg.RedirectURL, + Scopes: []string{ + "https://www.googleapis.com/auth/gmail.readonly", + "https://www.googleapis.com/auth/drive.readonly", + "https://www.googleapis.com/auth/calendar.readonly", + "https://www.googleapis.com/auth/contacts.readonly", + }, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://accounts.google.com/o/oauth2/v2/auth", + TokenURL: "https://oauth2.googleapis.com/token", + }, + } + case ProviderMicrosoft: + if s.cfg.MicrosoftClientID == "" || s.cfg.MicrosoftSecret == "" || s.cfg.RedirectURL == "" { + return nil + } + tenant := s.cfg.MicrosoftTenant + if tenant == "" { + tenant = "common" + } + return &oauth2.Config{ + ClientID: s.cfg.MicrosoftClientID, + ClientSecret: s.cfg.MicrosoftSecret, + RedirectURL: s.cfg.RedirectURL, + Scopes: []string{ + "offline_access", + "User.Read", + "Mail.Read", + "Files.Read.All", + "Calendars.Read", + "Contacts.Read", + }, + Endpoint: oauth2.Endpoint{ + AuthURL: fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/v2.0/authorize", tenant), + TokenURL: fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/v2.0/token", tenant), + }, + } + default: + return nil + } +} + +func (s *OAuthService) savePending(ctx context.Context, state string, pending PendingOAuth) error { + if s.rdb == nil { + return errors.New("oauth state store unavailable") + } + raw, err := json.Marshal(pending) + if err != nil { + return err + } + return s.rdb.Set(ctx, pendingKeyPrefix+state, raw, pendingTTL).Err() +} + +func (s *OAuthService) loadPending(ctx context.Context, state string) (PendingOAuth, error) { + if s.rdb == nil { + return PendingOAuth{}, errors.New("oauth state store unavailable") + } + raw, err := s.rdb.Get(ctx, pendingKeyPrefix+state).Bytes() + if err != nil { + if errors.Is(err, redis.Nil) { + return PendingOAuth{}, ErrUnknownState + } + return PendingOAuth{}, err + } + var pending PendingOAuth + if err := json.Unmarshal(raw, &pending); err != nil { + return PendingOAuth{}, err + } + return pending, nil +} diff --git a/internal/migration/oauth_admin_test.go b/internal/migration/oauth_admin_test.go new file mode 100644 index 0000000..449f772 --- /dev/null +++ b/internal/migration/oauth_admin_test.go @@ -0,0 +1,61 @@ +package migration + +import ( + "strings" + "testing" +) + +func TestAdminConsentURL(t *testing.T) { + svc := NewOAuthService(OAuthConfig{ + MicrosoftClientID: "client-id", + MicrosoftSecret: "secret", + MicrosoftTenant: "contoso.onmicrosoft.com", + RedirectURL: "https://suite.example.com/api/v1/migration/oauth/callback", + }, nil) + url, err := svc.AdminConsentURL("", EncodeAdminConsentState("550e8400-e29b-41d4-a716-446655440000")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(url, "adminconsent") { + t.Fatalf("expected adminconsent url, got %q", url) + } + if !strings.Contains(url, "client-id") { + t.Fatalf("missing client id: %q", url) + } + if !strings.Contains(url, "state=project%3A") { + t.Fatalf("missing project state: %q", url) + } +} + +func TestAdminConsentURLWithoutState(t *testing.T) { + svc := NewOAuthService(OAuthConfig{ + MicrosoftClientID: "client-id", + MicrosoftSecret: "secret", + RedirectURL: "https://suite.example.com/api/v1/migration/oauth/callback", + }, nil) + url, err := svc.AdminConsentURL("contoso.onmicrosoft.com", "") + if err != nil { + t.Fatal(err) + } + if strings.Contains(url, "state=") { + t.Fatalf("unexpected state in url: %q", url) + } +} + +func TestSanitizeMigrationUID(t *testing.T) { + uid := sanitizeMigrationUID("google", "people/abc123") + if uid != "google-people-abc123@ultimail.migrated" { + t.Fatalf("got %q", uid) + } +} + +func TestParseFlexibleTime(t *testing.T) { + tm := parseFlexibleTime("2024-05-01T10:00:00Z", "") + if tm.IsZero() { + t.Fatal("expected datetime parse") + } + tm = parseFlexibleTime("", "2024-05-01") + if tm.IsZero() { + t.Fatal("expected date parse") + } +} diff --git a/internal/migration/oauth_refresh.go b/internal/migration/oauth_refresh.go new file mode 100644 index 0000000..7c32975 --- /dev/null +++ b/internal/migration/oauth_refresh.go @@ -0,0 +1,78 @@ +package migration + +import ( + "context" + "fmt" + "time" + + "golang.org/x/oauth2" + + "github.com/ultisuite/ulti-backend/internal/mail/credentials" +) + +// RefreshCredential refreshes an expired migration OAuth token and persists it. +func (s *Service) RefreshCredential( + ctx context.Context, + oauth *OAuthService, + userID, projectID, provider string, + cred credentials.Credential, +) (credentials.Credential, error) { + if oauth == nil { + return cred, fmt.Errorf("migration oauth not configured") + } + if !cred.NeedsRefresh() { + return cred, nil + } + if cred.RefreshToken == "" { + return cred, fmt.Errorf("migration refresh token missing; re-run OAuth consent") + } + token, err := oauth.Refresh(ctx, Provider(provider), cred.RefreshToken) + if err != nil { + return cred, fmt.Errorf("refresh migration token: %w", err) + } + updated := applyOAuthTokenFromOAuth2(cred, token) + if err := s.SaveCredential(ctx, userID, projectID, provider, updated); err != nil { + return cred, err + } + return updated, nil +} + +// SaveCredential encrypts and stores migration OAuth credentials. +func (s *Service) SaveCredential(ctx context.Context, userID, projectID, provider string, cred credentials.Credential) error { + if s.creds == nil { + return fmt.Errorf("credential manager not configured") + } + cred.AuthType = credentials.AuthOAuth2 + cred.OAuthProvider = provider + enc, err := s.creds.EncryptCredential(cred) + if err != nil { + return err + } + var expiresAt *time.Time + if !cred.Expiry.IsZero() { + expiresAt = &cred.Expiry + } + _, err = s.db.Exec(ctx, ` + UPDATE migration_credentials SET + encrypted_token = $4, + expires_at = $5, + revoked_at = NULL + WHERE user_id = $1::uuid AND project_id = $2::uuid AND provider = $3 + `, userID, projectID, provider, enc, expiresAt) + return err +} + +func (s *Service) MicrosoftAdminConsentURL(tenant, projectID string) (string, error) { + if s.oauth == nil { + return "", fmt.Errorf("migration oauth not configured") + } + return s.oauth.AdminConsentURL(tenant, EncodeAdminConsentState(projectID)) +} + +func applyOAuthTokenFromOAuth2(cred credentials.Credential, token *oauth2.Token) credentials.Credential { + return applyOAuthToken(cred, &oauthToken{ + AccessToken: token.AccessToken, + RefreshToken: token.RefreshToken, + Expiry: token.Expiry, + }) +} diff --git a/internal/migration/onboarding.go b/internal/migration/onboarding.go new file mode 100644 index 0000000..c048c6a --- /dev/null +++ b/internal/migration/onboarding.go @@ -0,0 +1,93 @@ +package migration + +import ( + "context" + "strings" +) + +// OnboardingHints guides the user-facing migration setup flow. +type OnboardingHints struct { + NeedsUserOAuth bool `json:"needs_user_oauth"` + OAuthProvider string `json:"oauth_provider,omitempty"` + WaitingForAdmin bool `json:"waiting_for_admin"` + WaitingReason string `json:"waiting_reason,omitempty"` + HasMigrationCredentials bool `json:"has_migration_credentials"` + NeedsMicrosoftAdminConsent bool `json:"needs_microsoft_admin_consent,omitempty"` +} + +func (s *Service) BuildOnboardingHints(ctx context.Context, userID string, proj Project, invite Invite) OnboardingHints { + h := OnboardingHints{ + OAuthProvider: proj.SourceProvider, + } + + switch proj.Status { + case "active", "cutover": + // worker eligible + default: + h.WaitingForAdmin = true + if proj.Status == "draft" { + h.WaitingReason = "project_not_activated" + } else { + h.WaitingReason = "project_status_" + proj.Status + } + } + + if UsesUserOAuth(proj.SourceProvider, proj.AuthMode) { + h.NeedsUserOAuth = true + if userID != "" { + hasCred, err := s.hasMigrationCredential(ctx, userID, proj.ID, proj.SourceProvider) + if err == nil { + h.HasMigrationCredentials = hasCred + } + } + } + + if proj.SourceProvider == "microsoft" && proj.MicrosoftAdminConsentAt == nil { + h.NeedsMicrosoftAdminConsent = true + } + + if invite.Status == "claimed" && h.WaitingForAdmin { + h.WaitingReason = waitingReasonMessage(h.WaitingReason) + } + + return h +} + +func waitingReasonMessage(code string) string { + switch code { + case "project_not_activated": + return "project_not_activated" + default: + return code + } +} + +func (s *Service) hasMigrationCredential(ctx context.Context, userID, projectID, provider string) (bool, error) { + provider = strings.ToLower(strings.TrimSpace(provider)) + if provider == "" { + return false, nil + } + var exists bool + err := s.db.QueryRow(ctx, ` + SELECT EXISTS( + SELECT 1 FROM migration_credentials + WHERE user_id = $1::uuid AND project_id = $2::uuid AND provider = $3 AND revoked_at IS NULL + ) + `, userID, projectID, provider).Scan(&exists) + return exists, err +} + +func (s *Service) BuildInviteOnboardingHints(proj Project, invite Invite) OnboardingHints { + h := OnboardingHints{OAuthProvider: proj.SourceProvider} + if invite.Status == "claimed" { + h.NeedsUserOAuth = UsesUserOAuth(proj.SourceProvider, proj.AuthMode) + return h + } + if UsesUserOAuth(proj.SourceProvider, proj.AuthMode) { + h.NeedsUserOAuth = true + } + if proj.SourceProvider == "microsoft" && proj.MicrosoftAdminConsentAt == nil { + h.NeedsMicrosoftAdminConsent = true + } + return h +} diff --git a/internal/migration/onboarding_test.go b/internal/migration/onboarding_test.go new file mode 100644 index 0000000..067ef03 --- /dev/null +++ b/internal/migration/onboarding_test.go @@ -0,0 +1,75 @@ +package migration + +import "testing" + +func TestBuildOnboardingHintsGoogleDWD(t *testing.T) { + s := &Service{} + h := s.BuildOnboardingHints(t.Context(), "user-id", Project{ + ID: "p1", + SourceProvider: "google", + AuthMode: AuthModeGoogleDWD, + Status: "active", + }, Invite{Status: "claimed"}) + if h.NeedsUserOAuth { + t.Fatal("google dwd should not need user oauth") + } + if h.WaitingForAdmin { + t.Fatal("active project should not wait for admin") + } +} + +func TestBuildOnboardingHintsDraftProject(t *testing.T) { + s := &Service{} + h := s.BuildOnboardingHints(t.Context(), "", Project{ + ID: "p1", + SourceProvider: "google", + AuthMode: "oauth", + Status: "draft", + }, Invite{Status: "claimed"}) + if !h.WaitingForAdmin || h.WaitingReason != "project_not_activated" { + t.Fatalf("expected wait activate, got %#v", h) + } + if !h.NeedsUserOAuth { + t.Fatal("oauth mode needs user oauth") + } +} + +func TestBuildOnboardingHintsMicrosoftConsent(t *testing.T) { + s := &Service{} + h := s.BuildOnboardingHints(t.Context(), "", Project{ + ID: "p1", + SourceProvider: "microsoft", + AuthMode: "oauth", + Status: "active", + }, Invite{Status: "claimed"}) + if !h.NeedsMicrosoftAdminConsent { + t.Fatal("expected ms admin consent hint") + } +} + +func TestBuildOnboardingHintsMicrosoftApp(t *testing.T) { + s := &Service{} + h := s.BuildOnboardingHints(t.Context(), "user-id", Project{ + ID: "p1", + SourceProvider: "microsoft", + AuthMode: AuthModeMicrosoftApp, + Status: "active", + MicrosoftAdminConsentAt: strPtr("2026-01-01T00:00:00Z"), + }, Invite{Status: "claimed"}) + if h.NeedsUserOAuth { + t.Fatal("microsoft app should not need user oauth") + } +} + +func strPtr(s string) *string { return &s } + +func TestBuildInviteOnboardingHintsUnclaimed(t *testing.T) { + s := &Service{} + h := s.BuildInviteOnboardingHints(Project{ + SourceProvider: "google", + AuthMode: AuthModeGoogleDWD, + }, Invite{Status: "invited"}) + if h.NeedsUserOAuth { + t.Fatal("dwd invite should not prompt oauth before claim") + } +} diff --git a/internal/migration/pkce.go b/internal/migration/pkce.go new file mode 100644 index 0000000..b66df98 --- /dev/null +++ b/internal/migration/pkce.go @@ -0,0 +1,30 @@ +package migration + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base64" +) + +func base64URLEncode(b []byte) string { + return base64.RawURLEncoding.EncodeToString(b) +} + +func sha256Sum(input string) ([]byte, error) { + sum := sha256.Sum256([]byte(input)) + return sum[:], nil +} + +func newPKCE() (verifier, challenge string, err error) { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return "", "", err + } + verifier = base64.RawURLEncoding.EncodeToString(b) + sum, err := sha256Sum(verifier) + if err != nil { + return "", "", err + } + challenge = base64.RawURLEncoding.EncodeToString(sum) + return verifier, challenge, nil +} diff --git a/internal/migration/project_columns.go b/internal/migration/project_columns.go new file mode 100644 index 0000000..564f8bc --- /dev/null +++ b/internal/migration/project_columns.go @@ -0,0 +1,94 @@ +package migration + +import ( + "encoding/json" + "strings" + + "github.com/ultisuite/ulti-backend/internal/mail/hosted" +) + +var projectSelectColumns = []string{ + "id::text", + "COALESCE(domain_id::text, '')", + "name", + "source_provider", + "auth_mode", + "status", + "cutover_at::text", + "delta_mode", + "created_at::text", + "NULLIF(microsoft_tenant_id, '')", + "microsoft_admin_consent_at::text", + "COALESCE(NULLIF(microsoft_admin_consent_error, ''), '')", + "cutover_dns_json", +} + +func projectSelectSQL(tablePrefix string) string { + if tablePrefix != "" && !strings.HasSuffix(tablePrefix, ".") { + tablePrefix += "." + } + cols := make([]string, len(projectSelectColumns)) + for i, col := range projectSelectColumns { + cols[i] = tablePrefix + col + } + return strings.Join(cols, ", ") +} + +type projectScanner struct { + project Project + cutoverDNSRaw []byte +} + +func newProjectScanner() *projectScanner { + return &projectScanner{} +} + +func (s *projectScanner) targets() []any { + return []any{ + &s.project.ID, &s.project.DomainID, &s.project.Name, &s.project.SourceProvider, + &s.project.AuthMode, &s.project.Status, &s.project.CutoverAt, &s.project.DeltaMode, + &s.project.CreatedAt, &s.project.MicrosoftTenantID, &s.project.MicrosoftAdminConsentAt, + &s.project.MicrosoftAdminConsentError, &s.cutoverDNSRaw, + } +} + +func (s *projectScanner) result() Project { + applyCutoverDNS(&s.project, s.cutoverDNSRaw) + return s.project +} + +func applyCutoverDNS(p *Project, raw []byte) { + p.CutoverDNS = nil + raw = bytesTrimSpace(raw) + if len(raw) == 0 || string(raw) == "{}" || string(raw) == "null" { + return + } + var report hosted.DNSCheckReport + if err := json.Unmarshal(raw, &report); err != nil { + return + } + if !dnsReportHasContent(report) { + return + } + p.CutoverDNS = &report +} + +func dnsReportHasContent(r hosted.DNSCheckReport) bool { + if strings.TrimSpace(r.Domain) != "" { + return true + } + if len(r.Errors) > 0 || len(r.Warnings) > 0 { + return true + } + if len(r.MXRecords) > 0 || len(r.TXTRecords) > 0 || len(r.ExpectedMX) > 0 { + return true + } + if r.TXTVerified || r.MXVerified { + return true + } + return strings.TrimSpace(r.TXTExpected) != "" +} + +func bytesTrimSpace(b []byte) []byte { + return []byte(strings.TrimSpace(string(b))) +} diff --git a/internal/migration/project_columns_test.go b/internal/migration/project_columns_test.go new file mode 100644 index 0000000..544c265 --- /dev/null +++ b/internal/migration/project_columns_test.go @@ -0,0 +1,32 @@ +package migration + +import ( + "testing" + + "github.com/ultisuite/ulti-backend/internal/mail/hosted" +) + +func TestApplyCutoverDNSEmptyObject(t *testing.T) { + var p Project + applyCutoverDNS(&p, []byte(`{}`)) + if p.CutoverDNS != nil { + t.Fatal("expected nil cutover dns for empty object") + } +} + +func TestApplyCutoverDNSReport(t *testing.T) { + var p Project + applyCutoverDNS(&p, []byte(`{"domain":"acme.com","txt_verified":true,"mx_verified":false}`)) + if p.CutoverDNS == nil || p.CutoverDNS.Domain != "acme.com" { + t.Fatalf("got %#v", p.CutoverDNS) + } +} + +func TestDNSReportHasContent(t *testing.T) { + if dnsReportHasContent(hosted.DNSCheckReport{}) { + t.Fatal("empty report should have no content") + } + if !dnsReportHasContent(hosted.DNSCheckReport{Warnings: []string{"no domain_id"}}) { + t.Fatal("warnings should count as content") + } +} diff --git a/internal/migration/service.go b/internal/migration/service.go new file mode 100644 index 0000000..9238da6 --- /dev/null +++ b/internal/migration/service.go @@ -0,0 +1,499 @@ +package migration + +import ( + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "golang.org/x/oauth2" + + "github.com/ultisuite/ulti-backend/internal/mail/credentials" + "github.com/ultisuite/ulti-backend/internal/mail/hosted" +) + +var ( + ErrInviteNotFound = errors.New("migration invite not found") + ErrInviteClaimed = errors.New("migration invite already claimed") + ErrEmailMismatch = errors.New("email does not match invite") + ErrMigrationDomainNotActive = errors.New("migration project mail domain is not active") + ErrMigrationDomainMismatch = errors.New("invite email domain does not match migration project domain") +) + +type Service struct { + db *pgxpool.Pool + rdb *redis.Client + creds *credentials.Manager + hosted *hosted.Service + oauth *OAuthService + cutover CutoverConfig +} + +func NewService(db *pgxpool.Pool, rdb *redis.Client, creds *credentials.Manager, hostedSvc *hosted.Service, oauth *OAuthService) *Service { + return &Service{db: db, rdb: rdb, creds: creds, hosted: hostedSvc, oauth: oauth} +} + +func (s *Service) SetCutoverConfig(cfg CutoverConfig) { + s.cutover = cfg +} + +type Project struct { + ID string `json:"id"` + DomainID string `json:"domain_id,omitempty"` + Name string `json:"name"` + SourceProvider string `json:"source_provider"` + AuthMode string `json:"auth_mode"` + Status string `json:"status"` + CutoverAt *string `json:"cutover_at,omitempty"` + DeltaMode bool `json:"delta_mode"` + CreatedAt string `json:"created_at"` + MicrosoftTenantID string `json:"microsoft_tenant_id,omitempty"` + MicrosoftAdminConsentAt *string `json:"microsoft_admin_consent_at,omitempty"` + MicrosoftAdminConsentError string `json:"microsoft_admin_consent_error,omitempty"` + CutoverDNS *hosted.DNSCheckReport `json:"cutover_dns,omitempty"` +} + +type Invite struct { + ID string `json:"id"` + ProjectID string `json:"project_id"` + Email string `json:"email"` + AlternateEmails []string `json:"alternate_emails,omitempty"` + Token string `json:"token,omitempty"` + Status string `json:"status"` + ClaimedAt *string `json:"claimed_at,omitempty"` + UserID string `json:"user_id,omitempty"` +} + +type Job struct { + ID string `json:"id"` + ProjectID string `json:"project_id"` + UserID string `json:"user_id"` + Service string `json:"service"` + Status string `json:"status"` + CursorJSON map[string]any `json:"cursor_json"` + StatsJSON map[string]any `json:"stats_json"` + Error string `json:"error,omitempty"` + StartedAt *string `json:"started_at,omitempty"` + CompletedAt *string `json:"completed_at,omitempty"` +} + +type UserStatus struct { + Project Project `json:"project"` + Invite Invite `json:"invite,omitempty"` + Jobs []Job `json:"jobs"` + Onboarding OnboardingHints `json:"onboarding"` +} + +func (s *Service) CreateProject(ctx context.Context, name, sourceProvider, domainID, authMode string) (Project, error) { + name = strings.TrimSpace(name) + if name == "" { + return Project{}, fmt.Errorf("project name required") + } + sourceProvider = strings.ToLower(strings.TrimSpace(sourceProvider)) + if sourceProvider == "" { + sourceProvider = "google" + } + authMode = NormalizeAuthMode(sourceProvider, authMode) + sc := newProjectScanner() + err := s.db.QueryRow(ctx, ` + INSERT INTO migration_projects (name, source_provider, domain_id, auth_mode) + VALUES ($1, $2, NULLIF($3, '')::uuid, $4) + RETURNING `+projectSelectSQL("")+` + `, name, sourceProvider, domainID, authMode).Scan(sc.targets()...) + return sc.result(), err +} + +func (s *Service) ListProjects(ctx context.Context) ([]Project, error) { + rows, err := s.db.Query(ctx, ` + SELECT `+projectSelectSQL("")+` + FROM migration_projects ORDER BY created_at DESC + `) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Project + for rows.Next() { + sc := newProjectScanner() + if err := rows.Scan(sc.targets()...); err != nil { + return nil, err + } + out = append(out, sc.result()) + } + return out, rows.Err() +} + +func (s *Service) CreateInvite(ctx context.Context, projectID, email string, alternateEmails []string) (Invite, error) { + email = strings.ToLower(strings.TrimSpace(email)) + if email == "" { + return Invite{}, fmt.Errorf("email required") + } + alternates := normalizeAlternateEmails(email, alternateEmails) + token, err := hosted.NewInviteToken() + if err != nil { + return Invite{}, err + } + var row Invite + err = s.db.QueryRow(ctx, ` + INSERT INTO migration_invites (project_id, email, token, alternate_emails) + VALUES ($1::uuid, $2, $3, $4) + RETURNING id::text, project_id::text, email, token, status, claimed_at::text, COALESCE(user_id::text, ''), alternate_emails + `, projectID, email, token, alternates).Scan( + &row.ID, &row.ProjectID, &row.Email, &row.Token, &row.Status, &row.ClaimedAt, &row.UserID, &row.AlternateEmails, + ) + return row, err +} + +func normalizeAlternateEmails(inviteEmail string, alternateEmails []string) []string { + inviteEmail = normalizeInviteEmail(inviteEmail) + seen := map[string]struct{}{inviteEmail: {}} + var out []string + for _, raw := range alternateEmails { + email := normalizeInviteEmail(raw) + if email == "" || !isEmailAddress(email) { + continue + } + if _, ok := seen[email]; ok { + continue + } + seen[email] = struct{}{} + out = append(out, email) + } + return out +} + +func (s *Service) ImportInvites(ctx context.Context, projectID string, emails []string) (int, error) { + count := 0 + for _, email := range emails { + email = strings.ToLower(strings.TrimSpace(email)) + if email == "" { + continue + } + if _, err := s.CreateInvite(ctx, projectID, email, nil); err != nil { + return count, err + } + count++ + } + return count, nil +} + +func (s *Service) GetInviteByToken(ctx context.Context, token string) (Invite, Project, error) { + var inv Invite + sc := newProjectScanner() + scanArgs := append([]any{ + &inv.ID, &inv.ProjectID, &inv.Email, &inv.Status, &inv.ClaimedAt, &inv.UserID, &inv.AlternateEmails, + }, sc.targets()...) + err := s.db.QueryRow(ctx, ` + SELECT i.id::text, i.project_id::text, i.email, i.status, i.claimed_at::text, COALESCE(i.user_id::text, ''), i.alternate_emails, + `+projectSelectSQL("p")+` + FROM migration_invites i + JOIN migration_projects p ON p.id = i.project_id + WHERE i.token = $1 + `, token).Scan(scanArgs...) + if errors.Is(err, pgx.ErrNoRows) { + return Invite{}, Project{}, ErrInviteNotFound + } + return inv, sc.result(), err +} + +func (s *Service) ClaimInvite(ctx context.Context, token, userID string, identity ClaimIdentity, displayName, password string) (UserStatus, error) { + inv, proj, err := s.GetInviteByToken(ctx, token) + if err != nil { + return UserStatus{}, err + } + if inv.Status == "claimed" { + return UserStatus{}, ErrInviteClaimed + } + + projectDomain := "" + var hostedDomain *hosted.DomainRow + if strings.TrimSpace(proj.DomainID) != "" && s.hosted != nil { + domain, err := s.hosted.GetDomain(ctx, proj.DomainID) + if err != nil { + return UserStatus{}, fmt.Errorf("migration domain: %w", err) + } + hostedDomain = &domain + projectDomain = domain.Name + } + if !InviteEmailMatchesIdentity(inv.Email, inv.AlternateEmails, projectDomain, identity) { + return UserStatus{}, ErrEmailMismatch + } + mailboxEmail := normalizeInviteEmail(inv.Email) + + tx, err := s.db.Begin(ctx) + if err != nil { + return UserStatus{}, err + } + defer tx.Rollback(ctx) + + _, err = tx.Exec(ctx, ` + UPDATE migration_invites + SET status = 'claimed', claimed_at = NOW(), user_id = $1::uuid + WHERE id = $2::uuid AND status = 'invited' + `, userID, inv.ID) + if err != nil { + return UserStatus{}, err + } + + if s.hosted != nil { + provision := hosted.ProvisionMailboxInput{ + UserID: userID, + Email: mailboxEmail, + DisplayName: displayName, + Password: password, + QuotaBytes: 0, + } + if hostedDomain != nil { + at := strings.LastIndex(mailboxEmail, "@") + if at <= 0 || !strings.EqualFold(mailboxEmail[at+1:], hostedDomain.Name) { + return UserStatus{}, ErrMigrationDomainMismatch + } + if hostedDomain.Status != "active" && !hostedDomain.IsPlatformDomain { + return UserStatus{}, ErrMigrationDomainNotActive + } + provision.DomainID = proj.DomainID + } + _, err = s.hosted.ProvisionMailbox(ctx, provision) + if err != nil { + if errors.Is(err, hosted.ErrDomainNotActive) { + return UserStatus{}, ErrMigrationDomainNotActive + } + if !errors.Is(err, hosted.ErrAddressTaken) { + return UserStatus{}, err + } + } + } + + services := []string{"mail", "contacts", "calendar", "drive"} + for _, svc := range services { + _, err = tx.Exec(ctx, ` + INSERT INTO migration_jobs (project_id, user_id, service, status) + VALUES ($1::uuid, $2::uuid, $3, 'pending') + ON CONFLICT (project_id, user_id, service) DO NOTHING + `, proj.ID, userID, svc) + if err != nil { + return UserStatus{}, err + } + } + + if err := tx.Commit(ctx); err != nil { + return UserStatus{}, err + } + + return s.GetUserStatus(ctx, userID, proj.ID) +} + +func (s *Service) StoreMigrationToken(ctx context.Context, userID, projectID, provider string, token *oauth2.Token, scopes []string) error { + if s.creds == nil { + return fmt.Errorf("credential manager not configured") + } + payload, err := json.Marshal(map[string]any{ + "access_token": token.AccessToken, + "refresh_token": token.RefreshToken, + "expiry": token.Expiry.UTC().Format(time.RFC3339), + "token_type": token.TokenType, + }) + if err != nil { + return err + } + enc, err := s.creds.EncryptCredential(credentials.Credential{ + AuthType: credentials.AuthOAuth2, + AccessToken: token.AccessToken, + RefreshToken: token.RefreshToken, + Expiry: token.Expiry, + OAuthProvider: provider, + }) + if err != nil { + _ = payload + return err + } + var expiresAt *time.Time + if !token.Expiry.IsZero() { + expiresAt = &token.Expiry + } + _, err = s.db.Exec(ctx, ` + INSERT INTO migration_credentials (user_id, project_id, provider, encrypted_token, scopes, expires_at) + VALUES ($1::uuid, $2::uuid, $3, $4, $5, $6) + ON CONFLICT (user_id, project_id, provider) DO UPDATE SET + encrypted_token = EXCLUDED.encrypted_token, + scopes = EXCLUDED.scopes, + expires_at = EXCLUDED.expires_at, + revoked_at = NULL + `, userID, projectID, provider, enc, scopes, expiresAt) + return err +} + +func (s *Service) GetUserStatus(ctx context.Context, userID, projectID string) (UserStatus, error) { + sc := newProjectScanner() + err := s.db.QueryRow(ctx, ` + SELECT `+projectSelectSQL("")+` + FROM migration_projects WHERE id = $1::uuid + `, projectID).Scan(sc.targets()...) + proj := sc.result() + if err != nil { + return UserStatus{}, err + } + + var inv Invite + _ = s.db.QueryRow(ctx, ` + SELECT id::text, project_id::text, email, status, claimed_at::text, COALESCE(user_id::text, '') + FROM migration_invites WHERE project_id = $1::uuid AND user_id = $2::uuid + `, projectID, userID).Scan( + &inv.ID, &inv.ProjectID, &inv.Email, &inv.Status, &inv.ClaimedAt, &inv.UserID, + ) + + jobs, err := s.listJobs(ctx, projectID, userID) + if err != nil { + return UserStatus{}, err + } + return UserStatus{ + Project: proj, + Invite: inv, + Jobs: jobs, + Onboarding: s.BuildOnboardingHints(ctx, userID, proj, inv), + }, nil +} + +func (s *Service) GetActiveUserStatus(ctx context.Context, userID string) (UserStatus, error) { + var projectID string + err := s.db.QueryRow(ctx, ` + SELECT project_id::text FROM migration_invites + WHERE user_id = $1::uuid AND status = 'claimed' + ORDER BY claimed_at DESC NULLS LAST LIMIT 1 + `, userID).Scan(&projectID) + if errors.Is(err, pgx.ErrNoRows) { + return UserStatus{}, nil + } + if err != nil { + return UserStatus{}, err + } + return s.GetUserStatus(ctx, userID, projectID) +} + +func (s *Service) listJobs(ctx context.Context, projectID, userID string) ([]Job, error) { + rows, err := s.db.Query(ctx, ` + SELECT id::text, project_id::text, user_id::text, service, status, + cursor_json, stats_json, error, started_at::text, completed_at::text + FROM migration_jobs + WHERE project_id = $1::uuid AND user_id = $2::uuid + ORDER BY service ASC + `, projectID, userID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Job + for rows.Next() { + var row Job + var cursorRaw, statsRaw []byte + if err := rows.Scan( + &row.ID, &row.ProjectID, &row.UserID, &row.Service, &row.Status, + &cursorRaw, &statsRaw, &row.Error, &row.StartedAt, &row.CompletedAt, + ); err != nil { + return nil, err + } + _ = json.Unmarshal(cursorRaw, &row.CursorJSON) + _ = json.Unmarshal(statsRaw, &row.StatsJSON) + if row.CursorJSON == nil { + row.CursorJSON = map[string]any{} + } + if row.StatsJSON == nil { + row.StatsJSON = map[string]any{} + } + out = append(out, row) + } + return out, rows.Err() +} + +func (s *Service) PendingJobs(ctx context.Context, limit int) ([]Job, error) { + if limit <= 0 { + limit = 10 + } + rows, err := s.db.Query(ctx, ` + SELECT j.id::text, j.project_id::text, j.user_id::text, j.service, j.status, + j.cursor_json, j.stats_json, j.error, j.started_at::text, j.completed_at::text + FROM migration_jobs j + JOIN migration_projects p ON p.id = j.project_id + WHERE j.status IN ('pending', 'running') + AND p.status IN ('active', 'cutover') + ORDER BY j.updated_at ASC + LIMIT $1 + `, limit) + if err != nil { + return nil, err + } + defer rows.Close() + return scanJobs(rows) +} + +func scanJobs(rows pgx.Rows) ([]Job, error) { + var out []Job + for rows.Next() { + var row Job + var cursorRaw, statsRaw []byte + if err := rows.Scan( + &row.ID, &row.ProjectID, &row.UserID, &row.Service, &row.Status, + &cursorRaw, &statsRaw, &row.Error, &row.StartedAt, &row.CompletedAt, + ); err != nil { + return nil, err + } + _ = json.Unmarshal(cursorRaw, &row.CursorJSON) + _ = json.Unmarshal(statsRaw, &row.StatsJSON) + if row.CursorJSON == nil { + row.CursorJSON = map[string]any{} + } + if row.StatsJSON == nil { + row.StatsJSON = map[string]any{} + } + out = append(out, row) + } + return out, rows.Err() +} + +func (s *Service) UpdateJobProgress(ctx context.Context, jobID, status string, cursor, stats map[string]any, jobErr string) error { + cursorRaw, _ := json.Marshal(cursor) + statsRaw, _ := json.Marshal(stats) + _, err := s.db.Exec(ctx, ` + UPDATE migration_jobs SET + status = $2, + cursor_json = $3, + stats_json = $4, + error = $5, + started_at = COALESCE(started_at, CASE WHEN $2 = 'running' THEN NOW() ELSE NULL END), + completed_at = CASE WHEN $2 IN ('completed', 'failed') THEN NOW() ELSE completed_at END, + updated_at = NOW() + WHERE id = $1::uuid + `, jobID, status, cursorRaw, statsRaw, jobErr) + return err +} + +func (s *Service) ActivateProject(ctx context.Context, projectID string) (Project, error) { + sc := newProjectScanner() + err := s.db.QueryRow(ctx, ` + UPDATE migration_projects SET status = 'active', updated_at = NOW() + WHERE id = $1::uuid + RETURNING `+projectSelectSQL("")+` + `, projectID).Scan(sc.targets()...) + return sc.result(), err +} + +func (s *Service) LookupUserID(ctx context.Context, externalID string) (string, error) { + var userID string + err := s.db.QueryRow(ctx, `SELECT id::text FROM users WHERE external_id = $1`, externalID).Scan(&userID) + return userID, err +} + +func randomState() (string, error) { + b := make([]byte, 24) + if _, err := rand.Read(b); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(b), nil +} diff --git a/internal/migration/test_exports.go b/internal/migration/test_exports.go new file mode 100644 index 0000000..7a4febf --- /dev/null +++ b/internal/migration/test_exports.go @@ -0,0 +1,11 @@ +package migration + +// GmailUIDForTest exposes gmailUID for integration tests. +func GmailUIDForTest(gmailID string) int64 { + return gmailUID(gmailID) +} + +// RemoteMessageUIDForTest exposes remoteMessageUID for integration tests. +func RemoteMessageUIDForTest(graphID string) int64 { + return remoteMessageUID(graphID) +} diff --git a/internal/migration/worker.go b/internal/migration/worker.go new file mode 100644 index 0000000..301b1f1 --- /dev/null +++ b/internal/migration/worker.go @@ -0,0 +1,285 @@ +package migration + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "strings" + "sync" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/ultisuite/ulti-backend/internal/mail/credentials" + mailstorage "github.com/ultisuite/ulti-backend/internal/mail/storage" + "github.com/ultisuite/ulti-backend/internal/nextcloud" + "github.com/ultisuite/ulti-backend/internal/observability" +) + +type Worker struct { + db *pgxpool.Pool + svc *Service + oauth *OAuthService + creds *credentials.Manager + googleDWD *GoogleDWD + microsoftApp *MicrosoftApp + nc *nextcloud.Client + storage *mailstorage.Client + attachBucket string + concurrency int + jobLimit int + logger *slog.Logger + client *http.Client +} + +// WorkerConfig tunes migration worker parallelism and job pickup. +type WorkerConfig struct { + Concurrency int + JobLimit int +} + +func NewWorker(db *pgxpool.Pool, svc *Service, oauth *OAuthService, creds *credentials.Manager, googleDWD *GoogleDWD, microsoftApp *MicrosoftApp, nc *nextcloud.Client, storage *mailstorage.Client, attachBucket string, cfg WorkerConfig) *Worker { + concurrency := cfg.Concurrency + if concurrency <= 0 { + concurrency = 1 + } + jobLimit := cfg.JobLimit + if jobLimit <= 0 { + jobLimit = concurrency * 3 + if jobLimit < 5 { + jobLimit = 5 + } + } + return &Worker{ + db: db, + svc: svc, + oauth: oauth, + creds: creds, + googleDWD: googleDWD, + microsoftApp: microsoftApp, + nc: nc, + storage: storage, + attachBucket: attachBucket, + concurrency: concurrency, + jobLimit: jobLimit, + logger: slog.Default().With("component", "migration-worker"), + client: &http.Client{Timeout: 60 * time.Second}, + } +} + +func (w *Worker) Start(ctx context.Context, interval time.Duration) { + if interval <= 0 { + interval = 30 * time.Second + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + w.tick(ctx) + } + } +} + +func (w *Worker) tick(ctx context.Context) { + jobs, err := w.svc.PendingJobs(ctx, w.jobLimit) + if err != nil { + w.logger.Error("list pending migration jobs", "error", err) + return + } + observability.SetMigrationPendingJobs(len(jobs)) + if len(jobs) == 0 { + return + } + + sem := make(chan struct{}, w.concurrency) + var wg sync.WaitGroup + for _, job := range jobs { + wg.Add(1) + sem <- struct{}{} + go func(job Job) { + defer wg.Done() + defer func() { <-sem }() + if _, err := w.processJob(ctx, job); err != nil { + w.logger.Error("migration job failed", "job_id", job.ID, "service", job.Service, "error", err) + } + }(job) + } + wg.Wait() +} + +func (w *Worker) processJob(ctx context.Context, job Job) (string, error) { + start := time.Now() + outcome := "unknown" + defer func() { + observability.ObserveMigrationJob(job.Service, outcome, time.Since(start)) + }() + if err := w.svc.UpdateJobProgress(ctx, job.ID, "running", job.CursorJSON, job.StatsJSON, ""); err != nil { + outcome = "failed" + return outcome, err + } + + var provider string + var delta bool + var authMode string + err := w.db.QueryRow(ctx, ` + SELECT p.source_provider, p.delta_mode, p.auth_mode + FROM migration_projects p WHERE p.id = $1::uuid + `, job.ProjectID).Scan(&provider, &delta, &authMode) + if err != nil { + outcome = "failed" + return outcome, err + } + + accessToken, graphUserUPN, err := w.loadAccessToken(ctx, job, provider, authMode) + if err != nil { + outcome = "failed" + _ = w.svc.UpdateJobProgress(ctx, job.ID, "failed", job.CursorJSON, job.StatsJSON, err.Error()) + return outcome, err + } + + var lastStatus string + update := func(status string, cursor, stats map[string]any, jobErr string) error { + lastStatus = status + return w.svc.UpdateJobProgress(ctx, job.ID, status, cursor, stats, jobErr) + } + + var procErr error + var selfManaged bool + switch job.Service { + case "mail": + selfManaged = true + if provider == "google" { + procErr = NewGmailImporter(w.db).WithStorage(w.storage, w.attachBucket).ImportBatch(ctx, &job, accessToken, delta, update) + } else { + procErr = NewGraphImporter(w.db).WithUserPrincipal(graphUserUPN).ImportBatch(ctx, &job, accessToken, delta, update) + } + case "contacts": + selfManaged = true + procErr = NewContactsImporter(w.db, w.nc).WithUserPrincipal(graphUserUPN).ImportBatch(ctx, &job, accessToken, provider, delta, update) + case "calendar": + selfManaged = true + procErr = NewCalendarImporter(w.db, w.nc).WithUserPrincipal(graphUserUPN).ImportBatch(ctx, &job, accessToken, provider, delta, update) + case "drive": + selfManaged = true + procErr = NewDriveImporter(w.db, w.nc).WithUserPrincipal(graphUserUPN).ImportBatch(ctx, &job, accessToken, provider, delta, update) + default: + procErr = fmt.Errorf("unknown service %q", job.Service) + } + + if procErr != nil { + if IsRateLimitError(procErr) { + if job.StatsJSON == nil { + job.StatsJSON = map[string]any{} + } + job.StatsJSON["rate_limited"] = true + job.StatsJSON["rate_limit_at"] = time.Now().UTC().Format(time.RFC3339) + outcome = "rate_limited" + return outcome, w.svc.UpdateJobProgress(ctx, job.ID, "pending", job.CursorJSON, job.StatsJSON, procErr.Error()) + } + outcome = "failed" + return outcome, w.svc.UpdateJobProgress(ctx, job.ID, "failed", job.CursorJSON, job.StatsJSON, procErr.Error()) + } + if selfManaged { + switch lastStatus { + case "completed": + outcome = "completed" + case "pending": + outcome = "pending" + default: + outcome = "completed" + } + return outcome, nil + } + outcome = "completed" + return outcome, w.svc.UpdateJobProgress(ctx, job.ID, "completed", job.CursorJSON, job.StatsJSON, "") +} + +func (w *Worker) loadAccessToken(ctx context.Context, job Job, provider, authMode string) (accessToken, graphUserUPN string, err error) { + if provider == "google" && authMode == AuthModeGoogleDWD { + if w.googleDWD == nil || !w.googleDWD.Enabled() { + return "", "", fmt.Errorf("google domain-wide delegation not configured") + } + email, err := w.inviteEmail(ctx, job.ProjectID, job.UserID) + if err != nil { + return "", "", err + } + token, err := w.googleDWD.AccessToken(ctx, email) + return token, "", err + } + if provider == "microsoft" && authMode == AuthModeMicrosoftApp { + if w.microsoftApp == nil || !w.microsoftApp.Enabled() { + return "", "", fmt.Errorf("microsoft app-only auth not configured") + } + tenantID, err := w.projectMicrosoftTenant(ctx, job.ProjectID) + if err != nil { + return "", "", err + } + email, err := w.inviteEmail(ctx, job.ProjectID, job.UserID) + if err != nil { + return "", "", err + } + token, err := w.microsoftApp.AccessToken(ctx, tenantID) + return token, email, err + } + cred, err := w.loadToken(ctx, job.UserID, job.ProjectID, provider) + if err != nil { + return "", "", err + } + if w.oauth != nil && cred.NeedsRefresh() { + cred, err = w.svc.RefreshCredential(ctx, w.oauth, job.UserID, job.ProjectID, provider, cred) + if err != nil { + return "", "", err + } + } + return cred.AccessToken, "", nil +} + +func (w *Worker) projectMicrosoftTenant(ctx context.Context, projectID string) (string, error) { + var tenantID string + err := w.db.QueryRow(ctx, ` + SELECT COALESCE(NULLIF(microsoft_tenant_id, ''), '') + FROM migration_projects WHERE id = $1::uuid + `, projectID).Scan(&tenantID) + if err != nil { + return "", fmt.Errorf("migration project tenant lookup: %w", err) + } + if strings.TrimSpace(tenantID) == "" { + return "", fmt.Errorf("microsoft tenant id missing: complete admin consent first") + } + return tenantID, nil +} + +func (w *Worker) inviteEmail(ctx context.Context, projectID, userID string) (string, error) { + var email string + err := w.db.QueryRow(ctx, ` + SELECT email FROM migration_invites + WHERE project_id = $1::uuid AND user_id = $2::uuid + ORDER BY claimed_at DESC NULLS LAST LIMIT 1 + `, projectID, userID).Scan(&email) + if err != nil { + return "", fmt.Errorf("migration invite email missing for domain-wide delegation") + } + return email, nil +} + +func (w *Worker) loadToken(ctx context.Context, userID, projectID, provider string) (credentials.Credential, error) { + var blob []byte + err := w.db.QueryRow(ctx, ` + SELECT encrypted_token FROM migration_credentials + WHERE user_id = $1::uuid AND project_id = $2::uuid AND provider = $3 AND revoked_at IS NULL + `, userID, projectID, provider).Scan(&blob) + if err != nil { + return credentials.Credential{}, fmt.Errorf("migration credentials missing: run OAuth consent first") + } + cred, err := w.creds.DecryptCredential(blob) + if err != nil { + return credentials.Credential{}, err + } + cred.AuthType = credentials.AuthOAuth2 + cred.OAuthProvider = provider + return cred, nil +} diff --git a/internal/migration/worker_test.go b/internal/migration/worker_test.go new file mode 100644 index 0000000..c33a292 --- /dev/null +++ b/internal/migration/worker_test.go @@ -0,0 +1,26 @@ +package migration + +import "testing" + +func TestWorkerConfigDefaults(t *testing.T) { + w := NewWorker(nil, nil, nil, nil, nil, nil, nil, nil, "", WorkerConfig{}) + if w.concurrency != 1 { + t.Fatalf("concurrency = %d", w.concurrency) + } + if w.jobLimit != 5 { + t.Fatalf("jobLimit = %d", w.jobLimit) + } + + w = NewWorker(nil, nil, nil, nil, nil, nil, nil, nil, "", WorkerConfig{Concurrency: 4}) + if w.concurrency != 4 { + t.Fatalf("concurrency = %d", w.concurrency) + } + if w.jobLimit != 12 { + t.Fatalf("jobLimit = %d", w.jobLimit) + } + + w = NewWorker(nil, nil, nil, nil, nil, nil, nil, nil, "", WorkerConfig{Concurrency: 2, JobLimit: 8}) + if w.jobLimit != 8 { + t.Fatalf("jobLimit = %d", w.jobLimit) + } +} diff --git a/internal/observability/metrics.go b/internal/observability/metrics.go index dcce7ba..080d5fc 100644 --- a/internal/observability/metrics.go +++ b/internal/observability/metrics.go @@ -77,6 +77,27 @@ var ( Name: "ultid_webhook_payload_truncated_total", Help: "Total number of webhook payloads truncated in logs.", }) + + migrationJobsProcessedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "ultid_migration_jobs_processed_total", + Help: "Total number of migration worker job runs.", + }, []string{"service", "outcome"}) + + migrationJobDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "ultid_migration_job_duration_seconds", + Help: "Migration worker job run duration in seconds.", + Buckets: []float64{0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120, 300}, + }, []string{"service", "outcome"}) + + migrationPendingJobs = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "ultid_migration_pending_jobs", + Help: "Migration jobs picked up on the latest worker tick.", + }) + + migrationRateLimitRetriesTotal = promauto.NewCounter(prometheus.CounterOpts{ + Name: "ultid_migration_rate_limit_retries_total", + Help: "Total number of migration provider API 429 retries.", + }) ) type metricsResponseWriter struct { @@ -142,3 +163,22 @@ func IncWebhookDeadLetter() { func IncWebhookPayloadTruncated() { webhookPayloadTruncatedTotal.Inc() } + +func ObserveMigrationJob(service, outcome string, duration time.Duration) { + if service == "" { + service = "unknown" + } + if outcome == "" { + outcome = "unknown" + } + migrationJobsProcessedTotal.WithLabelValues(service, outcome).Inc() + migrationJobDurationSeconds.WithLabelValues(service, outcome).Observe(duration.Seconds()) +} + +func SetMigrationPendingJobs(count int) { + migrationPendingJobs.Set(float64(count)) +} + +func IncMigrationRateLimitRetry() { + migrationRateLimitRetriesTotal.Inc() +} diff --git a/internal/provision/authentik.go b/internal/provision/authentik.go new file mode 100644 index 0000000..68e2bb0 --- /dev/null +++ b/internal/provision/authentik.go @@ -0,0 +1,75 @@ +package provision + +import ( + "encoding/json" + "net/http" + "strconv" + "strings" +) + +func authorizeProvision(r *http.Request, secret string) bool { + if secret == "" { + return false + } + if r.Header.Get("X-Provision-Secret") == secret { + return true + } + if r.URL.Query().Get("secret") == secret { + return true + } + return false +} + +type authentikWebhookPayload struct { + Email string `json:"email"` + Username string `json:"username"` + Name string `json:"name"` + Password string `json:"password"` + ExternalID string `json:"external_id"` + Sub string `json:"sub"` + User struct { + Email string `json:"email"` + Username string `json:"username"` + Name string `json:"name"` + UUID string `json:"uuid"` + PK int `json:"pk"` + } `json:"user"` +} + +func decodeProvisionBody(r *http.Request) (provisionUserRequest, error) { + var ak authentikWebhookPayload + if err := json.NewDecoder(r.Body).Decode(&ak); err != nil { + return provisionUserRequest{}, err + } + + req := provisionUserRequest{ + Email: firstNonEmpty(ak.Email, ak.User.Email), + Username: firstNonEmpty(ak.Username, ak.User.Username), + Name: firstNonEmpty(ak.Name, ak.User.Name), + Password: ak.Password, + ExternalID: firstNonEmpty(ak.ExternalID, ak.Sub, ak.User.UUID), + } + if req.ExternalID == "" && ak.User.PK > 0 { + req.ExternalID = strconv.Itoa(ak.User.PK) + } + normalizeProvisionRequest(&req) + return req, nil +} + +func normalizeProvisionRequest(dst *provisionUserRequest) { + if dst.Email == "" { + dst.Email = strings.ToLower(strings.TrimSpace(dst.Username)) + } + if dst.Name == "" { + dst.Name = dst.Email + } +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} diff --git a/internal/provision/handler.go b/internal/provision/handler.go new file mode 100644 index 0000000..f9e4a5c --- /dev/null +++ b/internal/provision/handler.go @@ -0,0 +1,151 @@ +package provision + +import ( + "errors" + "log/slog" + "net/http" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/ultisuite/ulti-backend/internal/api/apiresponse" + "github.com/ultisuite/ulti-backend/internal/auth" + "github.com/ultisuite/ulti-backend/internal/mail/hosted" + "github.com/ultisuite/ulti-backend/internal/migration" + "github.com/ultisuite/ulti-backend/internal/nextcloud" + "github.com/ultisuite/ulti-backend/internal/users" +) + +type Handler struct { + secret string + platformDomain string + hosted *hosted.Service + nc *nextcloud.Client + db *pgxpool.Pool + logger *slog.Logger +} + +func NewHandler(secret, platformDomain string, hostedSvc *hosted.Service, nc *nextcloud.Client, db *pgxpool.Pool) *Handler { + return &Handler{ + secret: strings.TrimSpace(secret), + platformDomain: strings.ToLower(strings.TrimSpace(platformDomain)), + hosted: hostedSvc, + nc: nc, + db: db, + logger: slog.Default().With("component", "provision"), + } +} + +type provisionUserRequest struct { + Email string `json:"email"` + Username string `json:"username"` + Name string `json:"name"` + Password string `json:"password"` + ExternalID string `json:"external_id"` +} + +func (h *Handler) ProvisionUser(w http.ResponseWriter, r *http.Request) { + if h.secret == "" { + apiresponse.WriteError(w, r, http.StatusServiceUnavailable, "provision_disabled", "provisioning webhook not configured", nil) + return + } + if !authorizeProvision(r, h.secret) { + apiresponse.WriteError(w, r, http.StatusUnauthorized, "unauthorized", "invalid provision secret", nil) + return + } + + req, err := decodeProvisionBody(r) + if err != nil { + apiresponse.WriteError(w, r, http.StatusBadRequest, "invalid_json", "invalid request body", nil) + return + } + + email := strings.ToLower(strings.TrimSpace(req.Email)) + if email == "" { + apiresponse.WriteError(w, r, http.StatusBadRequest, "validation_error", "email required", nil) + return + } + + if h.platformDomain != "" && !strings.Contains(email, "@") { + email = email + "@" + h.platformDomain + } else if h.platformDomain != "" && !strings.HasSuffix(email, "@"+h.platformDomain) { + local := strings.Split(email, "@")[0] + email = local + "@" + h.platformDomain + } + + ctx := r.Context() + var userID string + externalID := strings.TrimSpace(req.ExternalID) + if externalID != "" { + err := h.db.QueryRow(ctx, `SELECT id::text FROM users WHERE external_id = $1`, externalID).Scan(&userID) + if errors.Is(err, pgx.ErrNoRows) { + userID, err = users.EnsureUser(ctx, h.db, &auth.Claims{ + Sub: externalID, + Email: email, + Name: req.Name, + }) + } + if err != nil { + h.logger.Error("ensure user", "error", err) + apiresponse.WriteError(w, r, http.StatusInternalServerError, "internal_error", "failed to provision user", nil) + return + } + } + + result, err := h.hosted.ProvisionMailbox(ctx, hosted.ProvisionMailboxInput{ + UserID: userID, + Email: email, + DisplayName: req.Name, + Password: req.Password, + }) + if err != nil { + if errors.Is(err, hosted.ErrAddressTaken) { + apiresponse.WriteError(w, r, http.StatusConflict, "address_taken", err.Error(), nil) + return + } + h.logger.Error("provision mailbox", "error", err, "email", email) + apiresponse.WriteError(w, r, http.StatusConflict, "provision_failed", err.Error(), nil) + return + } + + if userID != "" { + _ = migration.LinkHostedMailboxByEmail(ctx, h.db, userID, email) + } + + if h.nc != nil && userID != "" && externalID != "" { + if _, err := h.nc.EnsurePrincipal(ctx, email, externalID, req.Name); err != nil { + h.logger.Warn("nextcloud provision", "error", err) + } + } + + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{ + "user_id": userID, + "email": email, + "mailbox_id": result.Mailbox.ID, + "mail_account_id": result.MailAccountID, + }) +} + +// CheckAddress validates local part availability (Authentik expression policy or public API). +func (h *Handler) CheckAddress(w http.ResponseWriter, r *http.Request) { + if h.hosted == nil { + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"available": true}) + return + } + local := strings.TrimSpace(r.URL.Query().Get("local")) + domain := strings.TrimSpace(r.URL.Query().Get("domain")) + if domain == "" { + domain = h.platformDomain + } + if local == "" || domain == "" { + apiresponse.WriteError(w, r, http.StatusBadRequest, "validation_error", "local and domain required", nil) + return + } + available, err := h.hosted.IsAddressAvailable(r.Context(), domain, local) + if err != nil { + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"available": false, "reason": err.Error()}) + return + } + apiresponse.WriteJSON(w, http.StatusOK, map[string]any{"available": available}) +} diff --git a/internal/server/bootstrap.go b/internal/server/bootstrap.go index f9583e5..aa9deed 100644 --- a/internal/server/bootstrap.go +++ b/internal/server/bootstrap.go @@ -6,6 +6,7 @@ import ( "fmt" "log/slog" "net/http" + "strings" "time" "github.com/go-chi/chi/v5" @@ -22,6 +23,7 @@ import ( "github.com/ultisuite/ulti-backend/internal/api/docs" "github.com/ultisuite/ulti-backend/internal/api/drive" mailapi "github.com/ultisuite/ulti-backend/internal/api/mail" + migrationapi "github.com/ultisuite/ulti-backend/internal/api/migration" "github.com/ultisuite/ulti-backend/internal/api/mail/drivebridge" "github.com/ultisuite/ulti-backend/internal/api/mail/sendguard" meetapi "github.com/ultisuite/ulti-backend/internal/api/meet" @@ -40,6 +42,10 @@ import ( mailcredentials "github.com/ultisuite/ulti-backend/internal/mail/credentials" imapsync "github.com/ultisuite/ulti-backend/internal/mail/imap" mailoauth "github.com/ultisuite/ulti-backend/internal/mail/oauth" + "github.com/ultisuite/ulti-backend/internal/mail/hosted" + "github.com/ultisuite/ulti-backend/internal/mail/stalwart" + "github.com/ultisuite/ulti-backend/internal/migration" + "github.com/ultisuite/ulti-backend/internal/provision" "github.com/ultisuite/ulti-backend/internal/mail/rules" "github.com/ultisuite/ulti-backend/internal/mail/smtp" mailstorage "github.com/ultisuite/ulti-backend/internal/mail/storage" @@ -208,6 +214,80 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*App, error) { RedirectURL: oauthRedirect, }, rdb) + stlwClient := stalwart.NewClient(stalwart.Config{ + Enabled: cfg.StalwartEnabled, + BaseURL: cfg.StalwartAPIURL, + APIKey: cfg.StalwartAPIKey, + IMAPHost: cfg.StalwartIMAPHost, + IMAPPort: cfg.StalwartIMAPPort, + IMAPTLS: cfg.StalwartIMAPTLS, + SMTPHost: cfg.StalwartSMTPHost, + SMTPPort: cfg.StalwartSMTPPort, + SMTPTLS: cfg.StalwartSMTPTLS, + }) + hostedSvc := hosted.NewService(pool, stlwClient, credentialManager) + if cfg.PlatformMailDomain != "" { + if _, err := hostedSvc.EnsurePlatformDomain(workerCtx, cfg.PlatformMailDomain); err != nil { + slog.Warn("platform mail domain bootstrap failed", "domain", cfg.PlatformMailDomain, "error", err) + } + } + + migrationOAuthRedirect := cfg.MigrationOAuthRedirectURL + if migrationOAuthRedirect == "" { + migrationOAuthRedirect = fmt.Sprintf("http://localhost:%d/api/v1/migration/oauth/callback", cfg.Port) + if cfg.Domain != "" && cfg.Domain != "localhost" { + migrationOAuthRedirect = fmt.Sprintf("https://%s/api/v1/migration/oauth/callback", cfg.Domain) + } + } + migrationOAuthSvc := migration.NewOAuthService(migration.OAuthConfig{ + GoogleClientID: firstNonEmpty(cfg.MigrationGoogleOAuthClientID, cfg.MailGoogleOAuthClientID), + GoogleClientSecret: firstNonEmpty(cfg.MigrationGoogleOAuthClientSecret, cfg.MailGoogleOAuthClientSecret), + MicrosoftClientID: firstNonEmpty(cfg.MigrationMicrosoftOAuthClientID, cfg.MailMicrosoftOAuthClientID), + MicrosoftSecret: firstNonEmpty(cfg.MigrationMicrosoftOAuthSecret, cfg.MailMicrosoftOAuthSecret), + MicrosoftTenant: firstNonEmpty(cfg.MigrationMicrosoftOAuthTenant, cfg.MailMicrosoftOAuthTenant), + RedirectURL: migrationOAuthRedirect, + }, rdb) + migrationSvc := migration.NewService(pool, rdb, credentialManager, hostedSvc, migrationOAuthSvc) + migrationSvc.SetCutoverConfig(migration.CutoverConfig{ + ExpectedMXHosts: migration.ParseCutoverMXHosts( + cfg.MigrationCutoverMXHosts, + cfg.PlatformMailDomain, + cfg.StalwartIMAPHost, + ), + RequireMX: cfg.MigrationCutoverRequireMX, + }) + googleDWD, err := migration.NewGoogleDWD(cfg.MigrationGoogleServiceAccountJSON) + if err != nil { + return nil, fmt.Errorf("google dwd: %w", err) + } + microsoftApp, err := migration.NewMicrosoftApp(migration.MicrosoftAppConfig{ + ClientID: firstNonEmpty(cfg.MigrationMicrosoftOAuthClientID, cfg.MailMicrosoftOAuthClientID), + ClientSecret: firstNonEmpty(cfg.MigrationMicrosoftOAuthSecret, cfg.MailMicrosoftOAuthSecret), + DefaultTenant: firstNonEmpty(cfg.MigrationMicrosoftOAuthTenant, cfg.MailMicrosoftOAuthTenant), + }) + if err != nil { + return nil, fmt.Errorf("microsoft app: %w", err) + } + migration.ConfigureRateLimit(migration.RateLimitConfig{ + MaxRetries: cfg.MigrationRateLimitMaxRetries, + BaseDelay: cfg.MigrationRateLimitBaseDelay, + MaxDelay: cfg.MigrationRateLimitMaxDelay, + }) + migration.ConfigureImportBatch(migration.ImportBatchConfig{ + Mail: cfg.MigrationImportBatchSize, + Drive: cfg.MigrationDriveBatchSize, + }) + if !opts.WithoutWorkers { + go migration.NewWorker( + pool, migrationSvc, migrationOAuthSvc, credentialManager, googleDWD, microsoftApp, ncClient, + attachmentStorage, cfg.MailAttachmentsBucket, + migration.WorkerConfig{ + Concurrency: cfg.MigrationWorkerConcurrency, + JobLimit: cfg.MigrationWorkerJobLimit, + }, + ).Start(workerCtx, cfg.MigrationWorkerInterval) + } + orgPolicyLoader := orgpolicy.NewLoader(pool, cfg) fileScanner := filescan.NewScanner(orgPolicyLoader, slog.Default()) @@ -238,6 +318,9 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*App, error) { sendRateLimiter := sendguard.NewRateLimiter(cfg.MailSendRatePerMinute, cfg.MailSendBurst) mailHandler := mailapi.NewHandler(pool, auditLogger, credentialManager, attachmentStorage, cfg.MailAttachmentsBucket, sendRateLimiter, mailOAuthSvc, cfg.MailAppURL, sender) + mailHandler.SetHostedService(hostedSvc) + migrationHandler := migrationapi.NewHandler(migrationSvc, migrationOAuthSvc, cfg.MailAppURL) + provisionHandler := provision.NewHandler(cfg.ProvisionWebhookSecret, cfg.PlatformMailDomain, hostedSvc, ncClient, pool) mailHandler.SetFileScanner(fileScanner) if syncWorker != nil { mailHandler.SetAccountSync(syncWorker) @@ -265,6 +348,10 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*App, error) { r.Get("/ws", hub.HandleWS) r.Get("/api/v1/mail/accounts/oauth/callback", mailHandler.OAuthCallback) + r.Get("/api/v1/migration/oauth/callback", migrationHandler.OAuthCallback) + r.Get("/api/v1/mail/addresses/check", mailHandler.CheckAddressAvailability) + r.Get("/api/v1/migration/invite", migrationHandler.GetInvite) + r.Post("/internal/provision/user", provisionHandler.ProvisionUser) var driveHandler *drive.Handler var driveSvc *drive.Service @@ -325,7 +412,10 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*App, error) { r.Use(middleware.EnforceApiTokenPolicy()) r.Mount("/api/v1/users", usersapi.NewHandler(pool).Routes()) - r.Mount("/api/v1/admin", admin.NewHandler(pool, auditLogger, cfg, ncClient).Routes()) + adminHandler := admin.NewHandler(pool, auditLogger, cfg, ncClient) + adminHandler.SetHostedService(hostedSvc) + adminHandler.SetMigrationService(migrationSvc) + r.Mount("/api/v1/admin", adminHandler.Routes()) if driveHandler != nil { r.Mount("/api/v1/drive", driveHandler.Routes()) } @@ -333,6 +423,7 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*App, error) { r.Group(func(r chi.Router) { r.Use(middleware.RequireFullAccount) r.Mount("/api/v1/mail", mailHandler.Routes()) + r.Mount("/api/v1/migration", migrationHandler.Routes()) r.Get("/api/v1/search", search.NewHandler(pool, search.Options{ Nextcloud: ncClient, Engine: cfg.SearchEngine, @@ -400,3 +491,12 @@ func closeOwned(ownsPool bool, pool *pgxpool.Pool, ownsRedis bool, rdb *redis.Cl _ = rdb.Close() } } + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} diff --git a/internal/users/provision.go b/internal/users/provision.go index c0a6da4..5ebef39 100644 --- a/internal/users/provision.go +++ b/internal/users/provision.go @@ -8,6 +8,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/ultisuite/ulti-backend/internal/auth" + "github.com/ultisuite/ulti-backend/internal/migration" ) // ProvisionEmail returns the email stored for a newly provisioned user. @@ -58,5 +59,6 @@ func EnsureUser(ctx context.Context, db *pgxpool.Pool, claims *auth.Claims) (str return "", fmt.Errorf("bootstrap platform admin: %w", err) } } + _ = migration.LinkHostedMailboxByEmail(ctx, db, userID, email) return userID, nil } diff --git a/migrations/000040_hosted_mail_migration.down.sql b/migrations/000040_hosted_mail_migration.down.sql new file mode 100644 index 0000000..90aa0a0 --- /dev/null +++ b/migrations/000040_hosted_mail_migration.down.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS migration_credentials; +DROP TABLE IF EXISTS migration_jobs; +DROP TABLE IF EXISTS migration_invites; +DROP TABLE IF EXISTS migration_projects; +DROP TABLE IF EXISTS mailboxes; +DROP TABLE IF EXISTS mail_domains; diff --git a/migrations/000040_hosted_mail_migration.up.sql b/migrations/000040_hosted_mail_migration.up.sql new file mode 100644 index 0000000..d1329ae --- /dev/null +++ b/migrations/000040_hosted_mail_migration.up.sql @@ -0,0 +1,96 @@ +-- Hosted mail domains and mailboxes (Stalwart provisioning) +CREATE TABLE mail_domains ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + name TEXT NOT NULL UNIQUE, + status TEXT NOT NULL DEFAULT 'pending_verification', + verification_token TEXT NOT NULL DEFAULT '', + dkim_selector TEXT NOT NULL DEFAULT '', + dkim_public_key TEXT NOT NULL DEFAULT '', + stalwart_domain_id TEXT NOT NULL DEFAULT '', + is_platform_domain BOOLEAN NOT NULL DEFAULT false, + mx_verified_at TIMESTAMPTZ, + txt_verified_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_mail_domains_status ON mail_domains(status); + +CREATE TABLE mailboxes ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + domain_id UUID NOT NULL REFERENCES mail_domains(id) ON DELETE CASCADE, + local_part TEXT NOT NULL, + user_id UUID REFERENCES users(id) ON DELETE SET NULL, + mail_account_id UUID REFERENCES mail_accounts(id) ON DELETE SET NULL, + stalwart_account_id TEXT NOT NULL DEFAULT '', + quota_bytes BIGINT NOT NULL DEFAULT 5368709120, + status TEXT NOT NULL DEFAULT 'active', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(domain_id, local_part) +); + +CREATE INDEX idx_mailboxes_user ON mailboxes(user_id); +CREATE INDEX idx_mailboxes_mail_account ON mailboxes(mail_account_id); + +-- Migration projects (Google Workspace / Microsoft 365 transitions) +CREATE TABLE migration_projects ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + domain_id UUID REFERENCES mail_domains(id) ON DELETE SET NULL, + name TEXT NOT NULL DEFAULT '', + source_provider TEXT NOT NULL DEFAULT 'google', + status TEXT NOT NULL DEFAULT 'draft', + cutover_at TIMESTAMPTZ, + delta_mode BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_migration_projects_status ON migration_projects(status); + +CREATE TABLE migration_invites ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + project_id UUID NOT NULL REFERENCES migration_projects(id) ON DELETE CASCADE, + email TEXT NOT NULL, + token TEXT NOT NULL UNIQUE, + status TEXT NOT NULL DEFAULT 'invited', + claimed_at TIMESTAMPTZ, + user_id UUID REFERENCES users(id) ON DELETE SET NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(project_id, email) +); + +CREATE INDEX idx_migration_invites_token ON migration_invites(token); +CREATE INDEX idx_migration_invites_email ON migration_invites(email); + +CREATE TABLE migration_jobs ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + project_id UUID NOT NULL REFERENCES migration_projects(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + service TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + cursor_json JSONB NOT NULL DEFAULT '{}', + stats_json JSONB NOT NULL DEFAULT '{}', + error TEXT NOT NULL DEFAULT '', + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(project_id, user_id, service) +); + +CREATE INDEX idx_migration_jobs_status ON migration_jobs(status); + +CREATE TABLE migration_credentials ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + project_id UUID NOT NULL REFERENCES migration_projects(id) ON DELETE CASCADE, + provider TEXT NOT NULL, + encrypted_token BYTEA NOT NULL, + scopes TEXT[] NOT NULL DEFAULT '{}', + expires_at TIMESTAMPTZ, + revoked_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(user_id, project_id, provider) +); + +CREATE INDEX idx_migration_credentials_user ON migration_credentials(user_id); diff --git a/migrations/000041_migration_auth_mode.down.sql b/migrations/000041_migration_auth_mode.down.sql new file mode 100644 index 0000000..b8b1ab6 --- /dev/null +++ b/migrations/000041_migration_auth_mode.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE migration_projects DROP CONSTRAINT IF EXISTS migration_projects_auth_mode_check; +ALTER TABLE migration_projects DROP COLUMN IF EXISTS auth_mode; diff --git a/migrations/000041_migration_auth_mode.up.sql b/migrations/000041_migration_auth_mode.up.sql new file mode 100644 index 0000000..2c42bd2 --- /dev/null +++ b/migrations/000041_migration_auth_mode.up.sql @@ -0,0 +1,7 @@ +-- Migration auth mode: per-user OAuth vs Google domain-wide delegation +ALTER TABLE migration_projects + ADD COLUMN auth_mode TEXT NOT NULL DEFAULT 'oauth'; + +ALTER TABLE migration_projects + ADD CONSTRAINT migration_projects_auth_mode_check + CHECK (auth_mode IN ('oauth', 'google_dwd')); diff --git a/migrations/000042_migration_imported_items.down.sql b/migrations/000042_migration_imported_items.down.sql new file mode 100644 index 0000000..a51e418 --- /dev/null +++ b/migrations/000042_migration_imported_items.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS migration_imported_items; diff --git a/migrations/000042_migration_imported_items.up.sql b/migrations/000042_migration_imported_items.up.sql new file mode 100644 index 0000000..502c966 --- /dev/null +++ b/migrations/000042_migration_imported_items.up.sql @@ -0,0 +1,11 @@ +-- Track per-job imported source IDs and drive paths outside cursor_json. +CREATE TABLE migration_imported_items ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + job_id UUID NOT NULL REFERENCES migration_jobs(id) ON DELETE CASCADE, + source_id TEXT NOT NULL, + rel_path TEXT NOT NULL DEFAULT '', + imported_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(job_id, source_id) +); + +CREATE INDEX idx_migration_imported_items_job ON migration_imported_items(job_id); diff --git a/migrations/000043_migration_cutover_dns.down.sql b/migrations/000043_migration_cutover_dns.down.sql new file mode 100644 index 0000000..02c64cf --- /dev/null +++ b/migrations/000043_migration_cutover_dns.down.sql @@ -0,0 +1 @@ +ALTER TABLE migration_projects DROP COLUMN IF EXISTS cutover_dns_json; diff --git a/migrations/000043_migration_cutover_dns.up.sql b/migrations/000043_migration_cutover_dns.up.sql new file mode 100644 index 0000000..0cd4a53 --- /dev/null +++ b/migrations/000043_migration_cutover_dns.up.sql @@ -0,0 +1,3 @@ +-- Persist last migration cutover DNS check for admin dashboards. +ALTER TABLE migration_projects + ADD COLUMN IF NOT EXISTS cutover_dns_json JSONB NOT NULL DEFAULT '{}'; diff --git a/migrations/000044_migration_microsoft_admin_consent.down.sql b/migrations/000044_migration_microsoft_admin_consent.down.sql new file mode 100644 index 0000000..769fdfc --- /dev/null +++ b/migrations/000044_migration_microsoft_admin_consent.down.sql @@ -0,0 +1,6 @@ +ALTER TABLE migration_projects + DROP COLUMN IF EXISTS microsoft_admin_consent_error, + DROP COLUMN IF EXISTS microsoft_admin_consent_at, + DROP COLUMN IF EXISTS microsoft_tenant_id; + +DROP TABLE IF EXISTS migration_microsoft_admin_consents; diff --git a/migrations/000044_migration_microsoft_admin_consent.up.sql b/migrations/000044_migration_microsoft_admin_consent.up.sql new file mode 100644 index 0000000..e314bdb --- /dev/null +++ b/migrations/000044_migration_microsoft_admin_consent.up.sql @@ -0,0 +1,20 @@ +-- Persist Microsoft tenant admin consent for migration OAuth app registration. +CREATE TABLE IF NOT EXISTS migration_microsoft_admin_consents ( + tenant_id TEXT NOT NULL, + client_id TEXT NOT NULL, + project_id UUID REFERENCES migration_projects(id) ON DELETE SET NULL, + granted BOOLEAN NOT NULL DEFAULT false, + error_code TEXT NOT NULL DEFAULT '', + error_description TEXT NOT NULL DEFAULT '', + consented_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (tenant_id, client_id) +); + +CREATE INDEX IF NOT EXISTS idx_migration_ms_admin_consents_project + ON migration_microsoft_admin_consents(project_id); + +ALTER TABLE migration_projects + ADD COLUMN IF NOT EXISTS microsoft_tenant_id TEXT NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS microsoft_admin_consent_at TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS microsoft_admin_consent_error TEXT NOT NULL DEFAULT ''; diff --git a/migrations/000045_migration_invite_alternate_emails.down.sql b/migrations/000045_migration_invite_alternate_emails.down.sql new file mode 100644 index 0000000..7aacbf1 --- /dev/null +++ b/migrations/000045_migration_invite_alternate_emails.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE migration_invites + DROP COLUMN IF EXISTS alternate_emails; diff --git a/migrations/000045_migration_invite_alternate_emails.up.sql b/migrations/000045_migration_invite_alternate_emails.up.sql new file mode 100644 index 0000000..4642b54 --- /dev/null +++ b/migrations/000045_migration_invite_alternate_emails.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE migration_invites + ADD COLUMN alternate_emails TEXT[] NOT NULL DEFAULT '{}'; diff --git a/migrations/000046_migration_item_audit.down.sql b/migrations/000046_migration_item_audit.down.sql new file mode 100644 index 0000000..ae6dafb --- /dev/null +++ b/migrations/000046_migration_item_audit.down.sql @@ -0,0 +1,8 @@ +DROP INDEX IF EXISTS idx_migration_imported_items_job_status; + +ALTER TABLE migration_imported_items + DROP CONSTRAINT IF EXISTS migration_imported_items_status_check; + +ALTER TABLE migration_imported_items + DROP COLUMN IF EXISTS reason, + DROP COLUMN IF EXISTS status; diff --git a/migrations/000046_migration_item_audit.up.sql b/migrations/000046_migration_item_audit.up.sql new file mode 100644 index 0000000..2839509 --- /dev/null +++ b/migrations/000046_migration_item_audit.up.sql @@ -0,0 +1,11 @@ +-- Per-item migration audit: track success, failure, and skip with reason. +ALTER TABLE migration_imported_items + ADD COLUMN status TEXT NOT NULL DEFAULT 'imported', + ADD COLUMN reason TEXT NOT NULL DEFAULT ''; + +ALTER TABLE migration_imported_items + ADD CONSTRAINT migration_imported_items_status_check + CHECK (status IN ('imported', 'failed', 'skipped')); + +CREATE INDEX idx_migration_imported_items_job_status + ON migration_imported_items(job_id, status); diff --git a/migrations/000047_migration_microsoft_app_auth.down.sql b/migrations/000047_migration_microsoft_app_auth.down.sql new file mode 100644 index 0000000..f5148e9 --- /dev/null +++ b/migrations/000047_migration_microsoft_app_auth.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE migration_projects DROP CONSTRAINT IF EXISTS migration_projects_auth_mode_check; + +ALTER TABLE migration_projects + ADD CONSTRAINT migration_projects_auth_mode_check + CHECK (auth_mode IN ('oauth', 'google_dwd')); diff --git a/migrations/000047_migration_microsoft_app_auth.up.sql b/migrations/000047_migration_microsoft_app_auth.up.sql new file mode 100644 index 0000000..cfa722f --- /dev/null +++ b/migrations/000047_migration_microsoft_app_auth.up.sql @@ -0,0 +1,6 @@ +-- Microsoft app-only auth mode (client credentials, equivalent to Google DWD) +ALTER TABLE migration_projects DROP CONSTRAINT IF EXISTS migration_projects_auth_mode_check; + +ALTER TABLE migration_projects + ADD CONSTRAINT migration_projects_auth_mode_check + CHECK (auth_mode IN ('oauth', 'google_dwd', 'microsoft_app')); diff --git a/project-plan/checklist-execution.md b/project-plan/checklist-execution.md index 9111ff5..3e8c532 100644 --- a/project-plan/checklist-execution.md +++ b/project-plan/checklist-execution.md @@ -178,6 +178,34 @@ Objectif: transformer état actuel (partiellement implémenté) vers produit fon - [x] Ajouter quotas multi-service (mail/drive/photos). - [x] Ajouter pages stats exploitables + export audit. +### 2.8 Migration hébergée (Google Workspace / Microsoft 365) + +> **Avancement (2026-06-13)** — MVP migration B2B opérationnel (claim flexible, MS app-only, audit item-level) ; reste durcissement sécu claim, import Graph multi-dossiers, drive partagés/gros fichiers, roster, export audit, e2e OAuth réel. + +#### Déjà implémenté + +- [x] Projets migration CRUD, invitations tokenisées, claim + provision mailbox Stalwart avec `domain_id`. *(backend: `internal/migration/service.go`, `internal/api/migration/handlers.go`; frontend: `migration-projects-panel.tsx`, `/onboard/claim`)* +- [x] Worker asynchrone mail / contacts / calendar / drive avec OAuth utilisateur + Google DWD (`google_dwd`). *(backend: `internal/migration/worker.go`, `google_dwd.go`, `oauth.go`)* +- [x] Import et delta Gmail + Graph, pièces jointes Gmail, backoff HTTP 429, curseur `migration_imported_items` (table dédiée vs maps illimitées dans `cursor_json`). *(backend: `gmail_import.go`, `graph_import.go`, `gmail_attachments.go`, `http_retry.go`, `imported_items.go`)* +- [x] UI admin : sélecteur `auth_mode`, consentement admin Microsoft persisté, retry job / reset curseur / retry failed. *(frontend: `migration-projects-panel.tsx`; backend: `admin_jobs.go`, `microsoft_admin_consent.go`)* +- [x] Cutover MX/DNS avec pré-vérification live et blocage si MX non prêt (`migration_cutover_mx_not_ready`). *(backend: `cutover_dns.go`; frontend: panel admin DNS + bascule MX)* +- [x] Onboarding utilisateur claim → attente activation → hints OAuth / DWD / consent MS. *(backend: `onboarding.go`; frontend: `/onboard/migration`, `onboarding-ui.tsx`)* +- [x] Tests e2e Playwright parcours onboarding (API mockée). *(frontend: `e2e/onboard-migration.spec.ts`, `e2e/helpers/migration-onboarding.ts`)* +- [x] Claim alias flexible (email invite ≠ identité OIDC) — matching multi-claims + `alternate_emails` admin + règles UPN Microsoft. *(backend: `claim_email_match.go`, migration `000045`; durcissement sécu UPN/tenant : partiel)* +- [x] Auth Microsoft app-only (`microsoft_app`) — client credentials Graph, équivalent DWD Google. *(backend: `microsoft_app.go`, `auth_modes.go`, migration `000047`; validation tenant Azure pilote : partiel)* +- [x] Rapport d'audit migration par item (succès/échec/skip) — API paginée + panel admin. *(backend: migration `000046`, `audit_report.go`; export CSV/NDJSON : non fait)* + +#### Partiel / non fait + +- [ ] Durcissement claim — vérif `tid` OIDC vs `microsoft_tenant_id`, alias points/plus limités Gmail, UI admin `alternate_emails`. *(backend: `claim_email_match.go`; frontend: champ alternates absent du panel)* +- [ ] Import Graph mail folder-aware complet — import flat `/me/messages` ; métadonnées dossiers chargées mais pas d'itération par dossier. *(backend: `graph_import.go`)* +- [ ] Drive Shared Drives + fichiers >25 Mo — skip au-delà de `maxDriveFileBytes`, pas de shared drives Google. *(backend: `drive_import.go`)* +- [ ] Pré-provision roster (import CSV/liste utilisateurs avant claim individuel). *(non implémenté)* +- [ ] Export audit migration (CSV/NDJSON) pour ops cutover. *(API list/summary seulement)* +- [ ] Tests d'intégration sandbox Google/Microsoft réels — providers mockés via `httptest` en CI. *(backend: `internal/integrationtest/migration/`)* +- [ ] e2e Playwright OAuth réel (sans mocks API migration/auth). *(e2e actuels: `installMigrationApiMocks`)* +- [ ] Double chemin provision Authentik enrollment vs claim migration. *(risque doublons/orphelins non résolu)* + ## 3) Frontend web (`gmail-interface-clone`) > **Avancement (2026-05-23)** — migration offline-first API dans `gmail-interface-clone` : **14/22** items section 3 cochés. Reste : cleanup fichiers mock, recherche sans fallback offline, sync par compte, modules suite (§3.4), a11y/i18n/perf (§3.6).