//go:build integration package migration_test import ( "context" "net/http" "net/http/httptest" "strings" "testing" "github.com/google/uuid" "github.com/ultisuite/ulti-backend/internal/integrationtest" migr "github.com/ultisuite/ulti-backend/internal/migration" "github.com/ultisuite/ulti-backend/internal/users" ) func TestMigrationInviteClaimFlow(t *testing.T) { h := integrationtest.RequireHarness(t) ctx := context.Background() adminClient, adminClaims := integrationtest.RequireAdminClient(t, h) if _, err := users.EnsureUser(ctx, h.Pool, adminClaims); err != nil { t.Fatalf("ensure admin: %v", err) } if err := users.GrantPlatformAdmin(ctx, h.Pool, adminClaims.Sub); err != nil { t.Fatalf("grant admin: %v", err) } createResp, err := adminClient.Post("/api/v1/admin/migration/projects", map[string]any{ "name": "Test migration", "source_provider": "microsoft", }) integrationtest.FailIf(err, t, "create project") integrationtest.FailUnlessStatus(t, createResp, 201) var created struct { ID string `json:"id"` } integrationtest.DecodeJSON(t, createResp, &created) if created.ID == "" { t.Fatalf("missing project id") } actResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/activate", nil) integrationtest.FailIf(err, t, "activate project") integrationtest.FailUnlessStatus(t, actResp, 200) migrateeEmail := "migratee-" + uuid.NewString() + "@example.com" inviteResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/invites", map[string]string{ "email": migrateeEmail, }) integrationtest.FailIf(err, t, "create invite") integrationtest.FailUnlessStatus(t, inviteResp, 201) var invite struct { Token string `json:"token"` } integrationtest.DecodeJSON(t, inviteResp, &invite) if invite.Token == "" { t.Fatalf("missing invite token") } migrateeClaims := integrationtest.RegularUser(integrationtest.NewExternalID("migratee")) migrateeClaims.Email = migrateeEmail migrateeClient, err := h.Client(migrateeClaims) integrationtest.FailIf(err, t, "migratee client") userID, err := users.EnsureUser(ctx, h.Pool, migrateeClaims) integrationtest.FailIf(err, t, "ensure migratee") claimResp, err := migrateeClient.Post("/api/v1/migration/claim", map[string]string{ "token": invite.Token, "password": "test-password-123", }) integrationtest.FailIf(err, t, "claim invite") integrationtest.FailUnlessStatus(t, claimResp, 200) var status struct { Jobs []struct { Service string `json:"service"` Status string `json:"status"` } `json:"jobs"` } integrationtest.DecodeJSON(t, claimResp, &status) if len(status.Jobs) != 4 { t.Fatalf("expected 4 jobs, got %d", len(status.Jobs)) } var jobCount int if err := h.Pool.QueryRow(ctx, ` SELECT COUNT(*) FROM migration_jobs WHERE project_id = $1::uuid AND user_id = $2::uuid `, created.ID, userID).Scan(&jobCount); err != nil { t.Fatalf("count jobs: %v", err) } if jobCount != 4 { t.Fatalf("db job count = %d, want 4", jobCount) } jobsResp, err := adminClient.Get("/api/v1/admin/migration/projects/" + created.ID + "/jobs") integrationtest.FailIf(err, t, "list admin jobs") integrationtest.FailUnlessStatus(t, jobsResp, 200) var adminJobs struct { Jobs []struct { ID string `json:"id"` Service string `json:"service"` Status string `json:"status"` Email string `json:"user_email"` } `json:"jobs"` } integrationtest.DecodeJSON(t, jobsResp, &adminJobs) if len(adminJobs.Jobs) != 4 { t.Fatalf("admin jobs = %d, want 4", len(adminJobs.Jobs)) } for _, job := range adminJobs.Jobs { if job.Email != migrateeEmail { t.Fatalf("user_email = %q, want %q", job.Email, migrateeEmail) } } var mailJobID string for _, job := range adminJobs.Jobs { if job.Service == "mail" { mailJobID = job.ID break } } if mailJobID == "" { t.Fatal("mail job not found") } _, err = h.Pool.Exec(ctx, ` UPDATE migration_jobs SET status = 'failed', error = 'simulated failure', updated_at = NOW() WHERE id = $1::uuid `, mailJobID) integrationtest.FailIf(err, t, "mark job failed") retryResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/jobs/"+mailJobID+"/retry", nil) integrationtest.FailIf(err, t, "retry job") integrationtest.FailUnlessStatus(t, retryResp, 200) var retried struct { Status string `json:"status"` } integrationtest.DecodeJSON(t, retryResp, &retried) if retried.Status != "pending" { t.Fatalf("retried status = %q, want pending", retried.Status) } _, err = h.Pool.Exec(ctx, ` UPDATE migration_jobs SET cursor_json = '{"historyId":"123"}'::jsonb, stats_json = '{"imported":42}'::jsonb, status = 'completed', updated_at = NOW() WHERE id = $1::uuid `, mailJobID) integrationtest.FailIf(err, t, "seed job cursor") _, err = h.Pool.Exec(ctx, ` INSERT INTO migration_imported_items (job_id, source_id, status, reason) VALUES ($1::uuid, 'msg-abc', 'imported', ''), ($1::uuid, 'msg-fail', 'failed', 'upload timeout'), ($1::uuid, 'msg-skip', 'skipped', 'file too large') `, mailJobID) integrationtest.FailIf(err, t, "seed imported items") summaryResp, err := adminClient.Get("/api/v1/admin/migration/projects/" + created.ID + "/jobs/" + mailJobID + "/audit/summary") integrationtest.FailIf(err, t, "audit summary") integrationtest.FailUnlessStatus(t, summaryResp, 200) var auditSummary struct { Imported int64 `json:"imported"` Failed int64 `json:"failed"` Skipped int64 `json:"skipped"` Total int64 `json:"total"` Service string `json:"service"` } integrationtest.DecodeJSON(t, summaryResp, &auditSummary) if auditSummary.Imported != 1 || auditSummary.Failed != 1 || auditSummary.Skipped != 1 || auditSummary.Total != 3 { t.Fatalf("audit summary = %+v, want 1 imported / 1 failed / 1 skipped", auditSummary) } if auditSummary.Service != "mail" { t.Fatalf("audit service = %q, want mail", auditSummary.Service) } failedResp, err := adminClient.Get("/api/v1/admin/migration/projects/" + created.ID + "/jobs/" + mailJobID + "/audit?status=failed") integrationtest.FailIf(err, t, "audit failed list") integrationtest.FailUnlessStatus(t, failedResp, 200) var failedList struct { Items []struct { SourceID string `json:"source_id"` Status string `json:"status"` Reason string `json:"reason"` } `json:"items"` } integrationtest.DecodeJSON(t, failedResp, &failedList) if len(failedList.Items) != 1 || failedList.Items[0].SourceID != "msg-fail" { t.Fatalf("failed audit items = %+v", failedList.Items) } resetResp, err := adminClient.Post("/api/v1/admin/migration/projects/"+created.ID+"/jobs/"+mailJobID+"/reset-cursor", nil) integrationtest.FailIf(err, t, "reset cursor") integrationtest.FailUnlessStatus(t, resetResp, 200) var reset struct { Status string `json:"status"` CursorJSON map[string]any `json:"cursor_json"` StatsJSON map[string]any `json:"stats_json"` } integrationtest.DecodeJSON(t, resetResp, &reset) if reset.Status != "pending" { t.Fatalf("reset status = %q, want pending", reset.Status) } if len(reset.CursorJSON) != 0 { t.Fatalf("cursor not cleared: %#v", reset.CursorJSON) } if len(reset.StatsJSON) != 0 { t.Fatalf("stats not cleared: %#v", reset.StatsJSON) } var importedCount int if err := h.Pool.QueryRow(ctx, ` SELECT COUNT(*) FROM migration_imported_items WHERE job_id = $1::uuid `, mailJobID).Scan(&importedCount); err != nil { t.Fatalf("count imported items: %v", err) } if importedCount != 0 { t.Fatalf("imported items = %d, want 0", importedCount) } } func TestGraphImportWritesMessages(t *testing.T) { h := integrationtest.RequireHarness(t) ctx := context.Background() userID, err := users.EnsureUser(ctx, h.Pool, integrationtest.RegularUser(integrationtest.NewExternalID("graph-import"))) integrationtest.FailIf(err, t, "ensure user") var accountID string err = h.Pool.QueryRow(ctx, ` INSERT INTO mail_accounts (user_id, email, provider, is_active) VALUES ($1::uuid, 'graph-import@test.local', 'hosted', true) RETURNING id::text `, userID).Scan(&accountID) integrationtest.FailIf(err, t, "insert mail account") folderID := "inbox-folder-id" messagesListed := false srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case strings.Contains(r.URL.Path, "/mailFolders"): _, _ = w.Write([]byte(`{"value":[{"id":"` + folderID + `","displayName":"Inbox","wellKnownName":"inbox"}]}`)) case strings.Contains(r.URL.Path, "/messages"): messagesListed = true _, _ = w.Write([]byte(`{"value":[{ "id":"msg-1", "subject":"Hello Graph", "bodyPreview":"Preview text", "body":{"contentType":"text","content":"Body text"}, "from":{"emailAddress":{"name":"Alice","address":"alice@example.com"}}, "toRecipients":[{"emailAddress":{"name":"Bob","address":"bob@example.com"}}], "receivedDateTime":"2024-05-01T10:00:00Z", "parentFolderId":"` + folderID + `", "isRead":true, "internetMessageId":"" }]}`)) default: http.NotFound(w, r) } })) defer srv.Close() importer := migr.NewGraphImporter(h.Pool).WithBaseURL(srv.URL) job := &migr.Job{ UserID: userID, CursorJSON: map[string]any{}, StatsJSON: map[string]any{}, } err = importer.ImportBatch(ctx, job, "test-token", false, func(status string, cursor, stats map[string]any, jobErr string) error { if jobErr != "" { t.Fatalf("import error: %s", jobErr) } if status != "completed" { t.Fatalf("status = %q, want completed", status) } return nil }) integrationtest.FailIf(err, t, "import batch") if !messagesListed { t.Fatal("graph messages endpoint not called") } var count int if err := h.Pool.QueryRow(ctx, ` SELECT COUNT(*) FROM messages WHERE account_id = $1::uuid AND subject = 'Hello Graph' `, accountID).Scan(&count); err != nil { t.Fatalf("count messages: %v", err) } if count != 1 { t.Fatalf("message count = %d, want 1", count) } } func TestGmailImportStoresAttachments(t *testing.T) { h := integrationtest.RequireHarness(t) if h.AttachmentStorage == nil { t.Skip("attachment storage unavailable") } ctx := context.Background() userID, err := users.EnsureUser(ctx, h.Pool, integrationtest.RegularUser(integrationtest.NewExternalID("gmail-att-import"))) integrationtest.FailIf(err, t, "ensure user") var accountID string err = h.Pool.QueryRow(ctx, ` INSERT INTO mail_accounts (user_id, email, provider, is_active) VALUES ($1::uuid, 'gmail-att@test.local', 'hosted', true) RETURNING id::text `, userID).Scan(&accountID) integrationtest.FailIf(err, t, "insert mail account") gmailID := "msg-with-att" client := googleRewriteClient(t, func(w http.ResponseWriter, r *http.Request) { switch { case strings.Contains(r.URL.Path, "/users/me/profile"): _, _ = w.Write([]byte(`{"historyId":"12345"}`)) case strings.Contains(r.URL.Path, "/attachments/att-123"): _, _ = w.Write([]byte(`{"size":5,"data":"aGVsbG8="}`)) case strings.Contains(r.URL.Path, "/messages/"+gmailID): _, _ = w.Write([]byte(`{ "id":"` + gmailID + `", "threadId":"t1", "labelIds":["INBOX"], "snippet":"see attached", "payload":{ "mimeType":"multipart/mixed", "headers":[{"name":"Subject","value":"With attachment"}], "parts":[ {"mimeType":"text/plain","body":{"data":"dGV4dA=="}}, { "mimeType":"application/pdf", "headers":[{"name":"Content-Disposition","value":"attachment; filename=\"report.pdf\""}], "body":{"attachmentId":"att-123","size":5} } ] } }`)) case strings.HasSuffix(r.URL.Path, "/messages"): _, _ = w.Write([]byte(`{"messages":[{"id":"` + gmailID + `"}]}`)) default: http.NotFound(w, r) } }) importer := migr.NewGmailImporter(h.Pool). WithHTTPClient(client). WithStorage(h.AttachmentStorage, h.AttachmentsBucket) job := &migr.Job{ UserID: userID, CursorJSON: map[string]any{}, StatsJSON: map[string]any{}, } err = importer.ImportBatch(ctx, job, "token", false, func(status string, cursor, stats map[string]any, jobErr string) error { if jobErr != "" { t.Fatalf("import error: %s", jobErr) } if status != "completed" { t.Fatalf("status = %q, want completed", status) } return nil }) integrationtest.FailIf(err, t, "import batch") var attCount int var hasAttachments bool err = h.Pool.QueryRow(ctx, ` SELECT COUNT(*)::int, COALESCE(BOOL_OR(m.has_attachments), false) FROM attachments a JOIN messages m ON m.id = a.message_id WHERE m.account_id = $1::uuid AND a.filename = 'report.pdf' `, accountID).Scan(&attCount, &hasAttachments) integrationtest.FailIf(err, t, "count attachments") if attCount != 1 || !hasAttachments { t.Fatalf("attachments = %d has_attachments = %v", attCount, hasAttachments) } }