ultisuite-backend/internal/migration/roster.go
R3D347HR4Y 1ffd0817d8
Some checks are pending
CI / Go tests (push) Waiting to run
CI / Integration tests (push) Waiting to run
CI / DB migrations (push) Waiting to run
feat(migration): enhance migration API with roster and audit export features
- Added endpoints for listing and importing migration rosters.
- Introduced audit export functionality for migration jobs in CSV and NDJSON formats.
- Implemented tenant mismatch validation for Microsoft migration claims.
- Enhanced error handling for email claiming and migration processes.
- Added integration tests for roster import and claim workflows.
2026-06-13 13:11:30 +02:00

348 lines
8.8 KiB
Go

package migration
import (
"context"
"encoding/csv"
"errors"
"fmt"
"io"
"strings"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/ultisuite/ulti-backend/internal/mail/hosted"
)
const (
RosterStatusPending = "pending"
RosterStatusInvited = "invited"
RosterStatusClaimed = "claimed"
)
type RosterEntry struct {
ID string `json:"id"`
ProjectID string `json:"project_id"`
Email string `json:"email"`
DisplayName string `json:"display_name,omitempty"`
AlternateEmails []string `json:"alternate_emails,omitempty"`
Status string `json:"status"`
InviteID string `json:"invite_id,omitempty"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
type RosterRowInput struct {
Email string
DisplayName string
AlternateEmails []string
}
type RosterImportRowError struct {
Row int `json:"row"`
Email string `json:"email,omitempty"`
Message string `json:"message"`
}
type RosterImportResult struct {
Created int `json:"created"`
SkippedDuplicates int `json:"skipped_duplicates"`
Errors []RosterImportRowError `json:"errors,omitempty"`
}
var rosterHeaderAliases = map[string]string{
"email": "email",
"e-mail": "email",
"mail": "email",
"address": "email",
"display_name": "display_name",
"displayname": "display_name",
"name": "display_name",
"full_name": "display_name",
"alternate_emails": "alternate_emails",
"alternate_emails_": "alternate_emails",
"alternates": "alternate_emails",
"alias": "alternate_emails",
"aliases": "alternate_emails",
}
func ParseRosterCSV(r io.Reader) ([]RosterRowInput, error) {
reader := csv.NewReader(r)
reader.TrimLeadingSpace = true
reader.FieldsPerRecord = -1
var rows []RosterRowInput
lineNum := 0
emailCol := 0
displayCol := -1
alternateCol := -1
headerResolved := false
for {
record, err := reader.Read()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, fmt.Errorf("csv row %d: %w", lineNum+1, err)
}
lineNum++
if len(record) == 0 {
continue
}
for len(record) > 0 && strings.TrimSpace(record[len(record)-1]) == "" {
record = record[:len(record)-1]
}
if len(record) == 0 {
continue
}
if !headerResolved && looksLikeRosterHeader(record) {
for i, col := range record {
key := normalizeRosterHeader(col)
switch key {
case "email":
emailCol = i
case "display_name":
displayCol = i
case "alternate_emails":
alternateCol = i
}
}
headerResolved = true
continue
}
headerResolved = true
if emailCol >= len(record) {
continue
}
email := normalizeInviteEmail(record[emailCol])
if email == "" {
continue
}
if !isEmailAddress(email) {
return nil, fmt.Errorf("csv row %d: invalid email %q", lineNum, record[emailCol])
}
row := RosterRowInput{Email: email}
if displayCol >= 0 && displayCol < len(record) {
row.DisplayName = strings.TrimSpace(record[displayCol])
}
if alternateCol >= 0 && alternateCol < len(record) {
row.AlternateEmails = parseAlternateEmailsField(record[alternateCol])
} else if len(record) > 1 && displayCol < 0 && alternateCol < 0 {
// email,display_name or email,alternates without header
if len(record) > 1 {
second := strings.TrimSpace(record[1])
if strings.Contains(second, "@") {
row.AlternateEmails = parseAlternateEmailsField(second)
} else {
row.DisplayName = second
}
}
if len(record) > 2 {
row.AlternateEmails = parseAlternateEmailsField(record[2])
}
}
row.AlternateEmails = normalizeAlternateEmails(email, row.AlternateEmails)
rows = append(rows, row)
}
return rows, nil
}
func looksLikeRosterHeader(record []string) bool {
if len(record) == 0 {
return false
}
first := normalizeRosterHeader(record[0])
if first == "email" {
return true
}
for _, col := range record {
if normalizeRosterHeader(col) == "email" {
return true
}
}
return false
}
func normalizeRosterHeader(col string) string {
key := strings.ToLower(strings.TrimSpace(col))
key = strings.ReplaceAll(key, " ", "_")
key = strings.ReplaceAll(key, "-", "_")
if mapped, ok := rosterHeaderAliases[key]; ok {
return mapped
}
return key
}
func parseAlternateEmailsField(raw string) []string {
raw = strings.TrimSpace(raw)
if raw == "" {
return nil
}
raw = strings.Trim(raw, "\"'")
parts := strings.FieldsFunc(raw, func(r rune) bool {
return r == ';' || r == '|' || r == ','
})
var out []string
for _, p := range parts {
p = strings.TrimSpace(p)
if p != "" {
out = append(out, p)
}
}
return out
}
func (s *Service) ListRoster(ctx context.Context, projectID string) ([]RosterEntry, error) {
rows, err := s.db.Query(ctx, `
SELECT id::text, project_id::text, email, display_name, alternate_emails, status,
COALESCE(invite_id::text, ''), created_at::text, updated_at::text
FROM migration_roster
WHERE project_id = $1::uuid
ORDER BY email
`, projectID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []RosterEntry
for rows.Next() {
var row RosterEntry
if err := rows.Scan(
&row.ID, &row.ProjectID, &row.Email, &row.DisplayName, &row.AlternateEmails,
&row.Status, &row.InviteID, &row.CreatedAt, &row.UpdatedAt,
); err != nil {
return nil, err
}
out = append(out, row)
}
return out, rows.Err()
}
func (s *Service) ImportRoster(ctx context.Context, projectID string, inputs []RosterRowInput) (RosterImportResult, error) {
result := RosterImportResult{}
for i, input := range inputs {
rowNum := i + 1
email := normalizeInviteEmail(input.Email)
if email == "" || !isEmailAddress(email) {
result.Errors = append(result.Errors, RosterImportRowError{
Row: rowNum, Email: input.Email, Message: "invalid email",
})
continue
}
alternates := normalizeAlternateEmails(email, input.AlternateEmails)
displayName := strings.TrimSpace(input.DisplayName)
existingStatus, err := s.rosterStatusByEmail(ctx, projectID, email)
if err != nil {
return result, err
}
if existingStatus != "" {
result.SkippedDuplicates++
continue
}
tx, err := s.db.Begin(ctx)
if err != nil {
return result, err
}
var rosterID string
err = tx.QueryRow(ctx, `
INSERT INTO migration_roster (project_id, email, display_name, alternate_emails, status)
VALUES ($1::uuid, $2, $3, $4, $5)
RETURNING id::text
`, projectID, email, displayName, alternates, RosterStatusPending).Scan(&rosterID)
if err != nil {
_ = tx.Rollback(ctx)
if isUniqueViolation(err) {
result.SkippedDuplicates++
continue
}
result.Errors = append(result.Errors, RosterImportRowError{
Row: rowNum, Email: email, Message: err.Error(),
})
continue
}
token, err := hosted.NewInviteToken()
if err != nil {
_ = tx.Rollback(ctx)
return result, err
}
var inviteID string
err = tx.QueryRow(ctx, `
INSERT INTO migration_invites (project_id, email, token, alternate_emails)
VALUES ($1::uuid, $2, $3, $4)
RETURNING id::text
`, projectID, email, token, alternates).Scan(&inviteID)
if err != nil {
_ = tx.Rollback(ctx)
if isUniqueViolation(err) {
result.SkippedDuplicates++
continue
}
result.Errors = append(result.Errors, RosterImportRowError{
Row: rowNum, Email: email, Message: err.Error(),
})
continue
}
_, err = tx.Exec(ctx, `
UPDATE migration_roster
SET status = $1, invite_id = $2::uuid, updated_at = NOW()
WHERE id = $3::uuid
`, RosterStatusInvited, inviteID, rosterID)
if err != nil {
_ = tx.Rollback(ctx)
result.Errors = append(result.Errors, RosterImportRowError{
Row: rowNum, Email: email, Message: err.Error(),
})
continue
}
if err := tx.Commit(ctx); err != nil {
result.Errors = append(result.Errors, RosterImportRowError{
Row: rowNum, Email: email, Message: err.Error(),
})
continue
}
result.Created++
}
return result, nil
}
func (s *Service) rosterStatusByEmail(ctx context.Context, projectID, email string) (string, error) {
var status string
err := s.db.QueryRow(ctx, `
SELECT status FROM migration_roster
WHERE project_id = $1::uuid AND email = $2
`, projectID, email).Scan(&status)
if errors.Is(err, pgx.ErrNoRows) {
return "", nil
}
return status, err
}
func (s *Service) markRosterClaimed(ctx context.Context, tx pgx.Tx, projectID, inviteID, email string) error {
_, err := tx.Exec(ctx, `
UPDATE migration_roster
SET status = $1, updated_at = NOW()
WHERE project_id = $2::uuid
AND (invite_id = $3::uuid OR (invite_id IS NULL AND email = $4))
`, RosterStatusClaimed, projectID, inviteID, email)
return err
}
func isUniqueViolation(err error) bool {
var pgErr *pgconn.PgError
return errors.As(err, &pgErr) && pgErr.Code == "23505"
}