- Added endpoints for listing and importing migration rosters. - Introduced audit export functionality for migration jobs in CSV and NDJSON formats. - Implemented tenant mismatch validation for Microsoft migration claims. - Enhanced error handling for email claiming and migration processes. - Added integration tests for roster import and claim workflows.
348 lines
8.8 KiB
Go
348 lines
8.8 KiB
Go
package migration
|
|
|
|
import (
|
|
"context"
|
|
"encoding/csv"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgconn"
|
|
|
|
"github.com/ultisuite/ulti-backend/internal/mail/hosted"
|
|
)
|
|
|
|
const (
|
|
RosterStatusPending = "pending"
|
|
RosterStatusInvited = "invited"
|
|
RosterStatusClaimed = "claimed"
|
|
)
|
|
|
|
type RosterEntry struct {
|
|
ID string `json:"id"`
|
|
ProjectID string `json:"project_id"`
|
|
Email string `json:"email"`
|
|
DisplayName string `json:"display_name,omitempty"`
|
|
AlternateEmails []string `json:"alternate_emails,omitempty"`
|
|
Status string `json:"status"`
|
|
InviteID string `json:"invite_id,omitempty"`
|
|
CreatedAt string `json:"created_at"`
|
|
UpdatedAt string `json:"updated_at"`
|
|
}
|
|
|
|
type RosterRowInput struct {
|
|
Email string
|
|
DisplayName string
|
|
AlternateEmails []string
|
|
}
|
|
|
|
type RosterImportRowError struct {
|
|
Row int `json:"row"`
|
|
Email string `json:"email,omitempty"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
type RosterImportResult struct {
|
|
Created int `json:"created"`
|
|
SkippedDuplicates int `json:"skipped_duplicates"`
|
|
Errors []RosterImportRowError `json:"errors,omitempty"`
|
|
}
|
|
|
|
var rosterHeaderAliases = map[string]string{
|
|
"email": "email",
|
|
"e-mail": "email",
|
|
"mail": "email",
|
|
"address": "email",
|
|
"display_name": "display_name",
|
|
"displayname": "display_name",
|
|
"name": "display_name",
|
|
"full_name": "display_name",
|
|
"alternate_emails": "alternate_emails",
|
|
"alternate_emails_": "alternate_emails",
|
|
"alternates": "alternate_emails",
|
|
"alias": "alternate_emails",
|
|
"aliases": "alternate_emails",
|
|
}
|
|
|
|
func ParseRosterCSV(r io.Reader) ([]RosterRowInput, error) {
|
|
reader := csv.NewReader(r)
|
|
reader.TrimLeadingSpace = true
|
|
reader.FieldsPerRecord = -1
|
|
|
|
var rows []RosterRowInput
|
|
lineNum := 0
|
|
emailCol := 0
|
|
displayCol := -1
|
|
alternateCol := -1
|
|
headerResolved := false
|
|
|
|
for {
|
|
record, err := reader.Read()
|
|
if errors.Is(err, io.EOF) {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("csv row %d: %w", lineNum+1, err)
|
|
}
|
|
lineNum++
|
|
|
|
if len(record) == 0 {
|
|
continue
|
|
}
|
|
for len(record) > 0 && strings.TrimSpace(record[len(record)-1]) == "" {
|
|
record = record[:len(record)-1]
|
|
}
|
|
if len(record) == 0 {
|
|
continue
|
|
}
|
|
|
|
if !headerResolved && looksLikeRosterHeader(record) {
|
|
for i, col := range record {
|
|
key := normalizeRosterHeader(col)
|
|
switch key {
|
|
case "email":
|
|
emailCol = i
|
|
case "display_name":
|
|
displayCol = i
|
|
case "alternate_emails":
|
|
alternateCol = i
|
|
}
|
|
}
|
|
headerResolved = true
|
|
continue
|
|
}
|
|
headerResolved = true
|
|
|
|
if emailCol >= len(record) {
|
|
continue
|
|
}
|
|
email := normalizeInviteEmail(record[emailCol])
|
|
if email == "" {
|
|
continue
|
|
}
|
|
if !isEmailAddress(email) {
|
|
return nil, fmt.Errorf("csv row %d: invalid email %q", lineNum, record[emailCol])
|
|
}
|
|
|
|
row := RosterRowInput{Email: email}
|
|
if displayCol >= 0 && displayCol < len(record) {
|
|
row.DisplayName = strings.TrimSpace(record[displayCol])
|
|
}
|
|
if alternateCol >= 0 && alternateCol < len(record) {
|
|
row.AlternateEmails = parseAlternateEmailsField(record[alternateCol])
|
|
} else if len(record) > 1 && displayCol < 0 && alternateCol < 0 {
|
|
// email,display_name or email,alternates without header
|
|
if len(record) > 1 {
|
|
second := strings.TrimSpace(record[1])
|
|
if strings.Contains(second, "@") {
|
|
row.AlternateEmails = parseAlternateEmailsField(second)
|
|
} else {
|
|
row.DisplayName = second
|
|
}
|
|
}
|
|
if len(record) > 2 {
|
|
row.AlternateEmails = parseAlternateEmailsField(record[2])
|
|
}
|
|
}
|
|
row.AlternateEmails = normalizeAlternateEmails(email, row.AlternateEmails)
|
|
rows = append(rows, row)
|
|
}
|
|
|
|
return rows, nil
|
|
}
|
|
|
|
func looksLikeRosterHeader(record []string) bool {
|
|
if len(record) == 0 {
|
|
return false
|
|
}
|
|
first := normalizeRosterHeader(record[0])
|
|
if first == "email" {
|
|
return true
|
|
}
|
|
for _, col := range record {
|
|
if normalizeRosterHeader(col) == "email" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func normalizeRosterHeader(col string) string {
|
|
key := strings.ToLower(strings.TrimSpace(col))
|
|
key = strings.ReplaceAll(key, " ", "_")
|
|
key = strings.ReplaceAll(key, "-", "_")
|
|
if mapped, ok := rosterHeaderAliases[key]; ok {
|
|
return mapped
|
|
}
|
|
return key
|
|
}
|
|
|
|
func parseAlternateEmailsField(raw string) []string {
|
|
raw = strings.TrimSpace(raw)
|
|
if raw == "" {
|
|
return nil
|
|
}
|
|
raw = strings.Trim(raw, "\"'")
|
|
parts := strings.FieldsFunc(raw, func(r rune) bool {
|
|
return r == ';' || r == '|' || r == ','
|
|
})
|
|
var out []string
|
|
for _, p := range parts {
|
|
p = strings.TrimSpace(p)
|
|
if p != "" {
|
|
out = append(out, p)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (s *Service) ListRoster(ctx context.Context, projectID string) ([]RosterEntry, error) {
|
|
rows, err := s.db.Query(ctx, `
|
|
SELECT id::text, project_id::text, email, display_name, alternate_emails, status,
|
|
COALESCE(invite_id::text, ''), created_at::text, updated_at::text
|
|
FROM migration_roster
|
|
WHERE project_id = $1::uuid
|
|
ORDER BY email
|
|
`, projectID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var out []RosterEntry
|
|
for rows.Next() {
|
|
var row RosterEntry
|
|
if err := rows.Scan(
|
|
&row.ID, &row.ProjectID, &row.Email, &row.DisplayName, &row.AlternateEmails,
|
|
&row.Status, &row.InviteID, &row.CreatedAt, &row.UpdatedAt,
|
|
); err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, row)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
func (s *Service) ImportRoster(ctx context.Context, projectID string, inputs []RosterRowInput) (RosterImportResult, error) {
|
|
result := RosterImportResult{}
|
|
for i, input := range inputs {
|
|
rowNum := i + 1
|
|
email := normalizeInviteEmail(input.Email)
|
|
if email == "" || !isEmailAddress(email) {
|
|
result.Errors = append(result.Errors, RosterImportRowError{
|
|
Row: rowNum, Email: input.Email, Message: "invalid email",
|
|
})
|
|
continue
|
|
}
|
|
alternates := normalizeAlternateEmails(email, input.AlternateEmails)
|
|
displayName := strings.TrimSpace(input.DisplayName)
|
|
|
|
existingStatus, err := s.rosterStatusByEmail(ctx, projectID, email)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
if existingStatus != "" {
|
|
result.SkippedDuplicates++
|
|
continue
|
|
}
|
|
|
|
tx, err := s.db.Begin(ctx)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
|
|
var rosterID string
|
|
err = tx.QueryRow(ctx, `
|
|
INSERT INTO migration_roster (project_id, email, display_name, alternate_emails, status)
|
|
VALUES ($1::uuid, $2, $3, $4, $5)
|
|
RETURNING id::text
|
|
`, projectID, email, displayName, alternates, RosterStatusPending).Scan(&rosterID)
|
|
if err != nil {
|
|
_ = tx.Rollback(ctx)
|
|
if isUniqueViolation(err) {
|
|
result.SkippedDuplicates++
|
|
continue
|
|
}
|
|
result.Errors = append(result.Errors, RosterImportRowError{
|
|
Row: rowNum, Email: email, Message: err.Error(),
|
|
})
|
|
continue
|
|
}
|
|
|
|
token, err := hosted.NewInviteToken()
|
|
if err != nil {
|
|
_ = tx.Rollback(ctx)
|
|
return result, err
|
|
}
|
|
|
|
var inviteID string
|
|
err = tx.QueryRow(ctx, `
|
|
INSERT INTO migration_invites (project_id, email, token, alternate_emails)
|
|
VALUES ($1::uuid, $2, $3, $4)
|
|
RETURNING id::text
|
|
`, projectID, email, token, alternates).Scan(&inviteID)
|
|
if err != nil {
|
|
_ = tx.Rollback(ctx)
|
|
if isUniqueViolation(err) {
|
|
result.SkippedDuplicates++
|
|
continue
|
|
}
|
|
result.Errors = append(result.Errors, RosterImportRowError{
|
|
Row: rowNum, Email: email, Message: err.Error(),
|
|
})
|
|
continue
|
|
}
|
|
|
|
_, err = tx.Exec(ctx, `
|
|
UPDATE migration_roster
|
|
SET status = $1, invite_id = $2::uuid, updated_at = NOW()
|
|
WHERE id = $3::uuid
|
|
`, RosterStatusInvited, inviteID, rosterID)
|
|
if err != nil {
|
|
_ = tx.Rollback(ctx)
|
|
result.Errors = append(result.Errors, RosterImportRowError{
|
|
Row: rowNum, Email: email, Message: err.Error(),
|
|
})
|
|
continue
|
|
}
|
|
|
|
if err := tx.Commit(ctx); err != nil {
|
|
result.Errors = append(result.Errors, RosterImportRowError{
|
|
Row: rowNum, Email: email, Message: err.Error(),
|
|
})
|
|
continue
|
|
}
|
|
result.Created++
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (s *Service) rosterStatusByEmail(ctx context.Context, projectID, email string) (string, error) {
|
|
var status string
|
|
err := s.db.QueryRow(ctx, `
|
|
SELECT status FROM migration_roster
|
|
WHERE project_id = $1::uuid AND email = $2
|
|
`, projectID, email).Scan(&status)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return "", nil
|
|
}
|
|
return status, err
|
|
}
|
|
|
|
func (s *Service) markRosterClaimed(ctx context.Context, tx pgx.Tx, projectID, inviteID, email string) error {
|
|
_, err := tx.Exec(ctx, `
|
|
UPDATE migration_roster
|
|
SET status = $1, updated_at = NOW()
|
|
WHERE project_id = $2::uuid
|
|
AND (invite_id = $3::uuid OR (invite_id IS NULL AND email = $4))
|
|
`, RosterStatusClaimed, projectID, inviteID, email)
|
|
return err
|
|
}
|
|
|
|
func isUniqueViolation(err error) bool {
|
|
var pgErr *pgconn.PgError
|
|
return errors.As(err, &pgErr) && pgErr.Code == "23505"
|
|
}
|