package migration import ( "context" "encoding/json" "fmt" "net/http" "net/url" "strings" "github.com/jackc/pgx/v5/pgxpool" "github.com/ultisuite/ulti-backend/internal/nextcloud" ) const migrationContactsBookID = "migration-import" type ContactsImporter struct { db *pgxpool.Pool nc *nextcloud.Client client *http.Client userUPN string } func NewContactsImporter(db *pgxpool.Pool, nc *nextcloud.Client) *ContactsImporter { return &ContactsImporter{db: db, nc: nc, client: migrationHTTPClient()} } func (c *ContactsImporter) WithUserPrincipal(upn string) *ContactsImporter { c.userUPN = strings.TrimSpace(upn) return c } func (c *ContactsImporter) WithHTTPClient(client *http.Client) *ContactsImporter { if client != nil { c.client = client } return c } func (c *ContactsImporter) ImportBatch(ctx context.Context, job *Job, accessToken, provider string, delta bool, update progressUpdater) error { if c.nc == nil { return fmt.Errorf("nextcloud required for contacts migration") } user, err := resolveMigrationUser(ctx, c.db, job.UserID) if err != nil { return err } ncUserID := nextcloud.UserIDFromClaims(user.Email, user.ExternalID) if _, err := c.nc.EnsurePrincipal(ctx, user.Email, user.ExternalID, user.Name); err != nil { return fmt.Errorf("nextcloud user: %w", err) } bookPath := nextcloud.AddressBookPath(ncUserID, migrationContactsBookID) items, err := LoadImportedItemStore(ctx, c.db, job.ID, job.CursorJSON) if err != nil { return err } if delta { if provider == "google" { if syncToken, _ := job.CursorJSON["syncToken"].(string); strings.TrimSpace(syncToken) != "" { return c.importGoogleDelta(ctx, job, accessToken, ncUserID, bookPath, items, update) } } else if deltaLink, _ := job.CursorJSON["deltaLink"].(string); strings.TrimSpace(deltaLink) != "" { return c.importMicrosoftDelta(ctx, job, accessToken, ncUserID, bookPath, deltaLink, items, update) } else { return c.bootstrapMicrosoftDelta(ctx, job, accessToken, ncUserID, bookPath, update) } } switch provider { case "google": return c.importGoogleFull(ctx, job, accessToken, ncUserID, bookPath, delta, items, update) default: return c.importMicrosoftFull(ctx, job, accessToken, ncUserID, bookPath, delta, items, update) } } func (c *ContactsImporter) importGoogleFull(ctx context.Context, job *Job, accessToken, ncUserID, bookPath string, captureToken bool, items *ImportedItemStore, update progressUpdater) error { imported, _ := job.StatsJSON["imported"].(float64) batch := 0 pageToken, _ := job.CursorJSON["pageToken"].(string) listURL := "https://people.googleapis.com/v1/people/me/connections?pageSize=100&personFields=names,emailAddresses,phoneNumbers,organizations,metadata" if captureToken { listURL += "&requestSyncToken=true" } if pageToken != "" { listURL += "&pageToken=" + url.QueryEscape(pageToken) } body, err := apiGet(ctx, c.client, listURL, accessToken) if err != nil { return err } var listed struct { Connections []googlePerson `json:"connections"` NextPageToken string `json:"nextPageToken"` NextSyncToken string `json:"nextSyncToken"` } if err := json.Unmarshal(body, &listed); err != nil { return err } listIndex := int(jsonNumber(job.CursorJSON["listIndex"])) for i := listIndex; i < len(listed.Connections) && batch < mailImportBatchSize(); i++ { person := listed.Connections[i] sourceID := strings.TrimSpace(person.ResourceName) if sourceID == "" { sourceID = fmt.Sprintf("google-person-%d", i) } if alreadyImported(items, sourceID) { continue } contact := googlePersonToContact(sourceID, person) if contact.Email == "" && contact.FullName == "" { continue } if _, err := c.nc.CreateContact(ctx, ncUserID, bookPath, contact); err != nil { if markErr := items.MarkFailed(ctx, sourceID, err.Error(), ""); markErr != nil { return markErr } incJobStat(job.StatsJSON, "failed") batch++ continue } if err := items.MarkImported(ctx, sourceID); err != nil { return err } imported++ batch++ } job.StatsJSON["imported"] = imported if listIndex+batch < len(listed.Connections) { job.CursorJSON["listIndex"] = float64(listIndex + batch) return update("pending", job.CursorJSON, job.StatsJSON, "") } delete(job.CursorJSON, "listIndex") if listed.NextPageToken != "" { job.CursorJSON["pageToken"] = listed.NextPageToken return update("pending", job.CursorJSON, job.StatsJSON, "") } delete(job.CursorJSON, "pageToken") if listed.NextSyncToken != "" { job.CursorJSON["syncToken"] = listed.NextSyncToken } job.StatsJSON["phase"] = "imported" return update("completed", job.CursorJSON, job.StatsJSON, "") } func (c *ContactsImporter) importGoogleDelta(ctx context.Context, job *Job, accessToken, ncUserID, bookPath string, items *ImportedItemStore, update progressUpdater) error { syncToken, _ := job.CursorJSON["syncToken"].(string) listURL := "https://people.googleapis.com/v1/people/me/connections?syncToken=" + url.QueryEscape(syncToken) + "&personFields=names,emailAddresses,phoneNumbers,organizations,metadata&requestSyncToken=true" body, err := apiGet(ctx, c.client, listURL, accessToken) if err != nil { return err } var listed struct { Connections []googlePerson `json:"connections"` NextSyncToken string `json:"nextSyncToken"` } if err := json.Unmarshal(body, &listed); err != nil { return err } deltaCount, _ := job.StatsJSON["delta_imported"].(float64) deltaUpdated, _ := job.StatsJSON["delta_updated"].(float64) deleted, _ := job.StatsJSON["delta_deleted"].(float64) for _, person := range listed.Connections { sourceID := strings.TrimSpace(person.ResourceName) if sourceID == "" { continue } if person.Metadata != nil && person.Metadata.Deleted { contactPath := migrationContactPath(bookPath, "google", sourceID) if err := c.nc.DeleteContact(ctx, ncUserID, contactPath); err != nil && !isDeleteNotFound(err) { return err } if err := items.Unmark(ctx, sourceID); err != nil { return err } deleted++ continue } contact := googlePersonToContact(sourceID, person) if contact.Email == "" && contact.FullName == "" { continue } updated, err := c.upsertContact(ctx, ncUserID, bookPath, "google", sourceID, contact, items) if err != nil { return err } if updated { deltaUpdated++ } else if items.Has(sourceID) { deltaCount++ } else { incJobStat(job.StatsJSON, "failed") } } if listed.NextSyncToken != "" { job.CursorJSON["syncToken"] = listed.NextSyncToken } job.StatsJSON["delta_imported"] = deltaCount job.StatsJSON["delta_updated"] = deltaUpdated job.StatsJSON["delta_deleted"] = deleted job.StatsJSON["phase"] = "delta" return update("completed", job.CursorJSON, job.StatsJSON, "") } func (c *ContactsImporter) importMicrosoftFull(ctx context.Context, job *Job, accessToken, ncUserID, bookPath string, captureDelta bool, items *ImportedItemStore, update progressUpdater) error { imported, _ := job.StatsJSON["imported"].(float64) batch := 0 nextLink, _ := job.CursorJSON["nextLink"].(string) listURL := graphMicrosoftURL(c.userUPN, "/contacts?$top=100&$select=id,displayName,givenName,surname,emailAddresses,mobilePhone,businessPhones,companyName") if nextLink != "" { listURL = nextLink } body, err := apiGet(ctx, c.client, listURL, accessToken) if err != nil { return err } var listed struct { Value []graphContact `json:"value"` NextLink string `json:"@odata.nextLink"` } if err := json.Unmarshal(body, &listed); err != nil { return err } listIndex := int(jsonNumber(job.CursorJSON["listIndex"])) for i := listIndex; i < len(listed.Value) && batch < mailImportBatchSize(); i++ { item := listed.Value[i] sourceID := strings.TrimSpace(item.ID) if sourceID == "" { sourceID = fmt.Sprintf("graph-contact-%d", i) } if alreadyImported(items, sourceID) { continue } contact := graphContactToNC(sourceID, item) if contact.Email == "" && contact.FullName == "" { continue } if _, err := c.nc.CreateContact(ctx, ncUserID, bookPath, contact); err != nil { if markErr := items.MarkFailed(ctx, sourceID, err.Error(), ""); markErr != nil { return markErr } incJobStat(job.StatsJSON, "failed") batch++ continue } if err := items.MarkImported(ctx, sourceID); err != nil { return err } imported++ batch++ } job.StatsJSON["imported"] = imported if listIndex+batch < len(listed.Value) { job.CursorJSON["listIndex"] = float64(listIndex + batch) return update("pending", job.CursorJSON, job.StatsJSON, "") } delete(job.CursorJSON, "listIndex") if listed.NextLink != "" { job.CursorJSON["nextLink"] = listed.NextLink return update("pending", job.CursorJSON, job.StatsJSON, "") } delete(job.CursorJSON, "nextLink") if captureDelta { return c.bootstrapMicrosoftDelta(ctx, job, accessToken, ncUserID, bookPath, update) } job.StatsJSON["phase"] = "imported" return update("completed", job.CursorJSON, job.StatsJSON, "") } func (c *ContactsImporter) bootstrapMicrosoftDelta(ctx context.Context, job *Job, accessToken, ncUserID, bookPath string, update progressUpdater) error { body, err := apiGet(ctx, c.client, graphMicrosoftURL(c.userUPN, "/contacts/delta?$select=id,displayName,givenName,surname,emailAddresses,mobilePhone,businessPhones,companyName"), accessToken) if err != nil { return err } var listed struct { Value []graphContact `json:"value"` NextLink string `json:"@odata.nextLink"` DeltaLink string `json:"@odata.deltaLink"` } if err := json.Unmarshal(body, &listed); err != nil { return err } if listed.DeltaLink != "" { job.CursorJSON["deltaLink"] = listed.DeltaLink job.StatsJSON["phase"] = "delta_ready" return update("completed", job.CursorJSON, job.StatsJSON, "") } if listed.NextLink != "" { job.CursorJSON["nextLink"] = listed.NextLink return update("pending", job.CursorJSON, job.StatsJSON, "") } job.StatsJSON["phase"] = "imported" return update("completed", job.CursorJSON, job.StatsJSON, "") } func (c *ContactsImporter) importMicrosoftDelta(ctx context.Context, job *Job, accessToken, ncUserID, bookPath, deltaLink string, items *ImportedItemStore, update progressUpdater) error { body, err := apiGet(ctx, c.client, deltaLink, accessToken) if err != nil { return err } var listed struct { Value []graphContact `json:"value"` NextLink string `json:"@odata.nextLink"` DeltaLink string `json:"@odata.deltaLink"` } if err := json.Unmarshal(body, &listed); err != nil { return err } deltaCount, _ := job.StatsJSON["delta_imported"].(float64) deltaUpdated, _ := job.StatsJSON["delta_updated"].(float64) deleted, _ := job.StatsJSON["delta_deleted"].(float64) for _, item := range listed.Value { sourceID := strings.TrimSpace(item.ID) if sourceID == "" { continue } if item.Removed != nil { contactPath := migrationContactPath(bookPath, "microsoft", sourceID) if err := c.nc.DeleteContact(ctx, ncUserID, contactPath); err != nil && !isDeleteNotFound(err) { return err } if err := items.Unmark(ctx, sourceID); err != nil { return err } deleted++ continue } contact := graphContactToNC(sourceID, item) if contact.Email == "" && contact.FullName == "" { continue } updated, err := c.upsertContact(ctx, ncUserID, bookPath, "microsoft", sourceID, contact, items) if err != nil { return err } if updated { deltaUpdated++ } else if items.Has(sourceID) { deltaCount++ } else { incJobStat(job.StatsJSON, "failed") } } if listed.NextLink != "" { job.CursorJSON["deltaLink"] = listed.NextLink return update("pending", job.CursorJSON, job.StatsJSON, "") } if listed.DeltaLink != "" { job.CursorJSON["deltaLink"] = listed.DeltaLink } job.StatsJSON["delta_imported"] = deltaCount job.StatsJSON["delta_updated"] = deltaUpdated job.StatsJSON["delta_deleted"] = deleted job.StatsJSON["phase"] = "delta" return update("completed", job.CursorJSON, job.StatsJSON, "") } type googlePerson struct { ResourceName string `json:"resourceName"` Metadata *struct { Deleted bool `json:"deleted"` } `json:"metadata"` Names []struct { DisplayName string `json:"displayName"` } `json:"names"` EmailAddresses []struct { Value string `json:"value"` } `json:"emailAddresses"` PhoneNumbers []struct { Value string `json:"value"` } `json:"phoneNumbers"` Organizations []struct { Name string `json:"name"` } `json:"organizations"` } type graphContact struct { ID string `json:"id"` Removed *struct { Reason string `json:"reason"` } `json:"@removed"` DisplayName string `json:"displayName"` GivenName string `json:"givenName"` Surname string `json:"surname"` MobilePhone string `json:"mobilePhone"` BusinessPhones []string `json:"businessPhones"` CompanyName string `json:"companyName"` EmailAddresses []struct { Address string `json:"address"` } `json:"emailAddresses"` } func googlePersonToContact(sourceID string, p googlePerson) *nextcloud.Contact { name := "" if len(p.Names) > 0 { name = strings.TrimSpace(p.Names[0].DisplayName) } email := "" if len(p.EmailAddresses) > 0 { email = strings.ToLower(strings.TrimSpace(p.EmailAddresses[0].Value)) } phone := "" if len(p.PhoneNumbers) > 0 { phone = strings.TrimSpace(p.PhoneNumbers[0].Value) } org := "" if len(p.Organizations) > 0 { org = strings.TrimSpace(p.Organizations[0].Name) } return &nextcloud.Contact{ UID: sanitizeMigrationUID("google", sourceID), FullName: name, Email: email, Phone: phone, Org: org, } } func graphContactToNC(sourceID string, c graphContact) *nextcloud.Contact { name := strings.TrimSpace(c.DisplayName) if name == "" { name = strings.TrimSpace(strings.TrimSpace(c.GivenName + " " + c.Surname)) } email := "" if len(c.EmailAddresses) > 0 { email = strings.ToLower(strings.TrimSpace(c.EmailAddresses[0].Address)) } phone := strings.TrimSpace(c.MobilePhone) if phone == "" && len(c.BusinessPhones) > 0 { phone = strings.TrimSpace(c.BusinessPhones[0]) } return &nextcloud.Contact{ UID: sanitizeMigrationUID("microsoft", sourceID), FullName: name, Email: email, Phone: phone, Org: strings.TrimSpace(c.CompanyName), } } func (c *ContactsImporter) upsertContact( ctx context.Context, ncUserID, bookPath, provider, sourceID string, contact *nextcloud.Contact, items *ImportedItemStore, ) (updated bool, err error) { if alreadyImported(items, sourceID) { contactPath := migrationContactPath(bookPath, provider, sourceID) if _, err := c.nc.UpdateContact(ctx, ncUserID, contactPath, "", contact); err != nil { return false, err } return true, nil } if _, err := c.nc.CreateContact(ctx, ncUserID, bookPath, contact); err != nil { if markErr := items.MarkFailed(ctx, sourceID, err.Error(), ""); markErr != nil { return false, markErr } return false, nil } if err := items.MarkImported(ctx, sourceID); err != nil { return false, err } return false, nil }