feat(migration): graph childFolders, parent FK, B2B hardening
- Graph mail: discover nested childFolders, merge new folders into cached graphFolderQueue without breaking in-progress cursors - Add mail_folders.parent_id (migration 000050) and wire hierarchy on import - Shared drives: skip discovery on delta ticks, guard merge by project - Provision: remove platform-domain email rewrite on claim - Integration tests for nested folders, parent_id, delta childFolders mocks
This commit is contained in:
parent
1ffd0817d8
commit
951c88b1ca
@ -521,6 +521,10 @@ func TestGraphFolderDeltaDeletesRemoved(t *testing.T) {
|
|||||||
inboxID := "inbox-folder-id"
|
inboxID := "inbox-folder-id"
|
||||||
sentID := "sent-folder-id"
|
sentID := "sent-folder-id"
|
||||||
client := graphRewriteClient(t, func(w http.ResponseWriter, r *http.Request) {
|
client := graphRewriteClient(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if strings.Contains(r.URL.Path, "/childFolders") {
|
||||||
|
_, _ = w.Write([]byte(`{"value":[]}`))
|
||||||
|
return
|
||||||
|
}
|
||||||
if strings.HasSuffix(r.URL.Path, "/mailFolders") {
|
if strings.HasSuffix(r.URL.Path, "/mailFolders") {
|
||||||
_, _ = w.Write([]byte(`{"value":[
|
_, _ = w.Write([]byte(`{"value":[
|
||||||
{"id":"` + inboxID + `","displayName":"Inbox","wellKnownName":"inbox"},
|
{"id":"` + inboxID + `","displayName":"Inbox","wellKnownName":"inbox"},
|
||||||
|
|||||||
@ -284,6 +284,8 @@ func TestGraphImportWritesMessages(t *testing.T) {
|
|||||||
messagesListed := false
|
messagesListed := false
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
switch {
|
switch {
|
||||||
|
case strings.Contains(r.URL.Path, "/childFolders"):
|
||||||
|
_, _ = w.Write([]byte(`{"value":[]}`))
|
||||||
case strings.HasSuffix(r.URL.Path, "/mailFolders"):
|
case strings.HasSuffix(r.URL.Path, "/mailFolders"):
|
||||||
_, _ = w.Write([]byte(`{"value":[
|
_, _ = w.Write([]byte(`{"value":[
|
||||||
{"id":"` + folderID + `","displayName":"Inbox","wellKnownName":"inbox"},
|
{"id":"` + folderID + `","displayName":"Inbox","wellKnownName":"inbox"},
|
||||||
@ -349,6 +351,122 @@ func TestGraphImportWritesMessages(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGraphImportNestedFolderMessages(t *testing.T) {
|
||||||
|
h := integrationtest.RequireHarness(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
userID, err := users.EnsureUser(ctx, h.Pool, integrationtest.RegularUser(integrationtest.NewExternalID("graph-nested-import")))
|
||||||
|
integrationtest.FailIf(err, t, "ensure user")
|
||||||
|
|
||||||
|
var accountID string
|
||||||
|
err = h.Pool.QueryRow(ctx, `
|
||||||
|
INSERT INTO mail_accounts (user_id, email, provider, is_active)
|
||||||
|
VALUES ($1::uuid, 'graph-nested@test.local', 'hosted', true)
|
||||||
|
RETURNING id::text
|
||||||
|
`, userID).Scan(&accountID)
|
||||||
|
integrationtest.FailIf(err, t, "insert mail account")
|
||||||
|
|
||||||
|
inboxID := "inbox-folder-id"
|
||||||
|
projectsID := "projects-folder-id"
|
||||||
|
nestedListed := false
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/mailFolders"):
|
||||||
|
_, _ = w.Write([]byte(`{"value":[
|
||||||
|
{"id":"` + inboxID + `","displayName":"Inbox","wellKnownName":"inbox"}
|
||||||
|
]}`))
|
||||||
|
case strings.Contains(r.URL.Path, "/mailFolders/"+inboxID+"/childFolders"):
|
||||||
|
_, _ = w.Write([]byte(`{"value":[
|
||||||
|
{"id":"` + projectsID + `","displayName":"Projects","wellKnownName":""}
|
||||||
|
]}`))
|
||||||
|
case strings.Contains(r.URL.Path, "/mailFolders/"+projectsID+"/childFolders"):
|
||||||
|
_, _ = w.Write([]byte(`{"value":[]}`))
|
||||||
|
case strings.Contains(r.URL.Path, "/mailFolders/"+inboxID+"/messages"):
|
||||||
|
_, _ = w.Write([]byte(`{"value":[]}`))
|
||||||
|
case strings.Contains(r.URL.Path, "/mailFolders/"+projectsID+"/messages"):
|
||||||
|
nestedListed = true
|
||||||
|
_, _ = w.Write([]byte(`{"value":[{
|
||||||
|
"id":"nested-msg-1",
|
||||||
|
"subject":"Nested Graph",
|
||||||
|
"bodyPreview":"Nested preview",
|
||||||
|
"body":{"contentType":"text","content":"Nested body"},
|
||||||
|
"from":{"emailAddress":{"name":"Alice","address":"alice@example.com"}},
|
||||||
|
"toRecipients":[{"emailAddress":{"name":"Bob","address":"bob@example.com"}}],
|
||||||
|
"receivedDateTime":"2024-05-01T10:00:00Z",
|
||||||
|
"parentFolderId":"` + projectsID + `",
|
||||||
|
"isRead":true,
|
||||||
|
"internetMessageId":"<graph-nested@example.com>"
|
||||||
|
}]}`))
|
||||||
|
default:
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
importer := migr.NewGraphImporter(h.Pool).WithBaseURL(srv.URL)
|
||||||
|
job := &migr.Job{
|
||||||
|
UserID: userID,
|
||||||
|
CursorJSON: map[string]any{},
|
||||||
|
StatsJSON: map[string]any{},
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
var finalStatus string
|
||||||
|
err = importer.ImportBatch(ctx, job, "test-token", false, func(status string, cursor, stats map[string]any, jobErr string) error {
|
||||||
|
if jobErr != "" {
|
||||||
|
t.Fatalf("import error: %s", jobErr)
|
||||||
|
}
|
||||||
|
finalStatus = status
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
integrationtest.FailIf(err, t, "import batch")
|
||||||
|
if finalStatus == "completed" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if finalStatus != "pending" {
|
||||||
|
t.Fatalf("status = %q, want pending or completed", finalStatus)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !nestedListed {
|
||||||
|
t.Fatal("nested folder messages endpoint not called")
|
||||||
|
}
|
||||||
|
|
||||||
|
var msgCount int
|
||||||
|
if err := h.Pool.QueryRow(ctx, `
|
||||||
|
SELECT COUNT(*) FROM messages WHERE account_id = $1::uuid AND subject = 'Nested Graph'
|
||||||
|
`, accountID).Scan(&msgCount); err != nil {
|
||||||
|
t.Fatalf("count messages: %v", err)
|
||||||
|
}
|
||||||
|
if msgCount != 1 {
|
||||||
|
t.Fatalf("message count = %d, want 1", msgCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
var remoteName string
|
||||||
|
if err := h.Pool.QueryRow(ctx, `
|
||||||
|
SELECT f.remote_name FROM messages m
|
||||||
|
JOIN mail_folders f ON f.id = m.folder_id
|
||||||
|
WHERE m.account_id = $1::uuid AND m.subject = 'Nested Graph'
|
||||||
|
`, accountID).Scan(&remoteName); err != nil {
|
||||||
|
t.Fatalf("folder remote_name: %v", err)
|
||||||
|
}
|
||||||
|
if remoteName != "INBOX/PROJECTS" {
|
||||||
|
t.Fatalf("remote_name = %q, want INBOX/PROJECTS", remoteName)
|
||||||
|
}
|
||||||
|
|
||||||
|
var parentID *string
|
||||||
|
var inboxFolderID string
|
||||||
|
if err := h.Pool.QueryRow(ctx, `
|
||||||
|
SELECT f.parent_id, inbox.id::text
|
||||||
|
FROM mail_folders f
|
||||||
|
JOIN mail_folders inbox ON inbox.account_id = f.account_id AND inbox.remote_name = 'INBOX'
|
||||||
|
WHERE f.account_id = $1::uuid AND f.remote_name = 'INBOX/PROJECTS'
|
||||||
|
`, accountID).Scan(&parentID, &inboxFolderID); err != nil {
|
||||||
|
t.Fatalf("folder parent_id: %v", err)
|
||||||
|
}
|
||||||
|
if parentID == nil || *parentID != inboxFolderID {
|
||||||
|
t.Fatalf("parent_id = %v, want %s", parentID, inboxFolderID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestGmailImportStoresAttachments(t *testing.T) {
|
func TestGmailImportStoresAttachments(t *testing.T) {
|
||||||
h := integrationtest.RequireHarness(t)
|
h := integrationtest.RequireHarness(t)
|
||||||
if h.AttachmentStorage == nil {
|
if h.AttachmentStorage == nil {
|
||||||
|
|||||||
@ -88,12 +88,15 @@ func (d *DriveImporter) ImportBatch(ctx context.Context, job *Job, accessToken,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if provider == "google" && !jsonBool(job.CursorJSON["sharedDrivesBootstrapped"]) {
|
if provider == "google" && !jsonBool(job.CursorJSON["sharedDrivesBootstrapped"]) {
|
||||||
if err := d.bootstrapSharedDrives(ctx, job, accessToken); err != nil {
|
// Delta-only ticks already have a change token; skip shared-drive discovery API calls.
|
||||||
return err
|
if !(delta && d.hasDriveDeltaCursor(job, provider)) {
|
||||||
|
if err := d.bootstrapSharedDrives(ctx, job, accessToken); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
job.CursorJSON["sharedDrivesBootstrapped"] = true
|
job.CursorJSON["sharedDrivesBootstrapped"] = true
|
||||||
}
|
}
|
||||||
if provider == "google" {
|
if provider == "google" && strings.TrimSpace(job.ProjectID) != "" {
|
||||||
if err := d.mergeSharedDriveFolders(ctx, job, provider); err != nil {
|
if err := d.mergeSharedDriveFolders(ctx, job, provider); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -207,7 +207,7 @@ func (g *GmailImporter) importOne(ctx context.Context, accessToken, userID, acco
|
|||||||
}
|
}
|
||||||
|
|
||||||
remoteName, folderType := primaryGmailFolder(msg.LabelIDs)
|
remoteName, folderType := primaryGmailFolder(msg.LabelIDs)
|
||||||
folderID, err := ensureMailFolder(ctx, g.db, accountID, displayFolderName(remoteName, folderType), remoteName, folderType)
|
folderID, err := ensureMailFolder(ctx, g.db, accountID, displayFolderName(remoteName, folderType), remoteName, folderType, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
@ -474,22 +474,25 @@ func ensureDefaultMailFolders(ctx context.Context, db *pgxpool.Pool, accountID s
|
|||||||
{"Archives", "ARCHIVE", "archive"},
|
{"Archives", "ARCHIVE", "archive"},
|
||||||
}
|
}
|
||||||
for _, d := range defaults {
|
for _, d := range defaults {
|
||||||
if _, err := ensureMailFolder(ctx, db, accountID, d.name, d.remote, d.ftype); err != nil {
|
if _, err := ensureMailFolder(ctx, db, accountID, d.name, d.remote, d.ftype, nil); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func ensureMailFolder(ctx context.Context, db *pgxpool.Pool, accountID, name, remoteName, folderType string) (string, error) {
|
func ensureMailFolder(ctx context.Context, db *pgxpool.Pool, accountID, name, remoteName, folderType string, parentID *string) (string, error) {
|
||||||
var folderID string
|
var folderID string
|
||||||
err := db.QueryRow(ctx, `
|
err := db.QueryRow(ctx, `
|
||||||
INSERT INTO mail_folders (account_id, name, remote_name, folder_type)
|
INSERT INTO mail_folders (account_id, name, remote_name, folder_type, parent_id)
|
||||||
VALUES ($1, $2, $3, $4)
|
VALUES ($1, $2, $3, $4, $5)
|
||||||
ON CONFLICT (account_id, remote_name) DO UPDATE
|
ON CONFLICT (account_id, remote_name) DO UPDATE
|
||||||
SET name = EXCLUDED.name, folder_type = EXCLUDED.folder_type, updated_at = NOW()
|
SET name = EXCLUDED.name,
|
||||||
|
folder_type = EXCLUDED.folder_type,
|
||||||
|
parent_id = EXCLUDED.parent_id,
|
||||||
|
updated_at = NOW()
|
||||||
RETURNING id::text
|
RETURNING id::text
|
||||||
`, accountID, name, remoteName, folderType).Scan(&folderID)
|
`, accountID, name, remoteName, folderType, parentID).Scan(&folderID)
|
||||||
return folderID, err
|
return folderID, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -28,8 +28,9 @@ type GraphImporter struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type graphFolderMeta struct {
|
type graphFolderMeta struct {
|
||||||
RemoteName string
|
RemoteName string
|
||||||
FolderType string
|
FolderType string
|
||||||
|
ParentGraphID string
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewGraphImporter(db *pgxpool.Pool) *GraphImporter {
|
func NewGraphImporter(db *pgxpool.Pool) *GraphImporter {
|
||||||
@ -129,6 +130,9 @@ func (g *GraphImporter) ImportBatch(
|
|||||||
if err := g.ensureGraphFolders(ctx, accessToken); err != nil {
|
if err := g.ensureGraphFolders(ctx, accessToken); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if err := g.ensureGraphFolderRecords(ctx, accountID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
items, err := LoadImportedItemStore(ctx, g.db, job.ID, job.CursorJSON)
|
items, err := LoadImportedItemStore(ctx, g.db, job.ID, job.CursorJSON)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@ -443,16 +447,12 @@ func (g *GraphImporter) bootstrapFolderDeltaLinks(ctx context.Context, accessTok
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (g *GraphImporter) folderQueue(cursor map[string]any) []string {
|
func (g *GraphImporter) folderQueue(cursor map[string]any) []string {
|
||||||
if queue := readGraphFolderQueue(cursor); len(queue) > 0 {
|
|
||||||
return queue
|
|
||||||
}
|
|
||||||
ids := make([]string, 0, len(g.folders))
|
ids := make([]string, 0, len(g.folders))
|
||||||
for id := range g.folders {
|
for id := range g.folders {
|
||||||
ids = append(ids, id)
|
ids = append(ids, id)
|
||||||
}
|
}
|
||||||
sort.Strings(ids)
|
sort.Strings(ids)
|
||||||
writeGraphFolderQueue(cursor, ids)
|
return mergeGraphFolderQueue(cursor, ids)
|
||||||
return ids
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (g *GraphImporter) importOne(ctx context.Context, accountID string, msg graphMessage) (bool, error) {
|
func (g *GraphImporter) importOne(ctx context.Context, accountID string, msg graphMessage) (bool, error) {
|
||||||
@ -460,7 +460,11 @@ func (g *GraphImporter) importOne(ctx context.Context, accountID string, msg gra
|
|||||||
if meta.RemoteName == "" {
|
if meta.RemoteName == "" {
|
||||||
meta = graphFolderMeta{RemoteName: "ARCHIVE", FolderType: "archive"}
|
meta = graphFolderMeta{RemoteName: "ARCHIVE", FolderType: "archive"}
|
||||||
}
|
}
|
||||||
folderID, err := ensureMailFolder(ctx, g.db, accountID, displayFolderName(meta.RemoteName, meta.FolderType), meta.RemoteName, meta.FolderType)
|
parentID, err := g.parentFolderDBID(ctx, accountID, meta.ParentGraphID)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
folderID, err := ensureMailFolder(ctx, g.db, accountID, displayFolderName(meta.RemoteName, meta.FolderType), meta.RemoteName, meta.FolderType, parentID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
@ -548,36 +552,180 @@ func (g *GraphImporter) deleteByGraphID(ctx context.Context, accountID, graphID
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type graphFolderEntry struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
DisplayName string `json:"displayName"`
|
||||||
|
WellKnownName string `json:"wellKnownName"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type graphDiscoverEntry struct {
|
||||||
|
id string
|
||||||
|
parentRemote string
|
||||||
|
parentGraphID string
|
||||||
|
}
|
||||||
|
|
||||||
func (g *GraphImporter) ensureGraphFolders(ctx context.Context, accessToken string) error {
|
func (g *GraphImporter) ensureGraphFolders(ctx context.Context, accessToken string) error {
|
||||||
if len(g.folders) > 0 {
|
if len(g.folders) > 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
visited := map[string]struct{}{}
|
||||||
|
discover := make([]graphDiscoverEntry, 0, 16)
|
||||||
|
|
||||||
listURL := g.graphURL(g.userBase() + "/mailFolders?$top=100&$select=id,displayName,wellKnownName")
|
listURL := g.graphURL(g.userBase() + "/mailFolders?$top=100&$select=id,displayName,wellKnownName")
|
||||||
for listURL != "" {
|
for listURL != "" {
|
||||||
body, err := g.apiGet(ctx, listURL, accessToken)
|
entries, nextLink, err := g.listGraphMailFoldersPage(ctx, accessToken, listURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
var parsed struct {
|
for _, f := range entries {
|
||||||
Value []struct {
|
if _, ok := visited[f.ID]; ok {
|
||||||
ID string `json:"id"`
|
continue
|
||||||
DisplayName string `json:"displayName"`
|
}
|
||||||
WellKnownName string `json:"wellKnownName"`
|
visited[f.ID] = struct{}{}
|
||||||
} `json:"value"`
|
|
||||||
NextLink string `json:"@odata.nextLink"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, f := range parsed.Value {
|
|
||||||
remote, ftype := graphWellKnownFolder(f.WellKnownName, f.DisplayName)
|
remote, ftype := graphWellKnownFolder(f.WellKnownName, f.DisplayName)
|
||||||
g.folders[f.ID] = graphFolderMeta{RemoteName: remote, FolderType: ftype}
|
g.folders[f.ID] = graphFolderMeta{RemoteName: remote, FolderType: ftype}
|
||||||
|
discover = append(discover, graphDiscoverEntry{id: f.ID, parentRemote: remote})
|
||||||
}
|
}
|
||||||
listURL = parsed.NextLink
|
listURL = nextLink
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 0; i < len(discover); i++ {
|
||||||
|
entry := discover[i]
|
||||||
|
childDiscover, err := g.discoverGraphChildFolders(ctx, accessToken, entry.id, entry.parentRemote, visited)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
discover = append(discover, childDiscover...)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (g *GraphImporter) listGraphMailFoldersPage(ctx context.Context, accessToken, listURL string) ([]graphFolderEntry, string, error) {
|
||||||
|
body, err := g.apiGet(ctx, listURL, accessToken)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
var parsed struct {
|
||||||
|
Value []graphFolderEntry `json:"value"`
|
||||||
|
NextLink string `json:"@odata.nextLink"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
return parsed.Value, parsed.NextLink, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GraphImporter) discoverGraphChildFolders(
|
||||||
|
ctx context.Context,
|
||||||
|
accessToken, parentID, parentRemote string,
|
||||||
|
visited map[string]struct{},
|
||||||
|
) ([]graphDiscoverEntry, error) {
|
||||||
|
out := make([]graphDiscoverEntry, 0, 8)
|
||||||
|
listURL := g.graphURL(g.userBase() + "/mailFolders/" + url.PathEscape(parentID) +
|
||||||
|
"/childFolders?$top=100&$select=id,displayName,wellKnownName")
|
||||||
|
for listURL != "" {
|
||||||
|
entries, nextLink, err := g.listGraphMailFoldersPage(ctx, accessToken, listURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, f := range entries {
|
||||||
|
if _, ok := visited[f.ID]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
visited[f.ID] = struct{}{}
|
||||||
|
remote, ftype := graphNestedFolderMeta(parentRemote, f.WellKnownName, f.DisplayName)
|
||||||
|
g.folders[f.ID] = graphFolderMeta{RemoteName: remote, FolderType: ftype, ParentGraphID: parentID}
|
||||||
|
out = append(out, graphDiscoverEntry{id: f.ID, parentRemote: remote, parentGraphID: parentID})
|
||||||
|
}
|
||||||
|
listURL = nextLink
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GraphImporter) ensureGraphFolderRecords(ctx context.Context, accountID string) error {
|
||||||
|
type folderEntry struct {
|
||||||
|
graphID string
|
||||||
|
meta graphFolderMeta
|
||||||
|
depth int
|
||||||
|
}
|
||||||
|
entries := make([]folderEntry, 0, len(g.folders))
|
||||||
|
for graphID, meta := range g.folders {
|
||||||
|
depth := 0
|
||||||
|
if meta.RemoteName != "" {
|
||||||
|
depth = strings.Count(meta.RemoteName, "/")
|
||||||
|
}
|
||||||
|
entries = append(entries, folderEntry{graphID: graphID, meta: meta, depth: depth})
|
||||||
|
}
|
||||||
|
sort.Slice(entries, func(i, j int) bool {
|
||||||
|
if entries[i].depth != entries[j].depth {
|
||||||
|
return entries[i].depth < entries[j].depth
|
||||||
|
}
|
||||||
|
return entries[i].meta.RemoteName < entries[j].meta.RemoteName
|
||||||
|
})
|
||||||
|
|
||||||
|
graphToDB := make(map[string]string, len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
var parentDB *string
|
||||||
|
if entry.meta.ParentGraphID != "" {
|
||||||
|
if pid, ok := graphToDB[entry.meta.ParentGraphID]; ok {
|
||||||
|
parentDB = &pid
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dbID, err := ensureMailFolder(
|
||||||
|
ctx, g.db, accountID,
|
||||||
|
displayFolderName(entry.meta.RemoteName, entry.meta.FolderType),
|
||||||
|
entry.meta.RemoteName, entry.meta.FolderType,
|
||||||
|
parentDB,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
graphToDB[entry.graphID] = dbID
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *GraphImporter) parentFolderDBID(ctx context.Context, accountID, parentGraphID string) (*string, error) {
|
||||||
|
if parentGraphID == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
meta, ok := g.folders[parentGraphID]
|
||||||
|
if !ok || meta.RemoteName == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var id string
|
||||||
|
err := g.db.QueryRow(ctx, `
|
||||||
|
SELECT id::text FROM mail_folders WHERE account_id = $1::uuid AND remote_name = $2
|
||||||
|
`, accountID, meta.RemoteName).Scan(&id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &id, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func graphNestedFolderMeta(parentRemote, wellKnown, displayName string) (remoteName, folderType string) {
|
||||||
|
if strings.TrimSpace(wellKnown) != "" {
|
||||||
|
remote, ftype := graphWellKnownFolder(wellKnown, displayName)
|
||||||
|
if parentRemote != "" {
|
||||||
|
return parentRemote + "/" + remote, ftype
|
||||||
|
}
|
||||||
|
return remote, ftype
|
||||||
|
}
|
||||||
|
segment := graphCustomFolderSegment(displayName)
|
||||||
|
if parentRemote == "" {
|
||||||
|
return segment, "custom"
|
||||||
|
}
|
||||||
|
return parentRemote + "/" + segment, "custom"
|
||||||
|
}
|
||||||
|
|
||||||
|
func graphCustomFolderSegment(displayName string) string {
|
||||||
|
name := strings.TrimSpace(displayName)
|
||||||
|
if name == "" {
|
||||||
|
return "CUSTOM"
|
||||||
|
}
|
||||||
|
return strings.ToUpper(strings.ReplaceAll(name, " ", "_"))
|
||||||
|
}
|
||||||
|
|
||||||
func graphWellKnownFolder(wellKnown, displayName string) (remoteName, folderType string) {
|
func graphWellKnownFolder(wellKnown, displayName string) (remoteName, folderType string) {
|
||||||
switch strings.ToLower(strings.TrimSpace(wellKnown)) {
|
switch strings.ToLower(strings.TrimSpace(wellKnown)) {
|
||||||
case "inbox":
|
case "inbox":
|
||||||
@ -593,11 +741,7 @@ func graphWellKnownFolder(wellKnown, displayName string) (remoteName, folderType
|
|||||||
case "archive":
|
case "archive":
|
||||||
return "ARCHIVE", "archive"
|
return "ARCHIVE", "archive"
|
||||||
default:
|
default:
|
||||||
name := strings.TrimSpace(displayName)
|
return graphCustomFolderSegment(displayName), "custom"
|
||||||
if name == "" {
|
|
||||||
name = "CUSTOM"
|
|
||||||
}
|
|
||||||
return strings.ToUpper(strings.ReplaceAll(name, " ", "_")), "custom"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -87,6 +87,10 @@ func TestGraphFolderMessagesURLUsesMailFoldersPath(t *testing.T) {
|
|||||||
func TestGraphEnsureFoldersPaginates(t *testing.T) {
|
func TestGraphEnsureFoldersPaginates(t *testing.T) {
|
||||||
pages := 0
|
pages := 0
|
||||||
client := mockGraphHTTPClient(t, func(w http.ResponseWriter, r *http.Request) {
|
client := mockGraphHTTPClient(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if strings.Contains(r.URL.Path, "/childFolders") {
|
||||||
|
_, _ = w.Write([]byte(`{"value":[]}`))
|
||||||
|
return
|
||||||
|
}
|
||||||
if !strings.HasSuffix(r.URL.Path, "/mailFolders") {
|
if !strings.HasSuffix(r.URL.Path, "/mailFolders") {
|
||||||
http.NotFound(w, r)
|
http.NotFound(w, r)
|
||||||
return
|
return
|
||||||
@ -114,6 +118,163 @@ func TestGraphEnsureFoldersPaginates(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGraphEnsureFoldersDiscoversNestedChildFolders(t *testing.T) {
|
||||||
|
childCalls := 0
|
||||||
|
client := mockGraphHTTPClient(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/mailFolders"):
|
||||||
|
_, _ = w.Write([]byte(`{"value":[
|
||||||
|
{"id":"inbox-id","displayName":"Inbox","wellKnownName":"inbox"}
|
||||||
|
]}`))
|
||||||
|
case strings.Contains(r.URL.Path, "/mailFolders/inbox-id/childFolders"):
|
||||||
|
childCalls++
|
||||||
|
_, _ = w.Write([]byte(`{"value":[
|
||||||
|
{"id":"projects-id","displayName":"Projects","wellKnownName":""}
|
||||||
|
]}`))
|
||||||
|
case strings.Contains(r.URL.Path, "/mailFolders/projects-id/childFolders"):
|
||||||
|
childCalls++
|
||||||
|
_, _ = w.Write([]byte(`{"value":[
|
||||||
|
{"id":"nested-id","displayName":"2024","wellKnownName":""}
|
||||||
|
]}`))
|
||||||
|
case strings.Contains(r.URL.Path, "/mailFolders/nested-id/childFolders"):
|
||||||
|
childCalls++
|
||||||
|
_, _ = w.Write([]byte(`{"value":[]}`))
|
||||||
|
default:
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
g := NewGraphImporter(nil).WithHTTPClient(client)
|
||||||
|
if err := g.ensureGraphFolders(context.Background(), "token"); err != nil {
|
||||||
|
t.Fatalf("ensure folders: %v", err)
|
||||||
|
}
|
||||||
|
if childCalls != 3 {
|
||||||
|
t.Fatalf("childFolders calls = %d, want 3", childCalls)
|
||||||
|
}
|
||||||
|
if len(g.folders) != 3 {
|
||||||
|
t.Fatalf("folders = %d, want 3", len(g.folders))
|
||||||
|
}
|
||||||
|
projects := g.folders["projects-id"]
|
||||||
|
if projects.RemoteName != "INBOX/PROJECTS" || projects.FolderType != "custom" {
|
||||||
|
t.Fatalf("projects meta = %+v", projects)
|
||||||
|
}
|
||||||
|
nested := g.folders["nested-id"]
|
||||||
|
if nested.RemoteName != "INBOX/PROJECTS/2024" || nested.FolderType != "custom" {
|
||||||
|
t.Fatalf("nested meta = %+v", nested)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGraphEnsureFoldersChildFoldersPaginate(t *testing.T) {
|
||||||
|
pages := 0
|
||||||
|
client := mockGraphHTTPClient(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case strings.HasSuffix(r.URL.Path, "/mailFolders"):
|
||||||
|
_, _ = w.Write([]byte(`{"value":[
|
||||||
|
{"id":"inbox-id","displayName":"Inbox","wellKnownName":"inbox"}
|
||||||
|
]}`))
|
||||||
|
case strings.Contains(r.URL.Path, "/mailFolders/inbox-id/childFolders"):
|
||||||
|
pages++
|
||||||
|
if pages == 1 {
|
||||||
|
_, _ = w.Write([]byte(`{
|
||||||
|
"value":[{"id":"child-a","displayName":"Alpha","wellKnownName":""}],
|
||||||
|
"@odata.nextLink":"https://graph.microsoft.com/v1.0/me/mailFolders/inbox-id/childFolders?$skip=100"
|
||||||
|
}`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"value":[{"id":"child-b","displayName":"Beta","wellKnownName":""}]}`))
|
||||||
|
case strings.Contains(r.URL.Path, "/childFolders"):
|
||||||
|
_, _ = w.Write([]byte(`{"value":[]}`))
|
||||||
|
default:
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
g := NewGraphImporter(nil).WithHTTPClient(client)
|
||||||
|
if err := g.ensureGraphFolders(context.Background(), "token"); err != nil {
|
||||||
|
t.Fatalf("ensure folders: %v", err)
|
||||||
|
}
|
||||||
|
if pages != 2 {
|
||||||
|
t.Fatalf("child pages = %d, want 2", pages)
|
||||||
|
}
|
||||||
|
if g.folders["child-a"].RemoteName != "INBOX/ALPHA" {
|
||||||
|
t.Fatalf("child-a remote = %q", g.folders["child-a"].RemoteName)
|
||||||
|
}
|
||||||
|
if g.folders["child-b"].RemoteName != "INBOX/BETA" {
|
||||||
|
t.Fatalf("child-b remote = %q", g.folders["child-b"].RemoteName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGraphNestedFolderMeta(t *testing.T) {
|
||||||
|
remote, ftype := graphNestedFolderMeta("INBOX", "", "My Folder")
|
||||||
|
if remote != "INBOX/MY_FOLDER" || ftype != "custom" {
|
||||||
|
t.Fatalf("got %q / %q", remote, ftype)
|
||||||
|
}
|
||||||
|
remote, ftype = graphNestedFolderMeta("", "", "Top")
|
||||||
|
if remote != "TOP" || ftype != "custom" {
|
||||||
|
t.Fatalf("top-level custom: got %q / %q", remote, ftype)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGraphFolderQueueIncludesNestedFolders(t *testing.T) {
|
||||||
|
g := NewGraphImporter(nil)
|
||||||
|
g.folders = map[string]graphFolderMeta{
|
||||||
|
"inbox-id": {RemoteName: "INBOX", FolderType: "inbox"},
|
||||||
|
"projects-id": {RemoteName: "INBOX/PROJECTS", FolderType: "custom"},
|
||||||
|
"archive-id": {RemoteName: "ARCHIVE", FolderType: "archive"},
|
||||||
|
}
|
||||||
|
cursor := map[string]any{}
|
||||||
|
queue := g.folderQueue(cursor)
|
||||||
|
if len(queue) != 3 {
|
||||||
|
t.Fatalf("queue len = %d", len(queue))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGraphFolderQueueMergesLegacyCursor(t *testing.T) {
|
||||||
|
g := NewGraphImporter(nil)
|
||||||
|
g.folders = map[string]graphFolderMeta{
|
||||||
|
"inbox-id": {RemoteName: "INBOX", FolderType: "inbox"},
|
||||||
|
"sent-id": {RemoteName: "SENT", FolderType: "sent"},
|
||||||
|
"projects-id": {RemoteName: "INBOX/PROJECTS", FolderType: "custom", ParentGraphID: "inbox-id"},
|
||||||
|
}
|
||||||
|
cursor := map[string]any{
|
||||||
|
"graphFolderQueue": []any{"inbox-id", "sent-id"},
|
||||||
|
"folderIndex": float64(2),
|
||||||
|
}
|
||||||
|
queue := g.folderQueue(cursor)
|
||||||
|
if len(queue) != 3 {
|
||||||
|
t.Fatalf("queue len = %d, want 3", len(queue))
|
||||||
|
}
|
||||||
|
if queue[0] != "inbox-id" || queue[1] != "sent-id" || queue[2] != "projects-id" {
|
||||||
|
t.Fatalf("queue order = %v", queue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeGraphFolderQueuePreservesOrder(t *testing.T) {
|
||||||
|
cursor := map[string]any{
|
||||||
|
"graphFolderQueue": []any{"b-folder", "a-folder"},
|
||||||
|
}
|
||||||
|
merged := mergeGraphFolderQueue(cursor, []string{"a-folder", "b-folder", "c-folder"})
|
||||||
|
if len(merged) != 3 {
|
||||||
|
t.Fatalf("merged len = %d", len(merged))
|
||||||
|
}
|
||||||
|
if merged[0] != "b-folder" || merged[1] != "a-folder" || merged[2] != "c-folder" {
|
||||||
|
t.Fatalf("merged order = %v", merged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeGraphFolderQueueNoOpWhenComplete(t *testing.T) {
|
||||||
|
cursor := map[string]any{
|
||||||
|
"graphFolderQueue": []any{"inbox-id", "sent-id"},
|
||||||
|
}
|
||||||
|
merged := mergeGraphFolderQueue(cursor, []string{"inbox-id", "sent-id"})
|
||||||
|
if len(merged) != 2 {
|
||||||
|
t.Fatalf("merged len = %d", len(merged))
|
||||||
|
}
|
||||||
|
if merged[0] != "inbox-id" || merged[1] != "sent-id" {
|
||||||
|
t.Fatalf("merged = %v", merged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestGraphInitFolderDeltaLink(t *testing.T) {
|
func TestGraphInitFolderDeltaLink(t *testing.T) {
|
||||||
client := mockGraphHTTPClient(t, func(w http.ResponseWriter, r *http.Request) {
|
client := mockGraphHTTPClient(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
if strings.Contains(r.URL.Path, "/mailFolders/inbox-id/messages/delta") {
|
if strings.Contains(r.URL.Path, "/mailFolders/inbox-id/messages/delta") {
|
||||||
|
|||||||
@ -151,6 +151,36 @@ func writeGraphFolderQueue(cursor map[string]any, ids []string) {
|
|||||||
cursor["graphFolderQueue"] = queue
|
cursor["graphFolderQueue"] = queue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// mergeGraphFolderQueue extends a cached import queue with newly discovered folder
|
||||||
|
// IDs while preserving order for folders already in progress.
|
||||||
|
func mergeGraphFolderQueue(cursor map[string]any, discovered []string) []string {
|
||||||
|
existing := readGraphFolderQueue(cursor)
|
||||||
|
if len(existing) == 0 {
|
||||||
|
writeGraphFolderQueue(cursor, discovered)
|
||||||
|
return discovered
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(existing))
|
||||||
|
for _, id := range existing {
|
||||||
|
seen[id] = struct{}{}
|
||||||
|
}
|
||||||
|
merged := make([]string, len(existing), len(existing)+len(discovered))
|
||||||
|
copy(merged, existing)
|
||||||
|
for _, id := range discovered {
|
||||||
|
if id == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[id]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
merged = append(merged, id)
|
||||||
|
seen[id] = struct{}{}
|
||||||
|
}
|
||||||
|
if len(merged) != len(existing) {
|
||||||
|
writeGraphFolderQueue(cursor, merged)
|
||||||
|
}
|
||||||
|
return merged
|
||||||
|
}
|
||||||
|
|
||||||
func migrationContactPath(bookPath, provider, sourceID string) string {
|
func migrationContactPath(bookPath, provider, sourceID string) string {
|
||||||
uid := sanitizeMigrationUID(provider, sourceID)
|
uid := sanitizeMigrationUID(provider, sourceID)
|
||||||
return bookPath + uid + ".vcf"
|
return bookPath + uid + ".vcf"
|
||||||
|
|||||||
@ -67,9 +67,6 @@ func (h *Handler) ProvisionUser(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
if h.platformDomain != "" && !strings.Contains(email, "@") {
|
if h.platformDomain != "" && !strings.Contains(email, "@") {
|
||||||
email = email + "@" + h.platformDomain
|
email = email + "@" + h.platformDomain
|
||||||
} else if h.platformDomain != "" && !strings.HasSuffix(email, "@"+h.platformDomain) {
|
|
||||||
local := strings.Split(email, "@")[0]
|
|
||||||
email = local + "@" + h.platformDomain
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx := r.Context()
|
ctx := r.Context()
|
||||||
|
|||||||
2
migrations/000050_mail_folders_parent.down.sql
Normal file
2
migrations/000050_mail_folders_parent.down.sql
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
DROP INDEX IF EXISTS idx_mail_folders_parent;
|
||||||
|
ALTER TABLE mail_folders DROP COLUMN IF EXISTS parent_id;
|
||||||
5
migrations/000050_mail_folders_parent.up.sql
Normal file
5
migrations/000050_mail_folders_parent.up.sql
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
-- Parent hierarchy for mail_folders (Graph childFolders, IMAP nesting).
|
||||||
|
ALTER TABLE mail_folders
|
||||||
|
ADD COLUMN parent_id UUID NULL REFERENCES mail_folders(id) ON DELETE CASCADE;
|
||||||
|
|
||||||
|
CREATE INDEX idx_mail_folders_parent ON mail_folders(parent_id);
|
||||||
Loading…
Reference in New Issue
Block a user