perf(mail): fetch only headers for list/search (no body download)
list and search now fetch BODY.PEEK[HEADER] + BODYSTRUCTURE instead of the whole RFC822 message, so listing a large mailbox no longer downloads every message body and attachment. Header parsing reuses the same go-message path (RFC2047 decoding/formatting preserved); has_attachments is derived from the BODYSTRUCTURE tree. FetchFull keeps fetching the full message for get. Validated end-to-end against a live IMAP account: list/search/get output identical to the prior full-fetch behaviour, has_attachments correct. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+80
-37
@@ -4,6 +4,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/emersion/go-imap"
|
"github.com/emersion/go-imap"
|
||||||
@@ -66,18 +67,43 @@ func (c *Client) SelectFolder(folder string) (uint32, uint32, error) {
|
|||||||
return mbox.UidValidity, maxUID, nil
|
return mbox.UidValidity, maxUID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Client) fetchByUIDSet(folder string, set *imap.SeqSet, full bool) ([]Message, error) {
|
// hasAttachment reports whether a BODYSTRUCTURE tree contains a part marked
|
||||||
|
// Content-Disposition: attachment — the same notion of "attachment" that
|
||||||
|
// ParseMessage uses (go-message routes attachment-disposition parts to
|
||||||
|
// AttachmentHeader). This lets list/search report has_attachments without
|
||||||
|
// downloading any body.
|
||||||
|
func hasAttachment(bs *imap.BodyStructure) bool {
|
||||||
|
if bs == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if strings.EqualFold(bs.Disposition, "attachment") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, p := range bs.Parts {
|
||||||
|
if hasAttachment(p) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchHeadersByUIDSet fetches only message headers (BODY.PEEK[HEADER]) plus the
|
||||||
|
// BODYSTRUCTURE, never the body, so listing is cheap on large mailboxes.
|
||||||
|
func (c *Client) fetchHeadersByUIDSet(folder string, set *imap.SeqSet) ([]Header, error) {
|
||||||
if _, err := c.c.Select(folder, true); err != nil {
|
if _, err := c.c.Select(folder, true); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
section := &imap.BodySectionName{Peek: true} // entire message, no \Seen side-effect
|
section := &imap.BodySectionName{
|
||||||
items := []imap.FetchItem{imap.FetchUid, section.FetchItem()}
|
BodyPartName: imap.BodyPartName{Specifier: imap.HeaderSpecifier},
|
||||||
|
Peek: true, // BODY.PEEK[HEADER] — headers only, no \Seen side-effect
|
||||||
|
}
|
||||||
|
items := []imap.FetchItem{imap.FetchUid, imap.FetchBodyStructure, section.FetchItem()}
|
||||||
|
|
||||||
msgCh := make(chan *imap.Message, 16)
|
msgCh := make(chan *imap.Message, 16)
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- c.c.UidFetch(set, items, msgCh) }()
|
go func() { done <- c.c.UidFetch(set, items, msgCh) }()
|
||||||
|
|
||||||
var out []Message
|
var out []Header
|
||||||
for m := range msgCh {
|
for m := range msgCh {
|
||||||
r := m.GetBody(section)
|
r := m.GetBody(section)
|
||||||
if r == nil {
|
if r == nil {
|
||||||
@@ -85,25 +111,64 @@ func (c *Client) fetchByUIDSet(folder string, set *imap.SeqSet, full bool) ([]Me
|
|||||||
}
|
}
|
||||||
raw, err := io.ReadAll(r)
|
raw, err := io.ReadAll(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("uid %d: read body: %w", m.Uid, err)
|
return nil, fmt.Errorf("uid %d: read header: %w", m.Uid, err)
|
||||||
}
|
}
|
||||||
parsed, err := ParseMessage(m.Uid, raw)
|
h, err := ParseHeaderBytes(m.Uid, raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if !full {
|
h.HasAttachments = hasAttachment(m.BodyStructure)
|
||||||
parsed.Attachments = nil
|
out = append(out, h)
|
||||||
parsed.BodyText = ""
|
|
||||||
}
|
|
||||||
out = append(out, parsed)
|
|
||||||
}
|
}
|
||||||
if err := <-done; err != nil {
|
if err := <-done; err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i].Header.UID > out[j].Header.UID })
|
sort.Slice(out, func(i, j int) bool { return out[i].UID > out[j].UID })
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// fetchFullByUID fetches one message in full (body + attachments).
|
||||||
|
func (c *Client) fetchFullByUID(folder string, uid uint32) (Message, error) {
|
||||||
|
if _, err := c.c.Select(folder, true); err != nil {
|
||||||
|
return Message{}, err
|
||||||
|
}
|
||||||
|
section := &imap.BodySectionName{Peek: true} // entire message, no \Seen side-effect
|
||||||
|
items := []imap.FetchItem{imap.FetchUid, section.FetchItem()}
|
||||||
|
|
||||||
|
set := new(imap.SeqSet)
|
||||||
|
set.AddNum(uid)
|
||||||
|
msgCh := make(chan *imap.Message, 1)
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- c.c.UidFetch(set, items, msgCh) }()
|
||||||
|
|
||||||
|
var (
|
||||||
|
msg Message
|
||||||
|
found bool
|
||||||
|
)
|
||||||
|
for m := range msgCh {
|
||||||
|
r := m.GetBody(section)
|
||||||
|
if r == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, err := io.ReadAll(r)
|
||||||
|
if err != nil {
|
||||||
|
return Message{}, fmt.Errorf("uid %d: read body: %w", m.Uid, err)
|
||||||
|
}
|
||||||
|
parsed, err := ParseMessage(m.Uid, raw)
|
||||||
|
if err != nil {
|
||||||
|
return Message{}, err
|
||||||
|
}
|
||||||
|
msg, found = parsed, true
|
||||||
|
}
|
||||||
|
if err := <-done; err != nil {
|
||||||
|
return Message{}, err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return Message{}, fmt.Errorf("uid %d not found in %s", uid, folder)
|
||||||
|
}
|
||||||
|
return msg, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Client) FetchHeaders(folder string, uids []uint32) ([]Header, error) {
|
func (c *Client) FetchHeaders(folder string, uids []uint32) ([]Header, error) {
|
||||||
set := new(imap.SeqSet)
|
set := new(imap.SeqSet)
|
||||||
if len(uids) == 0 {
|
if len(uids) == 0 {
|
||||||
@@ -113,11 +178,7 @@ func (c *Client) FetchHeaders(folder string, uids []uint32) ([]Header, error) {
|
|||||||
set.AddNum(u)
|
set.AddNum(u)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
msgs, err := c.fetchByUIDSet(folder, set, false)
|
return c.fetchHeadersByUIDSet(folder, set)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return headersOf(msgs), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Client) FetchHeadersRange(folder string, sinceUID, beforeUID uint32, limit int) ([]Header, error) {
|
func (c *Client) FetchHeadersRange(folder string, sinceUID, beforeUID uint32, limit int) ([]Header, error) {
|
||||||
@@ -131,11 +192,10 @@ func (c *Client) FetchHeadersRange(folder string, sinceUID, beforeUID uint32, li
|
|||||||
hi = beforeUID - 1
|
hi = beforeUID - 1
|
||||||
}
|
}
|
||||||
set.AddRange(lo, hi)
|
set.AddRange(lo, hi)
|
||||||
msgs, err := c.fetchByUIDSet(folder, set, false)
|
h, err := c.fetchHeadersByUIDSet(folder, set)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
h := headersOf(msgs)
|
|
||||||
if limit > 0 && len(h) > limit {
|
if limit > 0 && len(h) > limit {
|
||||||
h = h[:limit]
|
h = h[:limit]
|
||||||
}
|
}
|
||||||
@@ -143,16 +203,7 @@ func (c *Client) FetchHeadersRange(folder string, sinceUID, beforeUID uint32, li
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *Client) FetchFull(folder string, uid uint32) (Message, error) {
|
func (c *Client) FetchFull(folder string, uid uint32) (Message, error) {
|
||||||
set := new(imap.SeqSet)
|
return c.fetchFullByUID(folder, uid)
|
||||||
set.AddNum(uid)
|
|
||||||
msgs, err := c.fetchByUIDSet(folder, set, true)
|
|
||||||
if err != nil {
|
|
||||||
return Message{}, err
|
|
||||||
}
|
|
||||||
if len(msgs) == 0 {
|
|
||||||
return Message{}, fmt.Errorf("uid %d not found in %s", uid, folder)
|
|
||||||
}
|
|
||||||
return msgs[0], nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type SearchCriteria struct {
|
type SearchCriteria struct {
|
||||||
@@ -195,11 +246,3 @@ func (c *Client) Search(folder string, sc SearchCriteria, limit int) ([]Header,
|
|||||||
}
|
}
|
||||||
return c.FetchHeaders(folder, uids)
|
return c.FetchHeaders(folder, uids)
|
||||||
}
|
}
|
||||||
|
|
||||||
func headersOf(msgs []Message) []Header {
|
|
||||||
out := make([]Header, 0, len(msgs))
|
|
||||||
for _, m := range msgs {
|
|
||||||
out = append(out, m.Header)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
package mail
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/emersion/go-imap"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHasAttachment(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
bs *imap.BodyStructure
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"nil", nil, false},
|
||||||
|
{
|
||||||
|
"single inline text",
|
||||||
|
&imap.BodyStructure{MIMEType: "text", MIMESubType: "plain", Disposition: "inline"},
|
||||||
|
false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"single attachment",
|
||||||
|
&imap.BodyStructure{MIMEType: "application", MIMESubType: "pdf", Disposition: "attachment"},
|
||||||
|
true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"disposition case-insensitive",
|
||||||
|
&imap.BodyStructure{MIMEType: "image", MIMESubType: "png", Disposition: "ATTACHMENT"},
|
||||||
|
true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"multipart with one attachment child",
|
||||||
|
&imap.BodyStructure{
|
||||||
|
MIMEType: "multipart",
|
||||||
|
MIMESubType: "mixed",
|
||||||
|
Parts: []*imap.BodyStructure{
|
||||||
|
{MIMEType: "text", MIMESubType: "plain", Disposition: "inline"},
|
||||||
|
{MIMEType: "application", MIMESubType: "octet-stream", Disposition: "attachment"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"multipart all inline",
|
||||||
|
&imap.BodyStructure{
|
||||||
|
MIMEType: "multipart",
|
||||||
|
MIMESubType: "alternative",
|
||||||
|
Parts: []*imap.BodyStructure{
|
||||||
|
{MIMEType: "text", MIMESubType: "plain", Disposition: "inline"},
|
||||||
|
{MIMEType: "text", MIMESubType: "html", Disposition: "inline"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"nested multipart with deep attachment",
|
||||||
|
&imap.BodyStructure{
|
||||||
|
MIMEType: "multipart",
|
||||||
|
MIMESubType: "mixed",
|
||||||
|
Parts: []*imap.BodyStructure{
|
||||||
|
{
|
||||||
|
MIMEType: "multipart",
|
||||||
|
MIMESubType: "alternative",
|
||||||
|
Parts: []*imap.BodyStructure{
|
||||||
|
{MIMEType: "text", MIMESubType: "plain", Disposition: "inline"},
|
||||||
|
{MIMEType: "application", MIMESubType: "pdf", Disposition: "attachment"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := hasAttachment(c.bs); got != c.want {
|
||||||
|
t.Errorf("%s: hasAttachment = %v, want %v", c.name, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -103,3 +103,16 @@ func ParseHeaderOnly(uid uint32, raw []byte) (Header, error) {
|
|||||||
}
|
}
|
||||||
return m.Header, nil
|
return m.Header, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParseHeaderBytes parses just a message's RFC822 header block (the bytes from
|
||||||
|
// an IMAP BODY[HEADER] fetch, with no body) into a Header, reusing the same
|
||||||
|
// RFC2047-decoding and formatting as ParseMessage. HasAttachments is left false:
|
||||||
|
// the caller, which holds the BODYSTRUCTURE, sets it. This lets list/search read
|
||||||
|
// headers without downloading message bodies.
|
||||||
|
func ParseHeaderBytes(uid uint32, raw []byte) (Header, error) {
|
||||||
|
mr, err := mail.CreateReader(bytes.NewReader(raw))
|
||||||
|
if err != nil {
|
||||||
|
return Header{}, err
|
||||||
|
}
|
||||||
|
return readHeader(mr, uid), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -58,6 +58,37 @@ func TestParseHeaderOnly(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseHeaderBytes(t *testing.T) {
|
||||||
|
// Header-only bytes (no body), as returned by an IMAP BODY[HEADER] fetch,
|
||||||
|
// including an RFC2047-encoded subject to confirm decoding is preserved.
|
||||||
|
raw := []byte("From: \"Bob\" <bob@trusted.com>\r\n" +
|
||||||
|
"To: me@example.com\r\n" +
|
||||||
|
"Subject: =?UTF-8?Q?Caf=C3=A9=20Invoice?=\r\n" +
|
||||||
|
"Date: Sat, 20 Jun 2026 10:00:00 +0000\r\n" +
|
||||||
|
"Message-ID: <abc123@trusted.com>\r\n" +
|
||||||
|
"\r\n")
|
||||||
|
h, err := ParseHeaderBytes(9, raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseHeaderBytes: %v", err)
|
||||||
|
}
|
||||||
|
if h.UID != 9 {
|
||||||
|
t.Fatalf("uid: %d", h.UID)
|
||||||
|
}
|
||||||
|
if h.Subject != "Café Invoice" {
|
||||||
|
t.Fatalf("subject not RFC2047-decoded: %q", h.Subject)
|
||||||
|
}
|
||||||
|
if h.From != `"Bob" <bob@trusted.com>` {
|
||||||
|
t.Fatalf("from: %q", h.From)
|
||||||
|
}
|
||||||
|
if h.MessageID != "abc123@trusted.com" && h.MessageID != "<abc123@trusted.com>" {
|
||||||
|
t.Fatalf("message-id: %q", h.MessageID)
|
||||||
|
}
|
||||||
|
// HasAttachments is left false — the IMAP layer sets it from BODYSTRUCTURE.
|
||||||
|
if h.HasAttachments {
|
||||||
|
t.Fatal("ParseHeaderBytes must not set HasAttachments")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func contains(s, sub string) bool {
|
func contains(s, sub string) bool {
|
||||||
return len(s) >= len(sub) && (func() bool {
|
return len(s) >= len(sub) && (func() bool {
|
||||||
for i := 0; i+len(sub) <= len(s); i++ {
|
for i := 0; i+len(sub) <= len(s); i++ {
|
||||||
|
|||||||
Reference in New Issue
Block a user