Fix leak of first recipient to other recipients because mail headers are also cached and re-used

README.md

@@ -34,27 +34,28 @@ Example:
 ```json
 {
   "api_fetch_interval": 600,
-  "datafile": "data.json",
   "enabled_api_endpoints": [
     "bay",
     "bund"
   ],
+  "datafile": "data.json",
   "loglevel": 2,
-  "recipients": [
+  "lists": [
     {
-      "address": "guenther@example.org",
+      "name": "Example List",
-      "include": [
+      "recipients": ["someone@example.org"],
-        {"classification": "kritisch"},
+      "filter": [
-        {"title_contains": "jQuery"}
+        {"classification": "hoch", "title_contains": "Microsoft"},
+        {"classification": "kritisch"}
       ]
     }
   ],
   "smtp": {
-    "from": "from@example.org",
+    "from": "user@localhost",
-    "host": "example.org",
+    "host": "127.0.0.1",
     "port": 587,
-    "user": "from@example.org",
+    "user": "user@localhost",
-    "password": "SiEhAbEnMiChInSgEsIcHtGeFiLmTdAsDüRfEnSiEnIcHt"
+    "password": "change me :)"
   },
   "template": {
     "subject": "",

config.go

@@ -11,7 +11,7 @@ type Config struct {
 	EnabledApiEndpoints []string `json:"enabled_api_endpoints"`
 	PersistentDataFilePath string `json:"datafile"`
 	LogLevel int `json:"loglevel"`
-	Recipients []Recipient `json:"recipients"`
+	Lists *[]NotifyList `json:"lists"`
 	SmtpConfiguration SmtpSettings `json:"smtp"`
 	Template MailTemplateConfig `json:"template"`
 }
@@ -23,12 +23,16 @@ func NewConfig() Config {
 		EnabledApiEndpoints: []string{"bay", "bund"},
 		PersistentDataFilePath: "data.json",
 		LogLevel: 2,
-		Recipients: []Recipient{},
+		Lists: &[]NotifyList{
+			{ Name: "Example List",
+			  Recipients: []string{},
+			  Filter: []Filter{},},
+		},
 		SmtpConfiguration: SmtpSettings{
-			From: "from@example.org",
+			From: "user@localhost",
-			User: "from@example.org",
+			User: "user@localhost",
-			Password: "SiEhAbEnMiChInSgEsIcHtGeFiLmTdAsDüRfEnSiEnIcHt",
+			Password: "change me :)",
-			ServerHost: "example.org",
+			ServerHost: "127.0.0.1",
 			ServerPort: 587},
 		Template: MailTemplateConfig{
 			SubjectTemplate: "",
@@ -39,18 +43,20 @@ func NewConfig() Config {
 }
 
 func checkConfig(config Config) {
-	if len(config.Recipients) < 1 {
+	if len(*config.Lists) < 1 {
 		logger.error("Configuration is incomplete")
-		panic(errors.New("no recipients are configured"))
+		panic(errors.New("no lists are configured"))
 	}
-	for _, r := range config.Recipients {
+	for _, l := range *config.Lists {
-		if !mailAddressIsValid(r.Address) {
+		if len(l.Filter) < 1 {
+			logger.error("Configuration is incomplete")
+			panic(errors.New("list " + l.Name + " has no filter defined - at least [{'any': true/false}] should be configured"))
+		}
+		for _, r := range l.Recipients {
+			if !mailAddressIsValid(r) {
 				logger.error("Configuration includes invalid data")
-			panic(errors.New("'" + r.Address + "' is not a valid e-mail address"))
+				panic(errors.New("'" + r + "' is not a valid e-mail address"))
 			}
-		if len(r.Filters) < 1 {
-			logger.error("Configuration is incomplete")
-			panic(errors.New("recipient " + r.Address + " has no filter defined - at least {'any': true/false} should be configured"))
 		}
 	}
 	if !mailAddressIsValid(config.SmtpConfiguration.From) {

go.mod

@@ -1,3 +1,3 @@
 module github.com/ChaoticByte/wid-notifier
 
-go 1.22
+go 1.24

mail.go

@@ -1,133 +0,0 @@
-// Copyright (c) 2023 Julian Müller (ChaoticByte)
-
-package main
-
-import (
-	"crypto/tls"
-	"encoding/base64"
-	"fmt"
-	"net/mail"
-	"net/smtp"
-	"time"
-)
-
-const MAIL_LINE_SEP = "\r\n"
-
-type MailContent struct {
-	Subject string
-	Body string
-}
-
-func (c MailContent) serializeValidMail(from string, to string) []byte {
-	// We'll send base64 encoded Subject & Body, because we Dschörmäns have umlauts
-	// and I'm too lazy to encode ä into =E4 and so on
-	subjectEncoded := base64.StdEncoding.EncodeToString([]byte(c.Subject))
-	bodyEncoded := base64.StdEncoding.EncodeToString([]byte(c.Body))
-	data := []byte(fmt.Sprintf(
-		"Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\nFrom: %v%vTo: %v%vSubject: =?utf-8?b?%v?=%v%v%v",
-		from, MAIL_LINE_SEP,
-		to, MAIL_LINE_SEP,
-		subjectEncoded, MAIL_LINE_SEP,
-		MAIL_LINE_SEP,
-		bodyEncoded))
-	// done, I guess
-	return data
-}
-
-type Recipient struct {
-	Address string `json:"address"`
-	// Must be a configured filter id
-	Filters []Filter `json:"include"`
-}
-
-type SmtpSettings struct {
-	From string `json:"from"`
-	ServerHost string `json:"host"`
-	ServerPort int `json:"port"`
-	User string `json:"user"`
-	Password string `json:"password"`
-}
-
-func (r Recipient) sendNotices(notices []WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, cache *map[string][]byte) error {
-	logger.debug("Generating and sending mails to " + r.Address + " ...")
-	cacheHits := 0
-	cacheMisses := 0
-	mails := [][]byte{}
-	for _, n := range notices {
-		var data []byte
-		cacheResult := (*cache)[n.Uuid]
-		if len(cacheResult) > 0 {
-			cacheHits++
-			data = cacheResult
-		} else {
-			cacheMisses++
-			mailContent, err := template.generate(TemplateData{n, Version})
-			if err != nil {
-				logger.error("Could not create mail from template")
-				logger.error(err)
-			}
-			// serialize & send mail
-			data = mailContent.serializeValidMail(smtpConfig.From, r.Address)
-			// add to cache
-			(*cache)[n.Uuid] = data
-		}
-		mails = append(mails, data)
-	}
-	logger.debug(fmt.Sprintf("%v mail cache hits, %v misses", cacheHits, cacheMisses))
-	err := sendMails(
-		smtpConfig,
-		auth,
-		r.Address,
-		mails,
-	)
-	if err != nil { return err }
-	logger.debug("Successfully sent all mails to " + r.Address)
-	return nil
-}
-
-func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, data [][]byte) error {
-	addr := fmt.Sprintf("%v:%v", smtpConf.ServerHost, smtpConf.ServerPort)
-	logger.debug("Connecting to mail server at " + addr + " ...")
-	connection, err := smtp.Dial(addr)
-	if err != nil { return err }
-	defer connection.Close()
-	// can leave out connection.Hello
-	hasTlsExt, _ := connection.Extension("starttls")
-	if hasTlsExt {
-		err = connection.StartTLS(&tls.Config{ServerName: smtpConf.ServerHost})
-		if err != nil { return err }
-		logger.debug("Mail Server supports TLS")
-	} else {
-		logger.debug("Mail Server doesn't support TLS")
-	}
-	logger.debug("Authenticating to mail server ...")
-	err = connection.Auth(auth)
-	if err != nil { return err }
-	if logger.LogLevel >= 3 {
-		fmt.Printf("DEBUG %v Sending mails to server ", time.Now().Format("2006/01/02 15:04:05.000000"))
-	}
-	for _, d := range data {
-		err = connection.Mail(smtpConf.From)
-		if err != nil { return err }
-		err = connection.Rcpt(to)
-		if err != nil { return err }
-		writer, err := connection.Data()
-		if err != nil { return err }
-		_, err = writer.Write(d)
-		if err != nil { return err }
-		err = writer.Close()
-		if err != nil { return err }
-		if logger.LogLevel >= 3 {
-			print(".")
-		}
-	}
-	if logger.LogLevel >= 3 {
-		print("\n")
-	}
-	return connection.Quit()
-}
-
-func mailAddressIsValid(address string) bool {
-	_, err := mail.ParseAddress(address);
-	return err == nil
-}

mail_common.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2023 Julian Müller (ChaoticByte)
+
+package main
+
+import (
+	"fmt"
+	"mime"
+	"mime/quotedprintable"
+	"net/mail"
+	"net/smtp"
+	"strings"
+)
+
+type MailContent struct {
+	Subject string
+	Body string
+}
+
+func (c MailContent) serializeValidMail(from string, to string) []byte {
+	// format subject using Q Encoding from RFC2047
+	subjectEncoded := mime.QEncoding.Encode("utf-8", c.Subject)
+	// format body using Quoted-Printable Encoding from RFC2045
+	var bodyEncoded strings.Builder
+	bew := quotedprintable.NewWriter(&bodyEncoded)
+	bew.Write([]byte(c.Body))
+	bew.Close()
+	// glue it all together
+	data := fmt.Appendf(nil, 
+		"Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: Quoted-Printable\r\nFrom: %v\r\nTo: %v\r\nSubject: %v\r\n\r\n%v",
+		from, to, subjectEncoded, bodyEncoded.String(),
+	)
+	return data
+}
+
+type NotifyList struct {
+	Name string `json:"name"`
+	Recipients []string `json:"recipients"`
+	// Must be a configured filter id
+	Filter []Filter `json:"filter"`
+}
+
+type SmtpSettings struct {
+	From string `json:"from"`
+	ServerHost string `json:"host"`
+	ServerPort int `json:"port"`
+	User string `json:"user"`
+	Password string `json:"password"`
+}
+
+func sendNotices(recipient string, notices []*WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, mailContentCache *map[string]*MailContent) error {
+	logger.debug("Generating and sending mails for recipient " + recipient + " ...")
+	cacheHits := 0
+	cacheMisses := 0
+	mails := []*MailContent{}
+	for _, n := range notices {
+		var mc *MailContent
+		cacheResult := (*mailContentCache)[n.Uuid]
+		if cacheResult != nil {
+			cacheHits++
+			mc = cacheResult
+		} else {
+			cacheMisses++
+			mc_, err := template.generate(TemplateData{n, Version})
+			if err != nil {
+				logger.error("Could not create mail from template")
+				logger.error(err)
+			} else {
+				mc = &mc_
+				// add to cache
+				(*mailContentCache)[n.Uuid] = mc
+			}
+		}
+		mails = append(mails, mc)
+	}
+	logger.debug(fmt.Sprintf("%v mail cache hits, %v misses", cacheHits, cacheMisses))
+	err := sendMails(
+		smtpConfig,
+		auth,
+		recipient,
+		mails,
+	)
+	if err != nil { return err }
+	logger.debug("Successfully sent all mails to " + recipient)
+	return nil
+}
+
+func mailAddressIsValid(address string) bool {
+	_, err := mail.ParseAddress(address);
+	return err == nil
+}

mail_transfer.go

@@ -0,0 +1,57 @@
+// +build !debug_mail_transfer
+// Copyright (c) 2023 Julian Müller (ChaoticByte)
+
+package main
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/smtp"
+	"time"
+)
+
+
+func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, mails []*MailContent) error {
+	addr := fmt.Sprintf("%v:%v", smtpConf.ServerHost, smtpConf.ServerPort)
+	logger.debug("Connecting to mail server at " + addr + " ...")
+	connection, err := smtp.Dial(addr)
+	if err != nil { return err }
+	defer connection.Close()
+	// can leave out connection.Hello
+	hasTlsExt, _ := connection.Extension("starttls")
+	if hasTlsExt {
+		err = connection.StartTLS(&tls.Config{ServerName: smtpConf.ServerHost})
+		if err != nil { return err }
+		logger.debug("Mail Server supports StartTLS")
+	} else {
+		logger.debug("Mail Server doesn't support StartTLS")
+	}
+	logger.debug("Authenticating to mail server ...")
+	err = connection.Auth(auth)
+	if err != nil { return err }
+	if logger.LogLevel >= 3 {
+		fmt.Printf("DEBUG %v Sending mails to server ", time.Now().Format("2006/01/02 15:04:05.000000"))
+	}
+	for _, mc := range mails {
+		// serialize mail
+		d := mc.serializeValidMail(smtpConf.From, to)
+		// send mail
+		err = connection.Mail(smtpConf.From)
+		if err != nil { return err }
+		err = connection.Rcpt(to)
+		if err != nil { return err }
+		writer, err := connection.Data()
+		if err != nil { return err }
+		_, err = writer.Write(d)
+		if err != nil { return err }
+		err = writer.Close()
+		if err != nil { return err }
+		if logger.LogLevel >= 3 {
+			print(".")
+		}
+	}
+	if logger.LogLevel >= 3 {
+		print("\n")
+	}
+	return connection.Quit()
+}

mail_transfer_debug.go

@@ -0,0 +1,26 @@
+// +build debug_mail_transfer
+// Copyright (c) 2023 Julian Müller (ChaoticByte)
+
+package main
+
+import (
+	"fmt"
+	"net/smtp"
+)
+
+func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, mails []*MailContent) error {
+	logger.warn("Mail Transfer Debugging is active. Not connecting.")
+	logger.info("MAIL TRANSFER: \n\n")
+	for _, mc := range mails {
+		// serialize mail
+		d := mc.serializeValidMail(smtpConf.From, to)
+		// output mail
+		fmt.Println("MAIL FROM:" + smtpConf.From)
+		fmt.Println("RCPT TO:" + to)
+		fmt.Println("DATA")
+		fmt.Println(string(d))
+		fmt.Println(".")
+	}
+	fmt.Print("\n\n")
+	return nil
+}

main.go

@@ -10,19 +10,39 @@ import (
 	"time"
 )
 
+var executableName string
 var logger Logger
 
+
+func showVersion() {
+	fmt.Printf("wid-notifier %s\n", Version)
+}
+
+func showHelp() {
+	fmt.Printf("Usage: %v <configfile>\n\nIf the config file doesn't exist, an incomplete \n" +
+			   "configuration with default values is created.\n\n",
+			   executableName)
+	showVersion()
+}
+
 func main() {
 	// get cli arguments
 	args := os.Args
+	executableName = args[0]
 	if len(args) < 2 {
-		fmt.Printf( "Usage: %v <configfile>\n\nIf the config file doesn't exist, an incomplete \n" +
+		showHelp()
-					"configuration with default values is created.\n\nVersion: %s\n",
-					args[0],
-					Version)
 		os.Exit(1)
 	}
-	configFilePath := os.Args[1]
+	for _, arg := range args {
+		if arg == "-h" || arg == "--help" {
+			showHelp()
+			os.Exit(0)
+		} else if arg == "--version" {
+			showVersion()
+			os.Exit(0)
+		}
+	}
+	configFilePath := args[1]
 	// create logger
 	logger = NewLogger(2)
 	// init
@@ -77,7 +97,8 @@ func main() {
 	for {
 		t1 := time.Now().UnixMilli()
 		newNotices := []WidNotice{}
-		cache := map[string][]byte{}
+		lastPublished := map[string]time.Time{} // endpoint id : last published timestamp
+		cache := map[string]*MailContent{}      // cache generated emails for reuse
 		for _, a := range enabledApiEndpoints {
 			logger.info("Querying endpoint '" + a.Id + "' for new notices ...")
 			n, t, err := a.getNotices(persistent.data.(PersistentData).LastPublished[a.Id])
@@ -93,35 +114,57 @@ func main() {
 				logger.error(err)
 			} else if len(n) > 0 {
 				newNotices = append(newNotices, n...)
-				persistent.data.(PersistentData).LastPublished[a.Id] = t
+				lastPublished[a.Id] = t
-				persistent.save()
 			}
 		}
 		logger.debug(fmt.Sprintf("Got %v new notices", len(newNotices)))
 		if len(newNotices) > 0 {
 			logger.info("Sending email notifications ...")
+			// mail recipient : pointer to slice of wid notices to be sent
+			noticesToBeSent := map[string][]*WidNotice{}
 			recipientsNotified := 0
-			for _, r := range config.Recipients {
+			var err error
-				// Filter notices for this recipient
+			for _, l := range *config.Lists {
-				filteredNotices := []WidNotice{}
+				// Filter notices for this list
-				for _, f := range r.Filters {
+				for _, f := range l.Filter {
 					for _, n := range f.filter(newNotices) {
-						if !noticeSliceContains(filteredNotices, n) {
+						np := &n
-							filteredNotices = append(filteredNotices, n)
+						for _, r := range l.Recipients {
+							if !noticeSliceContains(noticesToBeSent[r], np) {
+								noticesToBeSent[r] = append(noticesToBeSent[r], np)
 							}
 						}
 					}
-				slices.Reverse(filteredNotices)
+				}
-				logger.debug(fmt.Sprintf("Including %v of %v notices for recipient %v", len(filteredNotices), len(newNotices), r.Address))
+			}
-				// Send notices
+			for r, notices := range noticesToBeSent {
-				err := r.sendNotices(filteredNotices, mailTemplate, mailAuth, config.SmtpConfiguration, &cache)
+				// sort by publish date
+				slices.SortFunc(notices, func(a *WidNotice, b *WidNotice) int {
+					if a.Published == b.Published {
+						return 0
+					} else if a.Published.After(b.Published) {
+						return 1
+					} else {
+						return -1
+					}
+				})
+				// send
+				err = sendNotices(r, notices, mailTemplate, mailAuth, config.SmtpConfiguration, &cache)
 				if err != nil {
 					logger.error(err)
 				} else {
 					recipientsNotified++
 				}
 			}
-			logger.info(fmt.Sprintf("Email notifications sent to %v of %v recipients", recipientsNotified, len(config.Recipients)))
+			if recipientsNotified < 1 && err != nil {
+				logger.error("Couldn't send any mail notification!")
+			} else {
+				for id, t := range lastPublished {
+					persistent.data.(PersistentData).LastPublished[id] = t
+					persistent.save()
+				}
+				logger.info(fmt.Sprintf("Email notifications sent to %v of %v recipients", recipientsNotified, len(noticesToBeSent)))
+			}
 		}
 		dt := int(time.Now().UnixMilli() - t1)
 		time.Sleep(time.Millisecond * time.Duration((config.ApiFetchInterval * 1000) - dt))

notice.go

@@ -25,17 +25,7 @@ type WidNotice struct {
 	PortalUrl string
 }
 
-// func (n WidNotice) serialized() ([]byte, error) {
+func noticeSliceContains(notices []*WidNotice, notice *WidNotice) bool {
-// 	return json.Marshal(n)
-// }
-
-// func NewWidNoticeFromJSON(data []byte) (WidNotice, error) {
-// 	n := WidNotice{}
-// 	err := json.Unmarshal(data, &n)
-// 	return n, err
-// }
-
-func noticeSliceContains(notices []WidNotice, notice WidNotice) bool {
 	for _, x := range notices {
 		if x.Uuid == notice.Uuid {
 			return true

template.go

@@ -33,7 +33,7 @@ Sent by WidNotifier {{ .WidNotifierVersion }}
 `
 
 type TemplateData struct {
-	WidNotice
+	*WidNotice
 	WidNotifierVersion string
 }