diff --git a/README.md b/README.md index 1c6ddbd..e04f337 100644 --- a/README.md +++ b/README.md @@ -34,27 +34,28 @@ Example: ```json { "api_fetch_interval": 600, - "datafile": "data.json", "enabled_api_endpoints": [ "bay", "bund" ], + "datafile": "data.json", "loglevel": 2, - "recipients": [ + "lists": [ { - "address": "guenther@example.org", - "include": [ - {"classification": "kritisch"}, - {"title_contains": "jQuery"} + "name": "Example List", + "recipients": ["someone@example.org"], + "filter": [ + {"classification": "hoch", "title_contains": "Microsoft"}, + {"classification": "kritisch"} ] } ], "smtp": { - "from": "from@example.org", - "host": "example.org", + "from": "user@localhost", + "host": "127.0.0.1", "port": 587, - "user": "from@example.org", - "password": "SiEhAbEnMiChInSgEsIcHtGeFiLmTdAsDüRfEnSiEnIcHt" + "user": "user@localhost", + "password": "change me :)" }, "template": { "subject": "", diff --git a/config.go b/config.go index f808b22..8785375 100644 --- a/config.go +++ b/config.go @@ -11,7 +11,7 @@ type Config struct { EnabledApiEndpoints []string `json:"enabled_api_endpoints"` PersistentDataFilePath string `json:"datafile"` LogLevel int `json:"loglevel"` - Recipients []Recipient `json:"recipients"` + Lists *[]NotifyList `json:"lists"` SmtpConfiguration SmtpSettings `json:"smtp"` Template MailTemplateConfig `json:"template"` } @@ -23,12 +23,16 @@ func NewConfig() Config { EnabledApiEndpoints: []string{"bay", "bund"}, PersistentDataFilePath: "data.json", LogLevel: 2, - Recipients: []Recipient{}, + Lists: &[]NotifyList{ + { Name: "Example List", + Recipients: []string{}, + Filter: []Filter{},}, + }, SmtpConfiguration: SmtpSettings{ - From: "from@example.org", - User: "from@example.org", - Password: "SiEhAbEnMiChInSgEsIcHtGeFiLmTdAsDüRfEnSiEnIcHt", - ServerHost: "example.org", + From: "user@localhost", + User: "user@localhost", + Password: "change me :)", + ServerHost: "127.0.0.1", ServerPort: 587}, Template: MailTemplateConfig{ SubjectTemplate: "", @@ -39,18 +43,20 @@ func NewConfig() Config { } func checkConfig(config Config) { - if len(config.Recipients) < 1 { + if len(*config.Lists) < 1 { logger.error("Configuration is incomplete") - panic(errors.New("no recipients are configured")) + panic(errors.New("no lists are configured")) } - for _, r := range config.Recipients { - if !mailAddressIsValid(r.Address) { - logger.error("Configuration includes invalid data") - panic(errors.New("'" + r.Address + "' is not a valid e-mail address")) - } - if len(r.Filters) < 1 { + for _, l := range *config.Lists { + if len(l.Filter) < 1 { logger.error("Configuration is incomplete") - panic(errors.New("recipient " + r.Address + " has no filter defined - at least {'any': true/false} should be configured")) + panic(errors.New("list " + l.Name + " has no filter defined - at least [{'any': true/false}] should be configured")) + } + for _, r := range l.Recipients { + if !mailAddressIsValid(r) { + logger.error("Configuration includes invalid data") + panic(errors.New("'" + r + "' is not a valid e-mail address")) + } } } if !mailAddressIsValid(config.SmtpConfiguration.From) { diff --git a/go.mod b/go.mod index f31f7cb..acbf5d6 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/ChaoticByte/wid-notifier -go 1.22 +go 1.24 diff --git a/mail.go b/mail.go deleted file mode 100644 index e4c5890..0000000 --- a/mail.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) 2023 Julian Müller (ChaoticByte) - -package main - -import ( - "crypto/tls" - "encoding/base64" - "fmt" - "net/mail" - "net/smtp" - "time" -) - -const MAIL_LINE_SEP = "\r\n" - -type MailContent struct { - Subject string - Body string -} - -func (c MailContent) serializeValidMail(from string, to string) []byte { - // We'll send base64 encoded Subject & Body, because we Dschörmäns have umlauts - // and I'm too lazy to encode ä into =E4 and so on - subjectEncoded := base64.StdEncoding.EncodeToString([]byte(c.Subject)) - bodyEncoded := base64.StdEncoding.EncodeToString([]byte(c.Body)) - data := []byte(fmt.Sprintf( - "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\nFrom: %v%vTo: %v%vSubject: =?utf-8?b?%v?=%v%v%v", - from, MAIL_LINE_SEP, - to, MAIL_LINE_SEP, - subjectEncoded, MAIL_LINE_SEP, - MAIL_LINE_SEP, - bodyEncoded)) - // done, I guess - return data -} - -type Recipient struct { - Address string `json:"address"` - // Must be a configured filter id - Filters []Filter `json:"include"` -} - -type SmtpSettings struct { - From string `json:"from"` - ServerHost string `json:"host"` - ServerPort int `json:"port"` - User string `json:"user"` - Password string `json:"password"` -} - -func (r Recipient) sendNotices(notices []WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, cache *map[string][]byte) error { - logger.debug("Generating and sending mails to " + r.Address + " ...") - cacheHits := 0 - cacheMisses := 0 - mails := [][]byte{} - for _, n := range notices { - var data []byte - cacheResult := (*cache)[n.Uuid] - if len(cacheResult) > 0 { - cacheHits++ - data = cacheResult - } else { - cacheMisses++ - mailContent, err := template.generate(TemplateData{n, Version}) - if err != nil { - logger.error("Could not create mail from template") - logger.error(err) - } - // serialize & send mail - data = mailContent.serializeValidMail(smtpConfig.From, r.Address) - // add to cache - (*cache)[n.Uuid] = data - } - mails = append(mails, data) - } - logger.debug(fmt.Sprintf("%v mail cache hits, %v misses", cacheHits, cacheMisses)) - err := sendMails( - smtpConfig, - auth, - r.Address, - mails, - ) - if err != nil { return err } - logger.debug("Successfully sent all mails to " + r.Address) - return nil -} - -func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, data [][]byte) error { - addr := fmt.Sprintf("%v:%v", smtpConf.ServerHost, smtpConf.ServerPort) - logger.debug("Connecting to mail server at " + addr + " ...") - connection, err := smtp.Dial(addr) - if err != nil { return err } - defer connection.Close() - // can leave out connection.Hello - hasTlsExt, _ := connection.Extension("starttls") - if hasTlsExt { - err = connection.StartTLS(&tls.Config{ServerName: smtpConf.ServerHost}) - if err != nil { return err } - logger.debug("Mail Server supports TLS") - } else { - logger.debug("Mail Server doesn't support TLS") - } - logger.debug("Authenticating to mail server ...") - err = connection.Auth(auth) - if err != nil { return err } - if logger.LogLevel >= 3 { - fmt.Printf("DEBUG %v Sending mails to server ", time.Now().Format("2006/01/02 15:04:05.000000")) - } - for _, d := range data { - err = connection.Mail(smtpConf.From) - if err != nil { return err } - err = connection.Rcpt(to) - if err != nil { return err } - writer, err := connection.Data() - if err != nil { return err } - _, err = writer.Write(d) - if err != nil { return err } - err = writer.Close() - if err != nil { return err } - if logger.LogLevel >= 3 { - print(".") - } - } - if logger.LogLevel >= 3 { - print("\n") - } - return connection.Quit() -} - -func mailAddressIsValid(address string) bool { - _, err := mail.ParseAddress(address); - return err == nil -} diff --git a/mail_common.go b/mail_common.go new file mode 100644 index 0000000..71e29bf --- /dev/null +++ b/mail_common.go @@ -0,0 +1,90 @@ +// Copyright (c) 2023 Julian Müller (ChaoticByte) + +package main + +import ( + "fmt" + "mime" + "mime/quotedprintable" + "net/mail" + "net/smtp" + "strings" +) + +type MailContent struct { + Subject string + Body string +} + +func (c MailContent) serializeValidMail(from string, to string) []byte { + // format subject using Q Encoding from RFC2047 + subjectEncoded := mime.QEncoding.Encode("utf-8", c.Subject) + // format body using Quoted-Printable Encoding from RFC2045 + var bodyEncoded strings.Builder + bew := quotedprintable.NewWriter(&bodyEncoded) + bew.Write([]byte(c.Body)) + bew.Close() + // glue it all together + data := fmt.Appendf(nil, + "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: Quoted-Printable\r\nFrom: %v\r\nTo: %v\r\nSubject: %v\r\n\r\n%v", + from, to, subjectEncoded, bodyEncoded.String(), + ) + return data +} + +type NotifyList struct { + Name string `json:"name"` + Recipients []string `json:"recipients"` + // Must be a configured filter id + Filter []Filter `json:"filter"` +} + +type SmtpSettings struct { + From string `json:"from"` + ServerHost string `json:"host"` + ServerPort int `json:"port"` + User string `json:"user"` + Password string `json:"password"` +} + +func sendNotices(recipient string, notices []*WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, mailContentCache *map[string]*MailContent) error { + logger.debug("Generating and sending mails for recipient " + recipient + " ...") + cacheHits := 0 + cacheMisses := 0 + mails := []*MailContent{} + for _, n := range notices { + var mc *MailContent + cacheResult := (*mailContentCache)[n.Uuid] + if cacheResult != nil { + cacheHits++ + mc = cacheResult + } else { + cacheMisses++ + mc_, err := template.generate(TemplateData{n, Version}) + if err != nil { + logger.error("Could not create mail from template") + logger.error(err) + } else { + mc = &mc_ + // add to cache + (*mailContentCache)[n.Uuid] = mc + } + } + mails = append(mails, mc) + } + logger.debug(fmt.Sprintf("%v mail cache hits, %v misses", cacheHits, cacheMisses)) + err := sendMails( + smtpConfig, + auth, + recipient, + mails, + ) + if err != nil { return err } + logger.debug("Successfully sent all mails to " + recipient) + return nil +} + +func mailAddressIsValid(address string) bool { + _, err := mail.ParseAddress(address); + return err == nil +} diff --git a/mail_transfer.go b/mail_transfer.go new file mode 100644 index 0000000..45675d3 --- /dev/null +++ b/mail_transfer.go @@ -0,0 +1,57 @@ +// +build !debug_mail_transfer +// Copyright (c) 2023 Julian Müller (ChaoticByte) + +package main + +import ( + "crypto/tls" + "fmt" + "net/smtp" + "time" +) + + +func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, mails []*MailContent) error { + addr := fmt.Sprintf("%v:%v", smtpConf.ServerHost, smtpConf.ServerPort) + logger.debug("Connecting to mail server at " + addr + " ...") + connection, err := smtp.Dial(addr) + if err != nil { return err } + defer connection.Close() + // can leave out connection.Hello + hasTlsExt, _ := connection.Extension("starttls") + if hasTlsExt { + err = connection.StartTLS(&tls.Config{ServerName: smtpConf.ServerHost}) + if err != nil { return err } + logger.debug("Mail Server supports StartTLS") + } else { + logger.debug("Mail Server doesn't support StartTLS") + } + logger.debug("Authenticating to mail server ...") + err = connection.Auth(auth) + if err != nil { return err } + if logger.LogLevel >= 3 { + fmt.Printf("DEBUG %v Sending mails to server ", time.Now().Format("2006/01/02 15:04:05.000000")) + } + for _, mc := range mails { + // serialize mail + d := mc.serializeValidMail(smtpConf.From, to) + // send mail + err = connection.Mail(smtpConf.From) + if err != nil { return err } + err = connection.Rcpt(to) + if err != nil { return err } + writer, err := connection.Data() + if err != nil { return err } + _, err = writer.Write(d) + if err != nil { return err } + err = writer.Close() + if err != nil { return err } + if logger.LogLevel >= 3 { + print(".") + } + } + if logger.LogLevel >= 3 { + print("\n") + } + return connection.Quit() +} diff --git a/mail_transfer_debug.go b/mail_transfer_debug.go new file mode 100644 index 0000000..60fb2bd --- /dev/null +++ b/mail_transfer_debug.go @@ -0,0 +1,26 @@ +// +build debug_mail_transfer +// Copyright (c) 2023 Julian Müller (ChaoticByte) + +package main + +import ( + "fmt" + "net/smtp" +) + +func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, mails []*MailContent) error { + logger.warn("Mail Transfer Debugging is active. Not connecting.") + logger.info("MAIL TRANSFER: \n\n") + for _, mc := range mails { + // serialize mail + d := mc.serializeValidMail(smtpConf.From, to) + // output mail + fmt.Println("MAIL FROM:" + smtpConf.From) + fmt.Println("RCPT TO:" + to) + fmt.Println("DATA") + fmt.Println(string(d)) + fmt.Println(".") + } + fmt.Print("\n\n") + return nil +} diff --git a/main.go b/main.go index 47ce30b..d239bf0 100644 --- a/main.go +++ b/main.go @@ -10,19 +10,39 @@ import ( "time" ) +var executableName string var logger Logger + +func showVersion() { + fmt.Printf("wid-notifier %s\n", Version) +} + +func showHelp() { + fmt.Printf("Usage: %v \n\nIf the config file doesn't exist, an incomplete \n" + + "configuration with default values is created.\n\n", + executableName) + showVersion() +} + func main() { // get cli arguments args := os.Args + executableName = args[0] if len(args) < 2 { - fmt.Printf( "Usage: %v \n\nIf the config file doesn't exist, an incomplete \n" + - "configuration with default values is created.\n\nVersion: %s\n", - args[0], - Version) + showHelp() os.Exit(1) } - configFilePath := os.Args[1] + for _, arg := range args { + if arg == "-h" || arg == "--help" { + showHelp() + os.Exit(0) + } else if arg == "--version" { + showVersion() + os.Exit(0) + } + } + configFilePath := args[1] // create logger logger = NewLogger(2) // init @@ -77,7 +97,8 @@ func main() { for { t1 := time.Now().UnixMilli() newNotices := []WidNotice{} - cache := map[string][]byte{} + lastPublished := map[string]time.Time{} // endpoint id : last published timestamp + cache := map[string]*MailContent{} // cache generated emails for reuse for _, a := range enabledApiEndpoints { logger.info("Querying endpoint '" + a.Id + "' for new notices ...") n, t, err := a.getNotices(persistent.data.(PersistentData).LastPublished[a.Id]) @@ -93,35 +114,57 @@ func main() { logger.error(err) } else if len(n) > 0 { newNotices = append(newNotices, n...) - persistent.data.(PersistentData).LastPublished[a.Id] = t - persistent.save() + lastPublished[a.Id] = t } } logger.debug(fmt.Sprintf("Got %v new notices", len(newNotices))) if len(newNotices) > 0 { logger.info("Sending email notifications ...") + // mail recipient : pointer to slice of wid notices to be sent + noticesToBeSent := map[string][]*WidNotice{} recipientsNotified := 0 - for _, r := range config.Recipients { - // Filter notices for this recipient - filteredNotices := []WidNotice{} - for _, f := range r.Filters { + var err error + for _, l := range *config.Lists { + // Filter notices for this list + for _, f := range l.Filter { for _, n := range f.filter(newNotices) { - if !noticeSliceContains(filteredNotices, n) { - filteredNotices = append(filteredNotices, n) + np := &n + for _, r := range l.Recipients { + if !noticeSliceContains(noticesToBeSent[r], np) { + noticesToBeSent[r] = append(noticesToBeSent[r], np) + } } } } - slices.Reverse(filteredNotices) - logger.debug(fmt.Sprintf("Including %v of %v notices for recipient %v", len(filteredNotices), len(newNotices), r.Address)) - // Send notices - err := r.sendNotices(filteredNotices, mailTemplate, mailAuth, config.SmtpConfiguration, &cache) + } + for r, notices := range noticesToBeSent { + // sort by publish date + slices.SortFunc(notices, func(a *WidNotice, b *WidNotice) int { + if a.Published == b.Published { + return 0 + } else if a.Published.After(b.Published) { + return 1 + } else { + return -1 + } + }) + // send + err = sendNotices(r, notices, mailTemplate, mailAuth, config.SmtpConfiguration, &cache) if err != nil { logger.error(err) } else { recipientsNotified++ } } - logger.info(fmt.Sprintf("Email notifications sent to %v of %v recipients", recipientsNotified, len(config.Recipients))) + if recipientsNotified < 1 && err != nil { + logger.error("Couldn't send any mail notification!") + } else { + for id, t := range lastPublished { + persistent.data.(PersistentData).LastPublished[id] = t + persistent.save() + } + logger.info(fmt.Sprintf("Email notifications sent to %v of %v recipients", recipientsNotified, len(noticesToBeSent))) + } } dt := int(time.Now().UnixMilli() - t1) time.Sleep(time.Millisecond * time.Duration((config.ApiFetchInterval * 1000) - dt)) diff --git a/notice.go b/notice.go index 8967d20..393f2c9 100644 --- a/notice.go +++ b/notice.go @@ -25,17 +25,7 @@ type WidNotice struct { PortalUrl string } -// func (n WidNotice) serialized() ([]byte, error) { -// return json.Marshal(n) -// } - -// func NewWidNoticeFromJSON(data []byte) (WidNotice, error) { -// n := WidNotice{} -// err := json.Unmarshal(data, &n) -// return n, err -// } - -func noticeSliceContains(notices []WidNotice, notice WidNotice) bool { +func noticeSliceContains(notices []*WidNotice, notice *WidNotice) bool { for _, x := range notices { if x.Uuid == notice.Uuid { return true diff --git a/persistent_data.go b/persistent_data.go index 182c269..f3ce9ce 100644 --- a/persistent_data.go +++ b/persistent_data.go @@ -18,4 +18,4 @@ func NewPersistentData(c Config) PersistentData { d.LastPublished[e.Id] = time.Now().Add(-time.Hour * 24) // a day ago } return d -} \ No newline at end of file +} diff --git a/template.go b/template.go index 7d078e8..b07434c 100644 --- a/template.go +++ b/template.go @@ -33,7 +33,7 @@ Sent by WidNotifier {{ .WidNotifierVersion }} ` type TemplateData struct { - WidNotice + *WidNotice WidNotifierVersion string }