From 740862efa23c671077cbd0ae4a5bd472afc3a918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller=20=28ChaoticByte=29?= Date: Wed, 18 Oct 2023 20:17:27 +0200 Subject: [PATCH 01/15] Fixed a bug where all 1000 last notices where recognized as 'new'. This was caused by ApiEndpoint.getNotices() returning time.Time{} and a nil error in some cases. --- main.go | 2 +- widapi.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index ef85a52..c39f6bc 100644 --- a/main.go +++ b/main.go @@ -85,7 +85,7 @@ func main() { // ok then... logger.error("Couldn't query notices from API endpoint '" + a.Id + "'") logger.error(err) - } else { + } else if len(n) > 0 { newNotices = append(newNotices, n...) persistent.data.(PersistentData).LastPublished[a.Id] = t persistent.save() diff --git a/widapi.go b/widapi.go index 41638a0..f90af7d 100644 --- a/widapi.go +++ b/widapi.go @@ -41,7 +41,7 @@ type ApiEndpoint struct { } func (e ApiEndpoint) getNotices(since time.Time) ([]WidNotice, time.Time, error) { - // returns a slice of WidNotice and the 'published' field of the last notice + // returns a slice of WidNotice and the 'published' field of the last notice, and the error (or nil) var notices []WidNotice = []WidNotice{} var err error params := defaultParams @@ -58,19 +58,19 @@ func (e ApiEndpoint) getNotices(since time.Time) ([]WidNotice, time.Time, error) if res.StatusCode == 200 { resBody, err := io.ReadAll(res.Body) if err != nil { - return nil, time.Time{}, err + return []WidNotice{}, since, err } var decodedData map[string]interface{} if err = json.Unmarshal(resBody, &decodedData); err != nil { - return nil, time.Time{}, err + return []WidNotice{}, since, err } notices = parseApiResponse(decodedData, e) } else { logger.error(fmt.Sprintf("Get \"%v\": %v\n", url, res.Status)) - return nil, time.Time{}, err + return []WidNotice{}, since, err } } else { - return nil, time.Time{}, err + return []WidNotice{}, since, err } if len(notices) > 0 { // And here the filtering begins. yay -.- @@ -87,7 +87,7 @@ func (e ApiEndpoint) getNotices(since time.Time) ([]WidNotice, time.Time, error) } return noticesFiltered, lastPublished, nil } else { - return nil, time.Time{}, nil + return []WidNotice{}, since, nil } } From 676454972c2a334d470a4ed9d49d0a82212ad829 Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Thu, 19 Oct 2023 20:48:44 +0200 Subject: [PATCH 02/15] Changed the order of the api endpoints so that notifications from the LSI Bayern will be sent first --- widapi.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/widapi.go b/widapi.go index f90af7d..28fe259 100644 --- a/widapi.go +++ b/widapi.go @@ -13,16 +13,16 @@ import ( // known API endpoints var apiEndpoints []ApiEndpoint = []ApiEndpoint{ - { - Id: "bund", - EndpointUrl: "https://wid.cert-bund.de/content/public/securityAdvisory", - PortalUrl: "https://wid.cert-bund.de/portal/wid/securityadvisory", - }, { Id: "bay", EndpointUrl: "https://wid.lsi.bayern.de/content/public/securityAdvisory", PortalUrl: "https://wid.lsi.bayern.de/portal/wid/securityadvisory", }, + { + Id: "bund", + EndpointUrl: "https://wid.cert-bund.de/content/public/securityAdvisory", + PortalUrl: "https://wid.cert-bund.de/portal/wid/securityadvisory", + }, } const PUBLISHED_TIME_FORMAT = "2006-01-02T15:04:05.999-07:00" From 8c427cc5717cc05b113ce12725ccf7ab193bf07f Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Thu, 19 Oct 2023 20:50:32 +0200 Subject: [PATCH 03/15] Improved the default templates --- template.go | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/template.go b/template.go index 56cb46d..beda4ac 100644 --- a/template.go +++ b/template.go @@ -7,23 +7,26 @@ import ( "text/template" ) -const DEFAULT_SUBJECT_TEMPLATE = "{{ if .Status }}{{.Status}} {{ end }}[{{.Classification}}] {{.Title}}" -const DEFAULT_BODY_TEMPLATE = `{{.Name}} -{{.PortalUrl}} +const DEFAULT_SUBJECT_TEMPLATE = "[{{ .Classification }}] {{ .Title }}" +const DEFAULT_BODY_TEMPLATE = `{{ if .Status }}[{{ .Status }}] {{ end }}{{ .Name }} +-> {{ .PortalUrl }} +{{- if eq .NoPatch "true" }} -Published: {{.Published}} -{{ if gt .Basescore -1 }}Basescore: {{.Basescore}} -{{ end -}} -{{ if eq .NoPatch "true" }}There is no patch available at the moment! -{{ end }} -Affected Products: -{{ range $product := .ProductNames }} - {{ $product }} -{{ else }} unknown -{{ end }} -Assigned CVEs: -{{ range $cve := .Cves }} - {{ $cve }} -{{ else }} unknown -{{ end }}` +No patch available! +{{- end }} +{{ if gt .Basescore -1 }} +Basescore: {{ .Basescore }}{{- end }} +Published: {{ .Published }} +{{- if .ProductNames }} + +Affected Products:{{ range $product := .ProductNames }} + - {{ $product }} +{{- end }}{{ end }} +{{- if .Cves }} + +Assigned CVEs:{{ range $cve := .Cves }} + - {{ $cve }} -> https://www.cve.org/CVERecord?id={{ $cve }} +{{- end }}{{ end }}` type MailTemplateConfig struct { SubjectTemplate string `json:"subject"` From e5cc0ccbf11c1ca68881ea8aacb55f130cc4b063 Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Thu, 19 Oct 2023 21:01:29 +0200 Subject: [PATCH 04/15] Improved performance by manually handling mail server communication instead of using smtp.SendMail() and thus opening a new connection for each mail --- mail.go | 90 +++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/mail.go b/mail.go index 38a8281..af43e38 100644 --- a/mail.go +++ b/mail.go @@ -3,12 +3,13 @@ package main import ( + "crypto/tls" "encoding/base64" "fmt" "net/mail" "net/smtp" "slices" - "strings" + "time" ) const MAIL_LINE_SEP = "\r\n" @@ -22,13 +23,7 @@ func (c MailContent) serializeValidMail(from string, to string) []byte { // We'll send base64 encoded Subject & Body, because we Dschörmäns have umlauts // and I'm too lazy to encode ä into =E4 and so on subjectEncoded := base64.StdEncoding.EncodeToString([]byte(c.Subject)) - bodyEncoded := base64.StdEncoding.EncodeToString( - []byte( // ensure that all lines end with CRLF - strings.ReplaceAll( - strings.ReplaceAll(c.Body, "\n", MAIL_LINE_SEP), "\r\r", "\r", - ), - ), - ) + bodyEncoded := base64.StdEncoding.EncodeToString([]byte(c.Body)) data := []byte(fmt.Sprintf( "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\nFrom: %v%vTo: %v%vSubject: =?utf-8?b?%v?=%v%v%v", from, MAIL_LINE_SEP, @@ -68,6 +63,7 @@ func (r Recipient) filterAndSendNotices(notices []WidNotice, template MailTempla logger.debug("Generating and sending mails to " + r.Address + " ...") cacheHits := 0 cacheMisses := 0 + mails := [][]byte{} for _, n := range filteredNotices { var data []byte cacheResult := (*cache)[n.Uuid] @@ -86,20 +82,78 @@ func (r Recipient) filterAndSendNotices(notices []WidNotice, template MailTempla // add to cache (*cache)[n.Uuid] = data } - err := smtp.SendMail( - fmt.Sprintf("%v:%v", smtpConfig.ServerHost, smtpConfig.ServerPort), - auth, - smtpConfig.From, - []string{r.Address}, - data, - ) + mails = append(mails, data) + } + logger.debug(fmt.Sprintf("%v mail cache hits, %v misses", cacheHits, cacheMisses)) + err := sendMails( + smtpConfig, + auth, + r.Address, + mails, + ) + if err != nil { + return err + } + logger.debug("Successfully sent all mails to " + r.Address) + return nil +} + +func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, data [][]byte) error { + addr := fmt.Sprintf("%v:%v", smtpConf.ServerHost, smtpConf.ServerPort) + logger.debug("Connecting to mail server at " + addr + " ...") + connection, err := smtp.Dial(addr) + if err != nil { + return err + } + defer connection.Close() + // can leave out connection.Hello + hasTlsExt, _ := connection.Extension("starttls") + if hasTlsExt { + err = connection.StartTLS(&tls.Config{ServerName: smtpConf.ServerHost}) if err != nil { return err } + logger.debug("Mail Server supports TLS") + } else { + logger.debug("Mail Server doesn't support TLS") } - logger.debug(fmt.Sprintf("%v mail cache hits, %v misses", cacheHits, cacheMisses)) - logger.debug("Successfully sent all mails to " + r.Address) - return nil + logger.debug("Authenticating to mail server ...") + err = connection.Auth(auth) + if err != nil { + return err + } + if logger.LogLevel >= 3 { + fmt.Printf("DEBUG %v Sending mails to server ", time.Now().Format("2006/01/02 15:04:05.000000")) + } + for _, d := range data { + err = connection.Mail(smtpConf.From) + if err != nil { + return err + } + err = connection.Rcpt(to) + if err != nil { + return err + } + writer, err := connection.Data() + if err != nil { + return err + } + _, err = writer.Write(d) + if err != nil { + return err + } + err = writer.Close() + if err != nil { + return err + } + if logger.LogLevel >= 3 { + print(".") + } + } + if logger.LogLevel >= 3 { + print("\n") + } + return connection.Quit() } func mailAddressIsValid(address string) bool { From 368445e3a554288ea27b8a6e095bfb6cc9912cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller=20=28ChaoticByte=29?= Date: Thu, 11 Apr 2024 20:31:48 +0200 Subject: [PATCH 05/15] Updated to go 1.22.2 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 171c292..d8205ab 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/ChaoticByte/wid-notifier -go 1.21 +go 1.22.2 From e9f39d25b4105b91220b037286e617fe9ffb5225 Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Mon, 22 Jul 2024 18:41:22 +0200 Subject: [PATCH 06/15] Moved some code around and updated the README --- .gitignore | 3 ++ README.md | 18 +++++----- config.go | 60 ++++++++++++++++++++++++++++++++ datastore.go | 85 +++------------------------------------------- go.mod | 2 +- mail.go | 51 ++++++---------------------- main.go | 32 ++++++++++++----- persistent_data.go | 21 ++++++++++++ template.go | 8 ++--- widapi.go | 4 +-- 10 files changed, 135 insertions(+), 149 deletions(-) create mode 100644 config.go create mode 100644 persistent_data.go diff --git a/.gitignore b/.gitignore index b64f1e7..4ef31b5 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ dist/ data config *.bak + +# Config/data files +*.json diff --git a/README.md b/README.md index e584424..7b81e52 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ This Software only supports Linux. # Build To cross-compile the software for `i386`, `amd64`, `arm` and `arm64`, run `build.sh`. -You need a go version >= 1.21 and git. +You need Go 1.22.x and git. # Usage @@ -34,7 +34,7 @@ Example: ```json { "api_fetch_interval": 600, - "datafile": "data", + "datafile": "data.json", "enabled_api_endpoints": [ "bay", "bund" @@ -67,9 +67,9 @@ To show debug messages, set the `loglevel` to `3`. ## Filters -You define filters for notices to be sent per recipient. Multiple filters can be set per recipient and multiple criteria can be used per filter. The configuration field for those filters is `include`. See [Configuration](#configuration) for an example. +You define filters for notices to be sent (per recipient). Multiple filters can be set per recipient and multiple criteria can be used per filter. The configuration field for those filters is `include`. See [Configuration](#configuration) for an example. -It is determined by the following logic, if a notice is included: +If a notice is included is determined by the following logic: ``` {criteria, criteria, ... ALL APPLY} @@ -77,7 +77,7 @@ OR {criteria, criteria, ... ALL APPLY} OR ... ``` -The following criteria are available. Criteria marked with `*` are for optional fields that are not supported by every API endpoint (e.g. https://wid.lsi.bayern.de) - notices from those endpoints will therefore not be included when using those criteria in filters. +The following criteria are available. Criteria marked with * are optional fields that are not supported by every API endpoint (e.g. https://wid.lsi.bayern.de) - notices from those endpoints will therefore not be included when using those criteria in filters. ```json "include": [ @@ -122,7 +122,7 @@ Classification can be `"kritisch"`, `"hoch"`, `"mittel"` or `"niedrig"`. ``` If set to `""`, this criteria will be ignored. -### min_basescore `*` +### min_basescore * Include notices whose basescore (`0` - `100`) is >= `min_basescore`. @@ -131,7 +131,7 @@ Include notices whose basescore (`0` - `100`) is >= `min_basescore`. ``` This criteria will be ignored if set to `0`. -### status `*` +### status * Include notices with this status. This is usually either `NEU` or `UPDATE`. @@ -140,7 +140,7 @@ Include notices with this status. This is usually either `NEU` or `UPDATE`. ``` If set to `""`, this criteria will be ignored. -### products_contain `*` +### products_contain * Include notices whose product list contains this text. @@ -149,7 +149,7 @@ Include notices whose product list contains this text. ``` If set to `""`, this criteria will be ignored. -### no_patch `*` +### no_patch * If set to `"true"`, notices where no patch is available will be included. diff --git a/config.go b/config.go new file mode 100644 index 0000000..f808b22 --- /dev/null +++ b/config.go @@ -0,0 +1,60 @@ +// Copyright (c) 2024 Julian Müller (ChaoticByte) + +package main + +import ( + "errors" +) + +type Config struct { + ApiFetchInterval int `json:"api_fetch_interval"` // in seconds + EnabledApiEndpoints []string `json:"enabled_api_endpoints"` + PersistentDataFilePath string `json:"datafile"` + LogLevel int `json:"loglevel"` + Recipients []Recipient `json:"recipients"` + SmtpConfiguration SmtpSettings `json:"smtp"` + Template MailTemplateConfig `json:"template"` +} + +func NewConfig() Config { + // Initial config + c := Config{ + ApiFetchInterval: 60 * 10, // every 10 minutes, + EnabledApiEndpoints: []string{"bay", "bund"}, + PersistentDataFilePath: "data.json", + LogLevel: 2, + Recipients: []Recipient{}, + SmtpConfiguration: SmtpSettings{ + From: "from@example.org", + User: "from@example.org", + Password: "SiEhAbEnMiChInSgEsIcHtGeFiLmTdAsDüRfEnSiEnIcHt", + ServerHost: "example.org", + ServerPort: 587}, + Template: MailTemplateConfig{ + SubjectTemplate: "", + BodyTemplate: "", + }, + } + return c +} + +func checkConfig(config Config) { + if len(config.Recipients) < 1 { + logger.error("Configuration is incomplete") + panic(errors.New("no recipients are configured")) + } + for _, r := range config.Recipients { + if !mailAddressIsValid(r.Address) { + logger.error("Configuration includes invalid data") + panic(errors.New("'" + r.Address + "' is not a valid e-mail address")) + } + if len(r.Filters) < 1 { + logger.error("Configuration is incomplete") + panic(errors.New("recipient " + r.Address + " has no filter defined - at least {'any': true/false} should be configured")) + } + } + if !mailAddressIsValid(config.SmtpConfiguration.From) { + logger.error("Configuration includes invalid data") + panic(errors.New("'" + config.SmtpConfiguration.From + "' is not a valid e-mail address")) + } +} diff --git a/datastore.go b/datastore.go index fbc09e4..128aca9 100644 --- a/datastore.go +++ b/datastore.go @@ -4,79 +4,10 @@ package main import ( "encoding/json" - "errors" "io/fs" "os" - "time" ) -type Config struct { - ApiFetchInterval int `json:"api_fetch_interval"` // in seconds - EnabledApiEndpoints []string `json:"enabled_api_endpoints"` - PersistentDataFilePath string `json:"datafile"` - LogLevel int `json:"loglevel"` - Recipients []Recipient `json:"recipients"` - SmtpConfiguration SmtpSettings `json:"smtp"` - Template MailTemplateConfig `json:"template"` -} - -func NewConfig() Config { - // Initial config - c := Config{ - ApiFetchInterval: 60 * 10, // every 10 minutes, - EnabledApiEndpoints: []string{"bay", "bund"}, - PersistentDataFilePath: "data", - LogLevel: 2, - Recipients: []Recipient{}, - SmtpConfiguration: SmtpSettings{ - From: "from@example.org", - User: "from@example.org", - Password: "SiEhAbEnMiChInSgEsIcHtGeFiLmTdAsDüRfEnSiEnIcHt", - ServerHost: "example.org", - ServerPort: 587}, - Template: MailTemplateConfig{ - SubjectTemplate: "", - BodyTemplate: "", - }, - } - return c -} - -func checkConfig(config Config) { - if len(config.Recipients) < 1 { - logger.error("Configuration is incomplete") - panic(errors.New("no recipients are configured")) - } - for _, r := range config.Recipients { - if !mailAddressIsValid(r.Address) { - logger.error("Configuration includes invalid data") - panic(errors.New("'" + r.Address + "' is not a valid e-mail address")) - } - if len(r.Filters) < 1 { - logger.error("Configuration is incomplete") - panic(errors.New("recipient " + r.Address + " has no filter defined - at least {'any': true/false} should be configured")) - } - } - if !mailAddressIsValid(config.SmtpConfiguration.From) { - logger.error("Configuration includes invalid data") - panic(errors.New("'" + config.SmtpConfiguration.From + "' is not a valid e-mail address")) - } -} - -type PersistentData struct { - // {endpoint id 1: time last published, endpoint id 2: ..., ...} - LastPublished map[string]time.Time `json:"last_published"` -} - -func NewPersistentData(c Config) PersistentData { - // Initial persistent data - d := PersistentData{LastPublished: map[string]time.Time{}} - for _, e := range apiEndpoints { - d.LastPublished[e.Id] = time.Now().Add(-time.Hour * 24) // a day ago - } - return d -} - type DataStore struct { filepath string prettyJSON bool @@ -92,32 +23,24 @@ func (ds *DataStore) save() error { } else { data, err = json.Marshal(ds.data) } - if err != nil { - return err - } + if err != nil { return err } err = os.WriteFile(ds.filepath, data, ds.fileMode) return err } func (ds *DataStore) load() error { data, err := os.ReadFile(ds.filepath) - if err != nil { - return err - } + if err != nil { return err } switch ds.data.(type) { case Config: d, _ := ds.data.(Config); err = json.Unmarshal(data, &d) - if err != nil { - return err - } + if err != nil { return err } ds.data = d case PersistentData: d, _ := ds.data.(PersistentData); err = json.Unmarshal(data, &d) - if err != nil { - return err - } + if err != nil { return err } ds.data = d } return err diff --git a/go.mod b/go.mod index d8205ab..f31f7cb 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/ChaoticByte/wid-notifier -go 1.22.2 +go 1.22 diff --git a/mail.go b/mail.go index af43e38..df9763c 100644 --- a/mail.go +++ b/mail.go @@ -8,7 +8,6 @@ import ( "fmt" "net/mail" "net/smtp" - "slices" "time" ) @@ -49,22 +48,12 @@ type SmtpSettings struct { Password string `json:"password"` } -func (r Recipient) filterAndSendNotices(notices []WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, cache *map[string][]byte) error { - filteredNotices := []WidNotice{} - for _, f := range r.Filters { - for _, n := range f.filter(notices) { - if !noticeSliceContains(filteredNotices, n) { - filteredNotices = append(filteredNotices, n) - } - } - } - slices.Reverse(filteredNotices) - logger.debug(fmt.Sprintf("Including %v of %v notices for recipient %v", len(filteredNotices), len(notices), r.Address)) +func (r Recipient) sendNotices(notices []WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, cache *map[string][]byte) error { logger.debug("Generating and sending mails to " + r.Address + " ...") cacheHits := 0 cacheMisses := 0 mails := [][]byte{} - for _, n := range filteredNotices { + for _, n := range notices { var data []byte cacheResult := (*cache)[n.Uuid] if len(cacheResult) > 0 { @@ -91,9 +80,7 @@ func (r Recipient) filterAndSendNotices(notices []WidNotice, template MailTempla r.Address, mails, ) - if err != nil { - return err - } + if err != nil { return err } logger.debug("Successfully sent all mails to " + r.Address) return nil } @@ -102,50 +89,34 @@ func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, data [][]byte) addr := fmt.Sprintf("%v:%v", smtpConf.ServerHost, smtpConf.ServerPort) logger.debug("Connecting to mail server at " + addr + " ...") connection, err := smtp.Dial(addr) - if err != nil { - return err - } + if err != nil { return err } defer connection.Close() // can leave out connection.Hello hasTlsExt, _ := connection.Extension("starttls") if hasTlsExt { err = connection.StartTLS(&tls.Config{ServerName: smtpConf.ServerHost}) - if err != nil { - return err - } + if err != nil { return err } logger.debug("Mail Server supports TLS") } else { logger.debug("Mail Server doesn't support TLS") } logger.debug("Authenticating to mail server ...") err = connection.Auth(auth) - if err != nil { - return err - } + if err != nil { return err } if logger.LogLevel >= 3 { fmt.Printf("DEBUG %v Sending mails to server ", time.Now().Format("2006/01/02 15:04:05.000000")) } for _, d := range data { err = connection.Mail(smtpConf.From) - if err != nil { - return err - } + if err != nil { return err } err = connection.Rcpt(to) - if err != nil { - return err - } + if err != nil { return err } writer, err := connection.Data() - if err != nil { - return err - } + if err != nil { return err } _, err = writer.Write(d) - if err != nil { - return err - } + if err != nil { return err } err = writer.Close() - if err != nil { - return err - } + if err != nil { return err } if logger.LogLevel >= 3 { print(".") } diff --git a/main.go b/main.go index c39f6bc..cd9c725 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "fmt" "net/smtp" "os" + "slices" "time" ) @@ -24,17 +25,13 @@ func main() { // init logger.info("Initializing ...") defer logger.info("Exiting ...") + // open & check config config := NewDataStore( configFilePath, NewConfig(), true, 0600, ).data.(Config) - persistent := NewDataStore( - config.PersistentDataFilePath, - NewPersistentData(config), - false, - 0640) logger.LogLevel = config.LogLevel logger.debug("Checking configuration file ...") checkConfig(config) @@ -66,6 +63,12 @@ func main() { } } } + // open data file + persistent := NewDataStore( + config.PersistentDataFilePath, + NewPersistentData(config), + false, + 0640) // main loop logger.debug("Entering main loop ...") for { @@ -76,7 +79,7 @@ func main() { logger.info("Querying endpoint '" + a.Id + "' for new notices ...") n, t, err := a.getNotices(persistent.data.(PersistentData).LastPublished[a.Id]) if err != nil { - // retry + // retry (once) logger.warn("Couldn't query notices from API endpoint '" + a.Id + "'. Retrying ...") logger.warn(err) n, t, err = a.getNotices(persistent.data.(PersistentData).LastPublished[a.Id]) @@ -96,7 +99,19 @@ func main() { logger.info("Sending email notifications ...") recipientsNotified := 0 for _, r := range config.Recipients { - err := r.filterAndSendNotices(newNotices, mailTemplate, mailAuth, config.SmtpConfiguration, &cache) + // Filter notices for this recipient + filteredNotices := []WidNotice{} + for _, f := range r.Filters { + for _, n := range f.filter(newNotices) { + if !noticeSliceContains(filteredNotices, n) { + filteredNotices = append(filteredNotices, n) + } + } + } + slices.Reverse(filteredNotices) + logger.debug(fmt.Sprintf("Including %v of %v notices for recipient %v", len(filteredNotices), len(newNotices), r.Address)) + // Send notices + err := r.sendNotices(filteredNotices, mailTemplate, mailAuth, config.SmtpConfiguration, &cache) if err != nil { logger.error(err) } else { @@ -105,8 +120,7 @@ func main() { } logger.info(fmt.Sprintf("Email notifications sent to %v of %v recipients", recipientsNotified, len(config.Recipients))) } - t2 := time.Now().UnixMilli() - dt := int(t2 - t1) + dt := int(time.Now().UnixMilli() - t1) time.Sleep(time.Millisecond * time.Duration((config.ApiFetchInterval * 1000) - dt)) } } diff --git a/persistent_data.go b/persistent_data.go new file mode 100644 index 0000000..182c269 --- /dev/null +++ b/persistent_data.go @@ -0,0 +1,21 @@ +// Copyright (c) 2024 Julian Müller (ChaoticByte) + +package main + +import ( + "time" +) + +type PersistentData struct { + // {endpoint id 1: time last published, endpoint id 2: ..., ...} + LastPublished map[string]time.Time `json:"last_published"` +} + +func NewPersistentData(c Config) PersistentData { + // Initial persistent data + d := PersistentData{LastPublished: map[string]time.Time{}} + for _, e := range apiEndpoints { + d.LastPublished[e.Id] = time.Now().Add(-time.Hour * 24) // a day ago + } + return d +} \ No newline at end of file diff --git a/template.go b/template.go index beda4ac..7869e1a 100644 --- a/template.go +++ b/template.go @@ -42,15 +42,11 @@ func (t MailTemplate) generate(notice WidNotice) (MailContent, error) { c := MailContent{} buffer := &bytes.Buffer{} err := t.SubjectTemplate.Execute(buffer, notice) - if err != nil { - return c, err - } + if err != nil { return c, err } c.Subject = buffer.String() buffer.Truncate(0) // we can recycle our buffer err = t.BodyTemplate.Execute(buffer, notice) - if err != nil { - return c, err - } + if err != nil { return c, err } c.Body = buffer.String() return c, nil } diff --git a/widapi.go b/widapi.go index 28fe259..0ef572e 100644 --- a/widapi.go +++ b/widapi.go @@ -57,9 +57,7 @@ func (e ApiEndpoint) getNotices(since time.Time) ([]WidNotice, time.Time, error) if err == nil { if res.StatusCode == 200 { resBody, err := io.ReadAll(res.Body) - if err != nil { - return []WidNotice{}, since, err - } + if err != nil { return []WidNotice{}, since, err } var decodedData map[string]interface{} if err = json.Unmarshal(resBody, &decodedData); err != nil { return []WidNotice{}, since, err From 9f31bf557dd8e91fe78bee5730e8ce2d65099fc5 Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Mon, 22 Jul 2024 19:55:44 +0200 Subject: [PATCH 07/15] Added the software version to the mail default template and cli help text, updated README and build script --- README.md | 4 +++- build.sh | 19 +++++++++---------- mail.go | 2 +- main.go | 5 ++++- template.go | 17 +++++++++++++---- version.go | 3 +++ 6 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 version.go diff --git a/README.md b/README.md index 7b81e52..1c6ddbd 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# WID Notifier +# WidNotifier The German [BSI](https://www.bsi.bund.de/) and [LSI Bavaria](https://lsi.bayern.de/) each have a page listing current security notices. This software queries the APIs of these services for new security notices and sends configurable email notifications. @@ -202,4 +202,6 @@ type WidNotice struct { } ``` +Additionally, the field `WidNotifierVersion` holds the version of the software. + For an example, take a look at `DEFAULT_SUBJECT_TEMPLATE` and `DEFAULT_BODY_TEMPLATE` in [template.go](./template.go). diff --git a/build.sh b/build.sh index 7ac90ca..8418388 100755 --- a/build.sh +++ b/build.sh @@ -2,14 +2,13 @@ VERSION=$(git describe --tags) -# i386 -GOOS=linux GOARCH=386 go build -o dist/wid-notifier_${VERSION}_linux_i386 +function build() { + printf "Compiling version ${VERSION} for ${1}/${2}\t" + GOOS=${1} GOARCH=${2} go build -ldflags "-X 'main.Version=${VERSION}'" -o dist/wid-notifier_${VERSION}_${1}_${3} + echo "✅" +} -# amd64 -GOOS=linux GOARCH=amd64 go build -o dist/wid-notifier_${VERSION}_linux_amd64 - -# arm -GOOS=linux GOARCH=arm go build -o dist/wid-notifier_${VERSION}_linux_arm - -# arm64 -GOOS=linux GOARCH=arm64 go build -o dist/wid-notifier_${VERSION}_linux_arm64 +build linux "386" i386 +build linux amd64 amd64 +build linux arm arm +build linux arm64 arm64 diff --git a/mail.go b/mail.go index df9763c..e4c5890 100644 --- a/mail.go +++ b/mail.go @@ -61,7 +61,7 @@ func (r Recipient) sendNotices(notices []WidNotice, template MailTemplate, auth data = cacheResult } else { cacheMisses++ - mailContent, err := template.generate(n) + mailContent, err := template.generate(TemplateData{n, Version}) if err != nil { logger.error("Could not create mail from template") logger.error(err) diff --git a/main.go b/main.go index cd9c725..47ce30b 100644 --- a/main.go +++ b/main.go @@ -16,7 +16,10 @@ func main() { // get cli arguments args := os.Args if len(args) < 2 { - fmt.Printf("Usage: %v \nIf the config file doesn't exist, a incomplete configuration with default values is created.\n", args[0]) + fmt.Printf( "Usage: %v \n\nIf the config file doesn't exist, an incomplete \n" + + "configuration with default values is created.\n\nVersion: %s\n", + args[0], + Version) os.Exit(1) } configFilePath := os.Args[1] diff --git a/template.go b/template.go index 7869e1a..7d078e8 100644 --- a/template.go +++ b/template.go @@ -26,7 +26,16 @@ Affected Products:{{ range $product := .ProductNames }} Assigned CVEs:{{ range $cve := .Cves }} - {{ $cve }} -> https://www.cve.org/CVERecord?id={{ $cve }} -{{- end }}{{ end }}` +{{- end }}{{ end }} + + +Sent by WidNotifier {{ .WidNotifierVersion }} +` + +type TemplateData struct { + WidNotice + WidNotifierVersion string +} type MailTemplateConfig struct { SubjectTemplate string `json:"subject"` @@ -38,14 +47,14 @@ type MailTemplate struct { BodyTemplate template.Template } -func (t MailTemplate) generate(notice WidNotice) (MailContent, error) { +func (t MailTemplate) generate(data TemplateData) (MailContent, error) { c := MailContent{} buffer := &bytes.Buffer{} - err := t.SubjectTemplate.Execute(buffer, notice) + err := t.SubjectTemplate.Execute(buffer, data) if err != nil { return c, err } c.Subject = buffer.String() buffer.Truncate(0) // we can recycle our buffer - err = t.BodyTemplate.Execute(buffer, notice) + err = t.BodyTemplate.Execute(buffer, data) if err != nil { return c, err } c.Body = buffer.String() return c, nil diff --git a/version.go b/version.go new file mode 100644 index 0000000..f0ce3c0 --- /dev/null +++ b/version.go @@ -0,0 +1,3 @@ +package main + +var Version = "devel" From b3290c357d7d60033a6669002d83267095a140d3 Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Wed, 11 Jun 2025 17:29:14 +0200 Subject: [PATCH 08/15] Update to go 1.24, small code cleanup --- go.mod | 2 +- mail.go | 2 +- notice.go | 10 ---------- persistent_data.go | 2 +- 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index f31f7cb..acbf5d6 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/ChaoticByte/wid-notifier -go 1.22 +go 1.24 diff --git a/mail.go b/mail.go index e4c5890..07477ba 100644 --- a/mail.go +++ b/mail.go @@ -66,7 +66,7 @@ func (r Recipient) sendNotices(notices []WidNotice, template MailTemplate, auth logger.error("Could not create mail from template") logger.error(err) } - // serialize & send mail + // serialize mail data = mailContent.serializeValidMail(smtpConfig.From, r.Address) // add to cache (*cache)[n.Uuid] = data diff --git a/notice.go b/notice.go index 8967d20..758d85d 100644 --- a/notice.go +++ b/notice.go @@ -25,16 +25,6 @@ type WidNotice struct { PortalUrl string } -// func (n WidNotice) serialized() ([]byte, error) { -// return json.Marshal(n) -// } - -// func NewWidNoticeFromJSON(data []byte) (WidNotice, error) { -// n := WidNotice{} -// err := json.Unmarshal(data, &n) -// return n, err -// } - func noticeSliceContains(notices []WidNotice, notice WidNotice) bool { for _, x := range notices { if x.Uuid == notice.Uuid { diff --git a/persistent_data.go b/persistent_data.go index 182c269..f3ce9ce 100644 --- a/persistent_data.go +++ b/persistent_data.go @@ -18,4 +18,4 @@ func NewPersistentData(c Config) PersistentData { d.LastPublished[e.Id] = time.Now().Add(-time.Hour * 24) // a day ago } return d -} \ No newline at end of file +} From 2e1aee0313884d0edb573a6991a44a8e93a0cabe Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Wed, 11 Jun 2025 17:45:14 +0200 Subject: [PATCH 09/15] Parse -h, --help and --version and show help, version --- main.go | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index 47ce30b..e2fea3a 100644 --- a/main.go +++ b/main.go @@ -10,19 +10,39 @@ import ( "time" ) +var executableName string var logger Logger + +func showVersion() { + fmt.Printf("wid-notifier %s\n", Version) +} + +func showHelp() { + fmt.Printf("Usage: %v \n\nIf the config file doesn't exist, an incomplete \n" + + "configuration with default values is created.\n\n", + executableName) + showVersion() +} + func main() { // get cli arguments args := os.Args + executableName = args[0] if len(args) < 2 { - fmt.Printf( "Usage: %v \n\nIf the config file doesn't exist, an incomplete \n" + - "configuration with default values is created.\n\nVersion: %s\n", - args[0], - Version) + showHelp() os.Exit(1) } - configFilePath := os.Args[1] + for _, arg := range args { + if arg == "-h" || arg == "--help" { + showHelp() + os.Exit(0) + } else if arg == "--version" { + showVersion() + os.Exit(0) + } + } + configFilePath := args[1] // create logger logger = NewLogger(2) // init From ef24503214c05554973af86de38836207c5a7986 Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Wed, 11 Jun 2025 22:05:44 +0200 Subject: [PATCH 10/15] fix: don't save last published timestamp when email sending fails, so they can be sent in the next iteration --- main.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index e2fea3a..62de84a 100644 --- a/main.go +++ b/main.go @@ -97,7 +97,8 @@ func main() { for { t1 := time.Now().UnixMilli() newNotices := []WidNotice{} - cache := map[string][]byte{} + lastPublished := map[string]time.Time{} // endpoint id : last published timestamp + cache := map[string][]byte{} // cache generated emails for reuse for _, a := range enabledApiEndpoints { logger.info("Querying endpoint '" + a.Id + "' for new notices ...") n, t, err := a.getNotices(persistent.data.(PersistentData).LastPublished[a.Id]) @@ -113,14 +114,14 @@ func main() { logger.error(err) } else if len(n) > 0 { newNotices = append(newNotices, n...) - persistent.data.(PersistentData).LastPublished[a.Id] = t - persistent.save() + lastPublished[a.Id] = t } } logger.debug(fmt.Sprintf("Got %v new notices", len(newNotices))) if len(newNotices) > 0 { logger.info("Sending email notifications ...") recipientsNotified := 0 + var err error for _, r := range config.Recipients { // Filter notices for this recipient filteredNotices := []WidNotice{} @@ -134,14 +135,22 @@ func main() { slices.Reverse(filteredNotices) logger.debug(fmt.Sprintf("Including %v of %v notices for recipient %v", len(filteredNotices), len(newNotices), r.Address)) // Send notices - err := r.sendNotices(filteredNotices, mailTemplate, mailAuth, config.SmtpConfiguration, &cache) + err = r.sendNotices(filteredNotices, mailTemplate, mailAuth, config.SmtpConfiguration, &cache) if err != nil { logger.error(err) } else { recipientsNotified++ } } - logger.info(fmt.Sprintf("Email notifications sent to %v of %v recipients", recipientsNotified, len(config.Recipients))) + if recipientsNotified < 1 && err != nil { + logger.error("Couldn't send any mail notification!") + } else { + for id, t := range lastPublished { + persistent.data.(PersistentData).LastPublished[id] = t + persistent.save() + } + logger.info(fmt.Sprintf("Email notifications sent to %v of %v recipients", recipientsNotified, len(config.Recipients))) + } } dt := int(time.Now().UnixMilli() - t1) time.Sleep(time.Millisecond * time.Duration((config.ApiFetchInterval * 1000) - dt)) From 295ceec3de248a8f77f35c6113d43831569dc38a Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Thu, 12 Jun 2025 00:00:52 +0200 Subject: [PATCH 11/15] Improve config file structure by combining multiple recipients into lists (breaking) --- README.md | 21 +++++++++++---------- config.go | 36 +++++++++++++++++++++--------------- mail.go | 21 +++++++++++---------- main.go | 35 ++++++++++++++++++++++++----------- 4 files changed, 67 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 1c6ddbd..e04f337 100644 --- a/README.md +++ b/README.md @@ -34,27 +34,28 @@ Example: ```json { "api_fetch_interval": 600, - "datafile": "data.json", "enabled_api_endpoints": [ "bay", "bund" ], + "datafile": "data.json", "loglevel": 2, - "recipients": [ + "lists": [ { - "address": "guenther@example.org", - "include": [ - {"classification": "kritisch"}, - {"title_contains": "jQuery"} + "name": "Example List", + "recipients": ["someone@example.org"], + "filter": [ + {"classification": "hoch", "title_contains": "Microsoft"}, + {"classification": "kritisch"} ] } ], "smtp": { - "from": "from@example.org", - "host": "example.org", + "from": "user@localhost", + "host": "127.0.0.1", "port": 587, - "user": "from@example.org", - "password": "SiEhAbEnMiChInSgEsIcHtGeFiLmTdAsDüRfEnSiEnIcHt" + "user": "user@localhost", + "password": "change me :)" }, "template": { "subject": "", diff --git a/config.go b/config.go index f808b22..8785375 100644 --- a/config.go +++ b/config.go @@ -11,7 +11,7 @@ type Config struct { EnabledApiEndpoints []string `json:"enabled_api_endpoints"` PersistentDataFilePath string `json:"datafile"` LogLevel int `json:"loglevel"` - Recipients []Recipient `json:"recipients"` + Lists *[]NotifyList `json:"lists"` SmtpConfiguration SmtpSettings `json:"smtp"` Template MailTemplateConfig `json:"template"` } @@ -23,12 +23,16 @@ func NewConfig() Config { EnabledApiEndpoints: []string{"bay", "bund"}, PersistentDataFilePath: "data.json", LogLevel: 2, - Recipients: []Recipient{}, + Lists: &[]NotifyList{ + { Name: "Example List", + Recipients: []string{}, + Filter: []Filter{},}, + }, SmtpConfiguration: SmtpSettings{ - From: "from@example.org", - User: "from@example.org", - Password: "SiEhAbEnMiChInSgEsIcHtGeFiLmTdAsDüRfEnSiEnIcHt", - ServerHost: "example.org", + From: "user@localhost", + User: "user@localhost", + Password: "change me :)", + ServerHost: "127.0.0.1", ServerPort: 587}, Template: MailTemplateConfig{ SubjectTemplate: "", @@ -39,18 +43,20 @@ func NewConfig() Config { } func checkConfig(config Config) { - if len(config.Recipients) < 1 { + if len(*config.Lists) < 1 { logger.error("Configuration is incomplete") - panic(errors.New("no recipients are configured")) + panic(errors.New("no lists are configured")) } - for _, r := range config.Recipients { - if !mailAddressIsValid(r.Address) { - logger.error("Configuration includes invalid data") - panic(errors.New("'" + r.Address + "' is not a valid e-mail address")) - } - if len(r.Filters) < 1 { + for _, l := range *config.Lists { + if len(l.Filter) < 1 { logger.error("Configuration is incomplete") - panic(errors.New("recipient " + r.Address + " has no filter defined - at least {'any': true/false} should be configured")) + panic(errors.New("list " + l.Name + " has no filter defined - at least [{'any': true/false}] should be configured")) + } + for _, r := range l.Recipients { + if !mailAddressIsValid(r) { + logger.error("Configuration includes invalid data") + panic(errors.New("'" + r + "' is not a valid e-mail address")) + } } } if !mailAddressIsValid(config.SmtpConfiguration.From) { diff --git a/mail.go b/mail.go index 07477ba..d2f6ae4 100644 --- a/mail.go +++ b/mail.go @@ -23,21 +23,22 @@ func (c MailContent) serializeValidMail(from string, to string) []byte { // and I'm too lazy to encode ä into =E4 and so on subjectEncoded := base64.StdEncoding.EncodeToString([]byte(c.Subject)) bodyEncoded := base64.StdEncoding.EncodeToString([]byte(c.Body)) - data := []byte(fmt.Sprintf( + data := fmt.Appendf(nil, "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\nFrom: %v%vTo: %v%vSubject: =?utf-8?b?%v?=%v%v%v", from, MAIL_LINE_SEP, to, MAIL_LINE_SEP, subjectEncoded, MAIL_LINE_SEP, MAIL_LINE_SEP, - bodyEncoded)) + bodyEncoded) // done, I guess return data } -type Recipient struct { - Address string `json:"address"` +type NotifyList struct { + Name string `json:"name"` + Recipients []string `json:"recipients"` // Must be a configured filter id - Filters []Filter `json:"include"` + Filter []Filter `json:"filter"` } type SmtpSettings struct { @@ -48,8 +49,8 @@ type SmtpSettings struct { Password string `json:"password"` } -func (r Recipient) sendNotices(notices []WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, cache *map[string][]byte) error { - logger.debug("Generating and sending mails to " + r.Address + " ...") +func sendNotices(recipient string, notices []WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, cache *map[string][]byte) error { + logger.debug("Generating and sending mails for recipient " + recipient + " ...") cacheHits := 0 cacheMisses := 0 mails := [][]byte{} @@ -67,7 +68,7 @@ func (r Recipient) sendNotices(notices []WidNotice, template MailTemplate, auth logger.error(err) } // serialize mail - data = mailContent.serializeValidMail(smtpConfig.From, r.Address) + data = mailContent.serializeValidMail(smtpConfig.From, recipient) // add to cache (*cache)[n.Uuid] = data } @@ -77,11 +78,11 @@ func (r Recipient) sendNotices(notices []WidNotice, template MailTemplate, auth err := sendMails( smtpConfig, auth, - r.Address, + recipient, mails, ) if err != nil { return err } - logger.debug("Successfully sent all mails to " + r.Address) + logger.debug("Successfully sent all mails to " + recipient) return nil } diff --git a/main.go b/main.go index 62de84a..89456e3 100644 --- a/main.go +++ b/main.go @@ -120,22 +120,35 @@ func main() { logger.debug(fmt.Sprintf("Got %v new notices", len(newNotices))) if len(newNotices) > 0 { logger.info("Sending email notifications ...") + // mail recipient : pointer to slice of wid notices to be sent + noticesToBeSent := map[string][]WidNotice{} recipientsNotified := 0 var err error - for _, r := range config.Recipients { - // Filter notices for this recipient - filteredNotices := []WidNotice{} - for _, f := range r.Filters { + for _, l := range *config.Lists { + // Filter notices for this list + for _, f := range l.Filter { for _, n := range f.filter(newNotices) { - if !noticeSliceContains(filteredNotices, n) { - filteredNotices = append(filteredNotices, n) + for _, r := range l.Recipients { + if !noticeSliceContains(noticesToBeSent[r], n) { + noticesToBeSent[r] = append(noticesToBeSent[r], n) + } } } } - slices.Reverse(filteredNotices) - logger.debug(fmt.Sprintf("Including %v of %v notices for recipient %v", len(filteredNotices), len(newNotices), r.Address)) - // Send notices - err = r.sendNotices(filteredNotices, mailTemplate, mailAuth, config.SmtpConfiguration, &cache) + } + for r, notices := range noticesToBeSent { + // sort by publish date + slices.SortFunc(notices, func(a WidNotice, b WidNotice) int { + if a.Published == b.Published { + return 0 + } else if a.Published.After(b.Published) { + return 1 + } else { + return -1 + } + }) + // send + err = sendNotices(r, notices, mailTemplate, mailAuth, config.SmtpConfiguration, &cache) if err != nil { logger.error(err) } else { @@ -149,7 +162,7 @@ func main() { persistent.data.(PersistentData).LastPublished[id] = t persistent.save() } - logger.info(fmt.Sprintf("Email notifications sent to %v of %v recipients", recipientsNotified, len(config.Recipients))) + logger.info(fmt.Sprintf("Email notifications sent to %v of %v recipients", recipientsNotified, len(noticesToBeSent))) } } dt := int(time.Now().UnixMilli() - t1) From 9b00959fdb6d0c6a746b6cd83f4f041fba070acd Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Thu, 12 Jun 2025 17:23:08 +0200 Subject: [PATCH 12/15] Improve memory efficiency of noticesToBeSent by mapping pointers instead of WidNotice structs --- mail.go | 2 +- main.go | 9 +++++---- notice.go | 2 +- template.go | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/mail.go b/mail.go index d2f6ae4..7cefa09 100644 --- a/mail.go +++ b/mail.go @@ -49,7 +49,7 @@ type SmtpSettings struct { Password string `json:"password"` } -func sendNotices(recipient string, notices []WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, cache *map[string][]byte) error { +func sendNotices(recipient string, notices []*WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, cache *map[string][]byte) error { logger.debug("Generating and sending mails for recipient " + recipient + " ...") cacheHits := 0 cacheMisses := 0 diff --git a/main.go b/main.go index 89456e3..8fc4ab0 100644 --- a/main.go +++ b/main.go @@ -121,16 +121,17 @@ func main() { if len(newNotices) > 0 { logger.info("Sending email notifications ...") // mail recipient : pointer to slice of wid notices to be sent - noticesToBeSent := map[string][]WidNotice{} + noticesToBeSent := map[string][]*WidNotice{} recipientsNotified := 0 var err error for _, l := range *config.Lists { // Filter notices for this list for _, f := range l.Filter { for _, n := range f.filter(newNotices) { + np := &n for _, r := range l.Recipients { - if !noticeSliceContains(noticesToBeSent[r], n) { - noticesToBeSent[r] = append(noticesToBeSent[r], n) + if !noticeSliceContains(noticesToBeSent[r], np) { + noticesToBeSent[r] = append(noticesToBeSent[r], np) } } } @@ -138,7 +139,7 @@ func main() { } for r, notices := range noticesToBeSent { // sort by publish date - slices.SortFunc(notices, func(a WidNotice, b WidNotice) int { + slices.SortFunc(notices, func(a *WidNotice, b *WidNotice) int { if a.Published == b.Published { return 0 } else if a.Published.After(b.Published) { diff --git a/notice.go b/notice.go index 758d85d..393f2c9 100644 --- a/notice.go +++ b/notice.go @@ -25,7 +25,7 @@ type WidNotice struct { PortalUrl string } -func noticeSliceContains(notices []WidNotice, notice WidNotice) bool { +func noticeSliceContains(notices []*WidNotice, notice *WidNotice) bool { for _, x := range notices { if x.Uuid == notice.Uuid { return true diff --git a/template.go b/template.go index 7d078e8..b07434c 100644 --- a/template.go +++ b/template.go @@ -33,7 +33,7 @@ Sent by WidNotifier {{ .WidNotifierVersion }} ` type TemplateData struct { - WidNotice + *WidNotice WidNotifierVersion string } From 30bf7935980bb87e386020acb402d252fde6c89f Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Thu, 12 Jun 2025 20:13:08 +0200 Subject: [PATCH 13/15] Encode mail subject as rfc2047 Q-Encoding and body as rfc2045 Quoted-Printable Encoding --- mail.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/mail.go b/mail.go index 7cefa09..a37c933 100644 --- a/mail.go +++ b/mail.go @@ -4,33 +4,33 @@ package main import ( "crypto/tls" - "encoding/base64" "fmt" + "mime" + "mime/quotedprintable" "net/mail" "net/smtp" + "strings" "time" ) -const MAIL_LINE_SEP = "\r\n" - type MailContent struct { Subject string Body string } func (c MailContent) serializeValidMail(from string, to string) []byte { - // We'll send base64 encoded Subject & Body, because we Dschörmäns have umlauts - // and I'm too lazy to encode ä into =E4 and so on - subjectEncoded := base64.StdEncoding.EncodeToString([]byte(c.Subject)) - bodyEncoded := base64.StdEncoding.EncodeToString([]byte(c.Body)) + // format subject using Q Encoding from RFC2047 + subjectEncoded := mime.QEncoding.Encode("utf-8", c.Subject) + // format body using Quoted-Printable Encoding from RFC2045 + var bodyEncoded strings.Builder + bew := quotedprintable.NewWriter(&bodyEncoded) + bew.Write([]byte(c.Body)) + bew.Close() + // glue it all together data := fmt.Appendf(nil, - "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\nFrom: %v%vTo: %v%vSubject: =?utf-8?b?%v?=%v%v%v", - from, MAIL_LINE_SEP, - to, MAIL_LINE_SEP, - subjectEncoded, MAIL_LINE_SEP, - MAIL_LINE_SEP, - bodyEncoded) - // done, I guess + "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: Quoted-Printable\r\nFrom: %v\r\nTo: %v\r\nSubject: %v\r\n\r\n%v", + from, to, subjectEncoded, bodyEncoded.String(), + ) return data } From a4fd43e03d81334a36ab701b2c21c8a73e4bd5f0 Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Mon, 16 Jun 2025 17:00:58 +0200 Subject: [PATCH 14/15] Allow safe debugging of mail transfer by adding debug_mail_transfer build flag --- mail.go => mail_common.go | 44 ------------------------------- mail_transfer.go | 54 +++++++++++++++++++++++++++++++++++++++ mail_transfer_debug.go | 23 +++++++++++++++++ 3 files changed, 77 insertions(+), 44 deletions(-) rename mail.go => mail_common.go (63%) create mode 100644 mail_transfer.go create mode 100644 mail_transfer_debug.go diff --git a/mail.go b/mail_common.go similarity index 63% rename from mail.go rename to mail_common.go index a37c933..fee72f9 100644 --- a/mail.go +++ b/mail_common.go @@ -3,14 +3,12 @@ package main import ( - "crypto/tls" "fmt" "mime" "mime/quotedprintable" "net/mail" "net/smtp" "strings" - "time" ) type MailContent struct { @@ -86,48 +84,6 @@ func sendNotices(recipient string, notices []*WidNotice, template MailTemplate, return nil } -func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, data [][]byte) error { - addr := fmt.Sprintf("%v:%v", smtpConf.ServerHost, smtpConf.ServerPort) - logger.debug("Connecting to mail server at " + addr + " ...") - connection, err := smtp.Dial(addr) - if err != nil { return err } - defer connection.Close() - // can leave out connection.Hello - hasTlsExt, _ := connection.Extension("starttls") - if hasTlsExt { - err = connection.StartTLS(&tls.Config{ServerName: smtpConf.ServerHost}) - if err != nil { return err } - logger.debug("Mail Server supports TLS") - } else { - logger.debug("Mail Server doesn't support TLS") - } - logger.debug("Authenticating to mail server ...") - err = connection.Auth(auth) - if err != nil { return err } - if logger.LogLevel >= 3 { - fmt.Printf("DEBUG %v Sending mails to server ", time.Now().Format("2006/01/02 15:04:05.000000")) - } - for _, d := range data { - err = connection.Mail(smtpConf.From) - if err != nil { return err } - err = connection.Rcpt(to) - if err != nil { return err } - writer, err := connection.Data() - if err != nil { return err } - _, err = writer.Write(d) - if err != nil { return err } - err = writer.Close() - if err != nil { return err } - if logger.LogLevel >= 3 { - print(".") - } - } - if logger.LogLevel >= 3 { - print("\n") - } - return connection.Quit() -} - func mailAddressIsValid(address string) bool { _, err := mail.ParseAddress(address); return err == nil diff --git a/mail_transfer.go b/mail_transfer.go new file mode 100644 index 0000000..1013f0b --- /dev/null +++ b/mail_transfer.go @@ -0,0 +1,54 @@ +// +build !debug_mail_transfer +// Copyright (c) 2023 Julian Müller (ChaoticByte) + +package main + +import ( + "crypto/tls" + "fmt" + "net/smtp" + "time" +) + + +func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, data [][]byte) error { + addr := fmt.Sprintf("%v:%v", smtpConf.ServerHost, smtpConf.ServerPort) + logger.debug("Connecting to mail server at " + addr + " ...") + connection, err := smtp.Dial(addr) + if err != nil { return err } + defer connection.Close() + // can leave out connection.Hello + hasTlsExt, _ := connection.Extension("starttls") + if hasTlsExt { + err = connection.StartTLS(&tls.Config{ServerName: smtpConf.ServerHost}) + if err != nil { return err } + logger.debug("Mail Server supports StartTLS") + } else { + logger.debug("Mail Server doesn't support StartTLS") + } + logger.debug("Authenticating to mail server ...") + err = connection.Auth(auth) + if err != nil { return err } + if logger.LogLevel >= 3 { + fmt.Printf("DEBUG %v Sending mails to server ", time.Now().Format("2006/01/02 15:04:05.000000")) + } + for _, d := range data { + err = connection.Mail(smtpConf.From) + if err != nil { return err } + err = connection.Rcpt(to) + if err != nil { return err } + writer, err := connection.Data() + if err != nil { return err } + _, err = writer.Write(d) + if err != nil { return err } + err = writer.Close() + if err != nil { return err } + if logger.LogLevel >= 3 { + print(".") + } + } + if logger.LogLevel >= 3 { + print("\n") + } + return connection.Quit() +} diff --git a/mail_transfer_debug.go b/mail_transfer_debug.go new file mode 100644 index 0000000..d3846a3 --- /dev/null +++ b/mail_transfer_debug.go @@ -0,0 +1,23 @@ +// +build debug_mail_transfer +// Copyright (c) 2023 Julian Müller (ChaoticByte) + +package main + +import ( + "fmt" + "net/smtp" +) + +func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, data [][]byte) error { + logger.warn("Mail Transfer Debugging is active. Not connecting.") + logger.info("MAIL TRANSFER: \n\n") + for _, d := range data { + fmt.Println("MAIL FROM:" + smtpConf.From) + fmt.Println("RCPT TO:" + to) + fmt.Println("DATA") + fmt.Println(string(d)) + fmt.Println(".") + } + fmt.Print("\n\n") + return nil +} From c1b3d106a0005c62ed40c6d9454cf6484a4975a8 Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Mon, 16 Jun 2025 17:29:38 +0200 Subject: [PATCH 15/15] Fix leak of first recipient to other recipients because mail headers are also cached and re-used --- mail_common.go | 24 ++++++++++++------------ mail_transfer.go | 7 +++++-- mail_transfer_debug.go | 7 +++++-- main.go | 2 +- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/mail_common.go b/mail_common.go index fee72f9..71e29bf 100644 --- a/mail_common.go +++ b/mail_common.go @@ -47,30 +47,30 @@ type SmtpSettings struct { Password string `json:"password"` } -func sendNotices(recipient string, notices []*WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, cache *map[string][]byte) error { +func sendNotices(recipient string, notices []*WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, mailContentCache *map[string]*MailContent) error { logger.debug("Generating and sending mails for recipient " + recipient + " ...") cacheHits := 0 cacheMisses := 0 - mails := [][]byte{} + mails := []*MailContent{} for _, n := range notices { - var data []byte - cacheResult := (*cache)[n.Uuid] - if len(cacheResult) > 0 { + var mc *MailContent + cacheResult := (*mailContentCache)[n.Uuid] + if cacheResult != nil { cacheHits++ - data = cacheResult + mc = cacheResult } else { cacheMisses++ - mailContent, err := template.generate(TemplateData{n, Version}) + mc_, err := template.generate(TemplateData{n, Version}) if err != nil { logger.error("Could not create mail from template") logger.error(err) + } else { + mc = &mc_ + // add to cache + (*mailContentCache)[n.Uuid] = mc } - // serialize mail - data = mailContent.serializeValidMail(smtpConfig.From, recipient) - // add to cache - (*cache)[n.Uuid] = data } - mails = append(mails, data) + mails = append(mails, mc) } logger.debug(fmt.Sprintf("%v mail cache hits, %v misses", cacheHits, cacheMisses)) err := sendMails( diff --git a/mail_transfer.go b/mail_transfer.go index 1013f0b..45675d3 100644 --- a/mail_transfer.go +++ b/mail_transfer.go @@ -11,7 +11,7 @@ import ( ) -func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, data [][]byte) error { +func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, mails []*MailContent) error { addr := fmt.Sprintf("%v:%v", smtpConf.ServerHost, smtpConf.ServerPort) logger.debug("Connecting to mail server at " + addr + " ...") connection, err := smtp.Dial(addr) @@ -32,7 +32,10 @@ func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, data [][]byte) if logger.LogLevel >= 3 { fmt.Printf("DEBUG %v Sending mails to server ", time.Now().Format("2006/01/02 15:04:05.000000")) } - for _, d := range data { + for _, mc := range mails { + // serialize mail + d := mc.serializeValidMail(smtpConf.From, to) + // send mail err = connection.Mail(smtpConf.From) if err != nil { return err } err = connection.Rcpt(to) diff --git a/mail_transfer_debug.go b/mail_transfer_debug.go index d3846a3..60fb2bd 100644 --- a/mail_transfer_debug.go +++ b/mail_transfer_debug.go @@ -8,10 +8,13 @@ import ( "net/smtp" ) -func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, data [][]byte) error { +func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, mails []*MailContent) error { logger.warn("Mail Transfer Debugging is active. Not connecting.") logger.info("MAIL TRANSFER: \n\n") - for _, d := range data { + for _, mc := range mails { + // serialize mail + d := mc.serializeValidMail(smtpConf.From, to) + // output mail fmt.Println("MAIL FROM:" + smtpConf.From) fmt.Println("RCPT TO:" + to) fmt.Println("DATA") diff --git a/main.go b/main.go index 8fc4ab0..d239bf0 100644 --- a/main.go +++ b/main.go @@ -98,7 +98,7 @@ func main() { t1 := time.Now().UnixMilli() newNotices := []WidNotice{} lastPublished := map[string]time.Time{} // endpoint id : last published timestamp - cache := map[string][]byte{} // cache generated emails for reuse + cache := map[string]*MailContent{} // cache generated emails for reuse for _, a := range enabledApiEndpoints { logger.info("Querying endpoint '" + a.Id + "' for new notices ...") n, t, err := a.getNotices(persistent.data.(PersistentData).LastPublished[a.Id])