Moved some code around and updated the README

.gitignore

@@ -24,3 +24,6 @@ dist/
 data
 config
 *.bak
+
+# Config/data files
+*.json

README.md

@@ -17,7 +17,7 @@ This Software only supports Linux.
 # Build
 
 To cross-compile the software for `i386`, `amd64`, `arm` and `arm64`, run `build.sh`.  
-You need a go version >= 1.21 and git.
+You need Go 1.22.x and git.
 
 # Usage
 
@@ -34,7 +34,7 @@ Example:
 ```json
 {
   "api_fetch_interval": 600,
-  "datafile": "data",
+  "datafile": "data.json",
   "enabled_api_endpoints": [
     "bay",
     "bund"
@@ -67,9 +67,9 @@ To show debug messages, set the `loglevel` to `3`.
 
 ## Filters
 
-You define filters for notices to be sent per recipient. Multiple filters can be set per recipient and multiple criteria can be used per filter. The configuration field for those filters is `include`. See [Configuration](#configuration) for an example.
+You define filters for notices to be sent (per recipient). Multiple filters can be set per recipient and multiple criteria can be used per filter. The configuration field for those filters is `include`. See [Configuration](#configuration) for an example.
 
-It is determined by the following logic, if a notice is included:
+If a notice is included is determined by the following logic:
 
 ```
 {criteria, criteria, ... ALL APPLY}
@@ -77,7 +77,7 @@ OR {criteria, criteria, ... ALL APPLY}
 OR ...
 ```
 
-The following criteria are available. Criteria marked with `*` are for optional fields that are not supported by every API endpoint (e.g. https://wid.lsi.bayern.de) - notices from those endpoints will therefore not be included when using those criteria in filters.
+The following criteria are available. Criteria marked with * are optional fields that are not supported by every API endpoint (e.g. https://wid.lsi.bayern.de) - notices from those endpoints will therefore not be included when using those criteria in filters.
 
 ```json
 "include": [
@@ -122,7 +122,7 @@ Classification can be `"kritisch"`, `"hoch"`, `"mittel"` or `"niedrig"`.
 ```
 If set to `""`, this criteria will be ignored.
 
-### min_basescore `*`
+### min_basescore *
 
 Include notices whose basescore (`0` - `100`) is >= `min_basescore`.
 
@@ -131,7 +131,7 @@ Include notices whose basescore (`0` - `100`) is >= `min_basescore`.
 ```
 This criteria will be ignored if set to `0`.
 
-### status `*`
+### status *
 
 Include notices with this status. This is usually either `NEU` or `UPDATE`.
 
@@ -140,7 +140,7 @@ Include notices with this status. This is usually either `NEU` or `UPDATE`.
 ```
 If set to `""`, this criteria will be ignored.
 
-### products_contain `*`
+### products_contain *
 
 Include notices whose product list contains this text.
 
@@ -149,7 +149,7 @@ Include notices whose product list contains this text.
 ```
 If set to `""`, this criteria will be ignored.
 
-### no_patch `*`
+### no_patch *
 
 If set to `"true"`, notices where no patch is available will be included.
 

config.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2024 Julian Müller (ChaoticByte)
+
+package main
+
+import (
+	"errors"
+)
+
+type Config struct {
+	ApiFetchInterval int `json:"api_fetch_interval"` // in seconds
+	EnabledApiEndpoints []string `json:"enabled_api_endpoints"`
+	PersistentDataFilePath string `json:"datafile"`
+	LogLevel int `json:"loglevel"`
+	Recipients []Recipient `json:"recipients"`
+	SmtpConfiguration SmtpSettings `json:"smtp"`
+	Template MailTemplateConfig `json:"template"`
+}
+
+func NewConfig() Config {
+	// Initial config
+	c := Config{
+		ApiFetchInterval: 60 * 10, // every 10 minutes,
+		EnabledApiEndpoints: []string{"bay", "bund"},
+		PersistentDataFilePath: "data.json",
+		LogLevel: 2,
+		Recipients: []Recipient{},
+		SmtpConfiguration: SmtpSettings{
+			From: "from@example.org",
+			User: "from@example.org",
+			Password: "SiEhAbEnMiChInSgEsIcHtGeFiLmTdAsDüRfEnSiEnIcHt",
+			ServerHost: "example.org",
+			ServerPort: 587},
+		Template: MailTemplateConfig{
+			SubjectTemplate: "",
+			BodyTemplate: "",
+		},
+	}
+	return c
+}
+
+func checkConfig(config Config) {
+	if len(config.Recipients) < 1 {
+		logger.error("Configuration is incomplete")
+		panic(errors.New("no recipients are configured"))
+	}
+	for _, r := range config.Recipients {
+		if !mailAddressIsValid(r.Address) {
+			logger.error("Configuration includes invalid data")
+			panic(errors.New("'" + r.Address + "' is not a valid e-mail address"))
+		}
+		if len(r.Filters) < 1 {
+			logger.error("Configuration is incomplete")
+			panic(errors.New("recipient " + r.Address + " has no filter defined - at least {'any': true/false} should be configured"))
+		}
+	}
+	if !mailAddressIsValid(config.SmtpConfiguration.From) {
+		logger.error("Configuration includes invalid data")
+		panic(errors.New("'" + config.SmtpConfiguration.From + "' is not a valid e-mail address"))
+	}
+}

datastore.go

@@ -4,79 +4,10 @@ package main
 
 import (
 	"encoding/json"
-	"errors"
 	"io/fs"
 	"os"
-	"time"
 )
 
-type Config struct {
-	ApiFetchInterval int `json:"api_fetch_interval"` // in seconds
-	EnabledApiEndpoints []string `json:"enabled_api_endpoints"`
-	PersistentDataFilePath string `json:"datafile"`
-	LogLevel int `json:"loglevel"`
-	Recipients []Recipient `json:"recipients"`
-	SmtpConfiguration SmtpSettings `json:"smtp"`
-	Template MailTemplateConfig `json:"template"`
-}
-
-func NewConfig() Config {
-	// Initial config
-	c := Config{
-		ApiFetchInterval: 60 * 10, // every 10 minutes,
-		EnabledApiEndpoints: []string{"bay", "bund"},
-		PersistentDataFilePath: "data",
-		LogLevel: 2,
-		Recipients: []Recipient{},
-		SmtpConfiguration: SmtpSettings{
-			From: "from@example.org",
-			User: "from@example.org",
-			Password: "SiEhAbEnMiChInSgEsIcHtGeFiLmTdAsDüRfEnSiEnIcHt",
-			ServerHost: "example.org",
-			ServerPort: 587},
-		Template: MailTemplateConfig{
-			SubjectTemplate: "",
-			BodyTemplate: "",
-		},
-	}
-	return c
-}
-
-func checkConfig(config Config) {
-	if len(config.Recipients) < 1 {
-		logger.error("Configuration is incomplete")
-		panic(errors.New("no recipients are configured"))
-	}
-	for _, r := range config.Recipients {
-		if !mailAddressIsValid(r.Address) {
-			logger.error("Configuration includes invalid data")
-			panic(errors.New("'" + r.Address + "' is not a valid e-mail address"))
-		}
-		if len(r.Filters) < 1 {
-			logger.error("Configuration is incomplete")
-			panic(errors.New("recipient " + r.Address + " has no filter defined - at least {'any': true/false} should be configured"))
-		}
-	}
-	if !mailAddressIsValid(config.SmtpConfiguration.From) {
-		logger.error("Configuration includes invalid data")
-		panic(errors.New("'" + config.SmtpConfiguration.From + "' is not a valid e-mail address"))
-	}
-}
-
-type PersistentData struct {
-	// {endpoint id 1: time last published, endpoint id 2: ..., ...}
-	LastPublished map[string]time.Time `json:"last_published"`
-}
-
-func NewPersistentData(c Config) PersistentData {
-	// Initial persistent data
-	d := PersistentData{LastPublished: map[string]time.Time{}}
-	for _, e := range apiEndpoints {
-		d.LastPublished[e.Id] = time.Now().Add(-time.Hour * 24) // a day ago
-	}
-	return d
-}
-
 type DataStore struct {
 	filepath string
 	prettyJSON bool
@@ -92,32 +23,24 @@ func (ds *DataStore) save() error {
 	} else {
 		data, err = json.Marshal(ds.data)
 	}
-	if err != nil {
+	if err != nil { return err }
-		return err
-	}
 	err = os.WriteFile(ds.filepath, data, ds.fileMode)
 	return err
 }
 
 func (ds *DataStore) load() error {
 	data, err := os.ReadFile(ds.filepath)
-	if err != nil {
+	if err != nil { return err }
-		return err
-	}
 	switch ds.data.(type) {
 	case Config:
 		d, _ := ds.data.(Config);
 		err = json.Unmarshal(data, &d)
-		if err != nil {
+		if err != nil { return err }
-			return err
-		}
 		ds.data = d
 	case PersistentData:
 		d, _ := ds.data.(PersistentData);
 		err = json.Unmarshal(data, &d)
-		if err != nil {
+		if err != nil { return err }
-			return err
-		}
 		ds.data = d
 	}
 	return err

go.mod

@@ -1,3 +1,3 @@
 module github.com/ChaoticByte/wid-notifier
 
-go 1.22.2
+go 1.22

mail.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"net/mail"
 	"net/smtp"
-	"slices"
 	"time"
 )
 
@@ -49,22 +48,12 @@ type SmtpSettings struct {
 	Password string `json:"password"`
 }
 
-func (r Recipient) filterAndSendNotices(notices []WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, cache *map[string][]byte) error {
+func (r Recipient) sendNotices(notices []WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings, cache *map[string][]byte) error {
-	filteredNotices := []WidNotice{}
-	for _, f := range r.Filters {
-		for _, n := range f.filter(notices) {
-			if !noticeSliceContains(filteredNotices, n) {
-				filteredNotices = append(filteredNotices, n)
-			}
-		}
-	}
-	slices.Reverse(filteredNotices)
-	logger.debug(fmt.Sprintf("Including %v of %v notices for recipient %v", len(filteredNotices), len(notices), r.Address))
 	logger.debug("Generating and sending mails to " + r.Address + " ...")
 	cacheHits := 0
 	cacheMisses := 0
 	mails := [][]byte{}
-	for _, n := range filteredNotices {
+	for _, n := range notices {
 		var data []byte
 		cacheResult := (*cache)[n.Uuid]
 		if len(cacheResult) > 0 {
@@ -91,9 +80,7 @@ func (r Recipient) filterAndSendNotices(notices []WidNotice, template MailTempla
 		r.Address,
 		mails,
 	)
-	if err != nil {
+	if err != nil { return err }
-		return err
-	}
 	logger.debug("Successfully sent all mails to " + r.Address)
 	return nil
 }
@@ -102,50 +89,34 @@ func sendMails(smtpConf SmtpSettings, auth smtp.Auth, to string, data [][]byte)
 	addr := fmt.Sprintf("%v:%v", smtpConf.ServerHost, smtpConf.ServerPort)
 	logger.debug("Connecting to mail server at " + addr + " ...")
 	connection, err := smtp.Dial(addr)
-	if err != nil {
+	if err != nil { return err }
-		return err
-	}
 	defer connection.Close()
 	// can leave out connection.Hello
 	hasTlsExt, _ := connection.Extension("starttls")
 	if hasTlsExt {
 		err = connection.StartTLS(&tls.Config{ServerName: smtpConf.ServerHost})
-		if err != nil {
+		if err != nil { return err }
-			return err
-		}
 		logger.debug("Mail Server supports TLS")
 	} else {
 		logger.debug("Mail Server doesn't support TLS")
 	}
 	logger.debug("Authenticating to mail server ...")
 	err = connection.Auth(auth)
-	if err != nil {
+	if err != nil { return err }
-		return err
-	}
 	if logger.LogLevel >= 3 {
 		fmt.Printf("DEBUG %v Sending mails to server ", time.Now().Format("2006/01/02 15:04:05.000000"))
 	}
 	for _, d := range data {
 		err = connection.Mail(smtpConf.From)
-		if err != nil {
+		if err != nil { return err }
-			return err
-		}
 		err = connection.Rcpt(to)
-		if err != nil {
+		if err != nil { return err }
-			return err
-		}
 		writer, err := connection.Data()
-		if err != nil {
+		if err != nil { return err }
-			return err
-		}
 		_, err = writer.Write(d)
-		if err != nil {
+		if err != nil { return err }
-			return err
-		}
 		err = writer.Close()
-		if err != nil {
+		if err != nil { return err }
-			return err
-		}
 		if logger.LogLevel >= 3 {
 			print(".")
 		}

main.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/smtp"
 	"os"
+	"slices"
 	"time"
 )
 
@@ -24,17 +25,13 @@ func main() {
 	// init
 	logger.info("Initializing ...")
 	defer logger.info("Exiting ...")
+	// open & check config
 	config := NewDataStore(
 		configFilePath,
 		NewConfig(),
 		true,
 		0600,
 	).data.(Config)
-	persistent := NewDataStore(
-		config.PersistentDataFilePath,
-		NewPersistentData(config),
-		false,
-		0640)
 	logger.LogLevel = config.LogLevel
 	logger.debug("Checking configuration file ...")
 	checkConfig(config)
@@ -66,6 +63,12 @@ func main() {
 			}
 		}
 	}
+	// open data file
+	persistent := NewDataStore(
+		config.PersistentDataFilePath,
+		NewPersistentData(config),
+		false,
+		0640)
 	// main loop
 	logger.debug("Entering main loop ...")
 	for {
@@ -76,7 +79,7 @@ func main() {
 			logger.info("Querying endpoint '" + a.Id + "' for new notices ...")
 			n, t, err := a.getNotices(persistent.data.(PersistentData).LastPublished[a.Id])
 			if err != nil {
-				// retry
+				// retry (once)
 				logger.warn("Couldn't query notices from API endpoint '" + a.Id + "'. Retrying ...")
 				logger.warn(err)
 				n, t, err = a.getNotices(persistent.data.(PersistentData).LastPublished[a.Id])
@@ -96,7 +99,19 @@ func main() {
 			logger.info("Sending email notifications ...")
 			recipientsNotified := 0
 			for _, r := range config.Recipients {
-				err := r.filterAndSendNotices(newNotices, mailTemplate, mailAuth, config.SmtpConfiguration, &cache)
+				// Filter notices for this recipient
+				filteredNotices := []WidNotice{}
+				for _, f := range r.Filters {
+					for _, n := range f.filter(newNotices) {
+						if !noticeSliceContains(filteredNotices, n) {
+							filteredNotices = append(filteredNotices, n)
+						}
+					}
+				}
+				slices.Reverse(filteredNotices)
+				logger.debug(fmt.Sprintf("Including %v of %v notices for recipient %v", len(filteredNotices), len(newNotices), r.Address))
+				// Send notices
+				err := r.sendNotices(filteredNotices, mailTemplate, mailAuth, config.SmtpConfiguration, &cache)
 				if err != nil {
 					logger.error(err)
 				} else {
@@ -105,8 +120,7 @@ func main() {
 			}
 			logger.info(fmt.Sprintf("Email notifications sent to %v of %v recipients", recipientsNotified, len(config.Recipients)))
 		}
-		t2 := time.Now().UnixMilli()
+		dt := int(time.Now().UnixMilli() - t1)
-		dt := int(t2 - t1)
 		time.Sleep(time.Millisecond * time.Duration((config.ApiFetchInterval * 1000) - dt))
 	}
 }

persistent_data.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2024 Julian Müller (ChaoticByte)
+
+package main
+
+import (
+	"time"
+)
+
+type PersistentData struct {
+	// {endpoint id 1: time last published, endpoint id 2: ..., ...}
+	LastPublished map[string]time.Time `json:"last_published"`
+}
+
+func NewPersistentData(c Config) PersistentData {
+	// Initial persistent data
+	d := PersistentData{LastPublished: map[string]time.Time{}}
+	for _, e := range apiEndpoints {
+		d.LastPublished[e.Id] = time.Now().Add(-time.Hour * 24) // a day ago
+	}
+	return d
+}

template.go

@@ -42,15 +42,11 @@ func (t MailTemplate) generate(notice WidNotice) (MailContent, error) {
 	c := MailContent{}
 	buffer := &bytes.Buffer{}
 	err := t.SubjectTemplate.Execute(buffer, notice)
-	if err != nil {
+	if err != nil { return c, err }
-		return c, err
-	}
 	c.Subject = buffer.String()
 	buffer.Truncate(0) // we can recycle our buffer
 	err = t.BodyTemplate.Execute(buffer, notice)
-	if err != nil {
+	if err != nil { return c, err }
-		return c, err
-	}
 	c.Body = buffer.String()
 	return c, nil
 }

widapi.go

@@ -57,9 +57,7 @@ func (e ApiEndpoint) getNotices(since time.Time) ([]WidNotice, time.Time, error)
 	if err == nil {
 		if res.StatusCode == 200 {
 			resBody, err := io.ReadAll(res.Body)
-			if err != nil {
+			if err != nil { return []WidNotice{}, since, err }
-				return []WidNotice{}, since, err
-			}
 			var decodedData map[string]interface{}
 			if err = json.Unmarshal(resBody, &decodedData); err != nil {
 				return []WidNotice{}, since, err