From a6377b805d7949da0c325c3757fad4e378b20922 Mon Sep 17 00:00:00 2001 From: ChaoticByte Date: Wed, 11 Oct 2023 22:14:01 +0200 Subject: [PATCH] Initial commit --- .gitignore | 5 ++ README.md | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++- datastore.go | 148 ++++++++++++++++++++++++++++++++++++++++++++++ filter.go | 56 ++++++++++++++++++ go.mod | 3 + mail.go | 91 ++++++++++++++++++++++++++++ main.go | 83 ++++++++++++++++++++++++++ notice.go | 42 +++++++++++++ template.go | 71 ++++++++++++++++++++++ widapi.go | 138 +++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 799 insertions(+), 2 deletions(-) create mode 100644 datastore.go create mode 100644 filter.go create mode 100644 go.mod create mode 100644 mail.go create mode 100644 main.go create mode 100644 notice.go create mode 100644 template.go create mode 100644 widapi.go diff --git a/.gitignore b/.gitignore index 3b735ec..9f3237e 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,8 @@ # Go workspace file go.work + +data +config +*.bak +wid-notifier diff --git a/README.md b/README.md index b002441..d19bfdc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,162 @@ -# wid-notifier -A tool that sends configurable email notifications for wid.cert-bund.de and wid.lsi.bayern.de +# WID Notifier + +A tool that sends configurable email notifications for + +- https://wid.cert-bund.de/portal/wid/kurzinformationen +- https://wid.lsi.bayern.de/portal/wid/warnmeldungen + +## Supported Platforms + +This Software only supports Linux. + +# Config + +Example: + +```json +{ + "api_fetch_interval": 600, + "enabled_api_endpoints": [ + "bay", + "bund" + ], + "datafile": "data", + "recipients": [ + { + "address": "guenther@example.org",, + "include": [ + {"classification": "kritisch"}, + {"title_contains": "jQuery"} + ] + } + ], + "smtp": { + "from": "WID Notifier \u003cfrom@example.org\u003e", + "host": "example.org", + "port": 587, + "user": "from@example.org", + "password": "SiEhAbEnMiChInSgEsIcHtGeFiLmTdAsDüRfEnSiEnIcHt" + }, + "template": { + "subject": "", + "body": "" + } +} +``` + +## Filters + +You must filter the notices to be sent per user. Multiple filters can be set per user and multiple criteria can be defined per filter. + +```json +"include": [ + { + "any": false, + "title_contains": "", + "classification": "", + "min_basescore": 0, + "status": "", + "products_contain": "", + "no_patch": "" + }, + ... +] +``` + +The following filter criteria are supported. Criteria marked with `*` are for optional fields that are not supported by every API endpoint (e.g. https://wid.lsi.bayern.de) - notices from those endpoints will therefore not be included when using those filters. + +### any + +Includes all notices if set to `true`. + +```json +{"any": true} +``` + +### title_contains + +Include notices whose title contains this text. + +```json +{"title_contains": "Denial Of Service"} +``` +If set to `""`, this criteria will be ignored. + +### classification + +Include notices whose classification is in this list. +Classification can be `"kritisch"`, `"hoch"`, `"mittel"` or `"niedrig"`. + +```json +{"classification": "hoch"} +``` +If set to `""`, this criteria will be ignored. + +### min_basescore `*` + +Include notices whose basescore (`0` - `100`) is >= `min_basescore`. + +```json +{"min_basescore": 40} +``` +This criteria will be ignored if set to `0`. + +### status `*` + +Include notices with this status. This is usually either `NEU` or `UPDATE`. + +```json +{"status": "NEU"} +``` +If set to `""`, this criteria will be ignored. + +### products_contain `*` + +Include notices whose product list contains this text. + +```json +{"products_contain": "Debian Linux"} +``` +If set to `""`, this criteria will be ignored. + +### no_patch `*` + +If set to `"true"`, notices where no patch is available will be included. + +```json +{"no_patch": "true"} +``` + +If set to `"false"`, notices where no patch is available will be included. + +```json +{"no_patch": "false"} +``` + +If set to `""`, this criteria will be ignored. + +## Templates + +The syntax for the mail templates is described [here](https://pkg.go.dev/text/template). + +All fields from the WidNotice struct can be used. + +```go +type WidNotice struct { + Uuid string + Name string + Title string + Published time.Time + Classification string + // optional fields (only fully supported by cert-bund) + Basescore int // -1 = unknown + Status string // "" = unknown + ProductNames []string // empty = unknown + Cves []string // empty = unknown + NoPatch string // "" = unknown + // metadata + PortalUrl string +} +``` + +For an example, take a look at `DEFAULT_SUBJECT_TEMPLATE` and `DEFAULT_BODY_TEMPLATE` in [template.go](./template.go). diff --git a/datastore.go b/datastore.go new file mode 100644 index 0000000..8f9c888 --- /dev/null +++ b/datastore.go @@ -0,0 +1,148 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "time" +) + +type Config struct { + ApiFetchInterval int `json:"api_fetch_interval"` // in seconds + EnabledApiEndpoints []string `json:"enabled_api_endpoints"` + PersistentDataFilePath string `json:"datafile"` + Recipients []Recipient `json:"recipients"` + SmtpConfiguration SmtpSettings `json:"smtp"` + Template MailTemplateConfig `json:"template"` +} + +func NewConfig() Config { + // Initial config + c := Config{ + ApiFetchInterval: 60 * 10, // every 10 minutes, + EnabledApiEndpoints: []string{"bay", "bund"}, + PersistentDataFilePath: "data", + Recipients: []Recipient{}, + SmtpConfiguration: SmtpSettings{ + From: "WID Notifier ", + User: "from@example.org", + Password: "SiEhAbEnMiChInSgEsIcHtGeFiLmTdAsDüRfEnSiEnIcHt", + ServerHost: "example.org", + ServerPort: 587}, + Template: MailTemplateConfig{ + SubjectTemplate: "", + BodyTemplate: "", + }, + } + return c +} + +func checkConfig(config Config) { + if len(config.Recipients) < 1 { + fmt.Println("ERROR\tConfiguration is incomplete.") + panic(errors.New("no recipients are configured")) + } + for _, r := range config.Recipients { + if !mailAddressIsValid(r.Address) { + fmt.Println("ERROR\tConfiguration includes invalid data.") + panic(errors.New("'" + r.Address + "' is not a valid e-mail address")) + } + if len(r.Filters) < 1 { + fmt.Println("ERROR\tConfiguration is incomplete.") + panic(errors.New("recipient " + r.Address + " has no filter defined - at least {'any': true/false} should be configured")) + } + } + if !mailAddressIsValid(config.SmtpConfiguration.From) { + fmt.Println("ERROR\tConfiguration includes invalid data.") + panic(errors.New("'" + config.SmtpConfiguration.From + "' is not a valid e-mail address")) + } +} + +type PersistentData struct { + // {endpoint id 1: time last published, endpoint id 2: ..., ...} + LastPublished map[string]time.Time `json:"last_published"` +} + +func NewPersistentData(c Config) PersistentData { + // Initial persistent data + d := PersistentData{LastPublished: map[string]time.Time{}} + for _, e := range apiEndpoints { + d.LastPublished[e.Id] = time.Now().Add(-time.Hour * 24) // a day ago + } + return d +} + +type DataStore struct { + filepath string + prettyJSON bool + data any + fileMode fs.FileMode +} + +func (ds *DataStore) save() error { + var err error + var data []byte + if ds.prettyJSON { + data, err = json.MarshalIndent(ds.data, "", " ") + } else { + data, err = json.Marshal(ds.data) + } + if err != nil { + return err + } + err = os.WriteFile(ds.filepath, data, ds.fileMode) + return err +} + +func (ds *DataStore) load() error { + data, err := os.ReadFile(ds.filepath) + if err != nil { + return err + } + switch ds.data.(type) { + case Config: + d, _ := ds.data.(Config); + err = json.Unmarshal(data, &d) + if err != nil { + return err + } + ds.data = d + case PersistentData: + d, _ := ds.data.(PersistentData); + err = json.Unmarshal(data, &d) + if err != nil { + return err + } + ds.data = d + } + return err +} + +func (ds *DataStore) init() error { + err := ds.load() + if err != nil { + if !os.IsNotExist(err) { + return err + } else { + // Write initial data + err = ds.save() + } + } + return err +} + +func NewDataStore(savePath string, data any, prettyJSON bool, fileMode fs.FileMode) DataStore { + // Initial data + ds := DataStore{} + ds.filepath = savePath + ds.data = data + ds.prettyJSON = prettyJSON + ds.fileMode = fileMode + if err := ds.init(); err != nil { + // We don't like that, we panic + panic(err) + } + return ds +} diff --git a/filter.go b/filter.go new file mode 100644 index 0000000..8c32671 --- /dev/null +++ b/filter.go @@ -0,0 +1,56 @@ +package main + +import ( + "strings" +) + +type Filter struct { + Any bool `json:"any"` + TitleContains string `json:"title_contains"` + Classification string `json:"classification"` + MinBaseScore int `json:"min_basescore"` + Status string `json:"status"` + ProductsContain string `json:"products_contain"` + NoPatch string `json:"no_patch"` +} + +func (f Filter) filter(notices []WidNotice) []WidNotice { + filteredNotices := []WidNotice{} + for _, n := range notices { + matches := []bool{} + if f.Any { + matches = append(matches, true) + } else + { + if f.TitleContains != "" { + matches = append(matches, strings.Contains(n.Title, f.TitleContains)) + } + if f.Classification != "" { + matches = append(matches, f.Classification == n.Classification) + } + if f.MinBaseScore > 0 { + matches = append(matches, f.MinBaseScore <= n.Basescore) + } + if f.Status != "" { + matches = append(matches, f.Status == n.Status) + } + if f.ProductsContain != "" { + matches = append(matches, len(n.ProductNames) > 0) + } + if f.NoPatch != "" { + matches = append(matches, f.NoPatch == n.NoPatch) + } + } + allMatch := len(matches) > 0 + for _, m := range matches { + if !m { + allMatch = false + break + } + } + if allMatch { + filteredNotices = append(filteredNotices, n) + } + } + return filteredNotices +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5584a12 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/ChaoticByte/wid-notifier + +go 1.21.1 diff --git a/mail.go b/mail.go new file mode 100644 index 0000000..c26ec01 --- /dev/null +++ b/mail.go @@ -0,0 +1,91 @@ +package main + +import ( + "encoding/base64" + "fmt" + "net/mail" + "net/smtp" + "slices" + "strings" +) + +const MAIL_LINE_SEP = "\r\n" + +type MailContent struct { + Subject string + Body string +} + +func (c MailContent) serializeValidMail(from string, to string) []byte { + // We'll send base64 encoded Subject & Body, because we Dschörmäns have umlauts + // and I'm too lazy to encode ä into =E4 and so on + subjectEncoded := base64.StdEncoding.EncodeToString([]byte(c.Subject)) + bodyEncoded := base64.StdEncoding.EncodeToString( + []byte( // ensure that all lines end with CRLF + strings.ReplaceAll( + strings.ReplaceAll(c.Body, "\n", MAIL_LINE_SEP), "\r\r", "\r", + ), + ), + ) + data := []byte(fmt.Sprintf( + "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\nFrom: %v%vTo: %v%vSubject: =?utf-8?b?%v?=%v%v%v", + from, MAIL_LINE_SEP, + to, MAIL_LINE_SEP, + subjectEncoded, MAIL_LINE_SEP, + MAIL_LINE_SEP, + bodyEncoded)) + // done, I guess + return data +} + +type Recipient struct { + Address string `json:"address"` + // Must be a configured filter id + Filters []Filter `json:"include"` +} + +type SmtpSettings struct { + From string `json:"from"` + ServerHost string `json:"host"` + ServerPort int `json:"port"` + User string `json:"user"` + Password string `json:"password"` +} + +func (r Recipient) filterAndSendNotices(notices []WidNotice, template MailTemplate, auth smtp.Auth, smtpConfig SmtpSettings) error { + filteredNotices := []WidNotice{} + for _, f := range r.Filters { + for _, n := range f.filter(notices) { + if !noticeSliceContains(filteredNotices, n) { + filteredNotices = append(filteredNotices, n) + } + } + } + slices.Reverse(filteredNotices) + for _, n := range filteredNotices { + mailContent, err := template.generate(n) + if err != nil { + fmt.Println("ERROR\tCould not create mail from template.") + fmt.Println(err) + } + // serialize & send mail + data := mailContent.serializeValidMail(smtpConfig.From, r.Address) + err = smtp.SendMail( + fmt.Sprintf("%v:%v", smtpConfig.ServerHost, smtpConfig.ServerPort), + auth, + smtpConfig.From, + []string{r.Address}, + data, + ) + if err != nil { + return err + } + // fmt.Printf("%v", strings.ReplaceAll(string(data), "\r", "\\r")) + } + return nil +} + +func mailAddressIsValid(address string) bool { + _, err := mail.ParseAddress(address); + return err == nil +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..4a9bcbb --- /dev/null +++ b/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "fmt" + "net/smtp" + "os" + "time" +) + +func main() { + // get cli arguments + args := os.Args + if len(args) < 2 { + fmt.Printf("Usage: %v \nIf the config file doesn't exist, a incomplete configuration with default values is created.\n", args[0]) + os.Exit(1) + } + configFilePath := os.Args[1] + // init + println("INFO\tInitializing ...") + config := NewDataStore( + configFilePath, + NewConfig(), + true, + 0600, + ).data.(Config) + persistent := NewDataStore( + config.PersistentDataFilePath, + NewPersistentData(config), + false, + 0640) + // exit handler + defer println("INFO\tExiting ...") + // check config + checkConfig(config) + // create mail template from mail template config + if config.Template.SubjectTemplate == "" { + config.Template.SubjectTemplate = DEFAULT_SUBJECT_TEMPLATE + } + if config.Template.BodyTemplate == "" { + config.Template.BodyTemplate = DEFAULT_BODY_TEMPLATE + } + mailTemplate := NewTemplateFromTemplateConfig(config.Template) + // mail authentication from config + mailAuth := smtp.PlainAuth( + "", + config.SmtpConfiguration.User, + config.SmtpConfiguration.Password, + config.SmtpConfiguration.ServerHost, + ) + // filter out disabled api endpoints + enabledApiEndpoints := []ApiEndpoint{} + for _, a := range apiEndpoints { + for _, b := range config.EnabledApiEndpoints { + if a.Id == b { + enabledApiEndpoints = append(enabledApiEndpoints, a) + } + } + } + // main loop + for { + newNotices := []WidNotice{} + for _, a := range enabledApiEndpoints { + n, t, err := a.getNotices(persistent.data.(PersistentData).LastPublished[a.Id]) + if err != nil { + // retry + n, t, err = a.getNotices(persistent.data.(PersistentData).LastPublished[a.Id]) + } + if err != nil { + // ok then... + fmt.Print("ERROR\t", err) + } else { + newNotices = append(newNotices, n...) + persistent.data.(PersistentData).LastPublished[a.Id] = t + persistent.save() + } + } + // fmt.Println(newNotices) + for _, r := range config.Recipients { + r.filterAndSendNotices(newNotices, mailTemplate, mailAuth, config.SmtpConfiguration) + } + time.Sleep(time.Second * time.Duration(config.ApiFetchInterval)) + } +} diff --git a/notice.go b/notice.go new file mode 100644 index 0000000..6c01087 --- /dev/null +++ b/notice.go @@ -0,0 +1,42 @@ +package main + +import ( + // "encoding/json" + "time" +) + +type WidNotice struct { + // obligatory + Uuid string `json:"uuid"` + Name string `json:"name"` + Title string `json:"title"` + Published time.Time `json:"published"` + Classification string `json:"classification"` + // optional fields (only fully supported by cert-bund) + Basescore int `json:"basescore"` // -1 = unknown + Status string `json:"status"` // "" = unknown + ProductNames []string `json:"productNames"` // empty = unknown + Cves []string `json:"cves"` // empty = unknown + NoPatch string `json:"noPatch"` // "" = unknown + // metadata + PortalUrl string +} + +// func (n WidNotice) serialized() ([]byte, error) { +// return json.Marshal(n) +// } + +// func NewWidNoticeFromJSON(data []byte) (WidNotice, error) { +// n := WidNotice{} +// err := json.Unmarshal(data, &n) +// return n, err +// } + +func noticeSliceContains(notices []WidNotice, notice WidNotice) bool { + for _, x := range notices { + if x.Uuid == notice.Uuid { + return true + } + } + return false +} diff --git a/template.go b/template.go new file mode 100644 index 0000000..665eb07 --- /dev/null +++ b/template.go @@ -0,0 +1,71 @@ +package main + +import ( + "bytes" + "fmt" + "text/template" +) + +const DEFAULT_SUBJECT_TEMPLATE = "{{ if .Status }}{{.Status}} {{ end }}[{{.Classification}}] {{.Title}}" +const DEFAULT_BODY_TEMPLATE = `{{.Name}} - "{{.Title}}" with classification '{{.Classification}}' +Link: {{.PortalUrl}} + +Published: {{.Published}} +{{ if .Status }}Status: {{.Status}} +{{ end -}} +{{ if gt .Basescore -1 }}Basescore: {{.Basescore}} +{{ end -}} +{{ if eq .NoPatch "true" }}There is no patch available at the moment! +{{ end }} +Affected Products: +{{ range $product := .ProductNames }} - {{ $product }} +{{ else }} unknown +{{ end }} +Assigned CVEs: +{{ range $cve := .Cves }} - {{ $cve }} +{{ else }} unknown +{{ end }}` + +type MailTemplateConfig struct { + SubjectTemplate string `json:"subject"` + BodyTemplate string `json:"body"` +} + +type MailTemplate struct { + SubjectTemplate template.Template + BodyTemplate template.Template +} + +func (t MailTemplate) generate(notice WidNotice) (MailContent, error) { + c := MailContent{} + buffer := &bytes.Buffer{} + err := t.SubjectTemplate.Execute(buffer, notice) + if err != nil { + return c, err + } + c.Subject = buffer.String() + buffer.Truncate(0) // we can recycle our buffer + err = t.BodyTemplate.Execute(buffer, notice) + if err != nil { + return c, err + } + c.Body = buffer.String() + return c, nil +} + +func NewTemplateFromTemplateConfig(tc MailTemplateConfig) MailTemplate { + subjectTemplate, err := template.New("subject").Parse(tc.SubjectTemplate) + if err != nil { + fmt.Println("ERROR\tCould not parse template.") + panic(err) + } + bodyTemplate, err := template.New("body").Parse(tc.BodyTemplate) + if err != nil { + fmt.Println("ERROR\tCould not parse template.") + panic(err) + } + return MailTemplate{ + SubjectTemplate: *subjectTemplate, + BodyTemplate: *bodyTemplate, + } +} diff --git a/widapi.go b/widapi.go new file mode 100644 index 0000000..fa85110 --- /dev/null +++ b/widapi.go @@ -0,0 +1,138 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// known API endpoints +var apiEndpoints []ApiEndpoint = []ApiEndpoint{ + { + Id: "bund", + EndpointUrl: "https://wid.cert-bund.de/content/public/securityAdvisory", + PortalUrl: "https://wid.cert-bund.de/portal/wid/securityadvisory", + }, + { + Id: "bay", + EndpointUrl: "https://wid.lsi.bayern.de/content/public/securityAdvisory", + PortalUrl: "https://wid.lsi.bayern.de/portal/wid/securityadvisory", + }, +} + +const PUBLISHED_TIME_FORMAT = "2006-01-02T15:04:05.999-07:00" +const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0" + +var defaultParams = []string{ + "size=1000", // max backlog + "sort=published,desc", + "aboFilter=false", +} + +type ApiEndpoint struct { + Id string + EndpointUrl string + PortalUrl string +} + +func (e ApiEndpoint) getNotices(since time.Time) ([]WidNotice, time.Time, error) { + // returns a slice of WidNotice and the 'published' field of the last notice + var notices []WidNotice = []WidNotice{} + var err error + params := defaultParams + // params = append(params, "publishedFromFilter=" + publishedFrom.Format(PUBLISHED_FROM_FILTER_TIME_FORMAT)) + // ^ looks like the API is f***ed, 'publishedFromFilter=...' does only factor in the day (-2h because of the + // timezone), not the time of the day - echte Deutsche Wertarbeit mal wieder am Start + // -> we have to filter by hand (see below) + url := e.EndpointUrl + "?" + strings.Join(params, "&") + req, _ := http.NewRequest(http.MethodGet, url, nil) + req.Header.Set("User-Agent", USER_AGENT) + client := http.Client{} + res, err := client.Do(req) + if err == nil { + if res.StatusCode == 200 { + resBody, err := io.ReadAll(res.Body) + if err != nil { + return nil, time.Time{}, err + } + var decodedData map[string]interface{} + if err = json.Unmarshal(resBody, &decodedData); err != nil { + return nil, time.Time{}, err + } + notices = parseApiResponse(decodedData, e) + } else { + fmt.Printf("ERROR\tGet \"%v\": %v\n", url, res.Status) + return nil, time.Time{}, err + } + } else { + return nil, time.Time{}, err + } + if len(notices) > 0 { + // And here the filtering begins. yay -.- + noticesFiltered := []WidNotice{} + lastPublished := since + for _, n := range notices { + if n.Published.After(since) { + noticesFiltered = append(noticesFiltered, n) + // while we are at it, we can also find lastPublished + if n.Published.After(lastPublished) { + lastPublished = n.Published + } + } + } + return noticesFiltered, lastPublished, nil + } else { + return nil, time.Time{}, nil + } +} + +func parseApiResponse(data map[string]interface{}, apiEndpoint ApiEndpoint) []WidNotice { + var notices []WidNotice = []WidNotice{} + for _, d := range data["content"].([]interface{}) { + d := d.(map[string]interface{}) + notice := WidNotice{ + Uuid: d["uuid"].(string), + Name: d["name"].(string), + Title: d["title"].(string), + Classification: d["classification"].(string), + } + published, err := time.Parse(PUBLISHED_TIME_FORMAT, d["published"].(string)) + if err != nil { + fmt.Println("ERROR\t", err) + } + notice.Published = published + // optional fields + if v, ok := d["basescore"]; ok { + notice.Basescore = int(v.(float64)) + } else { + notice.Basescore = -1 + } + if v, ok := d["status"]; ok { + notice.Status = v.(string) + } + if v, ok := d["productNames"]; ok { + for _, n := range v.([]interface{}) { + notice.ProductNames = append(notice.ProductNames, n.(string)) + } + } + if v, ok := d["cves"]; ok { + for _, c := range v.([]interface{}) { + notice.Cves = append(notice.Cves, c.(string)) + } + } + if v, ok := d["noPatch"]; ok { + if v.(bool) { + notice.NoPatch = "true" + } else { + notice.NoPatch = "false" + } + } + // metadata + notice.PortalUrl = apiEndpoint.PortalUrl + "?name=" + notice.Name + notices = append(notices, notice) + } + return notices +}