Simple static blacklist support

This commit is contained in:
Denis Zheleztsov 2020-07-23 15:28:26 +03:00
parent fadc556cf8
commit e8bbdd1225
Signed by: Difrex
GPG Key ID: 41CF5135CF7FADB3
3 changed files with 58 additions and 6 deletions

2
go.mod
View File

@ -1,5 +1,7 @@
module gitea.difrex.ru/Umbrella/fetcher
go 1.14
require (
github.com/emirpasic/gods v1.12.0
github.com/google/uuid v1.0.0

3
go.sum
View File

@ -1,3 +1,4 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
@ -6,10 +7,12 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/idec-net/go-idec v0.0.0-20181106151523-61a006246343 h1:ABIlopLGU081SkX2KmXjho9vmm1MgPs38hxXCXC2BrM=
github.com/idec-net/go-idec v0.0.0-20181106151523-61a006246343/go.mod h1:XUvr43ZLN/4bTZT7TEhJA/rsfFLQxnggX6iU5TGXgIY=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

59
main.go
View File

@ -2,20 +2,24 @@ package main
import (
"flag"
"os"
"strings"
"strconv"
log "github.com/sirupsen/logrus"
"io/ioutil"
"gitea.difrex.ru/Umbrella/fetcher/i2es"
"gitea.difrex.ru/Umbrella/fetcher/idec"
)
var (
inode, es, esIndex, esType, addEchoes string
offset, limit int
all, reindex bool
inode, es, esIndex, esType, addEchoes, blacklistFile string
offset, limit int
all, reindex bool
denyMessages []string
)
// init ...
@ -29,11 +33,15 @@ func init() {
flag.StringVar(&esIndex, "esindex", "idec", "ES index")
flag.StringVar(&esType, "estype", "post", "ES document type")
flag.StringVar(&addEchoes, "echoes", "", "Additional echoes to fetch. Comma separated")
flag.StringVar(&blacklistFile, "blacklist", "/blacklist.txt", "Path to the blacklist file")
flag.Parse()
denyMessages = loadBlacklist()
}
func main() {
// extensions := idec.NewExtensions()
var fc idec.FetchConfig
fc.Node = inode
@ -128,11 +136,18 @@ func main() {
}
continue
}
esd, err = elastic.PutToIndex(esd)
if err != nil {
log.Error(err.Error())
if !isMessageBlacklisted(esd.MsgID) {
esd, err = elastic.PutToIndex(esd)
if err != nil {
log.Error(err.Error())
continue
}
} else {
log.Warnf("Skipping blacklisted message %s", esd.MsgID)
continue
}
log.Warn("Message ", m.ID, " added to index and to assignates list")
newMessages = append(newMessages, esd)
}
@ -142,3 +157,35 @@ func main() {
log.Error(err.Error())
}
}
func isMessageBlacklisted(id string) bool {
if denyMessages == nil {
return false
}
for i := range denyMessages {
if id == denyMessages[i] {
return true
}
}
return false
}
func loadBlacklist() []string {
if _, err := os.Stat(blacklistFile); os.IsNotExist(err) {
return nil
}
file, err := os.Open(blacklistFile)
if err != nil {
return nil
}
defer file.Close()
data, err := ioutil.ReadAll(file)
if err != nil {
return nil
}
return strings.Split(string(data), "\n")
}