Simple static blacklist support

go.mod

@@ -1,5 +1,7 @@
 module gitea.difrex.ru/Umbrella/fetcher
 
+go 1.14
+
 require (
 	github.com/emirpasic/gods v1.12.0
 	github.com/google/uuid v1.0.0

go.sum

@@ -1,3 +1,4 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
@@ -6,10 +7,12 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/idec-net/go-idec v0.0.0-20181106151523-61a006246343 h1:ABIlopLGU081SkX2KmXjho9vmm1MgPs38hxXCXC2BrM=
 github.com/idec-net/go-idec v0.0.0-20181106151523-61a006246343/go.mod h1:XUvr43ZLN/4bTZT7TEhJA/rsfFLQxnggX6iU5TGXgIY=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

main.go

@@ -2,20 +2,24 @@ package main
 
 import (
 	"flag"
+	"os"
 	"strings"
 
 	"strconv"
 
 	log "github.com/sirupsen/logrus"
 
+	"io/ioutil"
+
 	"gitea.difrex.ru/Umbrella/fetcher/i2es"
 	"gitea.difrex.ru/Umbrella/fetcher/idec"
 )
 
 var (
-	inode, es, esIndex, esType, addEchoes string
+	inode, es, esIndex, esType, addEchoes, blacklistFile string
-	offset, limit                         int
+	offset, limit                                        int
-	all, reindex                          bool
+	all, reindex                                         bool
+	denyMessages                                         []string
 )
 
 // init ...
@@ -29,11 +33,15 @@ func init() {
 	flag.StringVar(&esIndex, "esindex", "idec", "ES index")
 	flag.StringVar(&esType, "estype", "post", "ES document type")
 	flag.StringVar(&addEchoes, "echoes", "", "Additional echoes to fetch. Comma separated")
+	flag.StringVar(&blacklistFile, "blacklist", "/blacklist.txt", "Path to the blacklist file")
 	flag.Parse()
+
+	denyMessages = loadBlacklist()
 }
 
 func main() {
 	// extensions := idec.NewExtensions()
+
 	var fc idec.FetchConfig
 
 	fc.Node = inode
@@ -128,11 +136,18 @@ func main() {
 			}
 			continue
 		}
-		esd, err = elastic.PutToIndex(esd)
+
-		if err != nil {
+		if !isMessageBlacklisted(esd.MsgID) {
-			log.Error(err.Error())
+			esd, err = elastic.PutToIndex(esd)
+			if err != nil {
+				log.Error(err.Error())
+				continue
+			}
+		} else {
+			log.Warnf("Skipping blacklisted message %s", esd.MsgID)
 			continue
 		}
+
 		log.Warn("Message ", m.ID, " added to index and to assignates list")
 		newMessages = append(newMessages, esd)
 	}
@@ -142,3 +157,35 @@ func main() {
 		log.Error(err.Error())
 	}
 }
+
+func isMessageBlacklisted(id string) bool {
+	if denyMessages == nil {
+		return false
+	}
+
+	for i := range denyMessages {
+		if id == denyMessages[i] {
+			return true
+		}
+	}
+	return false
+}
+
+func loadBlacklist() []string {
+	if _, err := os.Stat(blacklistFile); os.IsNotExist(err) {
+		return nil
+	}
+
+	file, err := os.Open(blacklistFile)
+	if err != nil {
+		return nil
+	}
+	defer file.Close()
+
+	data, err := ioutil.ReadAll(file)
+	if err != nil {
+		return nil
+	}
+
+	return strings.Split(string(data), "\n")
+}