2017-02-11 19:53:20 +03:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"flag"
|
2018-11-08 14:50:15 +03:00
|
|
|
"strings"
|
2017-09-27 14:49:50 +03:00
|
|
|
|
|
|
|
"strconv"
|
|
|
|
|
2018-11-12 14:07:34 +03:00
|
|
|
log "github.com/sirupsen/logrus"
|
2017-09-27 14:49:50 +03:00
|
|
|
|
2017-02-11 19:53:20 +03:00
|
|
|
"gitea.difrex.ru/Umbrella/fetcher/i2es"
|
|
|
|
"gitea.difrex.ru/Umbrella/fetcher/idec"
|
|
|
|
)
|
|
|
|
|
2018-11-08 14:50:15 +03:00
|
|
|
var (
|
|
|
|
inode, es, esIndex, esType, addEchoes string
|
|
|
|
offset, limit int
|
|
|
|
all, reindex bool
|
|
|
|
)
|
2017-02-11 19:53:20 +03:00
|
|
|
|
|
|
|
// init ...
|
|
|
|
func init() {
|
|
|
|
flag.IntVar(&offset, "offset", 0, "<int>")
|
|
|
|
flag.IntVar(&limit, "limit", -1, "<int>")
|
|
|
|
flag.BoolVar(&all, "all", false, "Get all messages")
|
2018-01-05 17:05:45 +03:00
|
|
|
flag.BoolVar(&reindex, "reindex", false, "Force update an a message")
|
2017-02-11 19:53:20 +03:00
|
|
|
flag.StringVar(&inode, "inode", "https://ii-net.tk/ii/ii-point.php?q=/", "IDEC node URI")
|
|
|
|
flag.StringVar(&es, "es", "http://127.0.0.1:9200", "ES host")
|
|
|
|
flag.StringVar(&esIndex, "esindex", "idec", "ES index")
|
|
|
|
flag.StringVar(&esType, "estype", "post", "ES document type")
|
2018-11-08 14:50:15 +03:00
|
|
|
flag.StringVar(&addEchoes, "echoes", "", "Additional echoes to fetch. Comma separated")
|
2017-02-11 19:53:20 +03:00
|
|
|
flag.Parse()
|
|
|
|
}
|
|
|
|
|
|
|
|
func main() {
|
|
|
|
// extensions := idec.NewExtensions()
|
|
|
|
var fc idec.FetchConfig
|
|
|
|
|
|
|
|
fc.Node = inode
|
|
|
|
|
|
|
|
log.Print("Working for node: ", inode)
|
|
|
|
// trying get echo list
|
2018-11-08 14:50:15 +03:00
|
|
|
var echoes []string
|
2017-02-11 19:53:20 +03:00
|
|
|
e, err := fc.GetEchoList()
|
|
|
|
if err != nil {
|
2017-09-27 14:49:50 +03:00
|
|
|
log.Error(err.Error())
|
2018-11-08 14:50:15 +03:00
|
|
|
echoes = []string{"ii.14", "linux.14"}
|
2017-02-11 19:53:20 +03:00
|
|
|
} else {
|
|
|
|
for _, echo := range e {
|
|
|
|
echoes = append(echoes, echo.Name)
|
|
|
|
}
|
|
|
|
}
|
2018-11-08 14:50:15 +03:00
|
|
|
if addEchoes != "" {
|
|
|
|
echoes = append(echoes, strings.Split(addEchoes, ",")...)
|
|
|
|
}
|
|
|
|
fc.Echoes = echoes
|
2017-02-11 19:53:20 +03:00
|
|
|
|
|
|
|
log.Print("Receive echoelist: ", fc.Echoes)
|
|
|
|
|
|
|
|
fc.Num = 10
|
|
|
|
fc.Limit = limit
|
|
|
|
fc.Offset = offset
|
|
|
|
|
|
|
|
var elastic i2es.ESConf
|
|
|
|
elastic.Host = es
|
|
|
|
elastic.Index = esIndex
|
|
|
|
elastic.Type = esType
|
|
|
|
|
|
|
|
var ids []idec.ID
|
|
|
|
if all {
|
|
|
|
ids, err = fc.GetAllMessagesIDS()
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
ids, err = fc.GetMessagesIDS()
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-08 15:57:56 +03:00
|
|
|
var newMessages []i2es.ESDoc
|
2017-02-11 19:53:20 +03:00
|
|
|
var messages []idec.MSG
|
|
|
|
var stash []idec.ID
|
|
|
|
j := 0
|
|
|
|
for _, id := range ids {
|
|
|
|
stash = append(stash, id)
|
|
|
|
j++
|
2018-01-05 17:05:45 +03:00
|
|
|
if j == 50 {
|
2017-02-11 19:53:20 +03:00
|
|
|
m, err := fc.GetRawMessages(stash)
|
|
|
|
if err != nil {
|
|
|
|
log.Print(err)
|
|
|
|
}
|
|
|
|
for _, m := range m {
|
|
|
|
messages = append(messages, m)
|
|
|
|
}
|
|
|
|
stash = nil
|
|
|
|
j = 0
|
|
|
|
}
|
|
|
|
}
|
2017-12-07 17:07:46 +03:00
|
|
|
|
2017-02-11 19:53:20 +03:00
|
|
|
for _, m := range messages {
|
2017-12-07 17:07:46 +03:00
|
|
|
msg, err := idec.ParseMessage(m.Message)
|
|
|
|
if err != nil {
|
|
|
|
log.Error(err.Error())
|
2018-01-05 17:05:45 +03:00
|
|
|
log.Error(m.ID)
|
|
|
|
continue
|
2017-12-07 17:07:46 +03:00
|
|
|
}
|
2017-02-11 19:53:20 +03:00
|
|
|
|
2017-12-07 17:07:46 +03:00
|
|
|
var esd i2es.ESDoc
|
|
|
|
esd.Echo = msg.Echo
|
|
|
|
esd.Subg = msg.Subg
|
|
|
|
esd.Author = msg.From
|
|
|
|
esd.To = msg.To
|
|
|
|
ts := strconv.Itoa(msg.Timestamp)
|
|
|
|
esd.Date = ts
|
|
|
|
esd.Message = msg.Body
|
|
|
|
esd.MsgID = m.ID
|
|
|
|
esd.Repto = msg.Repto
|
|
|
|
esd.Address = msg.Address
|
2018-01-08 15:57:56 +03:00
|
|
|
esd.Misplaced = "no"
|
2018-01-05 17:05:45 +03:00
|
|
|
|
2017-12-07 17:07:46 +03:00
|
|
|
log.Debug("Check message ", m.ID)
|
|
|
|
_, err = elastic.CheckID(m.ID)
|
2018-01-05 17:05:45 +03:00
|
|
|
if err != nil && !reindex {
|
2018-01-05 17:39:37 +03:00
|
|
|
if reindex {
|
2018-01-08 15:57:56 +03:00
|
|
|
log.Warn("Reindexing " + m.ID)
|
2018-01-05 17:39:37 +03:00
|
|
|
}
|
2017-12-07 17:07:46 +03:00
|
|
|
continue
|
2017-02-11 19:53:20 +03:00
|
|
|
}
|
2018-01-08 15:57:56 +03:00
|
|
|
esd, err = elastic.PutToIndex(esd)
|
2017-12-07 17:07:46 +03:00
|
|
|
if err != nil {
|
|
|
|
log.Error(err.Error())
|
2018-01-05 17:05:45 +03:00
|
|
|
continue
|
2017-02-11 19:53:20 +03:00
|
|
|
}
|
2018-01-08 15:57:56 +03:00
|
|
|
log.Warn("Message ", m.ID, " added to index and to assignates list")
|
|
|
|
newMessages = append(newMessages, esd)
|
|
|
|
}
|
|
|
|
|
2018-11-12 15:39:24 +03:00
|
|
|
err = elastic.AssignTopics(newMessages)
|
|
|
|
if err != nil {
|
|
|
|
log.Error(err.Error())
|
|
|
|
}
|
2017-02-11 19:53:20 +03:00
|
|
|
}
|