139 lines
2.6 KiB
Go
139 lines
2.6 KiB
Go
package main
|
|
|
|
import (
|
|
"flag"
|
|
|
|
"strconv"
|
|
|
|
log "github.com/Sirupsen/logrus"
|
|
|
|
"gitea.difrex.ru/Umbrella/fetcher/i2es"
|
|
"gitea.difrex.ru/Umbrella/fetcher/idec"
|
|
)
|
|
|
|
var inode string
|
|
var offset int
|
|
var limit int
|
|
var all, reindex bool
|
|
var es string
|
|
var esIndex string
|
|
var esType string
|
|
|
|
// init ...
|
|
func init() {
|
|
flag.IntVar(&offset, "offset", 0, "<int>")
|
|
flag.IntVar(&limit, "limit", -1, "<int>")
|
|
flag.BoolVar(&all, "all", false, "Get all messages")
|
|
flag.BoolVar(&reindex, "reindex", false, "Force update an a message")
|
|
flag.StringVar(&inode, "inode", "https://ii-net.tk/ii/ii-point.php?q=/", "IDEC node URI")
|
|
flag.StringVar(&es, "es", "http://127.0.0.1:9200", "ES host")
|
|
flag.StringVar(&esIndex, "esindex", "idec", "ES index")
|
|
flag.StringVar(&esType, "estype", "post", "ES document type")
|
|
flag.Parse()
|
|
}
|
|
|
|
func main() {
|
|
// extensions := idec.NewExtensions()
|
|
var fc idec.FetchConfig
|
|
|
|
fc.Node = inode
|
|
|
|
log.Print("Working for node: ", inode)
|
|
// trying get echo list
|
|
e, err := fc.GetEchoList()
|
|
if err != nil {
|
|
log.Error(err.Error())
|
|
fc.Echoes = []string{"ii.14", "linux.14"}
|
|
} else {
|
|
var echoes []string
|
|
for _, echo := range e {
|
|
echoes = append(echoes, echo.Name)
|
|
}
|
|
fc.Echoes = echoes
|
|
}
|
|
|
|
log.Print("Receive echoelist: ", fc.Echoes)
|
|
|
|
fc.Num = 10
|
|
fc.Limit = limit
|
|
fc.Offset = offset
|
|
|
|
var elastic i2es.ESConf
|
|
elastic.Host = es
|
|
elastic.Index = esIndex
|
|
elastic.Type = esType
|
|
|
|
var ids []idec.ID
|
|
if all {
|
|
ids, err = fc.GetAllMessagesIDS()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
} else {
|
|
ids, err = fc.GetMessagesIDS()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
var messages []idec.MSG
|
|
var stash []idec.ID
|
|
j := 0
|
|
for _, id := range ids {
|
|
stash = append(stash, id)
|
|
j++
|
|
if j == 50 {
|
|
m, err := fc.GetRawMessages(stash)
|
|
if err != nil {
|
|
log.Print(err)
|
|
}
|
|
for _, m := range m {
|
|
messages = append(messages, m)
|
|
}
|
|
stash = nil
|
|
j = 0
|
|
}
|
|
}
|
|
|
|
for _, m := range messages {
|
|
msg, err := idec.ParseMessage(m.Message)
|
|
if err != nil {
|
|
log.Error(err.Error())
|
|
log.Error(m.ID)
|
|
continue
|
|
}
|
|
|
|
var esd i2es.ESDoc
|
|
esd.Echo = msg.Echo
|
|
esd.Subg = msg.Subg
|
|
esd.Author = msg.From
|
|
esd.To = msg.To
|
|
ts := strconv.Itoa(msg.Timestamp)
|
|
esd.Date = ts
|
|
esd.Message = msg.Body
|
|
esd.MsgID = m.ID
|
|
esd.Repto = msg.Repto
|
|
esd.Address = msg.Address
|
|
|
|
if !reindex {
|
|
log.Warn(msg.Repto)
|
|
continue
|
|
}
|
|
|
|
log.Debug("Check message ", m.ID)
|
|
_, err = elastic.CheckID(m.ID)
|
|
if err != nil && !reindex {
|
|
if reindex {
|
|
log.Warn("New thread message " + m.ID)
|
|
}
|
|
continue
|
|
}
|
|
err = elastic.PutToIndex(esd)
|
|
if err != nil {
|
|
log.Error(err.Error())
|
|
continue
|
|
}
|
|
log.Warn("Message ", m.ID, " added to index")
|
|
}
|
|
}
|