Umbrella/fetcher
Umbrella/fetcher
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Some changes

Browse Source
This commit is contained in:
Denis Zheleztsov 2018-01-05 17:57:38 +03:00
parent a0728b97c8
commit c88d5fcc4a
2 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
i2es/elastic.go
View File

@ -64,7 +64,6 @@ func doRequest(uri, data, method string) ([]byte, error) {
// findTopicID for msgid // findTopicID for msgid
func (t topicid) findTopicID(msgid string, enter int) (string, error) { func (t topicid) findTopicID(msgid string, enter int) (string, error) {
topicid := uuid.New().String()
searchURI := strings.Join([]string{t.es.Host, t.es.Index, t.es.Type, "_search"}, "/") searchURI := strings.Join([]string{t.es.Host, t.es.Index, t.es.Type, "_search"}, "/")
searchQ := strings.Join([]string{`{"sort": [{"date": {"order": "desc"}}, {"_score": { "order": "desc" }}], "query": { "query_string" : {"fields": ["msgid"], "query": "msgid.keyword:` + msgid + ` "}}, "size":' + show_rows + ', "from":' + from + '}`}, "") searchQ := strings.Join([]string{`{"sort": [{"date": {"order": "desc"}}, {"_score": { "order": "desc" }}], "query": { "query_string" : {"fields": ["msgid"], "query": "msgid.keyword:` + msgid + ` "}}, "size":' + show_rows + ', "from":' + from + '}`}, "")
@ -85,12 +84,13 @@ func (t topicid) findTopicID(msgid string, enter int) (string, error) {
} }
if res.Hits.Hits[0].Source.TopicID != "" { if res.Hits.Hits[0].Source.TopicID != "" {
topicid = res.Hits.Hits[0].Source.TopicID topicid := res.Hits.Hits[0].Source.TopicID
return topicid, nil return topicid, nil
} }
if res.Hits.Hits[0].Source.Repto != "" && res.Hits.Hits[0].Source.TopicID == "" { if res.Hits.Hits[0].Source.Repto != "" && res.Hits.Hits[0].Source.TopicID == "" {
if enter > 25 { if enter > 25 {
topicid := uuid.New().String()
log.Error("Maximum recurse depth. Top message not in index? Forse assign new topicID " + topicid) log.Error("Maximum recurse depth. Top message not in index? Forse assign new topicID " + topicid)
return topicid, nil return topicid, nil
} else { } else {
@ -102,6 +102,7 @@ func (t topicid) findTopicID(msgid string, enter int) (string, error) {
if res.Hits.Hits[0].Source.Repto == "" && res.Hits.Hits[0].Source.TopicID == "" { if res.Hits.Hits[0].Source.Repto == "" && res.Hits.Hits[0].Source.TopicID == "" {
// This is a old message without topicid => generate new topicid and update a top document // This is a old message without topicid => generate new topicid and update a top document
topicid := uuid.New().String()
log.Warn(topicid) log.Warn(topicid)
log.Warn("Top message " + res.Hits.Hits[0].Source.MsgID + " found but it is doesn not have a topicID") log.Warn("Top message " + res.Hits.Hits[0].Source.MsgID + " found but it is doesn not have a topicID")
log.Warn("Generated new topicID " + topicid) log.Warn("Generated new topicID " + topicid)
@ -124,10 +125,10 @@ func (t topicid) findTopicID(msgid string, enter int) (string, error) {
} }
log.Info(string(updateDocPlain)) log.Info(string(updateDocPlain))
}
return topicid, nil return topicid, nil
} }
return "", errors.New("Something went wrong")
}
// check if // check if
func (t topicid) checkTopicID(msg *ESDoc) func() string { func (t topicid) checkTopicID(msg *ESDoc) func() string {
@ -146,9 +147,10 @@ func (t topicid) checkTopicID(msg *ESDoc) func() string {
if msg.Repto != "" { if msg.Repto != "" {
var findAndSetTopicID = func() string { var findAndSetTopicID = func() string {
topicid, err := t.findTopicID(msg.MsgID, 0) topicid, err := t.findTopicID(msg.Repto, 0)
if err != nil { if err != nil {
log.Error(err.Error()) log.Error(err.Error())
return ""
} }
log.Info("Topic ID " + topicid + " found for message " + msg.MsgID) log.Info("Topic ID " + topicid + " found for message " + msg.MsgID)
return topicid return topicid
@ -175,7 +177,7 @@ func (es ESConf) PutToIndex(msg ESDoc) error {
var t topicid var t topicid
t.es = es t.es = es
topic := t.getOrCreate(&msg) topic := t.getOrCreate(&msg)
if msg.TopicID == "" { if msg.TopicID == "" && topic != "" {
msg.TopicID = topic msg.TopicID = topic
} }

2
main.go
View File

@ -115,7 +115,7 @@ func main() {
esd.Repto = msg.Repto esd.Repto = msg.Repto
esd.Address = msg.Address esd.Address = msg.Address
if reindex && msg.Repto != "" { if !reindex {
log.Warn(msg.Repto) log.Warn(msg.Repto)
continue continue
} }