From 07ec8e718b383a479d6eebe2491faa17ebd987bd Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Thu, 25 Mar 2021 18:11:00 +0300 Subject: [PATCH 1/7] [Static]: Initial --- go.mod | 10 ++++ go.sum | 36 ++++++++++++++ main.go | 6 +-- node/api.go | 4 ++ node/elastic.go | 97 +++++++++++++++++++++++++++---------- node/go.mod | 13 +++++ node/go.sum | 38 +++++++++++++++ node/ssr.go | 67 +++++++++++++++++++++++++ node/structs.go | 16 +++++- templates/echoes.html | 22 +++++++++ templates/footer.html | 8 +++ templates/header.html | 35 +++++++++++++ templates/latest_posts.html | 9 ++++ templates/post.html | 29 +++++++++++ templates/root.html | 17 +++++++ templates/style.html | 56 +++++++++++++++++++++ 16 files changed, 434 insertions(+), 29 deletions(-) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 node/go.mod create mode 100644 node/go.sum create mode 100644 node/ssr.go create mode 100644 templates/echoes.html create mode 100644 templates/footer.html create mode 100644 templates/header.html create mode 100644 templates/latest_posts.html create mode 100644 templates/post.html create mode 100644 templates/root.html create mode 100644 templates/style.html diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2273a10 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/idec-net/lessmore-node + +go 1.16 + +replace github.com/idec-net/lessmore-node/node => ./node + +require ( + github.com/idec-net/lessmore-node/node v0.0.0-00010101000000-000000000000 + github.com/sirupsen/logrus v1.8.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3aede04 --- /dev/null +++ b/go.sum @@ -0,0 +1,36 @@ +gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b h1:K0vLl90b8k+JaCcxoaqbKOfK0LpyTMHQg4a0ggI6HI0= +gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b/go.mod h1:rcNfqAtzWqj1MsvxDuqTuqTNiJ7r6f1reQvsuUaiHYY= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/idec-net/go-idec v0.0.0-20181106151523-61a006246343/go.mod h1:XUvr43ZLN/4bTZT7TEhJA/rsfFLQxnggX6iU5TGXgIY= +github.com/idec-net/go-idec v0.0.0-20190316125931-ba6681d1b33b h1:QnpZjlk1jtZwZzT8HKMSfFio+L/6QG16uz3zCbPTkLw= +github.com/idec-net/go-idec v0.0.0-20190316125931-ba6681d1b33b/go.mod h1:ST2XOvFc7oRd1FCiZPwYf78F43SV9D3r1S+J4OQMsUo= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/jarcoal/httpmock.v1 v1.0.0-20190304095222-3b6b0a8dbc05 h1:u9qyM/i6c8jhNxsMfz4qdKtZumvEhHWYu5jEeOn1SOA= +gopkg.in/jarcoal/httpmock.v1 v1.0.0-20190304095222-3b6b0a8dbc05/go.mod h1:d3R+NllX3X5e0zlG1Rful3uLvsGC/Q3OHut5464DEQw= diff --git a/main.go b/main.go index df4cd80..d3ab7a5 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,7 @@ import ( "flag" "os" - "gitea.difrex.ru/Umbrella/lessmore/node" + "github.com/idec-net/lessmore-node/node" log "github.com/sirupsen/logrus" ) @@ -22,8 +22,8 @@ var ( func init() { flag.StringVar(&listen, "listen", "127.0.0.1:15582", "Address to listen") flag.StringVar(&es, "es", "http://127.0.0.1:9200", "ES host") - flag.StringVar(&esMessagesIndex, "esindex", "idec3", "ES index") - flag.StringVar(&esMessagesType, "estype", "post", "ES index type") + flag.StringVar(&esMessagesIndex, "esindex", "", "ES index") + flag.StringVar(&esMessagesType, "estype", "", "ES index type") flag.StringVar(&add, "add", "", "User to add") flag.StringVar(&email, "email", "", "User email address") flag.BoolVar(&debug, "debug", false, "Debug output") diff --git a/node/api.go b/node/api.go index 244e08d..3af2103 100644 --- a/node/api.go +++ b/node/api.go @@ -230,6 +230,10 @@ func Serve(listen string, es ESConf) { // Point methods r.HandleFunc("/u/point", es.UPointHandler).Methods("POST") + // Simple and clean SSR UI + ssr := newSSR("./templates", es) + r.HandleFunc("/ssr", ssr.ssrRootHandler) + http.Handle("/", r) srv := http.Server{ diff --git a/node/elastic.go b/node/elastic.go index 25f608a..ed253eb 100644 --- a/node/elastic.go +++ b/node/elastic.go @@ -7,6 +7,7 @@ import ( "net/http" "strconv" "strings" + "time" "fmt" @@ -136,12 +137,6 @@ func (es ESConf) GetEchoMessageHashes(echo string) []string { // GetLimitedEchoMessageHashes ... func (es ESConf) GetLimitedEchoMessageHashes(echo string, offset int, limit int) []string { var hashes []string - var searchURI string - if es.Index != "" && es.Type != "" { - searchURI = strings.Join([]string{es.Host, es.Index, es.Type, "_search"}, "/") - } else { - searchURI = strings.Join([]string{es.Host, "search"}, "/") - } // Check offset var order string @@ -158,7 +153,7 @@ func (es ESConf) GetLimitedEchoMessageHashes(echo string, offset int, limit int) {"date":{ "order": "`, order, `" }},{ "_score":{ "order": "`, order, `" }}], "query": {"query_string" : {"fields": ["msgid", "echo"], "query":"`, echo, `"}}, "size":`, l, `}`}, "")) - req, err := http.NewRequest("POST", searchURI, bytes.NewBuffer(searchQ)) + req, err := http.NewRequest("POST", es.searchURI(), bytes.NewBuffer(searchQ)) if err != nil { log.Error(err.Error()) return hashes @@ -375,15 +370,57 @@ func (es ESConf) GetXC(echoes string) []string { return counts } -// GetListTXT ... -func (es ESConf) GetListTXT() []byte { - var searchURI string - if es.Index != "" && es.Type != "" { - searchURI = strings.Join([]string{es.Host, es.Index, "_search"}, "/") - } else { - searchURI = strings.Join([]string{es.Host, "search"}, "/") +func (es ESConf) GetLatestPosts(sum int) []i2es.ESDoc { + query := fmt.Sprintf(`{"sort": [{"date": {"order": "desc"}}, {"_score": {"order": "desc" }}], "size": %d}`, sum) + req, err := http.NewRequest("POST", es.searchURI(), bytes.NewBuffer([]byte(query))) + if err != nil { + log.Error(err.Error()) + return nil } - searchQ := []byte(`{ + req.Header.Add("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Error(err.Error()) + return nil + } + + defer resp.Body.Close() + + var esr ESSearchResp + err = json.NewDecoder(resp.Body).Decode(&esr) + if err != nil { + log.Error(err.Error()) + return nil + } + + var posts []i2es.ESDoc + for _, hit := range esr.Hits.Hits { + hit.Source.Date = parseTime(hit.Source.Date) + hit.Source.Message = strings.Trim(hit.Source.Message, "\n") + posts = append(posts, hit.Source) + } + + return posts +} + +func parseTime(t string) string { + i, err := strconv.ParseInt(t, 10, 64) + if err != nil { + return "" + } + ts := time.Unix(i, 0) + return ts.Format(time.UnixDate) +} + +type echo struct { + Name string + Docs int64 +} + +func (es ESConf) GetEchoesList() []echo { + searchQ := []byte(`{ "size": 0, "aggs": { "uniqueEcho": { @@ -399,12 +436,11 @@ func (es ESConf) GetListTXT() []byte { } } }`) - log.Print("Search URI: ", searchURI) - req, err := http.NewRequest("POST", searchURI, bytes.NewBuffer(searchQ)) + req, err := http.NewRequest("POST", es.searchURI(), bytes.NewBuffer(searchQ)) if err != nil { log.Error(err.Error()) - return []byte("") + return nil } req.Header.Add("Content-Type", "application/json") @@ -412,7 +448,7 @@ func (es ESConf) GetListTXT() []byte { resp, err := client.Do(req) if err != nil { log.Error(err.Error()) - return []byte("") + return nil } defer resp.Body.Close() @@ -421,15 +457,26 @@ func (es ESConf) GetListTXT() []byte { err = json.NewDecoder(resp.Body).Decode(&esr) if err != nil { log.Error(err.Error()) - return []byte("") + return nil } - log.Infof("%+v", esr) - var echoes []string + var echoes []echo for _, bucket := range esr.EchoAgg["echo"].Buckets { - echoes = append(echoes, fmt.Sprintf("%s:%d:", bucket.Key, bucket.DocCount)) + echoes = append(echoes, echo{bucket.Key, int64(bucket.DocCount)}) } - log.Print("Getting ", len(echoes), " echoes") - return []byte(strings.Join(echoes, "\n")) + return echoes +} + +// GetListTXT ... +func (es ESConf) GetListTXT() []byte { + var listTXT []string + echoes := es.GetEchoesList() + for _, echo := range echoes { + listTXT = append(listTXT, fmt.Sprintf("%s:%d:(TODO) description support", echo.Name, echo.Docs)) + } + // Add new line to be more compatible with fetchers + listTXT[len(listTXT)-1] = listTXT[len(listTXT)-1] + "\n" + + return []byte(strings.Join(listTXT, "\n")) } diff --git a/node/go.mod b/node/go.mod new file mode 100644 index 0000000..8f2f76d --- /dev/null +++ b/node/go.mod @@ -0,0 +1,13 @@ +module github.com/idec-net/lessmore-node/node + +go 1.16 + +require ( + gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b + github.com/google/uuid v1.2.0 // indirect + github.com/gorilla/mux v1.8.0 + github.com/idec-net/go-idec v0.0.0-20190316125931-ba6681d1b33b + github.com/sirupsen/logrus v1.8.1 + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 + golang.org/x/sys v0.0.0-20210324051608-47abb6519492 // indirect +) diff --git a/node/go.sum b/node/go.sum new file mode 100644 index 0000000..22a2149 --- /dev/null +++ b/node/go.sum @@ -0,0 +1,38 @@ +gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b h1:K0vLl90b8k+JaCcxoaqbKOfK0LpyTMHQg4a0ggI6HI0= +gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b/go.mod h1:rcNfqAtzWqj1MsvxDuqTuqTNiJ7r6f1reQvsuUaiHYY= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/idec-net/go-idec v0.0.0-20181106151523-61a006246343/go.mod h1:XUvr43ZLN/4bTZT7TEhJA/rsfFLQxnggX6iU5TGXgIY= +github.com/idec-net/go-idec v0.0.0-20190316125931-ba6681d1b33b h1:QnpZjlk1jtZwZzT8HKMSfFio+L/6QG16uz3zCbPTkLw= +github.com/idec-net/go-idec v0.0.0-20190316125931-ba6681d1b33b/go.mod h1:ST2XOvFc7oRd1FCiZPwYf78F43SV9D3r1S+J4OQMsUo= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/jarcoal/httpmock.v1 v1.0.0-20190304095222-3b6b0a8dbc05 h1:u9qyM/i6c8jhNxsMfz4qdKtZumvEhHWYu5jEeOn1SOA= +gopkg.in/jarcoal/httpmock.v1 v1.0.0-20190304095222-3b6b0a8dbc05/go.mod h1:d3R+NllX3X5e0zlG1Rful3uLvsGC/Q3OHut5464DEQw= diff --git a/node/ssr.go b/node/ssr.go new file mode 100644 index 0000000..eea1dab --- /dev/null +++ b/node/ssr.go @@ -0,0 +1,67 @@ +package node + +import ( + "html/template" + "net/http" + "os" + + "gitea.difrex.ru/Umbrella/fetcher/i2es" + log "github.com/sirupsen/logrus" +) + +type ssr struct { + es ESConf + templatesDir string +} + +func newSSR(templatesDir string, es ESConf) *ssr { + return &ssr{ + es: es, + templatesDir: templatesDir, + } +} + +func (s *ssr) templatePath(name string) string { + return s.templatesDir + string(os.PathSeparator) + name +} + +func (s *ssr) getTemplate(name string) (*template.Template, error) { + t := template.New("root.html") + + tpl, err := t.ParseFiles( + s.templatePath(name+".html"), + s.templatePath("style.html"), + s.templatePath("echoes.html"), + s.templatePath("post.html"), + s.templatePath("latest_posts.html"), + s.templatePath("header.html"), + s.templatePath("footer.html"), + ) + if err != nil { + return nil, err + } + + return tpl, nil +} + +func (s *ssr) ssrRootHandler(w http.ResponseWriter, r *http.Request) { + LogRequest(r) + + tpl, err := s.getTemplate("root") + if err != nil { + log.Error(err) + return + } + + var root struct { + Echoes []echo + CurrentPage string + Posts []i2es.ESDoc + } + root.Echoes = s.es.GetEchoesList() + root.Posts = s.es.GetLatestPosts(100) + + if err := tpl.Execute(w, root); err != nil { + log.Error(err) + } +} diff --git a/node/structs.go b/node/structs.go index 27a79d6..ebeaaf2 100644 --- a/node/structs.go +++ b/node/structs.go @@ -1,6 +1,10 @@ package node -import "gitea.difrex.ru/Umbrella/fetcher/i2es" +import ( + "strings" + + "gitea.difrex.ru/Umbrella/fetcher/i2es" +) // PointRequest with message type PointRequest struct { @@ -11,6 +15,16 @@ type PointRequest struct { // ESConf ... type ESConf i2es.ESConf +// searchURI returns an ElasticSearch search URL string +func (es ESConf) searchURI() (searchURI string) { + if es.Index != "" && es.Type != "" { + searchURI = strings.Join([]string{es.Host, es.Index, es.Type, "_search"}, "/") + } else { + searchURI = strings.Join([]string{es.Host, "search"}, "/") + } + return +} + // Bucket ... type Bucket struct { Key string `json:"key"` diff --git a/templates/echoes.html b/templates/echoes.html new file mode 100644 index 0000000..7f63f53 --- /dev/null +++ b/templates/echoes.html @@ -0,0 +1,22 @@ +{{ define "echoes" }} + +
+ {{ range .Echoes }} + +
+
+
+
+ {{ .Name }} +
+
+ {{ .Docs }} +
+
+
+
+ + {{ end }} +
+ +{{ end }} diff --git a/templates/footer.html b/templates/footer.html new file mode 100644 index 0000000..184d687 --- /dev/null +++ b/templates/footer.html @@ -0,0 +1,8 @@ +{{ define "footer" }} + + + + + + +{{ end }} diff --git a/templates/header.html b/templates/header.html new file mode 100644 index 0000000..594aa3c --- /dev/null +++ b/templates/header.html @@ -0,0 +1,35 @@ +{{ define "header" }} + + + {{ if .CurrentPage }}staic | {{ .CurrentPage }}{{ else }}dynamic{{ end }} + + + + + + + {{ end }} diff --git a/templates/latest_posts.html b/templates/latest_posts.html new file mode 100644 index 0000000..c17dee5 --- /dev/null +++ b/templates/latest_posts.html @@ -0,0 +1,9 @@ +{{ define "latest posts" }} + +{{ range .Posts }} + +{{ template "post" . }} + +{{ end }} + +{{ end }} diff --git a/templates/post.html b/templates/post.html new file mode 100644 index 0000000..f5a05a6 --- /dev/null +++ b/templates/post.html @@ -0,0 +1,29 @@ +{{ define "post" }} + +
+
+

+ {{ .Subg }} +

+
+

+ [{{ .Echo }}] + {{ .Date }} + @{{ .Author }} -> + {{ if .Repto }}{{ .To }} + {{ else }}{{ .To }}{{ end }} +

+
+
+ {{ .Message }} +
+
+
+ [#] [reply] +
+
+ +{{ end }} diff --git a/templates/root.html b/templates/root.html new file mode 100644 index 0000000..fc7f1e7 --- /dev/null +++ b/templates/root.html @@ -0,0 +1,17 @@ +{{ template "header" . }} + +{{ template "style" }} + +
+
+
+ {{ template "echoes" . }} +
+ +
+ {{ template "latest posts" . }} +
+
+
+ +{{ template "footer" . }} diff --git a/templates/style.html b/templates/style.html new file mode 100644 index 0000000..7043d17 --- /dev/null +++ b/templates/style.html @@ -0,0 +1,56 @@ +{{ define "style" }} + +{{ end }} From 5d507d2810430779f94a8e6d31fc48f37fc9ff44 Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Thu, 25 Mar 2021 18:25:40 +0300 Subject: [PATCH 2/7] [Static]: move to root --- node/api.go | 11 +++++------ node/elastic.go | 6 +----- templates/post.html | 6 ++++++ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/node/api.go b/node/api.go index 3af2103..5821929 100644 --- a/node/api.go +++ b/node/api.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "net/http" "strings" - "time" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" @@ -232,15 +231,15 @@ func Serve(listen string, es ESConf) { // Simple and clean SSR UI ssr := newSSR("./templates", es) - r.HandleFunc("/ssr", ssr.ssrRootHandler) + r.HandleFunc("/", ssr.ssrRootHandler) http.Handle("/", r) srv := http.Server{ - Handler: r, - Addr: listen, - WriteTimeout: 15 * time.Second, - ReadTimeout: 15 * time.Second, + Handler: r, + Addr: listen, + // WriteTimeout: 15 * time.Second, + // ReadTimeout: 15 * time.Second, } log.Print("Listening IDEC API on ", listen) diff --git a/node/elastic.go b/node/elastic.go index ed253eb..15aca14 100644 --- a/node/elastic.go +++ b/node/elastic.go @@ -111,12 +111,8 @@ func (es ESConf) GetEchoMessageHashes(echo string) []string { log.Error(err.Error()) return hashes } - defer resp.Body.Close() - content, _ := ioutil.ReadAll(resp.Body) - log.Info(string(content)) - var esr ESSearchResp err = json.NewDecoder(resp.Body).Decode(&esr) if err != nil { @@ -253,7 +249,7 @@ func (es ESConf) GetUEchoMessageHashes(echoes string) []string { o, err := strconv.Atoi(oflim[0]) l, err := strconv.Atoi(oflim[1]) if err != nil { - log.Print(err) + log.Error(err) } else { offset = o limit = l diff --git a/templates/post.html b/templates/post.html index f5a05a6..0cad48d 100644 --- a/templates/post.html +++ b/templates/post.html @@ -1,4 +1,10 @@ {{ define "post" }} +
From 400e8e2562c9c9133bf1b3050f85bad2c50febf5 Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Fri, 26 Mar 2021 11:11:14 +0300 Subject: [PATCH 3/7] [static]: simple html framework --- go.sum | 1 + node/api.go | 1 + node/elastic.go | 36 ++++++++++ node/go.sum | 2 - node/ssr.go | 53 ++++++-------- node/structs.go | 14 ++++ node/templates.go | 98 ++++++++++++++++++++++++++ templates/common/footer.html | 12 ++++ templates/{ => common}/header.html | 34 +++++---- templates/{ => common}/style.html | 0 templates/{ => components}/echoes.html | 2 +- templates/{ => components}/post.html | 8 ++- templates/footer.html | 8 --- templates/meta/forum.html | 29 ++++++++ templates/{ => meta}/latest_posts.html | 0 templates/views/forum.html | 17 +++++ templates/{ => views}/root.html | 0 17 files changed, 258 insertions(+), 57 deletions(-) create mode 100644 node/templates.go create mode 100644 templates/common/footer.html rename templates/{ => common}/header.html (57%) rename templates/{ => common}/style.html (100%) rename templates/{ => components}/echoes.html (88%) rename templates/{ => components}/post.html (76%) delete mode 100644 templates/footer.html create mode 100644 templates/meta/forum.html rename templates/{ => meta}/latest_posts.html (100%) create mode 100644 templates/views/forum.html rename templates/{ => views}/root.html (100%) diff --git a/go.sum b/go.sum index 3aede04..3ba0dcb 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b h1:K0vLl90b8 gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b/go.mod h1:rcNfqAtzWqj1MsvxDuqTuqTNiJ7r6f1reQvsuUaiHYY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elastic/go-elasticsearch/v6 v6.8.10/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= diff --git a/node/api.go b/node/api.go index 5821929..acfdd2d 100644 --- a/node/api.go +++ b/node/api.go @@ -232,6 +232,7 @@ func Serve(listen string, es ESConf) { // Simple and clean SSR UI ssr := newSSR("./templates", es) r.HandleFunc("/", ssr.ssrRootHandler) + r.HandleFunc("/forum", ssr.ssrForumHandler) http.Handle("/", r) diff --git a/node/elastic.go b/node/elastic.go index 15aca14..936e3da 100644 --- a/node/elastic.go +++ b/node/elastic.go @@ -366,6 +366,42 @@ func (es ESConf) GetXC(echoes string) []string { return counts } +type ThreadBucket struct { + DocCount int64 `json:"doc_count"` + Key string `json:"key"` + Post Hits +} + +func (es ESConf) GetThreads(echoes ...string) (posts []i2es.ESDoc) { + query := `{"sort":[{"date":{"order":"desc"}}],"aggs":{"topics":{"terms":{"field":"topicid.keyword","size":100},"aggs":{"post":{"top_hits":{"size":1,"sort":[{"date":{"order":"desc"}}],"_source":{"include":["subg","author","date","echo","topicid","address"]}}}}}},"query":{"bool":{"must":[{"range":{"date":{"from":"now-30d","to":"now-0d"}}},{"constant_score":{"filter":{"terms":{"echo.keyword":["idec.talks","pipe.2032","linux.14","develop.16","dynamic.local","std.club","std.hugeping","oldpc.51t.ru","difrex.blog","ii.test.14"]}}}}]}}}` + req, err := http.NewRequest("POST", es.searchURI(), bytes.NewReader([]byte(query))) + if err != nil { + log.Error(err) + return + } + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Error(err) + return + } + defer resp.Body.Close() + + var data ESAggsResp + if err := json.NewDecoder(resp.Body).Decode(&data); err != nil { + log.Error(err) + return + } + for _, bucket := range data.Aggregations.Topics.Buckets { + for _, post := range bucket.Post.Hits.Hits { + posts = append(posts, post.Source) + } + } + + return +} + func (es ESConf) GetLatestPosts(sum int) []i2es.ESDoc { query := fmt.Sprintf(`{"sort": [{"date": {"order": "desc"}}, {"_score": {"order": "desc" }}], "size": %d}`, sum) req, err := http.NewRequest("POST", es.searchURI(), bytes.NewBuffer([]byte(query))) diff --git a/node/go.sum b/node/go.sum index 22a2149..3aede04 100644 --- a/node/go.sum +++ b/node/go.sum @@ -3,7 +3,6 @@ gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b/go.mod h1:rc github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= -github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -27,7 +26,6 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/node/ssr.go b/node/ssr.go index eea1dab..a2b1d4d 100644 --- a/node/ssr.go +++ b/node/ssr.go @@ -1,9 +1,7 @@ package node import ( - "html/template" "net/http" - "os" "gitea.difrex.ru/Umbrella/fetcher/i2es" log "github.com/sirupsen/logrus" @@ -21,47 +19,40 @@ func newSSR(templatesDir string, es ESConf) *ssr { } } -func (s *ssr) templatePath(name string) string { - return s.templatesDir + string(os.PathSeparator) + name +type PageData struct { + Echoes []echo + CurrentPage string + Posts []i2es.ESDoc } -func (s *ssr) getTemplate(name string) (*template.Template, error) { - t := template.New("root.html") - - tpl, err := t.ParseFiles( - s.templatePath(name+".html"), - s.templatePath("style.html"), - s.templatePath("echoes.html"), - s.templatePath("post.html"), - s.templatePath("latest_posts.html"), - s.templatePath("header.html"), - s.templatePath("footer.html"), - ) - if err != nil { - return nil, err +func (s *ssr) newPageData(page string, posts []i2es.ESDoc) *PageData { + return &PageData{ + Echoes: s.es.GetEchoesList(), + Posts: posts, + CurrentPage: page, } - - return tpl, nil } func (s *ssr) ssrRootHandler(w http.ResponseWriter, r *http.Request) { - LogRequest(r) - tpl, err := s.getTemplate("root") if err != nil { log.Error(err) return } - var root struct { - Echoes []echo - CurrentPage string - Posts []i2es.ESDoc - } - root.Echoes = s.es.GetEchoesList() - root.Posts = s.es.GetLatestPosts(100) - - if err := tpl.Execute(w, root); err != nil { + if err := tpl.Execute(w, s.newPageData("", s.es.GetLatestPosts(50))); err != nil { + log.Error(err) + } +} + +func (s *ssr) ssrForumHandler(w http.ResponseWriter, r *http.Request) { + tpl, err := s.getTemplate("forum") + if err != nil { + log.Error(err) + return + } + + if err := tpl.Execute(w, s.newPageData("", s.es.GetThreads())); err != nil { log.Error(err) } } diff --git a/node/structs.go b/node/structs.go index ebeaaf2..26c1288 100644 --- a/node/structs.go +++ b/node/structs.go @@ -39,6 +39,20 @@ type ESSearchResp struct { Hits `json:"hits"` } +type ESAggsResp struct { + ESSearchResp + Aggregations struct { + Topics struct { + Buckets []struct { + Bucket + Post struct { + Hits Hits `json:"hits"` + } `json:"post"` + } `json:"buckets"` + } `json:"topics"` + } `json:"aggregations"` +} + type Hits struct { Total int64 `json:"total"` MaxScore float64 `json:"max_score"` diff --git a/node/templates.go b/node/templates.go new file mode 100644 index 0000000..b6d13ad --- /dev/null +++ b/node/templates.go @@ -0,0 +1,98 @@ +package node + +import ( + "os" + "strings" + "text/template" + + log "github.com/sirupsen/logrus" +) + +const ( + // Common templates + CommonTplDir = "common" + + // Reusable components + ComponentsTplDir = "components" + + // Something like component of components + MetaTplDir = "meta" + + // Main pages + ViewsTplDir = "views" +) + +func (s *ssr) tplDir(name string) string { + return strings.Join([]string{s.templatesDir, name}, string(os.PathSeparator)) +} + +func (s *ssr) readTplDir(name string) ([]string, error) { + var paths []string + + dir, err := os.ReadDir(s.templatesDir + string(os.PathSeparator) + name) + if err != nil { + return nil, err + } + + for _, entry := range dir { + if strings.HasSuffix(entry.Name(), ".html") { + paths = append(paths, strings.Join( + []string{s.templatesDir, name, entry.Name()}, + string(os.PathSeparator))) + } + } + + return paths, nil +} + +func (s *ssr) tplsPaths() ([]string, error) { + var paths []string + // Read common dir + commonTpls, err := s.readTplDir(CommonTplDir) + if err != nil { + return nil, err + } + paths = append(paths, commonTpls...) + + // Read components dir + compTpls, err := s.readTplDir(ComponentsTplDir) + if err != nil { + return nil, err + } + paths = append(paths, compTpls...) + + // Read meta dir + metaTpls, err := s.readTplDir(MetaTplDir) + if err != nil { + return nil, err + } + paths = append(paths, metaTpls...) + + return paths, nil +} + +func (s *ssr) componentsForTemplate(name string) []string { + paths := []string{s.tplDir(ViewsTplDir) + string(os.PathSeparator) + name + ".html"} + tpls, err := s.tplsPaths() + if err != nil { + log.Error(err) + return paths + } + + return append(paths, tpls...) +} + +func (s *ssr) templatePath(name string) string { + return s.templatesDir + string(os.PathSeparator) + name +} + +func (s *ssr) getTemplate(name string) (*template.Template, error) { + t := template.New(name + ".html") + + tpl, err := t.ParseFiles(s.componentsForTemplate(name)...) + if err != nil { + return nil, err + } + + return tpl, nil +} diff --git a/templates/common/footer.html b/templates/common/footer.html new file mode 100644 index 0000000..98e820a --- /dev/null +++ b/templates/common/footer.html @@ -0,0 +1,12 @@ +{{ define "footer" }} + + +

+ copyleft 2021 difrex at lessmore dot pw; + + source code +

+ + + +{{ end }} diff --git a/templates/header.html b/templates/common/header.html similarity index 57% rename from templates/header.html rename to templates/common/header.html index 594aa3c..6cdbb29 100644 --- a/templates/header.html +++ b/templates/common/header.html @@ -6,27 +6,37 @@ -
diff --git a/templates/style.html b/templates/common/style.html similarity index 100% rename from templates/style.html rename to templates/common/style.html diff --git a/templates/echoes.html b/templates/components/echoes.html similarity index 88% rename from templates/echoes.html rename to templates/components/echoes.html index 7f63f53..0ae1895 100644 --- a/templates/echoes.html +++ b/templates/components/echoes.html @@ -3,7 +3,7 @@
{{ range .Echoes }} -
+
diff --git a/templates/post.html b/templates/components/post.html similarity index 76% rename from templates/post.html rename to templates/components/post.html index 0cad48d..77e14e9 100644 --- a/templates/post.html +++ b/templates/components/post.html @@ -6,7 +6,9 @@ { template "post" .Posts[0] } --> -
+

{{ .Subg }} @@ -18,7 +20,7 @@ {{ .Date }} @{{ .Author }} -> + href="#/author/{{ .Author }}">{{ .Author }} -> {{ if .Repto }}{{ .To }} {{ else }}{{ .To }}{{ end }}

@@ -28,7 +30,7 @@

- [#] [reply] + [#] [reply]
diff --git a/templates/footer.html b/templates/footer.html deleted file mode 100644 index 184d687..0000000 --- a/templates/footer.html +++ /dev/null @@ -1,8 +0,0 @@ -{{ define "footer" }} - - - - - - -{{ end }} diff --git a/templates/meta/forum.html b/templates/meta/forum.html new file mode 100644 index 0000000..9518856 --- /dev/null +++ b/templates/meta/forum.html @@ -0,0 +1,29 @@ +{{ define "forum" }} + +
+
+ + + + + + + + + + + + {{ range .Posts }} + + + + + + + + {{ end }} + +
latest threads
echothreadlatestaddressdate
{{ .Echo }}{{ .Subg }}{{ .Author }}{{ .Address }}{{ .Date }}
+
+
+{{ end }} diff --git a/templates/latest_posts.html b/templates/meta/latest_posts.html similarity index 100% rename from templates/latest_posts.html rename to templates/meta/latest_posts.html diff --git a/templates/views/forum.html b/templates/views/forum.html new file mode 100644 index 0000000..e254ed5 --- /dev/null +++ b/templates/views/forum.html @@ -0,0 +1,17 @@ +{{ template "header" . }} + +{{ template "style" }} + +
+
+
+ {{ template "echoes" . }} +
+ +
+ {{ template "forum" . }} +
+
+
+ +{{ template "footer" . }} diff --git a/templates/root.html b/templates/views/root.html similarity index 100% rename from templates/root.html rename to templates/views/root.html From 0c29d2652956bd07327295025d5277676d48b718 Mon Sep 17 00:00:00 2001 From: Denis Zheleztsov Date: Tue, 30 Mar 2021 19:47:25 +0300 Subject: [PATCH 4/7] [ssr]: Read only is working --- node/api.go | 4 + node/elastic.go | 130 ++++++++++++++++++++++++++- node/ssr.go | 146 ++++++++++++++++++++++++++++++- templates/common/header.html | 15 ++-- templates/common/style.html | 3 + templates/components/echoes.html | 4 +- templates/components/post.html | 12 +-- templates/meta/forum.html | 20 +++-- templates/views/echo.html | 40 +++++++++ templates/views/message.html | 19 ++++ templates/views/root.html | 4 +- templates/views/search.html | 17 ++++ templates/views/thread.html | 17 ++++ 13 files changed, 406 insertions(+), 25 deletions(-) create mode 100644 templates/views/echo.html create mode 100644 templates/views/message.html create mode 100644 templates/views/search.html create mode 100644 templates/views/thread.html diff --git a/node/api.go b/node/api.go index acfdd2d..c4bb716 100644 --- a/node/api.go +++ b/node/api.go @@ -233,6 +233,10 @@ func Serve(listen string, es ESConf) { ssr := newSSR("./templates", es) r.HandleFunc("/", ssr.ssrRootHandler) r.HandleFunc("/forum", ssr.ssrForumHandler) + r.HandleFunc("/echo/{echo:[a-z0-9-_.]+}/page/{page:[0-9]+}", ssr.echoViewHandler) + r.HandleFunc("/thread/{topicid:[a-z0-9-]+}", ssr.threadViewHandler) + r.HandleFunc("/msg/{msgid:[a-zA-Z0-9]{20}}", ssr.singleMessageHandler) + r.HandleFunc("/find", ssr.searchHandler).Methods(http.MethodGet) http.Handle("/", r) diff --git a/node/elastic.go b/node/elastic.go index 936e3da..c070a37 100644 --- a/node/elastic.go +++ b/node/elastic.go @@ -178,6 +178,42 @@ func (es ESConf) GetLimitedEchoMessageHashes(echo string, offset int, limit int) return hashes } +func (es ESConf) DoSearch(query string) []i2es.ESDoc { + q := `{"sort": [ + {"date":{ "order": "desc" }},{ "_score":{ "order": "desc" }}], + "query": {"query_string" : {"fields": ["message", "subg"], "query":` + query + `}}, "size": 100}` + + req, err := http.NewRequest("POST", es.searchURI(), bytes.NewBuffer([]byte(q))) + if err != nil { + log.Error(err.Error()) + return nil + } + req.Header.Add("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Error(err.Error()) + return nil + } + + defer resp.Body.Close() + + var esr ESSearchResp + err = json.NewDecoder(resp.Body).Decode(&esr) + if err != nil { + log.Error(err.Error()) + return nil + } + + var posts []i2es.ESDoc + for _, hit := range esr.Hits.Hits { + posts = append(posts, hit.Source) + } + + return posts +} + func (es ESConf) GetUMMessages(msgs string) []string { var encodedMessages []string @@ -372,8 +408,97 @@ type ThreadBucket struct { Post Hits } -func (es ESConf) GetThreads(echoes ...string) (posts []i2es.ESDoc) { - query := `{"sort":[{"date":{"order":"desc"}}],"aggs":{"topics":{"terms":{"field":"topicid.keyword","size":100},"aggs":{"post":{"top_hits":{"size":1,"sort":[{"date":{"order":"desc"}}],"_source":{"include":["subg","author","date","echo","topicid","address"]}}}}}},"query":{"bool":{"must":[{"range":{"date":{"from":"now-30d","to":"now-0d"}}},{"constant_score":{"filter":{"terms":{"echo.keyword":["idec.talks","pipe.2032","linux.14","develop.16","dynamic.local","std.club","std.hugeping","oldpc.51t.ru","difrex.blog","ii.test.14"]}}}}]}}}` +var defaultEchoes = []string{`"idec.talks"`, `"pipe.2032"`, `"linux.14"`, `"develop.16"`, `"dynamic.local"`, `"std.club"`, `"std.hugeping"`, `"difrex.blog"`, `"ii.test.14"`} + +func (es ESConf) GetTopic(topicID string) (posts []i2es.ESDoc) { + query := []byte(strings.Join([]string{ + `{"sort": [{"date": {"order": "asc"}}, + {"_score": {"order": "desc" }}], "size":1000,"query": {"term": {"topicid.keyword": "`, topicID, `"}}}`}, "")) + + req, err := http.NewRequest("POST", es.searchURI(), bytes.NewReader([]byte(query))) + if err != nil { + log.Error(err) + return + } + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Error(err) + return + } + defer resp.Body.Close() + + var esr ESSearchResp + err = json.NewDecoder(resp.Body).Decode(&esr) + if err != nil { + log.Error(err.Error()) + return nil + } + + for _, hit := range esr.Hits.Hits { + hit.Source.Message = strings.Trim(hit.Source.Message, "\n") + hit.Source.Date = parseTime(hit.Source.Date) + posts = append(posts, hit.Source) + } + + return +} + +func (es ESConf) GetMessage(msgID string) (posts []i2es.ESDoc) { + query := []byte(strings.Join([]string{ + `{"sort": [{"date": {"order": "asc"}}, + {"_score": {"order": "desc" }}], "size":1000,"query": {"term": {"msgid.keyword": "`, msgID, `"}}}`}, "")) + + req, err := http.NewRequest("POST", es.searchURI(), bytes.NewReader([]byte(query))) + if err != nil { + log.Error(err) + return + } + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Error(err) + return + } + defer resp.Body.Close() + + var esr ESSearchResp + err = json.NewDecoder(resp.Body).Decode(&esr) + if err != nil { + log.Error(err.Error()) + return nil + } + + for _, hit := range esr.Hits.Hits { + hit.Source.Message = strings.Trim(hit.Source.Message, "\n") + hit.Source.Date = parseTime(hit.Source.Date) + posts = append(posts, hit.Source) + } + + return +} + +func (es ESConf) GetThreads(pageNum int, echoes ...string) (posts []i2es.ESDoc) { + ech := defaultEchoes + if len(echoes) > 0 { + ech = []string{} + for _, echo := range echoes { + ech = append(ech, fmt.Sprintf(`"%s"`, echo)) + } + } + rangeStr := `"from":"now-30d","to":"now-0d"` + if pageNum > 1 { + to := 30*pageNum - 30 + from := 30 * pageNum + rangeStr = fmt.Sprintf(`"from":"now-%dd","to":"now-%dd"`, from, to) + } + log.Debug(rangeStr) + + query := `{"sort":[{"date":{"order":"desc"}}],"aggs":{"topics":{"terms":{"field":"topicid.keyword","size":100},"aggs":{"post":{"top_hits":{"size":1,"sort":[{"date":{"order":"desc"}}],"_source":{"include": ["subg","author","date","echo","topicid","address"]}}}}}},"query":{"bool":{"must":[{"range":{"date":{` + rangeStr + `}}},{"constant_score":{"filter":{"terms":{"echo.keyword": [` + + strings.Join(ech, ",") + + `]}}}}]}}}` req, err := http.NewRequest("POST", es.searchURI(), bytes.NewReader([]byte(query))) if err != nil { log.Error(err) @@ -393,6 +518,7 @@ func (es ESConf) GetThreads(echoes ...string) (posts []i2es.ESDoc) { log.Error(err) return } + for _, bucket := range data.Aggregations.Topics.Buckets { for _, post := range bucket.Post.Hits.Hits { posts = append(posts, post.Source) diff --git a/node/ssr.go b/node/ssr.go index a2b1d4d..76e8246 100644 --- a/node/ssr.go +++ b/node/ssr.go @@ -1,9 +1,14 @@ package node import ( + "encoding/json" "net/http" + "time" + + "strconv" "gitea.difrex.ru/Umbrella/fetcher/i2es" + "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -22,14 +27,32 @@ func newSSR(templatesDir string, es ESConf) *ssr { type PageData struct { Echoes []echo CurrentPage string + PageNum int Posts []i2es.ESDoc } -func (s *ssr) newPageData(page string, posts []i2es.ESDoc) *PageData { +func (p *PageData) GetDate(date string) string { + d, err := strconv.ParseInt(date, 0, 64) + if err != nil { + return err.Error() + } + return time.Unix(d, 0).UTC().Format("02 Jan 06 15:04 MST") +} + +func (p *PageData) Inc() int { + return p.PageNum + 1 +} + +func (p *PageData) Dec() int { + return p.PageNum - 1 +} + +func (s *ssr) newPageData(page string, posts []i2es.ESDoc, num int) *PageData { return &PageData{ Echoes: s.es.GetEchoesList(), Posts: posts, CurrentPage: page, + PageNum: num, } } @@ -40,7 +63,7 @@ func (s *ssr) ssrRootHandler(w http.ResponseWriter, r *http.Request) { return } - if err := tpl.Execute(w, s.newPageData("", s.es.GetLatestPosts(50))); err != nil { + if err := tpl.Execute(w, s.newPageData("feed", s.es.GetLatestPosts(50), 1)); err != nil { log.Error(err) } } @@ -52,7 +75,124 @@ func (s *ssr) ssrForumHandler(w http.ResponseWriter, r *http.Request) { return } - if err := tpl.Execute(w, s.newPageData("", s.es.GetThreads())); err != nil { + vars := mux.Vars(r) + + var num int + if _, ok := vars["page"]; ok { + num = getPageNum(mux.Vars(r)["page"]) + } + + if err := tpl.Execute(w, s.newPageData("forum", s.es.GetThreads(num), num)); err != nil { log.Error(err) } } + +func (s *ssr) threadViewHandler(w http.ResponseWriter, r *http.Request) { + tpl, err := s.getTemplate("thread") + if err != nil { + log.Error(err) + return + } + + topicid, ok := mux.Vars(r)["topicid"] + if !ok { + log.Warn("empty topicid") + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("error: empty topicid")) + return + } + + posts := s.es.GetTopic(topicid) + thread := "nil" + if len(posts) > 0 { + thread = posts[0].Subg + } + + if err := tpl.Execute(w, s.newPageData(thread, posts, 1)); err != nil { + log.Error(err) + } +} + +func getPageNum(page string) int { + i, err := strconv.ParseInt(page, 0, 64) + if err != nil { + log.Error(err) + return 1 + } + if i < 1 { + return 1 + } + return int(i) +} + +func (s *ssr) echoViewHandler(w http.ResponseWriter, r *http.Request) { + tpl, err := s.getTemplate("echo") + if err != nil { + log.Error(err) + return + } + + vars := mux.Vars(r) + echo, ok := vars["echo"] + if !ok { + log.Warn("empty echo") + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("error: empty echo")) + return + } + + page := 1 + if _, ok := vars["page"]; ok { + page = getPageNum(vars["page"]) + } + + if err := tpl.Execute(w, s.newPageData(echo, s.es.GetThreads(page, echo), page)); err != nil { + log.Error(err) + } +} + +func (s *ssr) singleMessageHandler(w http.ResponseWriter, r *http.Request) { + tpl, err := s.getTemplate("message") + if err != nil { + log.Error(err) + return + } + + msgid, ok := mux.Vars(r)["msgid"] + if !ok { + log.Warn("empty msgid") + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("error: empty msgid")) + return + } + + if err := tpl.Execute(w, s.newPageData(msgid, s.es.GetMessage(msgid), 1)); err != nil { + log.Error(err) + } +} + +func (s *ssr) searchHandler(w http.ResponseWriter, r *http.Request) { + tpl, err := s.getTemplate("search") + if err != nil { + log.Error(err) + return + } + + q := r.URL.Query().Get("query") + if q != "" { + m, err := json.Marshal(q) + if err != nil { + log.Error(err) + m = []byte("") + } + + posts := s.es.DoSearch(string(m)) + for i := range posts { + posts[i].Date = parseTime(posts[i].Date) + } + + if err := tpl.Execute(w, s.newPageData("search", posts, 1)); err != nil { + log.Error(err) + } + } +} diff --git a/templates/common/header.html b/templates/common/header.html index 6cdbb29..20c1bd9 100644 --- a/templates/common/header.html +++ b/templates/common/header.html @@ -1,14 +1,14 @@ {{ define "header" }} - {{ if .CurrentPage }}staic | {{ .CurrentPage }}{{ else }}dynamic{{ end }} + {{ if .CurrentPage }}static | {{ .CurrentPage }}{{ else }}static{{ end }} -