[Quotes]: quotes parser

Dockerfile

@@ -0,0 +1,19 @@
+FROM golang:1.16 AS builder
+
+ENV CGO_ENABLED=0
+
+COPY . /build
+
+WORKDIR /build
+
+RUN go build -o /lessmore -v . && strip /lessmore
+
+FROM alpine AS final
+
+COPY --from=builder /lessmore /usr/local/bin/lessmore
+COPY --from=builder /build/templates /usr/local/share/lessmore/templates
+COPY --from=builder /build/static /usr/local/share/lessmore/static
+
+USER nobody
+
+ENTRYPOINT ["/usr/local/bin/lessmore"]

Dockerfile.app

@@ -1,14 +0,0 @@
-FROM alpine
-
-# Usage:
-# Required artifact from lessmore_builder in $(pwd)/out/
-# Build:
-# docker build -t lessmore -f Dockerfile.app .
-# Run:
-# docker run -ti -p 15582:15582 lessmore -listen 0.0.0.0:15582 -es http://$ES:9200 -esindex idec -estype idec
-
-MAINTAINER Denis Zheleztsov <difrex.punk@gmail.com>
-
-ADD out/lessmore /usr/bin
-
-ENTRYPOINT ["lessmore"]

Dockerfile.builder

@@ -1,25 +0,0 @@
-FROM alpine
-
-MAINTAINER Denis Zheleztsov <difrex.punk@gmail.com>
-
-# Usage:
-# Build docker image:
-# docker build -t lessmore_builder -f Dockerfile.builder .
-# Build binary artifact:
-# docker run -ti -v $(pwd)/out:/out/ lessmore_builder
-
-# Install depends
-RUN apk update && apk add git go
-
-ENV GOPATH /usr
-
-# Get go depends
-RUN go get gitea.difrex.ru/Umbrella/fetcher
-RUN go get gitea.difrex.ru/Umbrella/lessmore
-RUN go install gitea.difrex.ru/Umbrella/lessmore
-
-# Check build result
-RUN echo -ne "Check build result\n==============="
-RUN /usr/bin/lessmore --help || [[ $? -eq 2 ]]
-
-ENTRYPOINT mv /usr/bin/lessmore /out/

go.mod

@@ -0,0 +1,10 @@
+module github.com/idec-net/lessmore-node
+
+go 1.16
+
+replace github.com/idec-net/lessmore-node/node => ./node
+
+require (
+	github.com/idec-net/lessmore-node/node v0.0.0-00010101000000-000000000000
+	github.com/sirupsen/logrus v1.8.1
+)

go.sum

@@ -0,0 +1,37 @@
+gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b h1:K0vLl90b8k+JaCcxoaqbKOfK0LpyTMHQg4a0ggI6HI0=
+gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b/go.mod h1:rcNfqAtzWqj1MsvxDuqTuqTNiJ7r6f1reQvsuUaiHYY=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elastic/go-elasticsearch/v6 v6.8.10/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
+github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/idec-net/go-idec v0.0.0-20181106151523-61a006246343/go.mod h1:XUvr43ZLN/4bTZT7TEhJA/rsfFLQxnggX6iU5TGXgIY=
+github.com/idec-net/go-idec v0.0.0-20190316125931-ba6681d1b33b h1:QnpZjlk1jtZwZzT8HKMSfFio+L/6QG16uz3zCbPTkLw=
+github.com/idec-net/go-idec v0.0.0-20190316125931-ba6681d1b33b/go.mod h1:ST2XOvFc7oRd1FCiZPwYf78F43SV9D3r1S+J4OQMsUo=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg=
+golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/jarcoal/httpmock.v1 v1.0.0-20190304095222-3b6b0a8dbc05 h1:u9qyM/i6c8jhNxsMfz4qdKtZumvEhHWYu5jEeOn1SOA=
+gopkg.in/jarcoal/httpmock.v1 v1.0.0-20190304095222-3b6b0a8dbc05/go.mod h1:d3R+NllX3X5e0zlG1Rful3uLvsGC/Q3OHut5464DEQw=

main.go

@@ -4,7 +4,7 @@ import (
 	"flag"
 	"os"
 
-	"gitea.difrex.ru/Umbrella/lessmore/node"
+	"github.com/idec-net/lessmore-node/node"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -15,17 +15,28 @@ var (
 	esMessagesType  string
 	add             string
 	email           string
-	debug           bool
+
+	templatesDir string
+
+	serveStatic bool
+	staticDir   string
+
+	debug bool
 )
 
 // init ...
 func init() {
 	flag.StringVar(&listen, "listen", "127.0.0.1:15582", "Address to listen")
 	flag.StringVar(&es, "es", "http://127.0.0.1:9200", "ES host")
-	flag.StringVar(&esMessagesIndex, "esindex", "idec3", "ES index")
+	flag.StringVar(&esMessagesIndex, "esindex", "", "ES index")
-	flag.StringVar(&esMessagesType, "estype", "post", "ES index type")
+	flag.StringVar(&esMessagesType, "estype", "", "ES index type")
 	flag.StringVar(&add, "add", "", "User to add")
 	flag.StringVar(&email, "email", "", "User email address")
+	flag.StringVar(&templatesDir, "templates-dir", "/usr/local/share/lessmore/templates", "Path to templates dir")
+
+	flag.StringVar(&staticDir, "static-dir", "/usr/local/share/lessmore/static", "Path to static dir")
+	flag.BoolVar(&serveStatic, "static-serve", false, "Serve static files")
+
 	flag.BoolVar(&debug, "debug", false, "Debug output")
 	flag.Parse()
 
@@ -43,7 +54,16 @@ func main() {
 	if add != "" {
 		addUser(add, esconf)
 	}
-	node.Serve(listen, esconf)
+
+	opts := &node.ServeOpts{
+		Listen:       listen,
+		ES:           esconf,
+		TemplatesDir: templatesDir,
+		ServeStatic:  serveStatic,
+		StaticDir:    staticDir,
+	}
+
+	node.Serve(opts)
 }
 
 func addUser(name string, esconf node.ESConf) {

node/api.go

@@ -5,7 +5,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strings"
-	"time"
 
 	"github.com/gorilla/mux"
 	log "github.com/sirupsen/logrus"
@@ -81,13 +80,8 @@ func (es ESConf) UEHandler(w http.ResponseWriter, r *http.Request) {
 
 	LogRequest(r)
 
-	ch := make(chan []string)
 	// Get echolist
-	go func() {
+	messages := es.GetUEchoMessageHashes(e)
-		ch <- es.GetUEchoMessageHashes(e)
-	}()
-
-	messages := <-ch
 
 	w.WriteHeader(200)
 	w.Write([]byte(strings.Join(messages, "\n")))
@@ -211,34 +205,63 @@ func (es ESConf) UPointHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("msg ok"))
 }
 
+type ServeOpts struct {
+	Listen       string
+	TemplatesDir string
+	ServeStatic  bool
+	StaticDir    string
+	ES           ESConf
+}
+
 // Serve ...
-func Serve(listen string, es ESConf) {
+func Serve(opts *ServeOpts) {
 	r := mux.NewRouter()
-	r.HandleFunc("/list.txt", es.ListTXTHandler).Methods("GET")
+	r.HandleFunc("/list.txt", opts.ES.ListTXTHandler).Methods("GET")
-	r.HandleFunc("/blacklist.txt", es.BlacklistTXT).Methods("GET")
+	r.HandleFunc("/blacklist.txt", opts.ES.BlacklistTXT).Methods("GET")
 	r.HandleFunc("/x/features", XFeaturesHandler).Methods("GET")
 
 	// Standart schemas
-	r.HandleFunc("/e/{echo}", es.EHandler).Methods("GET")
+	r.HandleFunc("/e/{echo}", opts.ES.EHandler).Methods("GET")
-	r.HandleFunc("/m/{msgid}", es.MHandler).Methods("GET")
+	r.HandleFunc("/m/{msgid}", opts.ES.MHandler).Methods("GET")
 
 	// Extensions
-	r.HandleFunc("/u/e/{echoes:[a-z0-9-_/.:]+}", es.UEHandler).Methods("GET")
+	r.HandleFunc("/u/e/{echoes:[a-z0-9-_/.:]+}", opts.ES.UEHandler).Methods("GET")
-	r.HandleFunc("/u/m/{ids:[a-zA-Z0-9-_/.:]+}", es.UMHandler).Methods("GET")
+	r.HandleFunc("/u/m/{ids:[a-zA-Z0-9-_/.:]+}", opts.ES.UMHandler).Methods("GET")
-	r.HandleFunc("/x/c/{echoes:[a-zA-Z0-9-_/.:]+}", es.XCHandler).Methods("GET")
+	r.HandleFunc("/x/c/{echoes:[a-zA-Z0-9-_/.:]+}", opts.ES.XCHandler).Methods("GET")
 
 	// Point methods
-	r.HandleFunc("/u/point", es.UPointHandler).Methods("POST")
+	r.HandleFunc("/u/point", opts.ES.UPointHandler).Methods("POST")
+
+	// Simple and clean SSR UI
+	ssr := newSSR(opts.TemplatesDir, opts.ES)
+	r.HandleFunc("/", ssr.ssrRootHandler)
+	// Forum
+	r.HandleFunc("/forum/page/{page:[0-9]+}", ssr.ssrForumHandler)
+	// Echo
+	r.HandleFunc("/echo/{echo:[a-z0-9-_.]+}/page/{page:[0-9]+}", ssr.echoViewHandler)
+	// Thread
+	r.HandleFunc("/thread/{topicid:[a-z0-9-]+}", ssr.threadViewHandler)
+	// Single message
+	r.HandleFunc("/msg/{msgid:[a-zA-Z0-9]{20}}", ssr.singleMessageHandler)
+	// Search
+	r.HandleFunc("/find", ssr.searchHandler).Methods(http.MethodGet)
+	// Docs
+	// formatting
+	r.HandleFunc("/docs/formatting", ssr.docsFormattingHandler)
+
+	if opts.ServeStatic {
+		r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(opts.StaticDir))))
+	}
 
 	http.Handle("/", r)
 
 	srv := http.Server{
-		Handler:      r,
+		Handler: r,
-		Addr:         listen,
+		Addr:    opts.Listen,
-		WriteTimeout: 15 * time.Second,
+		// WriteTimeout: 15 * time.Second,
-		ReadTimeout:  15 * time.Second,
+		// ReadTimeout:  15 * time.Second,
 	}
 
-	log.Print("Listening IDEC API on ", listen)
+	log.Print("Listening IDEC API on ", opts.Listen)
 	log.Fatal(srv.ListenAndServe())
 }

node/elastic.go

@@ -3,10 +3,12 @@ package node
 import (
 	"bytes"
 	"encoding/json"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"strconv"
 	"strings"
+	"time"
 
 	"fmt"
 
@@ -95,7 +97,7 @@ func (es ESConf) GetEchoMessageHashes(echo string) []string {
 	searchQ := []byte(strings.Join([]string{
 		`{"sort": [
             {"date":{ "order": "desc" }},{ "_score":{ "order": "desc" }}],
-          "query": {"query_string" : {"fields": ["echo"], "query":"`, echo, `"}}, "size": 500}`}, ""))
+          "query": {"query_string" : {"fields": ["echo.keyword"], "query":"`, echo, `"}}, "size": 500}`}, ""))
 
 	req, err := http.NewRequest("POST", searchURI, bytes.NewBuffer(searchQ))
 	if err != nil {
@@ -110,17 +112,18 @@ func (es ESConf) GetEchoMessageHashes(echo string) []string {
 		log.Error(err.Error())
 		return hashes
 	}
-
 	defer resp.Body.Close()
 
-	content, _ := ioutil.ReadAll(resp.Body)
+	content, err := ioutil.ReadAll(resp.Body)
-	log.Info(string(content))
+	if err != nil {
+		log.Error(err.Error())
+		hashes = append(hashes, "error: Internal error")
+		return hashes
+	}
 
 	var esr ESSearchResp
-	err = json.NewDecoder(resp.Body).Decode(&esr)
+	err = json.Unmarshal(content, &esr)
 	if err != nil {
-		b, _ := ioutil.ReadAll(resp.Body)
-		log.Error(string(b))
 		log.Error(err.Error())
 		hashes = append(hashes, "error: Internal error")
 		return hashes
@@ -136,12 +139,6 @@ func (es ESConf) GetEchoMessageHashes(echo string) []string {
 // GetLimitedEchoMessageHashes ...
 func (es ESConf) GetLimitedEchoMessageHashes(echo string, offset int, limit int) []string {
 	var hashes []string
-	var searchURI string
-	if es.Index != "" && es.Type != "" {
-		searchURI = strings.Join([]string{es.Host, es.Index, es.Type, "_search"}, "/")
-	} else {
-		searchURI = strings.Join([]string{es.Host, "search"}, "/")
-	}
 
 	// Check offset
 	var order string
@@ -156,9 +153,9 @@ func (es ESConf) GetLimitedEchoMessageHashes(echo string, offset int, limit int)
 	searchQ := []byte(strings.Join([]string{
 		`{"sort": [
             {"date":{ "order": "`, order, `" }},{ "_score":{ "order": "`, order, `" }}],
-          "query": {"query_string" : {"fields": ["msgid", "echo"], "query":"`, echo, `"}}, "size":`, l, `}`}, ""))
+          "query": {"query_string" : {"fields": ["msgid.keyword", "echo.keyword"], "query":"`, echo, `"}}, "size":`, l, `}`}, ""))
 
-	req, err := http.NewRequest("POST", searchURI, bytes.NewBuffer(searchQ))
+	req, err := http.NewRequest("POST", es.searchURI(), bytes.NewBuffer(searchQ))
 	if err != nil {
 		log.Error(err.Error())
 		return hashes
@@ -187,6 +184,42 @@ func (es ESConf) GetLimitedEchoMessageHashes(echo string, offset int, limit int)
 	return hashes
 }
 
+func (es ESConf) DoSearch(query string) []i2es.ESDoc {
+	q := `{"sort": [
+            {"date":{ "order": "desc" }},{ "_score":{ "order": "desc" }}],
+          "query": {"query_string" : {"fields": ["message", "subg"], "query":` + query + `}}, "size": 100}`
+
+	req, err := http.NewRequest("POST", es.searchURI(), bytes.NewBuffer([]byte(q)))
+	if err != nil {
+		log.Error(err.Error())
+		return nil
+	}
+	req.Header.Add("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Error(err.Error())
+		return nil
+	}
+
+	defer resp.Body.Close()
+
+	var esr ESSearchResp
+	err = json.NewDecoder(resp.Body).Decode(&esr)
+	if err != nil {
+		log.Error(err.Error())
+		return nil
+	}
+
+	var posts []i2es.ESDoc
+	for _, hit := range esr.Hits.Hits {
+		posts = append(posts, hit.Source)
+	}
+
+	return posts
+}
+
 func (es ESConf) GetUMMessages(msgs string) []string {
 	var encodedMessages []string
 
@@ -202,7 +235,7 @@ func (es ESConf) GetUMMessages(msgs string) []string {
 {
   "query": {
     "query_string" : {
-      "fields": ["msgid"],
+      "fields": ["msgid.keyword"],
       "query":"` + strings.Join(messages, " OR ") + `"
     }
   },
@@ -258,7 +291,7 @@ func (es ESConf) GetUEchoMessageHashes(echoes string) []string {
 		o, err := strconv.Atoi(oflim[0])
 		l, err := strconv.Atoi(oflim[1])
 		if err != nil {
-			log.Print(err)
+			log.Error(err)
 		} else {
 			offset = o
 			limit = l
@@ -297,7 +330,7 @@ func (es ESConf) GetUEchoMessageHashes(echoes string) []string {
 	// Some20SimbolsHash333
 	for k, v := range eh {
 		echohashes = append(echohashes, k)
-		if k == "" {
+		if k == "" || k == "\n" {
 			continue
 		}
 		for _, e := range v {
@@ -308,7 +341,7 @@ func (es ESConf) GetUEchoMessageHashes(echoes string) []string {
 		}
 	}
 
-	return echohashes
+	return addNewLineToLastWord(echohashes)
 }
 
 // GetXC implements /x/c
@@ -325,7 +358,7 @@ func (es ESConf) GetXC(echoes string) []string {
 {
     "query": {
         "query_string" : {
-            "fields": ["echo"],
+            "fields": ["echo.keyword"],
             "query": "` + strings.Join(strings.Split(echoes, "/"), " OR ") + `"
         }
     },
@@ -333,12 +366,12 @@ func (es ESConf) GetXC(echoes string) []string {
     "aggs": {
         "uniqueEcho": {
             "cardinality": {
-                "field": "echo"
+                "field": "echo.keyword"
             }
         },
         "echo": {
             "terms": {
-                "field": "echo",
+                "field": "echo.keyword",
                 "size": 1000
             }
         }
@@ -375,36 +408,270 @@ func (es ESConf) GetXC(echoes string) []string {
 	return counts
 }
 
-// GetListTXT ...
+type ThreadBucket struct {
-func (es ESConf) GetListTXT() []byte {
+	DocCount int64  `json:"doc_count"`
-	var searchURI string
+	Key      string `json:"key"`
-	if es.Index != "" && es.Type != "" {
+	Post     Hits
-		searchURI = strings.Join([]string{es.Host, es.Index, "_search"}, "/")
+}
-	} else {
+
-		searchURI = strings.Join([]string{es.Host, "search"}, "/")
+var defaultEchoes = []string{`"idec.talks"`, `"pipe.2032"`, `"linux.14"`, `"develop.16"`, `"dynamic.local"`, `"std.club"`, `"std.hugeping"`, `"difrex.blog"`, `"ii.test.14"`}
+
+func (es ESConf) GetTopic(topicID string) (posts []i2es.ESDoc) {
+	query := []byte(strings.Join([]string{
+		`{"sort": [{"date": {"order": "asc"}},
+         {"_score": {"order": "desc" }}], "size":1000,"query": {"term": {"topicid.keyword": "`, topicID, `"}}}`}, ""))
+
+	req, err := http.NewRequest("POST", es.searchURI(), bytes.NewReader([]byte(query)))
+	if err != nil {
+		log.Error(err)
+		return
 	}
-	searchQ := []byte(`{                                                   
+	req.Header.Add("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+	defer resp.Body.Close()
+
+	var esr ESSearchResp
+	err = json.NewDecoder(resp.Body).Decode(&esr)
+	if err != nil {
+		log.Error(err.Error())
+		return nil
+	}
+
+	for _, hit := range esr.Hits.Hits {
+		hit.Source.Message = strings.Trim(hit.Source.Message, "\n")
+		hit.Source.Date = parseTime(hit.Source.Date)
+		posts = append(posts, hit.Source)
+	}
+
+	return
+}
+
+func (es ESConf) GetMessage(msgID string) (posts []i2es.ESDoc) {
+	query := []byte(strings.Join([]string{
+		`{"sort": [{"date": {"order": "asc"}},
+         {"_score": {"order": "desc" }}], "size":1000,"query": {"term": {"msgid.keyword": "`, msgID, `"}}}`}, ""))
+
+	req, err := http.NewRequest("POST", es.searchURI(), bytes.NewReader([]byte(query)))
+	if err != nil {
+		log.Error(err)
+		return
+	}
+	req.Header.Add("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+	defer resp.Body.Close()
+
+	var esr ESSearchResp
+	err = json.NewDecoder(resp.Body).Decode(&esr)
+	if err != nil {
+		log.Error(err.Error())
+		return nil
+	}
+
+	for _, hit := range esr.Hits.Hits {
+		hit.Source.Message = strings.Trim(hit.Source.Message, "\n")
+		hit.Source.Date = parseTime(hit.Source.Date)
+		posts = append(posts, hit.Source)
+	}
+
+	return
+}
+
+func (es ESConf) GetThreads(pageNum int, echoes ...string) (posts []i2es.ESDoc) {
+	ech := defaultEchoes
+	if len(echoes) > 0 {
+		ech = []string{}
+		for _, echo := range echoes {
+			ech = append(ech, fmt.Sprintf(`"%s"`, echo))
+		}
+	}
+	rangeStr := `"from":"now-30d","to":"now-0d"`
+	if pageNum > 1 {
+		to := 30*pageNum - 30
+		from := 30 * pageNum
+		rangeStr = fmt.Sprintf(`"from":"now-%dd","to":"now-%dd"`, from, to)
+	}
+
+	log.Debug(rangeStr)
+
+	query := `{"sort":[{"date":{"order":"desc"}}],"aggs":{"topics":{"terms":{"field":"topicid.keyword","size":100},"aggs":{"post":{"top_hits":{"size":1,"sort":[{"date":{"order":"desc"}}],"_source":{"include": ["subg","author","date","echo","topicid","address", "repto"]}}}}}},"query":{"bool":{"must":[{"range":{"date":{` + rangeStr + `}}},{"constant_score":{"filter":{"terms":{"echo.keyword": [` +
+		strings.Join(ech, ",") +
+		`]}}}}]}}}`
+	log.Debug("Run: ", query)
+
+	req, err := http.NewRequest("POST", es.searchURI(), bytes.NewReader([]byte(query)))
+	if err != nil {
+		log.Error(err)
+		return
+	}
+	req.Header.Add("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+	defer resp.Body.Close()
+
+	var data ESAggsResp
+	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+		log.Error(err)
+		return
+	}
+
+	for _, bucket := range data.Aggregations.Topics.Buckets {
+		// Empty topicid
+		if bucket.Key == "" {
+			continue
+		}
+		for _, post := range bucket.Post.Hits.Hits {
+			posts = append(posts, post.Source)
+		}
+	}
+
+	return
+}
+
+func (es ESConf) GetThreadsYear(pageNum int, echoes ...string) (posts []i2es.ESDoc) {
+	ech := defaultEchoes
+	if len(echoes) > 0 {
+		ech = []string{}
+		for _, echo := range echoes {
+			ech = append(ech, fmt.Sprintf(`"%s"`, echo))
+		}
+	}
+
+	query := `{"sort":[{"date":{"order":"desc"}}],"aggs":{"topics":{"terms":{"field":"topicid.keyword","size":500},"aggs":{"post":{"top_hits":{"size":1,"sort":[{"date":{"order":"desc"}}],"_source":{"include": ["subg","author","date","echo","topicid","address", "repto"]}}}}}},"query":{"bool":{"must":[{"range":{"date":{"from": "now-365d", "to": "now"}}}, {"constant_score":{"filter":{"terms":{"echo.keyword": [` +
+		strings.Join(ech, ",") +
+		`]}}}}]}}}`
+	log.Debug("Run: ", query)
+
+	req, err := http.NewRequest("POST", es.searchURI(), bytes.NewReader([]byte(query)))
+	if err != nil {
+		log.Error(err)
+		return
+	}
+	req.Header.Add("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode > 200 {
+		d, _ := io.ReadAll(resp.Body)
+		log.Debug(string(d))
+	}
+
+	var data ESAggsResp
+	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+		log.Error(err)
+		return
+	}
+
+	for _, bucket := range data.Aggregations.Topics.Buckets {
+		// Empty topicid
+		if bucket.Key == "" {
+			continue
+		}
+		for _, post := range bucket.Post.Hits.Hits {
+			posts = append(posts, post.Source)
+		}
+	}
+
+	return
+}
+
+func (es ESConf) GetLatestPosts(sum int) []i2es.ESDoc {
+	log.Debug(sum)
+	query := fmt.Sprintf(`{"sort": [{"date": {"order": "desc"}}, {"_score": {"order": "desc" }}], "size": %d}`, sum)
+	log.Debugf("Do %s request", query)
+
+	req, err := http.NewRequest("POST", es.searchURI(), bytes.NewBuffer([]byte(query)))
+	if err != nil {
+		log.Error(err.Error())
+		return nil
+	}
+
+	req.Header.Add("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Error(err.Error())
+		return nil
+	}
+
+	defer resp.Body.Close()
+
+	var esr ESSearchResp
+	err = json.NewDecoder(resp.Body).Decode(&esr)
+	if err != nil {
+		log.Error(err.Error())
+		return nil
+	}
+
+	var posts []i2es.ESDoc
+	for _, hit := range esr.Hits.Hits {
+		hit.Source.Date = parseTime(hit.Source.Date)
+		hit.Source.Message = strings.Trim(hit.Source.Message, "\n")
+		posts = append(posts, hit.Source)
+	}
+
+	return posts
+}
+
+func parseTime(t string) string {
+	i, err := strconv.ParseInt(t, 10, 64)
+	if err != nil {
+		return ""
+	}
+	ts := time.Unix(i, 0)
+	return ts.Format(time.UnixDate)
+}
+
+type echo struct {
+	Name string
+	Docs int64
+}
+
+func (es ESConf) GetEchoesList() []echo {
+	searchQ := []byte(`{
  "size": 0,
  "aggs": {
    "uniqueEcho": {
      "cardinality": {
-       "field": "echo"
+       "field": "echo.keyword"
      }
    },
    "echo": {
      "terms": {
-       "field": "echo",
+       "field": "echo.keyword",
        "size": 1000
      }
    }
  }
 }`)
-	log.Print("Search URI: ", searchURI)
+	log.Debugf("Do %s request", searchQ)
-
+	req, err := http.NewRequest("POST", es.searchURI(), bytes.NewBuffer(searchQ))
-	req, err := http.NewRequest("POST", searchURI, bytes.NewBuffer(searchQ))
 	if err != nil {
 		log.Error(err.Error())
-		return []byte("")
+		return nil
 	}
 	req.Header.Add("Content-Type", "application/json")
 
@@ -412,7 +679,7 @@ func (es ESConf) GetListTXT() []byte {
 	resp, err := client.Do(req)
 	if err != nil {
 		log.Error(err.Error())
-		return []byte("")
+		return nil
 	}
 
 	defer resp.Body.Close()
@@ -421,15 +688,26 @@ func (es ESConf) GetListTXT() []byte {
 	err = json.NewDecoder(resp.Body).Decode(&esr)
 	if err != nil {
 		log.Error(err.Error())
-		return []byte("")
+		return nil
 	}
-	log.Infof("%+v", esr)
 
-	var echoes []string
+	var echoes []echo
 	for _, bucket := range esr.EchoAgg["echo"].Buckets {
-		echoes = append(echoes, fmt.Sprintf("%s:%d:", bucket.Key, bucket.DocCount))
+		echoes = append(echoes, echo{bucket.Key, int64(bucket.DocCount)})
 	}
-	log.Print("Getting ", len(echoes), " echoes")
 
-	return []byte(strings.Join(echoes, "\n"))
+	return echoes
+}
+
+// GetListTXT ...
+func (es ESConf) GetListTXT() []byte {
+	var listTXT []string
+	echoes := es.GetEchoesList()
+	for _, echo := range echoes {
+		listTXT = append(listTXT, fmt.Sprintf("%s:%d:(TODO) description support", echo.Name, echo.Docs))
+	}
+	// Add new line to be more compatible with fetchers
+	listTXT[len(listTXT)-1] = listTXT[len(listTXT)-1] + "\n"
+
+	return []byte(strings.Join(listTXT, "\n"))
 }

node/go.mod

@@ -0,0 +1,13 @@
+module github.com/idec-net/lessmore-node/node
+
+go 1.16
+
+require (
+	gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b
+	github.com/google/uuid v1.2.0 // indirect
+	github.com/gorilla/mux v1.8.0
+	github.com/idec-net/go-idec v0.0.0-20190316125931-ba6681d1b33b
+	github.com/sirupsen/logrus v1.8.1
+	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
+	golang.org/x/sys v0.0.0-20210324051608-47abb6519492 // indirect
+)

node/go.sum

@@ -0,0 +1,36 @@
+gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b h1:K0vLl90b8k+JaCcxoaqbKOfK0LpyTMHQg4a0ggI6HI0=
+gitea.difrex.ru/Umbrella/fetcher v0.0.0-20200723122826-e8bbdd12256b/go.mod h1:rcNfqAtzWqj1MsvxDuqTuqTNiJ7r6f1reQvsuUaiHYY=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/idec-net/go-idec v0.0.0-20181106151523-61a006246343/go.mod h1:XUvr43ZLN/4bTZT7TEhJA/rsfFLQxnggX6iU5TGXgIY=
+github.com/idec-net/go-idec v0.0.0-20190316125931-ba6681d1b33b h1:QnpZjlk1jtZwZzT8HKMSfFio+L/6QG16uz3zCbPTkLw=
+github.com/idec-net/go-idec v0.0.0-20190316125931-ba6681d1b33b/go.mod h1:ST2XOvFc7oRd1FCiZPwYf78F43SV9D3r1S+J4OQMsUo=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg=
+golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/jarcoal/httpmock.v1 v1.0.0-20190304095222-3b6b0a8dbc05 h1:u9qyM/i6c8jhNxsMfz4qdKtZumvEhHWYu5jEeOn1SOA=
+gopkg.in/jarcoal/httpmock.v1 v1.0.0-20190304095222-3b6b0a8dbc05/go.mod h1:d3R+NllX3X5e0zlG1Rful3uLvsGC/Q3OHut5464DEQw=

node/helpers.go

@@ -11,6 +11,16 @@ import (
 	"golang.org/x/crypto/bcrypt"
 )
 
+func addNewLineToLastWord(slice []string) []string {
+	l := len(slice) - 1
+	word := slice[l]
+	if strings.Contains(word, "\n") {
+		return slice
+	}
+	slice[l] = word + "\n"
+	return slice
+}
+
 func hashAndSalt(authString []byte) string {
 	hash, err := bcrypt.GenerateFromPassword(authString, bcrypt.MinCost)
 	if err != nil {

node/ssr.go

@@ -0,0 +1,283 @@
+package node
+
+import (
+	"encoding/json"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	"strconv"
+
+	"gitea.difrex.ru/Umbrella/fetcher/i2es"
+	"github.com/gorilla/mux"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	quotesRE = `^(\w+)?>(.*)$`
+)
+
+type ssr struct {
+	es           ESConf
+	templatesDir string
+}
+
+func newSSR(templatesDir string, es ESConf) *ssr {
+	return &ssr{
+		es:           es,
+		templatesDir: templatesDir,
+	}
+}
+
+type PageData struct {
+	Echoes        []echo
+	CurrentPage   string
+	PageNum       int
+	Posts         []i2es.ESDoc
+	ShowPaginator bool
+}
+
+func (p *PageData) GetDate(date string) string {
+	d, err := strconv.ParseInt(date, 0, 64)
+	if err != nil {
+		return err.Error()
+	}
+	return time.Unix(d, 0).UTC().Format("02 Jan 06 15:04 MST")
+}
+
+func (p *PageData) Inc() int {
+	return p.PageNum + 1
+}
+
+func (p *PageData) Dec() int {
+	return p.PageNum - 1
+}
+
+func (p *PageData) RenderMessage(message string) (msg string) {
+	return parseQuotes(message)
+}
+
+func parseQuotes(message string) string {
+	var parsed []string
+	r, err := regexp.Compile(quotesRE)
+	if err != nil {
+		log.Error(err)
+		return message
+	}
+
+	for _, line := range strings.Split(message, "\n") {
+		if r.MatchString(line) {
+			author := r.FindAllStringSubmatch(line, 1)
+			quote := "<figure class=\"text-end quote\"><blockquote class=\"blockquote\"><small>" + author[0][2] + "</small></blockquote>"
+			if len(author) > 0 && author[0][1] != "" {
+				quote += `<figcaption class="blockquote-footer">` + author[0][1] + `</figcaption>`
+			}
+			quote += "</figure>"
+
+			parsed = append(parsed, quote)
+		} else {
+			parsed = append(parsed, line)
+		}
+	}
+
+	return strings.Join(parsed, "\n")
+}
+
+func (s *ssr) newPageData(page string, posts []i2es.ESDoc, num int, showPaginator bool) *PageData {
+	return &PageData{
+		Echoes:        s.es.GetEchoesList(),
+		Posts:         posts,
+		CurrentPage:   page,
+		PageNum:       num,
+		ShowPaginator: showPaginator,
+	}
+}
+
+func (s *ssr) ssrRootHandler(w http.ResponseWriter, r *http.Request) {
+	tpl, err := s.getTemplate("root")
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	posts := s.es.GetLatestPosts(50)
+	for i := range posts {
+		posts[i].Message = parseQuotes(posts[i].Message)
+	}
+
+	if err := tpl.Execute(w, s.newPageData("feed", posts, 1, true)); err != nil {
+		log.Error(err)
+	}
+}
+
+func (s *ssr) ssrForumHandler(w http.ResponseWriter, r *http.Request) {
+	tpl, err := s.getTemplate("forum")
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	vars := mux.Vars(r)
+
+	var num int
+	if _, ok := vars["page"]; ok {
+		num = getPageNum(mux.Vars(r)["page"])
+	}
+
+	bcVal := r.URL.Query().Get("year")
+	bc, _ := strconv.ParseBool(bcVal)
+
+	if bc {
+		if err := tpl.Execute(w, s.newPageData("forum", s.es.GetThreadsYear(num), num, false)); err != nil {
+			log.Error(err)
+		}
+		return
+	}
+
+	if err := tpl.Execute(w, s.newPageData("forum", s.es.GetThreads(num), num, true)); err != nil {
+		log.Error(err)
+	}
+}
+
+func (s *ssr) threadViewHandler(w http.ResponseWriter, r *http.Request) {
+	tpl, err := s.getTemplate("thread")
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	topicid, ok := mux.Vars(r)["topicid"]
+	if !ok {
+		log.Warn("empty topicid")
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte("error: empty topicid"))
+		return
+	}
+
+	posts := s.es.GetTopic(topicid)
+	thread := "nil"
+	if len(posts) > 0 {
+		thread = posts[0].Subg
+	}
+
+	for i := range posts {
+		posts[i].Message = parseQuotes(posts[i].Message)
+	}
+
+	if err := tpl.Execute(w, s.newPageData(thread, posts, 1, true)); err != nil {
+		log.Error(err)
+	}
+}
+
+func getPageNum(page string) int {
+	i, err := strconv.ParseInt(page, 0, 64)
+	if err != nil {
+		log.Error(err)
+		return 1
+	}
+	if i < 1 {
+		return 1
+	}
+	return int(i)
+}
+
+func (s *ssr) echoViewHandler(w http.ResponseWriter, r *http.Request) {
+	tpl, err := s.getTemplate("echo")
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	vars := mux.Vars(r)
+	echo, ok := vars["echo"]
+	if !ok {
+		log.Warn("empty echo")
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte("error: empty echo"))
+		return
+	}
+
+	page := 1
+	if _, ok := vars["page"]; ok {
+		page = getPageNum(vars["page"])
+	}
+
+	bcVal := r.URL.Query().Get("year")
+	bc, _ := strconv.ParseBool(bcVal)
+
+	if bc {
+		if err := tpl.Execute(w, s.newPageData("forum", s.es.GetThreadsYear(page, echo), page, false)); err != nil {
+			log.Error(err)
+		}
+		return
+	}
+
+	if err := tpl.Execute(w, s.newPageData(echo, s.es.GetThreads(page, echo), page, true)); err != nil {
+		log.Error(err)
+	}
+}
+
+func (s *ssr) docsFormattingHandler(w http.ResponseWriter, r *http.Request) {
+	tpl, err := s.getTemplate("docs_formatting")
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	if err := tpl.Execute(w, s.newPageData("docs: formatting", nil, 1, false)); err != nil {
+		log.Error(err)
+	}
+}
+
+func (s *ssr) singleMessageHandler(w http.ResponseWriter, r *http.Request) {
+	tpl, err := s.getTemplate("message")
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	msgid, ok := mux.Vars(r)["msgid"]
+	if !ok {
+		log.Warn("empty msgid")
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte("error: empty msgid"))
+		return
+	}
+
+	post := s.es.GetMessage(msgid)
+	if len(post) > 0 {
+		post[0].Message = parseQuotes(post[0].Message)
+	}
+
+	if err := tpl.Execute(w, s.newPageData(msgid, post, 1, false)); err != nil {
+		log.Error(err)
+	}
+}
+
+func (s *ssr) searchHandler(w http.ResponseWriter, r *http.Request) {
+	tpl, err := s.getTemplate("search")
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	q := r.URL.Query().Get("query")
+	if q != "" {
+		m, err := json.Marshal(q)
+		if err != nil {
+			log.Error(err)
+			m = []byte("")
+		}
+
+		posts := s.es.DoSearch(string(m))
+		for i := range posts {
+			posts[i].Date = parseTime(posts[i].Date)
+			posts[i].Message = parseQuotes(posts[i].Message)
+		}
+
+		if err := tpl.Execute(w, s.newPageData("search", posts, 1, false)); err != nil {
+			log.Error(err)
+		}
+	}
+}

node/structs.go

@@ -1,6 +1,10 @@
 package node
 
-import "gitea.difrex.ru/Umbrella/fetcher/i2es"
+import (
+	"strings"
+
+	"gitea.difrex.ru/Umbrella/fetcher/i2es"
+)
 
 // PointRequest with message
 type PointRequest struct {
@@ -11,6 +15,16 @@ type PointRequest struct {
 // ESConf ...
 type ESConf i2es.ESConf
 
+// searchURI returns an ElasticSearch search URL string
+func (es ESConf) searchURI() (searchURI string) {
+	if es.Index != "" && es.Type != "" {
+		searchURI = strings.Join([]string{es.Host, es.Index, es.Type, "_search"}, "/")
+	} else {
+		searchURI = strings.Join([]string{es.Host, "search"}, "/")
+	}
+	return
+}
+
 // Bucket ...
 type Bucket struct {
 	Key      string `json:"key"`
@@ -25,6 +39,20 @@ type ESSearchResp struct {
 	Hits           `json:"hits"`
 }
 
+type ESAggsResp struct {
+	ESSearchResp
+	Aggregations struct {
+		Topics struct {
+			Buckets []struct {
+				Bucket
+				Post struct {
+					Hits Hits `json:"hits"`
+				} `json:"post"`
+			} `json:"buckets"`
+		} `json:"topics"`
+	} `json:"aggregations"`
+}
+
 type Hits struct {
 	Total    int64   `json:"total"`
 	MaxScore float64 `json:"max_score"`

node/templates.go

@@ -0,0 +1,98 @@
+package node
+
+import (
+	"os"
+	"strings"
+	"text/template"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	// Common templates
+	CommonTplDir = "common"
+
+	// Reusable components
+	ComponentsTplDir = "components"
+
+	// Something like component of components
+	MetaTplDir = "meta"
+
+	// Main pages
+	ViewsTplDir = "views"
+)
+
+func (s *ssr) tplDir(name string) string {
+	return strings.Join([]string{s.templatesDir, name}, string(os.PathSeparator))
+}
+
+func (s *ssr) readTplDir(name string) ([]string, error) {
+	var paths []string
+
+	dir, err := os.ReadDir(s.templatesDir + string(os.PathSeparator) + name)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, entry := range dir {
+		if strings.HasSuffix(entry.Name(), ".html") {
+			paths = append(paths, strings.Join(
+				[]string{s.templatesDir, name, entry.Name()},
+				string(os.PathSeparator)))
+		}
+	}
+
+	return paths, nil
+}
+
+func (s *ssr) tplsPaths() ([]string, error) {
+	var paths []string
+	// Read common dir
+	commonTpls, err := s.readTplDir(CommonTplDir)
+	if err != nil {
+		return nil, err
+	}
+	paths = append(paths, commonTpls...)
+
+	// Read components dir
+	compTpls, err := s.readTplDir(ComponentsTplDir)
+	if err != nil {
+		return nil, err
+	}
+	paths = append(paths, compTpls...)
+
+	// Read meta dir
+	metaTpls, err := s.readTplDir(MetaTplDir)
+	if err != nil {
+		return nil, err
+	}
+	paths = append(paths, metaTpls...)
+
+	return paths, nil
+}
+
+func (s *ssr) componentsForTemplate(name string) []string {
+	paths := []string{s.tplDir(ViewsTplDir) + string(os.PathSeparator) + name + ".html"}
+	tpls, err := s.tplsPaths()
+	if err != nil {
+		log.Error(err)
+		return paths
+	}
+
+	return append(paths, tpls...)
+}
+
+func (s *ssr) templatePath(name string) string {
+	return s.templatesDir + string(os.PathSeparator) + name
+}
+
+func (s *ssr) getTemplate(name string) (*template.Template, error) {
+	t := template.New(name + ".html")
+
+	tpl, err := t.ParseFiles(s.componentsForTemplate(name)...)
+	if err != nil {
+		return nil, err
+	}
+
+	return tpl, nil
+}

templates/common/footer.html

@@ -0,0 +1,12 @@
+{{ define "footer" }}
+
+<!-- <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/js/bootstrap.bundle.min.js" integrity="sha384-b5kHyXgcpbZJO/tY9Ul7kGkf1S0CWuKcCD38l8YkeH8z8QjE0GmW1gYU5S9FOnJ0" crossorigin="anonymous"></script> -->
+<p class="text-center">
+  copyleft 2021 difrex at lessmore dot pw;
+
+  <a href="#">source code</a>
+</p>
+</body>
+</html>
+
+{{ end }}

templates/common/header.html

@@ -0,0 +1,60 @@
+{{ define "header" }}
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>{{ if .CurrentPage }}static | {{ .CurrentPage }}{{ else }}static{{ end }}</title>
+    <link href="/static/css/bootstrap.min.css" rel="stylesheet">
+    <link rel="icon" type="image/png" href="/static/images/favicon.png">
+  </head>
+  <body class="dynamic-bg">
+
+    <!-- Panel -->
+    <nav class="navbar navbar-expand-sm sticky-top navbar-dark dynamic-panel mb-2 shadow lg">
+      <div class="container-fluid">
+        <span class="navbar-brand">{{ if .CurrentPage }}static | {{ .CurrentPage }}{{ else }}static{{ end }}</span>
+
+        <div class="collapse navbar-collapse" id="navbarSupportedContent">
+          <ul class="navbar-nav me-auto mb-2 mb-lg-0">
+            <div class="nav-item">
+              <a class="nav-link"
+                 href="/"
+                 role="button">
+                feed
+              </a>
+            </div>
+            <div class="nav-item">
+              <a class="nav-link"
+                 href="/forum/page/1"
+                 role="button">
+                threads
+              </a>
+            </div>
+            <div class="nav-item">
+              <a class="nav-link"
+                 href="#"
+                 role="button">
+                new
+              </a>
+            </div>
+            <div class="nav-item">
+              <a class="nav-link"
+                 href="/docs/formatting"
+                 role="button">
+                doc
+              </a>
+            </div>
+          </ul>
+          <form method="get" action="/find" class="d-flex">
+            <input
+              class="form-control me-2 base2 border-0"
+              type="search"
+              placeholder='search query'
+              aria-label="Search"
+              name="query">
+            <button class="btn text-white-50" type="submit">Search</button>
+          </form>
+        </div>
+      </div>
+    </nav>
+
+    {{ end }}

templates/common/style.html

@@ -0,0 +1,111 @@
+{{ define "style" }}
+<style>
+  body {
+      background-image: url("/static/images/bg.webp");
+      /* Full height */
+      height: 100%;
+
+      /* Center and scale the image nicely */
+      background-position: center;
+      background-repeat: no-repeat;
+      background-size: cover;
+      background-attachment: fixed;
+
+      background-color: #002b36;
+
+      color: #eee8d5;
+  }
+
+  pre {
+      color: #2aa198;
+  }
+
+  .base2 {
+      background-color: #073642;
+      color: #eee8d5;
+  }
+
+  .dynamic-echo {
+      background-color: #002b36;
+  }
+
+  .dynamic-bg {
+      background-color: #002b36;
+  }
+
+  .dynamic-panel {
+      opacity: 0.75;
+      background-color: #002b36;
+  }
+
+  .dynamic-post-title {
+      color: #839496;
+  }
+
+  .dynamic-post-text {
+      white-space: pre-line;
+      color: #93a1a1;
+  }
+
+  .dynamic-opacity-1 {
+      opacity: 1 !important;
+  }
+
+  .dynamic-opacity-95 {
+      opacity: 0.95;
+  }
+
+  .dynamic-post {
+      margin-bottom: 0.5em;
+      background-color: #002b36;
+      box-shadow: 0px 0px 15px #002b36;
+  }
+
+  .docs a {
+      color: #268bd2;
+  }
+
+  .docs-subheader a {
+      text-decoration: none;
+  }
+
+  @media only screen and (max-width: 1000px) {
+    .dynamic-post {
+        margin-bottom: 0.5em;
+        background-color: #002b36;
+        box-shadow: 0px 0px 15px #002b36;
+    }
+  }
+
+  .dynamic-post a {
+      color: #268bd2;
+  }
+
+  .text-link {
+      color: #268bd2;
+  }
+
+  .echo {
+  }
+
+  .echo-card:hover {
+      background-color: #073642;
+  }
+
+  .quote {
+      color: #859900;
+      border: 0;
+      border-right: 1px;
+      border-style: solid;
+      border-color: #586e75;
+      padding-right: 0.5em;
+  }
+
+  @media only screen and (max-width: 1000px) {
+    .hideit {
+        display: none;
+    }
+  }
+
+</style>
+{{ end }}

templates/components/echoes.html

@@ -0,0 +1,37 @@
+{{ define "echoes" }}
+
+<!--
+    Echoes list component
+
+    Usage:
+
+    { template "echoes" . }
+  -->
+
+<div class="hideit echo">
+  {{ range .Echoes }}
+
+  <a
+    style="text-decoration:none;"
+    href="/echo/{{ .Name }}/page/1">
+    <div class="card echo-card dynamic-echo dynamic-opacity-95 mb-1 mt-1 p-1 shadow">
+
+      <div class="card-title">
+
+        <strong class="text-link">
+          {{ .Name }}
+        </strong>
+
+        <div class="card-subtitle text-white-50">
+          <span>{{ .Docs }} messages</span>
+        </div>
+
+      </div>
+
+    </div>
+  </a>
+
+  {{ end }}
+</div>
+
+{{ end }}

templates/components/post.html

@@ -0,0 +1,37 @@
+{{ define "post" }}
+<!--
+    Post component.
+
+    How to use:
+    { template "post" .Posts[0] }
+-->
+
+<div
+  id="{{ .MsgID }}"
+  class="card container dynamic-post p-3 mb-3 rounded">
+  <div class="card-body p-3">
+    <h3 class="card-title">
+      <a class="dynamic-post-title" href="/thread/{{ .TopicID }}#{{ .MsgID }}">{{ .Subg }}</a>
+    </h3>
+    <div class="card-subtitle text-white-50">
+      <p>
+        [<a href="/echo/{{ .Echo }}/page/1"
+            title="Go to {{ .Echo }} echo">{{ .Echo }}</a>]
+        {{ .Date }}
+        @<a
+           title="{{ .Author }} posts"
+           href="/find?query=author:{{ .Author }}">{{ .Author }}</a> ->
+        {{ if .Repto }}<a href="/msg/{{.Repto}}">{{ .To }}</a>
+        {{ else }}{{ .To }}{{ end }}
+      </p>
+    </div>
+    <div class="card-text dynamic-post-text">
+      {{ .Message }}
+    </div>
+  </div>
+  <div class="card-footer text-white-50">
+    [<a href="/msg/{{ .MsgID }}">#</a>] [<a href="#{{ .MsgID }}">reply</a>]
+  </div>
+</div>
+
+{{ end }}

templates/meta/forum.html

@@ -0,0 +1,60 @@
+{{ define "forum" }}
+
+<div class="container">
+  <div class="row container-fluid">
+    <table class="table table-responsive-sm table-borderless dynamic-bg dynamic-post-title dynamic-post dynamic-opacity-95 rounded">
+
+      <caption class="dynamic-bg dynamic-opacity-95 rounded text-center">
+        {{ if .ShowPaginator }}
+        {{ $prevPage := (.Dec) }}
+        {{ $nextPage := (.Inc) }}
+
+        <a href="{{ $nextPage }}">prev</a>
+
+        {{ if (gt .PageNum 1) }}
+        <a href="{{ $prevPage }}">next</a>
+        {{ end }}
+
+        {{ end }}
+      </caption>
+
+      <thead>
+        <th scope="col">echo</th>
+        <th scope="col">thread</th>
+        <th scope="col">latest</th>
+        <th class="hideit" scope="col">address</th>
+        <th class="hideit" scope="col">date</th>
+      </thead>
+      <tbody>
+        {{ $page := . }}
+        {{ range $i, $p := .Posts }}
+        {{ if (eq $i 0) }}
+        <div class="text-white-50">
+          {{ ($page.GetDate $p.Date) }}
+        </div>
+        {{ end }}
+        <tr>
+          <td>
+            <a
+              class="text-white-50"
+              style="text-decoration:none;"
+              href="/echo/{{ .Echo }}/page/1">
+              {{ $p.Echo }}
+            </a>
+          </td>
+          <td><strong><a href="/thread/{{ .TopicID }}">{{ $p.Subg }}</a></strong></td>
+          <td><strong>{{ $p.Author }}</strong></td>
+          <td class="hideit">{{ $p.Address }}</td>
+          <td class="hideit">{{ ($page.GetDate $p.Date) }}</td>
+        </tr>
+        {{ end }}
+        <tr>
+          <td>
+            <a href="?year=true">year</a>
+          </td>
+        </tr>
+      </tbody>
+    </table>
+  </div>
+</div>
+{{ end }}

templates/meta/latest_posts.html

@@ -0,0 +1,9 @@
+{{ define "latest posts" }}
+
+{{ range .Posts }}
+
+{{ template "post" . }}
+
+{{ end }}
+
+{{ end }}

templates/views/docs.html

@@ -0,0 +1,12 @@
+{{ template "header" . }}
+
+{{ template "style" }}
+
+<div class="container-fluid mt-5 docs">
+  <div class="container dynamic-bg dynamic-post">
+    <h2>message formating convention</h2>
+
+  </div>
+</div>
+
+{{ template "footer" }}

templates/views/docs_formatting.html

@@ -0,0 +1,80 @@
+{{ template "header" . }}
+
+{{ template "style" }}
+
+<div class="container-fluid mt-5 docs">
+  <div class="container dynamic-bg dynamic-post">
+    <h2>message formating convention</h2>
+
+    <div class="docs-subheader" id="code_blocks">
+      <h3><a href="#code_blocks">Code blocks</a></h3>
+      <p>
+        Put your code betwen <i>====</i>
+        <div id="code_example1">
+          <pre>
+====
+for i in $(seq 1 25); do
+  sleep 1
+done
+====
+          </pre>
+        </div>
+      </p>
+    </div>
+
+    <div class="docs-subheader" id="headers">
+      <h3><a href="#headers">Headers</a></h3>
+      <p>
+        Put your header after <i>==</i> from the beginning of line
+        <div id="code_example2">
+          <pre>
+== This is my header!
+
+Regular text
+          </pre>
+        </div>
+      </p>
+    </div>
+
+    <div class="docs-subheader" id="quotes">
+      <h3><a href="#quotes">Quotes</a></h3>
+      <p>
+        Simple quoting
+        <pre>
+&gt;&gt; Double quote
+&gt; Quoted text
+
+Answer
+        </pre>
+      </p>
+      <p>
+        Quoting with an author name
+        <pre>
+Author&gt; Quoted text
+
+Answer
+        </pre>
+      </p>
+    </div>
+
+    <div class="docs-subheader" id="links">
+      <h3><a href="#links">Links</a></h3>
+      <p>
+        Links to the message
+        <pre>
+ii://LzGnRCl9R0pwz6JHdShE
+        </pre>
+      </p>
+      <p>
+        Links to the echo
+        <pre>
+ii://pipe.2032
+        </pre>
+      </p>
+    </div>
+
+
+  </div>
+</div>
+
+{{ template "footer" . }}

templates/views/echo.html

@@ -0,0 +1,22 @@
+{{ template "header" . }}
+
+{{ template "style" }}
+
+<div class="container-fluid mt-5">
+  <div class="row">
+    <div class="col-2 hideit">
+      {{ template "echoes" . }}
+    </div>
+
+    <div class="col-sm col-9 text-primary">
+
+      <div class="container">
+
+        {{ template  "forum" . }}
+
+      </div>
+    </div>
+  </div>
+</div>
+
+{{ template "footer" . }}

templates/views/forum.html

@@ -0,0 +1,17 @@
+{{ template "header" . }}
+
+{{ template "style" }}
+
+<div class="container-fluid mt-5">
+  <div class="row">
+    <div class="col-2 hideit">
+      {{ template "echoes" . }}
+    </div>
+
+    <div class="col-sm col-9 text-primary">
+      {{ template  "forum" . }}
+    </div>
+  </div>
+</div>
+
+{{ template "footer" . }}

templates/views/message.html

@@ -0,0 +1,19 @@
+{{ template "header" . }}
+
+{{ template "style" }}
+
+<div class="container-fluid mt-5">
+  <div class="row">
+    <div class="col-2 hideit">
+      {{ template "echoes" . }}
+    </div>
+
+    <div class="col-sm col-9 text-primary">
+      {{ range .Posts}}
+      {{ template  "post" . }}
+      {{ end }}
+    </div>
+  </div>
+</div>
+
+{{ template "footer" . }}

templates/views/root.html

@@ -0,0 +1,17 @@
+{{ template "header" . }}
+
+{{ template "style" }}
+
+<div class="container-fluid mt-5">
+  <div class="row">
+    <div class="col-2 hideit">
+      {{ template "echoes" . }}
+    </div>
+
+    <div class="col-sm col-9 text-primary">
+      {{ template "latest posts" . }}
+    </div>
+  </div>
+</div>
+
+{{ template "footer" . }}

templates/views/search.html

@@ -0,0 +1,17 @@
+ {{ template "header" . }}
+
+{{ template "style" }}
+
+<div class="container-fluid mt-5">
+  <div class="row">
+    <div class="col-2 hideit">
+      {{ template "echoes" . }}
+    </div>
+
+    <div class="col-sm col-9 text-primary">
+      {{ template "latest posts" . }}
+    </div>
+  </div>
+</div>
+
+{{ template "footer" . }}

templates/views/thread.html

@@ -0,0 +1,17 @@
+{{ template "header" . }}
+
+{{ template "style" }}
+
+<div class="container-fluid mt-5">
+  <div class="row">
+    <div class="col-2 hideit">
+      {{ template "echoes" . }}
+    </div>
+
+    <div class="col-sm col-9 text-primary">
+      {{ template "latest posts" . }}
+    </div>
+  </div>
+</div>
+
+{{ template "footer" . }}