Umbrella/lessmore
Umbrella/lessmore
1
0
Fork 0
Code Issues Pull Requests 1 Packages Releases Wiki Activity

WIP: Node. New generation #1

Draft
Difrex wants to merge 18 commits from ssr into master
pull from: ssr
merge into: Umbrella:master
Conversation 0 Commits 18 Files Changed 32 +406 -25
13 changed files with 406 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit ec6d70721f - Show all commits

4
node/api.go
View File

@ -228,6 +228,10 @@ func Serve(listen string, es ESConf) {
ssr := newSSR("./templates", es)
r.HandleFunc("/", ssr.ssrRootHandler)
r.HandleFunc("/forum", ssr.ssrForumHandler)
r.HandleFunc("/echo/{echo:[a-z0-9-_.]+}/page/{page:[0-9]+}", ssr.echoViewHandler)
r.HandleFunc("/thread/{topicid:[a-z0-9-]+}", ssr.threadViewHandler)
r.HandleFunc("/msg/{msgid:[a-zA-Z0-9]{20}}", ssr.singleMessageHandler)
r.HandleFunc("/find", ssr.searchHandler).Methods(http.MethodGet)
http.Handle("/", r)

130
node/elastic.go
View File

@ -183,6 +183,42 @@ func (es ESConf) GetLimitedEchoMessageHashes(echo string, offset int, limit int)
return hashes
}
func (es ESConf) DoSearch(query string) []i2es.ESDoc {
q := `{"sort": [
{"date":{ "order": "desc" }},{ "_score":{ "order": "desc" }}],
"query": {"query_string" : {"fields": ["message", "subg"], "query":` + query + `}}, "size": 100}`
req, err := http.NewRequest("POST", es.searchURI(), bytes.NewBuffer([]byte(q)))
if err != nil {
log.Error(err.Error())
return nil
}
req.Header.Add("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Error(err.Error())
return nil
}
defer resp.Body.Close()
var esr ESSearchResp
err = json.NewDecoder(resp.Body).Decode(&esr)
if err != nil {
log.Error(err.Error())
return nil
}
var posts []i2es.ESDoc
for _, hit := range esr.Hits.Hits {
posts = append(posts, hit.Source)
}
return posts
}
func (es ESConf) GetUMMessages(msgs string) []string {
var encodedMessages []string
@ -377,8 +413,97 @@ type ThreadBucket struct {
Post Hits
}
func (es ESConf) GetThreads(echoes ...string) (posts []i2es.ESDoc) {
query := `{"sort":[{"date":{"order":"desc"}}],"aggs":{"topics":{"terms":{"field":"topicid.keyword","size":100},"aggs":{"post":{"top_hits":{"size":1,"sort":[{"date":{"order":"desc"}}],"_source":{"include":["subg","author","date","echo","topicid","address"]}}}}}},"query":{"bool":{"must":[{"range":{"date":{"from":"now-30d","to":"now-0d"}}},{"constant_score":{"filter":{"terms":{"echo.keyword":["idec.talks","pipe.2032","linux.14","develop.16","dynamic.local","std.club","std.hugeping","oldpc.51t.ru","difrex.blog","ii.test.14"]}}}}]}}}`
var defaultEchoes = []string{`"idec.talks"`, `"pipe.2032"`, `"linux.14"`, `"develop.16"`, `"dynamic.local"`, `"std.club"`, `"std.hugeping"`, `"difrex.blog"`, `"ii.test.14"`}
func (es ESConf) GetTopic(topicID string) (posts []i2es.ESDoc) {
query := []byte(strings.Join([]string{
`{"sort": [{"date": {"order": "asc"}},
{"_score": {"order": "desc" }}], "size":1000,"query": {"term": {"topicid.keyword": "`, topicID, `"}}}`}, ""))
req, err := http.NewRequest("POST", es.searchURI(), bytes.NewReader([]byte(query)))
if err != nil {
log.Error(err)
return
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Error(err)
return
}
defer resp.Body.Close()
var esr ESSearchResp
err = json.NewDecoder(resp.Body).Decode(&esr)
if err != nil {
log.Error(err.Error())
return nil
}
for _, hit := range esr.Hits.Hits {
hit.Source.Message = strings.Trim(hit.Source.Message, "\n")
hit.Source.Date = parseTime(hit.Source.Date)
posts = append(posts, hit.Source)
}
return
}
func (es ESConf) GetMessage(msgID string) (posts []i2es.ESDoc) {
query := []byte(strings.Join([]string{
`{"sort": [{"date": {"order": "asc"}},
{"_score": {"order": "desc" }}], "size":1000,"query": {"term": {"msgid.keyword": "`, msgID, `"}}}`}, ""))
req, err := http.NewRequest("POST", es.searchURI(), bytes.NewReader([]byte(query)))
if err != nil {
log.Error(err)
return
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Error(err)
return
}
defer resp.Body.Close()
var esr ESSearchResp
err = json.NewDecoder(resp.Body).Decode(&esr)
if err != nil {
log.Error(err.Error())
return nil
}
for _, hit := range esr.Hits.Hits {
hit.Source.Message = strings.Trim(hit.Source.Message, "\n")
hit.Source.Date = parseTime(hit.Source.Date)
posts = append(posts, hit.Source)
}
return
}
func (es ESConf) GetThreads(pageNum int, echoes ...string) (posts []i2es.ESDoc) {
ech := defaultEchoes
if len(echoes) > 0 {
ech = []string{}
for _, echo := range echoes {
ech = append(ech, fmt.Sprintf(`"%s"`, echo))
}
}
rangeStr := `"from":"now-30d","to":"now-0d"`
if pageNum > 1 {
to := 30*pageNum - 30
from := 30 * pageNum
rangeStr = fmt.Sprintf(`"from":"now-%dd","to":"now-%dd"`, from, to)
}
log.Debug(rangeStr)
query := `{"sort":[{"date":{"order":"desc"}}],"aggs":{"topics":{"terms":{"field":"topicid.keyword","size":100},"aggs":{"post":{"top_hits":{"size":1,"sort":[{"date":{"order":"desc"}}],"_source":{"include": ["subg","author","date","echo","topicid","address"]}}}}}},"query":{"bool":{"must":[{"range":{"date":{` + rangeStr + `}}},{"constant_score":{"filter":{"terms":{"echo.keyword": [` +
strings.Join(ech, ",") +
`]}}}}]}}}`
req, err := http.NewRequest("POST", es.searchURI(), bytes.NewReader([]byte(query)))
if err != nil {
log.Error(err)
@ -398,6 +523,7 @@ func (es ESConf) GetThreads(echoes ...string) (posts []i2es.ESDoc) {
log.Error(err)
return
}
for _, bucket := range data.Aggregations.Topics.Buckets {
for _, post := range bucket.Post.Hits.Hits {
posts = append(posts, post.Source)

146
node/ssr.go
View File

@ -1,9 +1,14 @@
package node
import (
"encoding/json"
"net/http"
"time"
"strconv"
"gitea.difrex.ru/Umbrella/fetcher/i2es"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
)
@ -22,14 +27,32 @@ func newSSR(templatesDir string, es ESConf) *ssr {
type PageData struct {
Echoes []echo
CurrentPage string
PageNum int
Posts []i2es.ESDoc
}
func (s *ssr) newPageData(page string, posts []i2es.ESDoc) *PageData {
func (p *PageData) GetDate(date string) string {
d, err := strconv.ParseInt(date, 0, 64)
if err != nil {
return err.Error()
}
return time.Unix(d, 0).UTC().Format("02 Jan 06 15:04 MST")
}
func (p *PageData) Inc() int {
return p.PageNum + 1
}
func (p *PageData) Dec() int {
return p.PageNum - 1
}
func (s *ssr) newPageData(page string, posts []i2es.ESDoc, num int) *PageData {
return &PageData{
Echoes: s.es.GetEchoesList(),
Posts: posts,
CurrentPage: page,
PageNum: num,
}
}
@ -40,7 +63,7 @@ func (s *ssr) ssrRootHandler(w http.ResponseWriter, r *http.Request) {
return
}
if err := tpl.Execute(w, s.newPageData("", s.es.GetLatestPosts(50))); err != nil {
if err := tpl.Execute(w, s.newPageData("feed", s.es.GetLatestPosts(50), 1)); err != nil {
log.Error(err)
}
}
@ -52,7 +75,124 @@ func (s *ssr) ssrForumHandler(w http.ResponseWriter, r *http.Request) {
return
}
if err := tpl.Execute(w, s.newPageData("", s.es.GetThreads())); err != nil {
vars := mux.Vars(r)
var num int
if _, ok := vars["page"]; ok {
num = getPageNum(mux.Vars(r)["page"])
}
if err := tpl.Execute(w, s.newPageData("forum", s.es.GetThreads(num), num)); err != nil {
log.Error(err)
}
}
func (s *ssr) threadViewHandler(w http.ResponseWriter, r *http.Request) {
tpl, err := s.getTemplate("thread")
if err != nil {
log.Error(err)
return
}
topicid, ok := mux.Vars(r)["topicid"]
if !ok {
log.Warn("empty topicid")
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("error: empty topicid"))
return
}
posts := s.es.GetTopic(topicid)
thread := "nil"
if len(posts) > 0 {
thread = posts[0].Subg
}
if err := tpl.Execute(w, s.newPageData(thread, posts, 1)); err != nil {
log.Error(err)
}
}
func getPageNum(page string) int {
i, err := strconv.ParseInt(page, 0, 64)
if err != nil {
log.Error(err)
return 1
}
if i < 1 {
return 1
}
return int(i)
}
func (s *ssr) echoViewHandler(w http.ResponseWriter, r *http.Request) {
tpl, err := s.getTemplate("echo")
if err != nil {
log.Error(err)
return
}
vars := mux.Vars(r)
echo, ok := vars["echo"]
if !ok {
log.Warn("empty echo")
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("error: empty echo"))
return
}
page := 1
if _, ok := vars["page"]; ok {
page = getPageNum(vars["page"])
}
if err := tpl.Execute(w, s.newPageData(echo, s.es.GetThreads(page, echo), page)); err != nil {
log.Error(err)
}
}
func (s *ssr) singleMessageHandler(w http.ResponseWriter, r *http.Request) {
tpl, err := s.getTemplate("message")
if err != nil {
log.Error(err)
return
}
msgid, ok := mux.Vars(r)["msgid"]
if !ok {
log.Warn("empty msgid")
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("error: empty msgid"))
return
}
if err := tpl.Execute(w, s.newPageData(msgid, s.es.GetMessage(msgid), 1)); err != nil {
log.Error(err)
}
}
func (s *ssr) searchHandler(w http.ResponseWriter, r *http.Request) {
tpl, err := s.getTemplate("search")
if err != nil {
log.Error(err)
return
}
q := r.URL.Query().Get("query")
if q != "" {
m, err := json.Marshal(q)
if err != nil {
log.Error(err)
m = []byte("")
}
posts := s.es.DoSearch(string(m))
for i := range posts {
posts[i].Date = parseTime(posts[i].Date)
}
if err := tpl.Execute(w, s.newPageData("search", posts, 1)); err != nil {
log.Error(err)
}
}
}

15
templates/common/header.html
View File

@ -1,14 +1,14 @@
{{ define "header" }}
<!DOCTYPE html>
<html>
<title>{{ if .CurrentPage }}staic | {{ .CurrentPage }}{{ else }}dynamic{{ end }}</title>
<title>{{ if .CurrentPage }}static | {{ .CurrentPage }}{{ else }}static{{ end }}</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-BmbxuPwQa2lc/FVzBcNJ7UAyJxM6wuqIj61tLrc4wSX0szH/Ev+nYRRuWlolflfl" crossorigin="anonymous">
<body class="dynamic-bg">
<!-- Panel -->
<nav class="navbar navbar-expand-lg position-static navbar-dark dynamic-panel mb-2 shadow">
<nav class="navbar navbar-expand-lg sticky-top navbar-dark dynamic-panel mb-2 shadow">
<div class="container-fluid">
<a class="navbar-brand" href="#">static | more</a>
<a class="navbar-brand" href="/">{{ if .CurrentPage }}static | {{ .CurrentPage }}{{ else }}static{{ end }}</a>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav me-auto mb-2 mb-lg-0">
@ -34,8 +34,13 @@
</a>
</div>
</ul>
<form class="d-flex">
<input class="form-control me-2 bg-dark text-black-50 border-0" type="search" placeholder='search query' aria-label="Search">
<form method="get" action="/find" class="d-flex">
<input
class="form-control me-2 bg-dark text-white-50 border-0"
type="search"
placeholder='search query'
aria-label="Search"
name="query">
<button class="btn btn-outline-light" type="submit">Search</button>
</form>
</div>

3
templates/common/style.html
View File

@ -11,6 +11,8 @@
background-size: cover;
background-attachment: fixed;
background-color: #002b36;
color: #657b83;
}
@ -47,6 +49,7 @@
.dynamic-post {
margin-bottom: 0.5em;
background-color: #002b36;
box-shadow: 0px 0px 15px #002b36;
}
.dynamic-post a {

4
templates/components/echoes.html
View File

@ -3,11 +3,11 @@
<div class="echoList">
{{ range .Echoes }}
<div class="card dynamic-echo dynamic-opacity-95 mb-1">
<div class="card dynamic-echo dynamic-opacity-95 mb-1 mt-1">
<div class="card-header text-info container-fluid">
<div class="row">
<div class="col-9">
<a class="text-white-50" href="/ssr#{{ .Name }}">{{ .Name }}</a>
<a class="text-white-50" href="/echo/{{ .Name }}/page/1">{{ .Name }}</a>
</div>
<div class="col-3 text-end">
<span class="text-white-50">{{ .Docs }}</span>

12
templates/components/post.html
View File

@ -8,20 +8,20 @@
<div
id="{{ .MsgID }}"
class="card container dynamic-post dynamic-opacity-95 p-3 mb-3">
class="card container dynamic-post p-3 mb-3 rounded">
<div class="card-body p-3">
<h3 class="card-title">
<a class="dynamic-post-title" href="/ssr#{{ .TopicID }}">{{ .Subg }}</a>
<a class="dynamic-post-title" href="/thread/{{ .TopicID }}#{{ .MsgID }}">{{ .Subg }}</a>
</h3>
<div class="card-subtitle text-white-50">
<p>
[<a href="/ssr#{{ .Echo }}"
[<a href="/echo/{{ .Echo }}/page/1"
title="Go to {{ .Echo }} echo">{{ .Echo }}</a>]
{{ .Date }}
@<a
title="{{ .Author }} posts"
href="#/author/{{ .Author }}">{{ .Author }}</a> ->
{{ if .Repto }}<a href="/ssr#{{.Repto}}">{{ .To }}</a>
href="/find?query=author:{{ .Author }}">{{ .Author }}</a> ->
{{ if .Repto }}<a href="/msg/{{.Repto}}">{{ .To }}</a>
{{ else }}{{ .To }}{{ end }}
</p>
</div>
@ -30,7 +30,7 @@
</div>
</div>
<div class="card-footer text-white-50">
[<a href="#{{ .MsgID }}">#</a>] [<a href="/ssr#{{ .MsgID }}">reply</a>]
[<a href="/msg/{{ .MsgID }}">#</a>] [<a href="#{{ .MsgID }}">reply</a>]
</div>
</div>

20
templates/meta/forum.html
View File

@ -2,8 +2,17 @@
<div class="container">
<div class="row container-fluid">
<table class="table table-sm border-0 dynamic-bg dynamic-post-title dynamic-opacity-95 shadow">
<caption>latest threads</caption>
<table class="table table-sm border-0 dynamic-bg dynamic-post-title dynamic-post dynamic-opacity-95 rounded">
<caption class="dynamic-bg dynamic-opacity-95 rounded">
{{ $prevPage := (.Dec) }}
{{ $nextPage := (.Inc) }}
{{ if (gt .PageNum 1) }}
<a href="{{ $prevPage }}">prev</a>
{{ end }}
<a href="{{ $nextPage }}">next</a>
</caption>
<thead>
<th scope="col">echo</th>
<th scope="col">thread</th>
@ -13,13 +22,14 @@
<!-- <th scope="col">comments</th> -->
</thead>
<tbody>
{{ $page := . }}
{{ range .Posts }}
<tr>
<td>{{ .Echo }}</td>
<td>{{ .Subg }}</td>
<td><a class="text-white-50" href="/echo/{{ .Echo }}/page/1">{{ .Echo }}</a></td>
<td><strong><a href="/thread/{{ .TopicID }}">{{ .Subg }}</a></strong></td>
<td>{{ .Author }}</td>
<td>{{ .Address }}</td>
<td>{{ .Date }}</td>
<td>{{ ($page.GetDate .Date) }}</td>
</tr>
{{ end }}
</tbody>

40
templates/views/echo.html Normal file
View File

@ -0,0 +1,40 @@
{{ template "header" . }}
{{ template "style" }}
<div class="container-fluid mt-5">
<div class="row">
<div class="col-2">
{{ template "echoes" . }}
</div>
<div class="col-9 text-primary">
<div class="container">
{{ $prevPage := (.Dec) }}
{{ $nextPage := (.Inc) }}
<div class="row">
{{ if (gt .PageNum 1) }}
<div class="col">
<div class="text-start">
<a href="/echo/pipe.2032/page/{{ $prevPage }}">prev</a>
</div>
</div>
{{ end }}
<div class="col">
<div class="text-end">
<a href="/echo/pipe.2032/page/{{ $nextPage }}">next</a>
</div>
</div>
</div>
{{ template "forum" . }}
</div>
</div>
</div>
</div>
{{ template "footer" . }}

19
templates/views/message.html Normal file
View File

@ -0,0 +1,19 @@
{{ template "header" . }}
{{ template "style" }}
<div class="container-fluid">
<div class="row">
<div class="col-2">
{{ template "echoes" . }}
</div>
<div class="col-9 text-primary">
{{ range .Posts}}
{{ template "post" . }}
{{ end }}
</div>
</div>
</div>
{{ template "footer" . }}

4
templates/views/root.html
View File

@ -1,8 +1,8 @@
{{ template "header" . }}
{{ template "header" . }}
{{ template "style" }}
<div class="container-fluid">
<div class="container-fluid mt-5">
<div class="row">
<div class="col-2">
{{ template "echoes" . }}

17
templates/views/search.html Normal file
View File

@ -0,0 +1,17 @@
{{ template "header" . }}
{{ template "style" }}
<div class="container-fluid mt-5">
<div class="row">
<div class="col-2">
{{ template "echoes" . }}
</div>
<div class="col-9 text-primary">
{{ template "latest posts" . }}
</div>
</div>
</div>
{{ template "footer" . }}

17
templates/views/thread.html Normal file
View File

@ -0,0 +1,17 @@
{{ template "header" . }}
{{ template "style" }}
<div class="container-fluid">
<div class="row">
<div class="col-2">
{{ template "echoes" . }}
</div>
<div class="col-9 text-primary">
{{ template "latest posts" . }}
</div>
</div>
</div>
{{ template "footer" . }}