Working fetcher

This commit is contained in:
Denis Zheleztsov 2017-02-12 00:09:58 +03:00
parent f18258c593
commit 635056c7c6
8 changed files with 2 additions and 721 deletions

View File

@ -1,72 +0,0 @@
package II::Config;
use Config::Tiny;
use LWP::UserAgent;
use HTTP::Request;
use Data::Dumper;
sub new {
my $class = shift;
my $self = { _file => 'config.ini', };
bless $self, $class;
return $self;
}
# Load configuration
sub load {
my ($self) = @_;
my $file = $self->{_file};
my $tiny = Config::Tiny->new();
$config = $tiny->read($file);
my $key = $config->{auth}->{key};
my $nick = $config->{auth}->{nick};
my $host = $config->{node}->{host};
my @echoareas = split /,/, $config->{node}->{echoareas};
my $name = $config->{node}->{name};
if ( $config->{node}->{list} eq 'yes' ) {
@echoareas = echo_from_list($host);
}
$c = {
nick => $nick,
key => $key,
host => $host,
echoareas => [@echoareas],
name => $name,
elastic_host => $config->{elastic}->{host},
elastic_index => $config->{elastic}->{index},
};
return $c;
}
# Make echoareas list from list.txt
sub echo_from_list {
my ($host) = @_;
my $list = $host . 'list.txt';
my @echoes;
my $ua = LWP::UserAgent->new();
$ua->agent("Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0)");
my $req = HTTP::Request->new( GET => $list );
my $res = $ua->request($req);
if ( $res->is_success ) {
my @e = split /\n/, $res->content();
foreach my $d (@e) {
my @description = split /:/, $d;
push @echoes, $description[0];
}
}
return @echoes;
}
1;

333
II/DB.pm
View File

@ -1,333 +0,0 @@
package II::DB;
use SQL::Abstract;
use DBI;
use Data::Dumper;
sub new {
my $class = shift;
my $dbh = DBI->connect( "dbi:SQLite:dbname=iinet2.sql", "", "" );
my $sql = SQL::Abstract->new();
my $self = {
_dbh => $dbh,
_sql => $sql,
};
bless $self, $class;
return $self;
}
sub check_hash {
my ( $self, $hash, $echo ) = @_;
my $dbh = $self->{_dbh};
my $q = "select hash from echo where hash='$hash' and echo='$echo'";
my $sth = $dbh->prepare($q);
$sth->execute();
while ( my @h = $sth->fetchrow_array() ) {
my ($base_hash) = @h;
if ( $hash eq $base_hash ) {
return 1;
}
else {
return 0;
}
}
}
sub begin {
my ($self) = @_;
my $dbh = $self->{_dbh};
# Begin transaction
$dbh->do('BEGIN');
}
sub commit {
my ($self) = @_;
my $dbh = $self->{_dbh};
# Commmit transaction
$dbh->do('COMMIT');
}
sub write_echo {
my ( $self, %data ) = @_;
my $dbh = $self->{_dbh};
my $sql = $self->{_sql};
my ( $stmt, @bind ) = $sql->insert( 'echo', \%data );
my $sth = $dbh->prepare($stmt);
$sth->execute(@bind);
$sth->finish();
}
sub write_out {
my ( $self, %data ) = @_;
my $dbh = $self->{_dbh};
my $sql = $self->{_sql};
my ( $stmt, @bind ) = $sql->insert( 'output', \%data );
my $sth = $dbh->prepare($stmt);
$sth->execute(@bind);
print "Message writed to DB!\n";
}
sub update_out {
my ( $self, $hash ) = @_;
my $dbh = $self->{_dbh};
my $q = "update output set send=1 where hash='$hash'";
my $sth = $dbh->prepare($q);
$sth->execute();
}
sub write {
my ( $self, %data ) = @_;
my $dbh = $self->{_dbh};
my $sql = $self->{_sql};
my ( $stmt, @bind ) = $sql->insert( 'messages', \%data );
my $sth = $dbh->prepare($stmt);
$sth->execute(@bind);
$sth->finish();
}
sub select_out {
my ($self) = @_;
my $dbh = $self->{_dbh};
my $q
= "select from_user, to_user, subg, time, echo, post, hash, base64 from output where send=0";
my $sth = $dbh->prepare($q);
$sth->execute();
my @posts;
while ( my @hash = $sth->fetchrow_array() ) {
my ( $from, $to, $subg, $time, $echo, $post, $h, $base64 ) = @hash;
my $data = {
from => "$from",
to => "$to",
subg => "$subg",
time => $time,
echo => "$echo",
post => "$post",
hash => $h,
base64 => $base64,
};
push( @posts, $data );
}
return @posts;
}
sub select_index {
my ( $self, $limit ) = @_;
my $dbh = $self->{_dbh};
my $q = "select hash from messages order by time desc limit $limit";
my $sth = $dbh->prepare($q);
$sth->execute();
my @hashes;
while ( my @hash = $sth->fetchrow_array() ) {
my ($h) = @hash;
push( @hashes, $h );
}
return @hashes;
}
sub select_subg {
my ( $self, $echo ) = @_;
}
# Select user messages
sub select_user {
my ( $self, $user ) = @_;
my $dbh = $self->{_dbh};
my $q
= "select from_user, to_user, subg, time, echo, post, hash from messages where from_user='$user' order by time desc";
my $sth = $dbh->prepare($q);
$sth->execute();
my @posts;
while ( my @hash = $sth->fetchrow_array() ) {
my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
my $data = {
from => "$from",
to => "$to",
subg => "$subg",
time => $time,
echo => "$echo",
post => "$post",
hash => $h,
};
push( @posts, $data );
}
return @posts;
}
sub from_me {
my ( $self, $config ) = @_;
my $dbh = $self->{_dbh};
my $nick = $config->{nick};
# print Dumper($config);
# print "NICK: $nick\n";
my $q
= "select from_user, to_user, subg, time, echo, post, hash from messages where from_user='$nick'";
my $sth = $dbh->prepare($q);
$sth->execute();
my @posts;
while ( my @hash = $sth->fetchrow_array() ) {
my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
my $data = {
from => "$from",
to => "$to",
subg => "$subg",
time => $time,
echo => "$echo",
post => "$post",
hash => $h,
};
push( @posts, $data );
}
return @posts;
}
sub thread {
my ( $self, $subg, $echo ) = @_;
my $dbh = $self->{_dbh};
my $q
= "select from_user, to_user, subg, time, echo, post, hash from messages where echo='$echo' and subg like '%$subg%' order by time";
my $sth = $dbh->prepare($q);
$sth->execute();
my @posts;
while ( my @hash = $sth->fetchrow_array() ) {
my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
my $data = {
from => "$from",
to => "$to",
subg => "$subg",
time => $time,
echo => "$echo",
post => "$post",
hash => $h,
};
push( @posts, $data );
}
return @posts;
}
sub echoes {
my ( $self, $echo ) = @_;
my $dbh = $self->{_dbh};
my $q
= "select from_user, to_user, subg, time, echo, post, hash from messages where echo='$echo' order by time desc";
my $sth = $dbh->prepare($q);
$sth->execute();
my @posts;
while ( my @hash = $sth->fetchrow_array() ) {
my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
my $data = {
from => "$from",
to => "$to",
subg => "$subg",
time => $time,
echo => "$echo",
post => "$post",
hash => $h,
};
push( @posts, $data );
}
return @posts;
}
sub to_me {
my ( $self, $config ) = @_;
my $dbh = $self->{_dbh};
my $nick = $config->{nick};
# print Dumper($config);
# print "NICK: $nick\n";
my $q
= "select from_user, to_user, subg, time, echo, post, hash from messages where to_user='$nick' order by time desc";
my $sth = $dbh->prepare($q);
$sth->execute();
my @posts;
while ( my @hash = $sth->fetchrow_array() ) {
my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
my $data = {
from => "$from",
to => "$to",
subg => "$subg",
time => $time,
echo => "$echo",
post => "$post",
hash => "$h",
};
push( @posts, $data );
}
return @posts;
}
sub select_new {
my ( $self, $msg ) = @_;
my $dbh = $self->{_dbh};
my $q
= "select from_user, to_user, subg, time, echo, post, hash from messages where hash='$msg'";
my $sth = $dbh->prepare($q);
$sth->execute();
my ( $from, $to, $subg, $time, $echo, $post );
while ( my @hash = $sth->fetchrow_array() ) {
( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
}
my $data = {
from => "$from",
to => "$to",
subg => "$subg",
time => $time,
echo => "$echo",
post => "$post",
hash => "$h",
};
return $data;
}
1;

View File

@ -1,75 +0,0 @@
package II::Enc;
use II::DB;
use MIME::Base64;
sub new {
my $class = shift;
my $db = II::DB->new();
my $self = {
_config => shift,
_data => shift,
_db => $db,
};
bless $self, $class;
return $self;
}
sub decrypt {
my ( $self, $base64 ) = @_;
return decode_base64($base64);
}
sub encode {
my ($self) = @_;
my $config = $self->{_config};
my $data = $self->{_data};
my $db = $self->{_db};
my $hash = II::Enc->new_hash();
# Make base64 message
my $message = $data->{echo} . "\n";
$message .= $data->{to} . "\n";
$message .= $data->{subg} . "\n\n";
$message .= '@repto:' . $data->{hash} . "\n" if defined( $data->{hash} );
$message .= $data->{post};
# my $encoded = `echo "$message" | base64`;
my $encoded = encode_base64($message);
$encoded =~ s/\//_/g;
$encoded =~ s/\+/-/g;
# Preparsing
my $post = II::T->in_pre($data->{post});
# Make data
my %out = (
hash => $hash,
time => $data->{time},
echo => $data->{echo},
from_user => $data->{from},
to_user => $data->{to},
subg => $data->{subg},
post => $post,
base64 => $encoded,
send => 0,
);
$db->write_out(%out);
return 0;
}
sub new_hash {
my @chars = ( "A" .. "Z", "a" .. "z", 0 .. 9 );
my $string;
$string .= $chars[ rand @chars ] for 1 .. 21;
return $string;
}
1;

174
II/Get.pm
View File

@ -1,174 +0,0 @@
package II::Get;
use LWP::UserAgent;
use HTTP::Request;
use II::DB;
use II::Enc;
use Data::Dumper;
use utf8;
sub new {
my $class = shift;
my $ua = LWP::UserAgent->new();
$ua->agent("Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0)");
my $db = II::DB->new();
my $self = {
_config => shift,
_ua => $ua,
_db => $db,
};
bless $self, $class;
return $self;
}
sub get_echo {
my ($self) = @_;
my $config = $self->{_config};
my $echoareas = $config->{echoareas};
my $host = $config->{host};
my $ua = $self->{_ua};
my $db = $self->{_db};
my $echo_url = 'u/e/';
my $msg_url = 'u/m/';
my $msgs;
my @messages;
my $base64;
my @messages_hash;
foreach my $echo (@$echoareas) {
# Get echo message hashes
my $req_echo = HTTP::Request->new( GET => "$host$echo_url$echo" );
my $res_echo = $ua->request($req_echo);
print "$host$echo_url$echo\n";
my @new;
$db->begin();
if ( $res_echo->is_success ) {
print "Request $echo is success\n";
my @mes = split /\n/, $res_echo->content();
while (<@mes>) {
if ( $_ =~ /.{20}/ ) {
if ( $db->check_hash( $_, $echo ) == 0 ) {
my $echo_hash = {
echo => $echo,
hash => $_,
};
my %e_write = (
echo => $echo,
hash => $_,
);
# Write new echo message
$db->write_echo(%e_write);
$msgs .= $_ . "\n";
push( @new, $echo_hash );
}
}
}
}
else {
print $res_echo->status_line, "\n";
}
$db->commit();
# Get messages and populate hash
my $count = 0;
while ( $count < @new ) {
my $new_messages_url = "$host$msg_url" . $new[$count]->{hash};
print "URI: $new_messages_url\n";
my $req_msg = HTTP::Request->new( GET => $new_messages_url );
my $res_msg = $ua->request($req_msg);
if ( $res_msg->is_success() ) {
my ( $hash, $m ) = split(':', $res_msg->content());
push( @messages_hash, { hash => $hash, base64 => $m } );
}
else {
print $res_msg->status_line, "\n";
}
$count++;
}
print "Sleep for 5 seconds\n";
sleep(5);
}
my $new_messages
= "<!DOCTYPE html><meta charset=utf8><body><h1>Новые сообщения</h1>\n";
if ( defined($msgs) ) {
# Begin transaction
print localtime() . ": writing messages\n";
$db->begin();
my $c = 0;
while ( $c < @messages_hash ) {
my $mes_hash = $messages_hash[$c]->{hash};
my $text = II::Enc->decrypt( $messages_hash[$c]->{base64} );
open my $m, "<", \$text
or die "Cannot open message: $!\n";
my @mes;
while (<$m>) {
push( @mes, $_ );
}
close $m;
my $count = 7;
my $post;
while ( $count < @mes ) {
$post .= $mes[$count];
$count++;
}
chomp( $mes[2] );
chomp( $mes[1] );
chomp( $mes[3] );
chomp( $mes[5] );
chomp( $mes[6] );
# Make data
my %data = (
hash => $mes_hash,
time => $mes[2],
echo => $mes[1],
from_user => $mes[3],
to_user => $mes[5],
subg => $mes[6],
post => "$post",
read => 0,
);
# Write message to DB
$db->write(%data);
push(@messages, {
hash => $mes_hash,
time => $mes[2],
echo => $mes[1],
from_user => $mes[3],
to_user => $mes[5],
subg => $mes[6],
post => "$post",
read => 0,
}
);
$c++;
}
# Commit transaction
$db->commit();
print localtime() . ": messages writen to DB!\n";
}
return @messages;
}
1;

View File

@ -1,15 +0,0 @@
[auth]
key =
nick =
[node]
host = http://ii-net.tk/ii/ii-point.php?q=/
list = yes
;host = http://spline.rooker.ru/i/ii-point.php?q=/
echoareas = ii.14,pipe.2032,lit.14,linux.14,lor-opennet.15,ru.humor.14,music.14,bone.14,lenta.dark.14,mlp.15,habra.16,ifiction.15,game.rogue.14,vk-news.14,lenta.rss.15.4,piratemedia.rss.15,younglinux.info.14,iing.15,python.15,develop.16
;echoareas = python.15
[elastic]
host = 127.0.0.1:9200
index = iinet4

View File

@ -1,50 +0,0 @@
#!/usr/bin/env perl
# (c) 2015-2015 Difrex <difrex.punk@gmail.com>
use strict;
use warnings;
use Search::Elasticsearch;
use II::Config;
use II::Get;
use Encode qw(decode encode);
my $config = II::Config->new()->load();
my $get = II::Get->new($config);
my @data = $get->get_echo();
# Connect to localhost:9200:
my $e = Search::Elasticsearch->new(
nodes => [$config->{elastic_host}]
);
foreach my $message (@data) {
if ($message) {
my $body = {
post => decode("UTF-8", $message->{post}),
subg => decode("UTF-8", $message->{subg}),
message => decode("UTF-8", $message->{post}),
date => $message->{time},
author => decode("UTF-8", $message->{from_user}),
to => decode("UTF-8", $message->{to_user}),
echo => $message->{echo},
msgid => $message->{hash}
};
# Index post
print localtime . ": Indexing message" . $message->{hash} . "\n";
$e->index(
index => $config->{elastic_index},
type => 'post',
id => $message->{hash},
body => $body
);
}
}
print localtime . " Done\n";

View File

@ -12,7 +12,7 @@ import (
// ESDoc Elasticsearch document structure
type ESDoc struct {
Echo string `json:"echo"`
Subg string `json:"subj"`
Subg string `json:"subg"`
To string `json:"to"`
Author string `json:"author"`
Message string `json:"message"`

View File

@ -108,7 +108,7 @@ func main() {
log.Print("Check message ", m.ID)
_, err = elastic.CheckID(m.ID)
if err != nil {
log.Print("Alredy in index ", err)
log.Print(err)
continue
}
err = elastic.PutToIndex(esd)