Fetcher
This commit is contained in:
Difrex
parent
7148d971fc
commit
0bb271cefd
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
*.swp
|
||||
41
II/Config.pm
Normal file
41
II/Config.pm
Normal file
@ -0,0 +1,41 @@
|
||||
package II::Config;
|
||||
|
||||
use Config::Tiny;
|
||||
|
||||
sub new {
|
||||
my $class = shift;
|
||||
|
||||
my $self = { _file => 'config.ini', };
|
||||
|
||||
bless $self, $class;
|
||||
return $self;
|
||||
}
|
||||
|
||||
# Load configuration
|
||||
sub load {
|
||||
my ($self) = @_;
|
||||
my $file = $self->{_file};
|
||||
|
||||
my $tiny = Config::Tiny->new();
|
||||
$config = $tiny->read($file);
|
||||
|
||||
my $key = $config->{auth}->{key};
|
||||
my $nick = $config->{auth}->{nick};
|
||||
my $host = $config->{node}->{host};
|
||||
my @echoareas = split /,/, $config->{node}->{echoareas};
|
||||
my $name = $config->{node}->{name};
|
||||
|
||||
$c = {
|
||||
nick => $nick,
|
||||
key => $key,
|
||||
host => $host,
|
||||
echoareas => [@echoareas],
|
||||
name => $name,
|
||||
elastic_host => $config->{elastic}->{host},
|
||||
elastic_index => $config->{elastic}->{index},
|
||||
};
|
||||
|
||||
return $c;
|
||||
}
|
||||
|
||||
1;
|
||||
333
II/DB.pm
Normal file
333
II/DB.pm
Normal file
@ -0,0 +1,333 @@
|
||||
package II::DB;
|
||||
|
||||
use SQL::Abstract;
|
||||
use DBI;
|
||||
|
||||
use Data::Dumper;
|
||||
|
||||
sub new {
|
||||
my $class = shift;
|
||||
|
||||
my $dbh = DBI->connect( "dbi:SQLite:dbname=iinet2.sql", "", "" );
|
||||
my $sql = SQL::Abstract->new();
|
||||
|
||||
my $self = {
|
||||
_dbh => $dbh,
|
||||
_sql => $sql,
|
||||
};
|
||||
|
||||
bless $self, $class;
|
||||
return $self;
|
||||
}
|
||||
|
||||
sub check_hash {
|
||||
my ( $self, $hash, $echo ) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
|
||||
my $q = "select hash from echo where hash='$hash' and echo='$echo'";
|
||||
my $sth = $dbh->prepare($q);
|
||||
$sth->execute();
|
||||
|
||||
while ( my @h = $sth->fetchrow_array() ) {
|
||||
my ($base_hash) = @h;
|
||||
if ( $hash eq $base_hash ) {
|
||||
return 1;
|
||||
}
|
||||
else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub begin {
|
||||
my ($self) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
|
||||
# Begin transaction
|
||||
$dbh->do('BEGIN');
|
||||
}
|
||||
|
||||
sub commit {
|
||||
my ($self) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
|
||||
# Commmit transaction
|
||||
$dbh->do('COMMIT');
|
||||
}
|
||||
|
||||
sub write_echo {
|
||||
my ( $self, %data ) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
my $sql = $self->{_sql};
|
||||
|
||||
my ( $stmt, @bind ) = $sql->insert( 'echo', \%data );
|
||||
|
||||
my $sth = $dbh->prepare($stmt);
|
||||
$sth->execute(@bind);
|
||||
$sth->finish();
|
||||
}
|
||||
|
||||
sub write_out {
|
||||
my ( $self, %data ) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
my $sql = $self->{_sql};
|
||||
|
||||
my ( $stmt, @bind ) = $sql->insert( 'output', \%data );
|
||||
|
||||
my $sth = $dbh->prepare($stmt);
|
||||
$sth->execute(@bind);
|
||||
|
||||
print "Message writed to DB!\n";
|
||||
}
|
||||
|
||||
sub update_out {
|
||||
my ( $self, $hash ) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
|
||||
my $q = "update output set send=1 where hash='$hash'";
|
||||
my $sth = $dbh->prepare($q);
|
||||
$sth->execute();
|
||||
}
|
||||
|
||||
sub write {
|
||||
my ( $self, %data ) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
my $sql = $self->{_sql};
|
||||
|
||||
my ( $stmt, @bind ) = $sql->insert( 'messages', \%data );
|
||||
|
||||
my $sth = $dbh->prepare($stmt);
|
||||
$sth->execute(@bind);
|
||||
$sth->finish();
|
||||
}
|
||||
|
||||
sub select_out {
|
||||
my ($self) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
|
||||
my $q
|
||||
= "select from_user, to_user, subg, time, echo, post, hash, base64 from output where send=0";
|
||||
|
||||
my $sth = $dbh->prepare($q);
|
||||
$sth->execute();
|
||||
|
||||
my @posts;
|
||||
while ( my @hash = $sth->fetchrow_array() ) {
|
||||
my ( $from, $to, $subg, $time, $echo, $post, $h, $base64 ) = @hash;
|
||||
my $data = {
|
||||
from => "$from",
|
||||
to => "$to",
|
||||
subg => "$subg",
|
||||
time => $time,
|
||||
echo => "$echo",
|
||||
post => "$post",
|
||||
hash => $h,
|
||||
base64 => $base64,
|
||||
};
|
||||
push( @posts, $data );
|
||||
}
|
||||
|
||||
return @posts;
|
||||
}
|
||||
|
||||
sub select_index {
|
||||
my ( $self, $limit ) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
|
||||
my $q = "select hash from messages order by time desc limit $limit";
|
||||
|
||||
my $sth = $dbh->prepare($q);
|
||||
$sth->execute();
|
||||
|
||||
my @hashes;
|
||||
while ( my @hash = $sth->fetchrow_array() ) {
|
||||
my ($h) = @hash;
|
||||
push( @hashes, $h );
|
||||
}
|
||||
|
||||
return @hashes;
|
||||
}
|
||||
|
||||
sub select_subg {
|
||||
my ( $self, $echo ) = @_;
|
||||
|
||||
}
|
||||
|
||||
# Select user messages
|
||||
sub select_user {
|
||||
my ( $self, $user ) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
|
||||
my $q
|
||||
= "select from_user, to_user, subg, time, echo, post, hash from messages where from_user='$user' order by time desc";
|
||||
|
||||
my $sth = $dbh->prepare($q);
|
||||
$sth->execute();
|
||||
|
||||
my @posts;
|
||||
while ( my @hash = $sth->fetchrow_array() ) {
|
||||
my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
|
||||
my $data = {
|
||||
from => "$from",
|
||||
to => "$to",
|
||||
subg => "$subg",
|
||||
time => $time,
|
||||
echo => "$echo",
|
||||
post => "$post",
|
||||
hash => $h,
|
||||
};
|
||||
push( @posts, $data );
|
||||
}
|
||||
|
||||
return @posts;
|
||||
}
|
||||
|
||||
sub from_me {
|
||||
my ( $self, $config ) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
my $nick = $config->{nick};
|
||||
|
||||
# print Dumper($config);
|
||||
# print "NICK: $nick\n";
|
||||
|
||||
my $q
|
||||
= "select from_user, to_user, subg, time, echo, post, hash from messages where from_user='$nick'";
|
||||
|
||||
my $sth = $dbh->prepare($q);
|
||||
$sth->execute();
|
||||
|
||||
my @posts;
|
||||
while ( my @hash = $sth->fetchrow_array() ) {
|
||||
my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
|
||||
my $data = {
|
||||
from => "$from",
|
||||
to => "$to",
|
||||
subg => "$subg",
|
||||
time => $time,
|
||||
echo => "$echo",
|
||||
post => "$post",
|
||||
hash => $h,
|
||||
};
|
||||
push( @posts, $data );
|
||||
}
|
||||
|
||||
return @posts;
|
||||
}
|
||||
|
||||
sub thread {
|
||||
my ( $self, $subg, $echo ) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
|
||||
my $q
|
||||
= "select from_user, to_user, subg, time, echo, post, hash from messages where echo='$echo' and subg like '%$subg%' order by time";
|
||||
|
||||
my $sth = $dbh->prepare($q);
|
||||
$sth->execute();
|
||||
|
||||
my @posts;
|
||||
while ( my @hash = $sth->fetchrow_array() ) {
|
||||
my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
|
||||
my $data = {
|
||||
from => "$from",
|
||||
to => "$to",
|
||||
subg => "$subg",
|
||||
time => $time,
|
||||
echo => "$echo",
|
||||
post => "$post",
|
||||
hash => $h,
|
||||
};
|
||||
push( @posts, $data );
|
||||
}
|
||||
|
||||
return @posts;
|
||||
}
|
||||
|
||||
sub echoes {
|
||||
my ( $self, $echo ) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
|
||||
my $q
|
||||
= "select from_user, to_user, subg, time, echo, post, hash from messages where echo='$echo' order by time desc";
|
||||
|
||||
my $sth = $dbh->prepare($q);
|
||||
$sth->execute();
|
||||
|
||||
my @posts;
|
||||
while ( my @hash = $sth->fetchrow_array() ) {
|
||||
my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
|
||||
my $data = {
|
||||
from => "$from",
|
||||
to => "$to",
|
||||
subg => "$subg",
|
||||
time => $time,
|
||||
echo => "$echo",
|
||||
post => "$post",
|
||||
hash => $h,
|
||||
};
|
||||
push( @posts, $data );
|
||||
}
|
||||
|
||||
return @posts;
|
||||
}
|
||||
|
||||
sub to_me {
|
||||
my ( $self, $config ) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
my $nick = $config->{nick};
|
||||
|
||||
# print Dumper($config);
|
||||
# print "NICK: $nick\n";
|
||||
|
||||
my $q
|
||||
= "select from_user, to_user, subg, time, echo, post, hash from messages where to_user='$nick' order by time desc";
|
||||
|
||||
my $sth = $dbh->prepare($q);
|
||||
$sth->execute();
|
||||
|
||||
my @posts;
|
||||
while ( my @hash = $sth->fetchrow_array() ) {
|
||||
my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
|
||||
my $data = {
|
||||
from => "$from",
|
||||
to => "$to",
|
||||
subg => "$subg",
|
||||
time => $time,
|
||||
echo => "$echo",
|
||||
post => "$post",
|
||||
hash => "$h",
|
||||
};
|
||||
push( @posts, $data );
|
||||
}
|
||||
|
||||
return @posts;
|
||||
}
|
||||
|
||||
sub select_new {
|
||||
my ( $self, $msg ) = @_;
|
||||
my $dbh = $self->{_dbh};
|
||||
|
||||
my $q
|
||||
= "select from_user, to_user, subg, time, echo, post, hash from messages where hash='$msg'";
|
||||
|
||||
my $sth = $dbh->prepare($q);
|
||||
$sth->execute();
|
||||
my ( $from, $to, $subg, $time, $echo, $post );
|
||||
|
||||
while ( my @hash = $sth->fetchrow_array() ) {
|
||||
( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
|
||||
}
|
||||
|
||||
my $data = {
|
||||
from => "$from",
|
||||
to => "$to",
|
||||
subg => "$subg",
|
||||
time => $time,
|
||||
echo => "$echo",
|
||||
post => "$post",
|
||||
hash => "$h",
|
||||
};
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
1;
|
||||
75
II/Enc.pm
Normal file
75
II/Enc.pm
Normal file
@ -0,0 +1,75 @@
|
||||
package II::Enc;
|
||||
|
||||
use II::DB;
|
||||
use MIME::Base64;
|
||||
|
||||
sub new {
|
||||
my $class = shift;
|
||||
|
||||
my $db = II::DB->new();
|
||||
|
||||
my $self = {
|
||||
_config => shift,
|
||||
_data => shift,
|
||||
_db => $db,
|
||||
};
|
||||
|
||||
bless $self, $class;
|
||||
return $self;
|
||||
}
|
||||
|
||||
sub decrypt {
|
||||
my ( $self, $base64 ) = @_;
|
||||
|
||||
return decode_base64($base64);
|
||||
}
|
||||
|
||||
sub encode {
|
||||
my ($self) = @_;
|
||||
my $config = $self->{_config};
|
||||
my $data = $self->{_data};
|
||||
my $db = $self->{_db};
|
||||
my $hash = II::Enc->new_hash();
|
||||
|
||||
# Make base64 message
|
||||
my $message = $data->{echo} . "\n";
|
||||
$message .= $data->{to} . "\n";
|
||||
$message .= $data->{subg} . "\n\n";
|
||||
$message .= '@repto:' . $data->{hash} . "\n" if defined( $data->{hash} );
|
||||
$message .= $data->{post};
|
||||
|
||||
# my $encoded = `echo "$message" | base64`;
|
||||
my $encoded = encode_base64($message);
|
||||
$encoded =~ s/\//_/g;
|
||||
$encoded =~ s/\+/-/g;
|
||||
|
||||
# Preparsing
|
||||
my $post = II::T->in_pre($data->{post});
|
||||
|
||||
# Make data
|
||||
my %out = (
|
||||
hash => $hash,
|
||||
time => $data->{time},
|
||||
echo => $data->{echo},
|
||||
from_user => $data->{from},
|
||||
to_user => $data->{to},
|
||||
subg => $data->{subg},
|
||||
post => $post,
|
||||
base64 => $encoded,
|
||||
send => 0,
|
||||
);
|
||||
|
||||
$db->write_out(%out);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
sub new_hash {
|
||||
my @chars = ( "A" .. "Z", "a" .. "z", 0 .. 9 );
|
||||
my $string;
|
||||
$string .= $chars[ rand @chars ] for 1 .. 21;
|
||||
|
||||
return $string;
|
||||
}
|
||||
|
||||
1;
|
||||
174
II/Get.pm
Normal file
174
II/Get.pm
Normal file
@ -0,0 +1,174 @@
|
||||
package II::Get;
|
||||
use LWP::UserAgent;
|
||||
use HTTP::Request;
|
||||
|
||||
use II::DB;
|
||||
use II::Enc;
|
||||
|
||||
use Data::Dumper;
|
||||
use utf8;
|
||||
|
||||
sub new {
|
||||
my $class = shift;
|
||||
|
||||
my $ua = LWP::UserAgent->new();
|
||||
$ua->agent("Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0)");
|
||||
my $db = II::DB->new();
|
||||
my $self = {
|
||||
_config => shift,
|
||||
_ua => $ua,
|
||||
_db => $db,
|
||||
};
|
||||
|
||||
bless $self, $class;
|
||||
return $self;
|
||||
}
|
||||
|
||||
sub get_echo {
|
||||
my ($self) = @_;
|
||||
my $config = $self->{_config};
|
||||
my $echoareas = $config->{echoareas};
|
||||
my $host = $config->{host};
|
||||
my $ua = $self->{_ua};
|
||||
my $db = $self->{_db};
|
||||
|
||||
my $echo_url = 'u/e/';
|
||||
my $msg_url = 'u/m/';
|
||||
|
||||
my $msgs;
|
||||
my @messages;
|
||||
my $base64;
|
||||
my @messages_hash;
|
||||
foreach my $echo (@$echoareas) {
|
||||
|
||||
# Get echo message hashes
|
||||
my $req_echo = HTTP::Request->new( GET => "$host$echo_url$echo" );
|
||||
my $res_echo = $ua->request($req_echo);
|
||||
|
||||
print "$host$echo_url$echo\n";
|
||||
|
||||
my @new;
|
||||
$db->begin();
|
||||
if ( $res_echo->is_success ) {
|
||||
print "Request $echo is success\n";
|
||||
my @mes = split /\n/, $res_echo->content();
|
||||
while (<@mes>) {
|
||||
if ( $_ =~ /.{20}/ ) {
|
||||
if ( $db->check_hash( $_, $echo ) == 0 ) {
|
||||
my $echo_hash = {
|
||||
echo => $echo,
|
||||
hash => $_,
|
||||
};
|
||||
my %e_write = (
|
||||
echo => $echo,
|
||||
hash => $_,
|
||||
);
|
||||
|
||||
# Write new echo message
|
||||
$db->write_echo(%e_write);
|
||||
$msgs .= $_ . "\n";
|
||||
push( @new, $echo_hash );
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
print $res_echo->status_line, "\n";
|
||||
}
|
||||
$db->commit();
|
||||
|
||||
# Get messages and populate hash
|
||||
my $count = 0;
|
||||
while ( $count < @new ) {
|
||||
my $new_messages_url = "$host$msg_url" . $new[$count]->{hash};
|
||||
print "URI: $new_messages_url\n";
|
||||
my $req_msg = HTTP::Request->new( GET => $new_messages_url );
|
||||
my $res_msg = $ua->request($req_msg);
|
||||
|
||||
if ( $res_msg->is_success() ) {
|
||||
my ( $hash, $m ) = split(':', $res_msg->content());
|
||||
push( @messages_hash, { hash => $hash, base64 => $m } );
|
||||
}
|
||||
else {
|
||||
print $res_msg->status_line, "\n";
|
||||
}
|
||||
$count++;
|
||||
}
|
||||
|
||||
print "Sleep for 5 seconds\n";
|
||||
sleep(5);
|
||||
}
|
||||
|
||||
my $new_messages
|
||||
= "<!DOCTYPE html><meta charset=utf8><body><h1>Новые сообщения</h1>\n";
|
||||
if ( defined($msgs) ) {
|
||||
|
||||
# Begin transaction
|
||||
print localtime() . ": writing messages\n";
|
||||
$db->begin();
|
||||
|
||||
my $c = 0;
|
||||
while ( $c < @messages_hash ) {
|
||||
my $mes_hash = $messages_hash[$c]->{hash};
|
||||
my $text = II::Enc->decrypt( $messages_hash[$c]->{base64} );
|
||||
|
||||
open my $m, "<", \$text
|
||||
or die "Cannot open message: $!\n";
|
||||
|
||||
my @mes;
|
||||
while (<$m>) {
|
||||
push( @mes, $_ );
|
||||
}
|
||||
close $m;
|
||||
|
||||
my $count = 7;
|
||||
my $post;
|
||||
while ( $count < @mes ) {
|
||||
$post .= $mes[$count];
|
||||
$count++;
|
||||
}
|
||||
|
||||
chomp( $mes[2] );
|
||||
chomp( $mes[1] );
|
||||
chomp( $mes[3] );
|
||||
chomp( $mes[5] );
|
||||
chomp( $mes[6] );
|
||||
|
||||
# Make data
|
||||
my %data = (
|
||||
hash => $mes_hash,
|
||||
time => $mes[2],
|
||||
echo => $mes[1],
|
||||
from_user => $mes[3],
|
||||
to_user => $mes[5],
|
||||
subg => $mes[6],
|
||||
post => "$post",
|
||||
read => 0,
|
||||
);
|
||||
|
||||
# Write message to DB
|
||||
$db->write(%data);
|
||||
|
||||
push(@messages, {
|
||||
hash => $mes_hash,
|
||||
time => $mes[2],
|
||||
echo => $mes[1],
|
||||
from_user => $mes[3],
|
||||
to_user => $mes[5],
|
||||
subg => $mes[6],
|
||||
post => "$post",
|
||||
read => 0,
|
||||
}
|
||||
);
|
||||
$c++;
|
||||
}
|
||||
|
||||
# Commit transaction
|
||||
$db->commit();
|
||||
print localtime() . ": messages writen to DB!\n";
|
||||
|
||||
}
|
||||
return @messages;
|
||||
}
|
||||
|
||||
1;
|
||||
14
config.ini
Normal file
14
config.ini
Normal file
@ -0,0 +1,14 @@
|
||||
[auth]
|
||||
key =
|
||||
nick =
|
||||
|
||||
[node]
|
||||
host = http://ii-net.tk/ii/ii-point.php?q=/
|
||||
;host = http://spline.rooker.ru/i/ii-point.php?q=/
|
||||
echoareas = ii.14,pipe.2032,lit.14,linux.14,lor-opennet.15,ru.humor.14,music.14,bone.14,lenta.dark.14,mlp.15,habra.16,ifiction.15,game.rogue.14,vk-news.14,lenta.rss.15.4,piratemedia.rss.15,younglinux.info.14,iing.15,python.15
|
||||
;echoareas = python.15
|
||||
|
||||
[elastic]
|
||||
host = 127.0.0.1:9200
|
||||
index = iinet4
|
||||
|
||||
50
fetch.pl
Executable file
50
fetch.pl
Executable file
@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env perl
|
||||
# (c) 2015-2015 Difrex <difrex.punk@gmail.com>
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use Search::Elasticsearch;
|
||||
|
||||
use II::Config;
|
||||
use II::Get;
|
||||
use Encode qw(decode encode);
|
||||
|
||||
my $config = II::Config->new()->load();
|
||||
my $get = II::Get->new($config);
|
||||
my @data = $get->get_echo();
|
||||
|
||||
|
||||
# Connect to localhost:9200:
|
||||
my $e = Search::Elasticsearch->new(
|
||||
nodes => [$config->{elastic_host}]
|
||||
);
|
||||
|
||||
foreach my $message (@data) {
|
||||
if ($message) {
|
||||
my $body = {
|
||||
post => decode("UTF-8", $message->{post}),
|
||||
subg => decode("UTF-8", $message->{subg}),
|
||||
message => decode("UTF-8", $message->{post}),
|
||||
date => $message->{time},
|
||||
author => decode("UTF-8", $message->{from_user}),
|
||||
to => decode("UTF-8", $message->{to_user}),
|
||||
echo => $message->{echo},
|
||||
msgid => $message->{hash}
|
||||
|
||||
};
|
||||
|
||||
# Index post
|
||||
print localtime . ": Indexing message" . $message->{hash} . "\n";
|
||||
$e->index(
|
||||
index => $config->{elastic_index},
|
||||
type => 'post',
|
||||
id => $message->{hash},
|
||||
body => $body
|
||||
);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
print localtime . " Done\n";
|
||||
|
||||
Loading…
Reference in New Issue
Block a user