diff --git a/II/Config.pm b/II/Config.pm
deleted file mode 100644
index d6efd5c..0000000
--- a/II/Config.pm
+++ /dev/null
@@ -1,72 +0,0 @@
-package II::Config;
-
-use Config::Tiny;
-use LWP::UserAgent;
-use HTTP::Request;
-use Data::Dumper;
-
-sub new {
- my $class = shift;
-
- my $self = { _file => 'config.ini', };
-
- bless $self, $class;
- return $self;
-}
-
-# Load configuration
-sub load {
- my ($self) = @_;
- my $file = $self->{_file};
-
- my $tiny = Config::Tiny->new();
- $config = $tiny->read($file);
-
- my $key = $config->{auth}->{key};
- my $nick = $config->{auth}->{nick};
- my $host = $config->{node}->{host};
- my @echoareas = split /,/, $config->{node}->{echoareas};
- my $name = $config->{node}->{name};
-
- if ( $config->{node}->{list} eq 'yes' ) {
- @echoareas = echo_from_list($host);
- }
-
- $c = {
- nick => $nick,
- key => $key,
- host => $host,
- echoareas => [@echoareas],
- name => $name,
- elastic_host => $config->{elastic}->{host},
- elastic_index => $config->{elastic}->{index},
- };
-
- return $c;
-}
-
-# Make echoareas list from list.txt
-sub echo_from_list {
- my ($host) = @_;
-
- my $list = $host . 'list.txt';
- my @echoes;
-
- my $ua = LWP::UserAgent->new();
- $ua->agent("Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0)");
-
- my $req = HTTP::Request->new( GET => $list );
- my $res = $ua->request($req);
-
- if ( $res->is_success ) {
- my @e = split /\n/, $res->content();
- foreach my $d (@e) {
- my @description = split /:/, $d;
- push @echoes, $description[0];
- }
- }
-
- return @echoes;
-}
-
-1;
diff --git a/II/DB.pm b/II/DB.pm
deleted file mode 100644
index 2d11ef8..0000000
--- a/II/DB.pm
+++ /dev/null
@@ -1,333 +0,0 @@
-package II::DB;
-
-use SQL::Abstract;
-use DBI;
-
-use Data::Dumper;
-
-sub new {
- my $class = shift;
-
- my $dbh = DBI->connect( "dbi:SQLite:dbname=iinet2.sql", "", "" );
- my $sql = SQL::Abstract->new();
-
- my $self = {
- _dbh => $dbh,
- _sql => $sql,
- };
-
- bless $self, $class;
- return $self;
-}
-
-sub check_hash {
- my ( $self, $hash, $echo ) = @_;
- my $dbh = $self->{_dbh};
-
- my $q = "select hash from echo where hash='$hash' and echo='$echo'";
- my $sth = $dbh->prepare($q);
- $sth->execute();
-
- while ( my @h = $sth->fetchrow_array() ) {
- my ($base_hash) = @h;
- if ( $hash eq $base_hash ) {
- return 1;
- }
- else {
- return 0;
- }
- }
-}
-
-sub begin {
- my ($self) = @_;
- my $dbh = $self->{_dbh};
-
- # Begin transaction
- $dbh->do('BEGIN');
-}
-
-sub commit {
- my ($self) = @_;
- my $dbh = $self->{_dbh};
-
- # Commmit transaction
- $dbh->do('COMMIT');
-}
-
-sub write_echo {
- my ( $self, %data ) = @_;
- my $dbh = $self->{_dbh};
- my $sql = $self->{_sql};
-
- my ( $stmt, @bind ) = $sql->insert( 'echo', \%data );
-
- my $sth = $dbh->prepare($stmt);
- $sth->execute(@bind);
- $sth->finish();
-}
-
-sub write_out {
- my ( $self, %data ) = @_;
- my $dbh = $self->{_dbh};
- my $sql = $self->{_sql};
-
- my ( $stmt, @bind ) = $sql->insert( 'output', \%data );
-
- my $sth = $dbh->prepare($stmt);
- $sth->execute(@bind);
-
- print "Message writed to DB!\n";
-}
-
-sub update_out {
- my ( $self, $hash ) = @_;
- my $dbh = $self->{_dbh};
-
- my $q = "update output set send=1 where hash='$hash'";
- my $sth = $dbh->prepare($q);
- $sth->execute();
-}
-
-sub write {
- my ( $self, %data ) = @_;
- my $dbh = $self->{_dbh};
- my $sql = $self->{_sql};
-
- my ( $stmt, @bind ) = $sql->insert( 'messages', \%data );
-
- my $sth = $dbh->prepare($stmt);
- $sth->execute(@bind);
- $sth->finish();
-}
-
-sub select_out {
- my ($self) = @_;
- my $dbh = $self->{_dbh};
-
- my $q
- = "select from_user, to_user, subg, time, echo, post, hash, base64 from output where send=0";
-
- my $sth = $dbh->prepare($q);
- $sth->execute();
-
- my @posts;
- while ( my @hash = $sth->fetchrow_array() ) {
- my ( $from, $to, $subg, $time, $echo, $post, $h, $base64 ) = @hash;
- my $data = {
- from => "$from",
- to => "$to",
- subg => "$subg",
- time => $time,
- echo => "$echo",
- post => "$post",
- hash => $h,
- base64 => $base64,
- };
- push( @posts, $data );
- }
-
- return @posts;
-}
-
-sub select_index {
- my ( $self, $limit ) = @_;
- my $dbh = $self->{_dbh};
-
- my $q = "select hash from messages order by time desc limit $limit";
-
- my $sth = $dbh->prepare($q);
- $sth->execute();
-
- my @hashes;
- while ( my @hash = $sth->fetchrow_array() ) {
- my ($h) = @hash;
- push( @hashes, $h );
- }
-
- return @hashes;
-}
-
-sub select_subg {
- my ( $self, $echo ) = @_;
-
-}
-
-# Select user messages
-sub select_user {
- my ( $self, $user ) = @_;
- my $dbh = $self->{_dbh};
-
- my $q
- = "select from_user, to_user, subg, time, echo, post, hash from messages where from_user='$user' order by time desc";
-
- my $sth = $dbh->prepare($q);
- $sth->execute();
-
- my @posts;
- while ( my @hash = $sth->fetchrow_array() ) {
- my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
- my $data = {
- from => "$from",
- to => "$to",
- subg => "$subg",
- time => $time,
- echo => "$echo",
- post => "$post",
- hash => $h,
- };
- push( @posts, $data );
- }
-
- return @posts;
-}
-
-sub from_me {
- my ( $self, $config ) = @_;
- my $dbh = $self->{_dbh};
- my $nick = $config->{nick};
-
- # print Dumper($config);
- # print "NICK: $nick\n";
-
- my $q
- = "select from_user, to_user, subg, time, echo, post, hash from messages where from_user='$nick'";
-
- my $sth = $dbh->prepare($q);
- $sth->execute();
-
- my @posts;
- while ( my @hash = $sth->fetchrow_array() ) {
- my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
- my $data = {
- from => "$from",
- to => "$to",
- subg => "$subg",
- time => $time,
- echo => "$echo",
- post => "$post",
- hash => $h,
- };
- push( @posts, $data );
- }
-
- return @posts;
-}
-
-sub thread {
- my ( $self, $subg, $echo ) = @_;
- my $dbh = $self->{_dbh};
-
- my $q
- = "select from_user, to_user, subg, time, echo, post, hash from messages where echo='$echo' and subg like '%$subg%' order by time";
-
- my $sth = $dbh->prepare($q);
- $sth->execute();
-
- my @posts;
- while ( my @hash = $sth->fetchrow_array() ) {
- my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
- my $data = {
- from => "$from",
- to => "$to",
- subg => "$subg",
- time => $time,
- echo => "$echo",
- post => "$post",
- hash => $h,
- };
- push( @posts, $data );
- }
-
- return @posts;
-}
-
-sub echoes {
- my ( $self, $echo ) = @_;
- my $dbh = $self->{_dbh};
-
- my $q
- = "select from_user, to_user, subg, time, echo, post, hash from messages where echo='$echo' order by time desc";
-
- my $sth = $dbh->prepare($q);
- $sth->execute();
-
- my @posts;
- while ( my @hash = $sth->fetchrow_array() ) {
- my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
- my $data = {
- from => "$from",
- to => "$to",
- subg => "$subg",
- time => $time,
- echo => "$echo",
- post => "$post",
- hash => $h,
- };
- push( @posts, $data );
- }
-
- return @posts;
-}
-
-sub to_me {
- my ( $self, $config ) = @_;
- my $dbh = $self->{_dbh};
- my $nick = $config->{nick};
-
- # print Dumper($config);
- # print "NICK: $nick\n";
-
- my $q
- = "select from_user, to_user, subg, time, echo, post, hash from messages where to_user='$nick' order by time desc";
-
- my $sth = $dbh->prepare($q);
- $sth->execute();
-
- my @posts;
- while ( my @hash = $sth->fetchrow_array() ) {
- my ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
- my $data = {
- from => "$from",
- to => "$to",
- subg => "$subg",
- time => $time,
- echo => "$echo",
- post => "$post",
- hash => "$h",
- };
- push( @posts, $data );
- }
-
- return @posts;
-}
-
-sub select_new {
- my ( $self, $msg ) = @_;
- my $dbh = $self->{_dbh};
-
- my $q
- = "select from_user, to_user, subg, time, echo, post, hash from messages where hash='$msg'";
-
- my $sth = $dbh->prepare($q);
- $sth->execute();
- my ( $from, $to, $subg, $time, $echo, $post );
-
- while ( my @hash = $sth->fetchrow_array() ) {
- ( $from, $to, $subg, $time, $echo, $post, $h ) = @hash;
- }
-
- my $data = {
- from => "$from",
- to => "$to",
- subg => "$subg",
- time => $time,
- echo => "$echo",
- post => "$post",
- hash => "$h",
- };
-
- return $data;
-}
-
-1;
diff --git a/II/Enc.pm b/II/Enc.pm
deleted file mode 100644
index feb7acc..0000000
--- a/II/Enc.pm
+++ /dev/null
@@ -1,75 +0,0 @@
-package II::Enc;
-
-use II::DB;
-use MIME::Base64;
-
-sub new {
- my $class = shift;
-
- my $db = II::DB->new();
-
- my $self = {
- _config => shift,
- _data => shift,
- _db => $db,
- };
-
- bless $self, $class;
- return $self;
-}
-
-sub decrypt {
- my ( $self, $base64 ) = @_;
-
- return decode_base64($base64);
-}
-
-sub encode {
- my ($self) = @_;
- my $config = $self->{_config};
- my $data = $self->{_data};
- my $db = $self->{_db};
- my $hash = II::Enc->new_hash();
-
- # Make base64 message
- my $message = $data->{echo} . "\n";
- $message .= $data->{to} . "\n";
- $message .= $data->{subg} . "\n\n";
- $message .= '@repto:' . $data->{hash} . "\n" if defined( $data->{hash} );
- $message .= $data->{post};
-
- # my $encoded = `echo "$message" | base64`;
- my $encoded = encode_base64($message);
- $encoded =~ s/\//_/g;
- $encoded =~ s/\+/-/g;
-
- # Preparsing
- my $post = II::T->in_pre($data->{post});
-
- # Make data
- my %out = (
- hash => $hash,
- time => $data->{time},
- echo => $data->{echo},
- from_user => $data->{from},
- to_user => $data->{to},
- subg => $data->{subg},
- post => $post,
- base64 => $encoded,
- send => 0,
- );
-
- $db->write_out(%out);
-
- return 0;
-}
-
-sub new_hash {
- my @chars = ( "A" .. "Z", "a" .. "z", 0 .. 9 );
- my $string;
- $string .= $chars[ rand @chars ] for 1 .. 21;
-
- return $string;
-}
-
-1;
diff --git a/II/Get.pm b/II/Get.pm
deleted file mode 100644
index 11749de..0000000
--- a/II/Get.pm
+++ /dev/null
@@ -1,174 +0,0 @@
-package II::Get;
-use LWP::UserAgent;
-use HTTP::Request;
-
-use II::DB;
-use II::Enc;
-
-use Data::Dumper;
-use utf8;
-
-sub new {
- my $class = shift;
-
- my $ua = LWP::UserAgent->new();
- $ua->agent("Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0)");
- my $db = II::DB->new();
- my $self = {
- _config => shift,
- _ua => $ua,
- _db => $db,
- };
-
- bless $self, $class;
- return $self;
-}
-
-sub get_echo {
- my ($self) = @_;
- my $config = $self->{_config};
- my $echoareas = $config->{echoareas};
- my $host = $config->{host};
- my $ua = $self->{_ua};
- my $db = $self->{_db};
-
- my $echo_url = 'u/e/';
- my $msg_url = 'u/m/';
-
- my $msgs;
- my @messages;
- my $base64;
- my @messages_hash;
- foreach my $echo (@$echoareas) {
-
- # Get echo message hashes
- my $req_echo = HTTP::Request->new( GET => "$host$echo_url$echo" );
- my $res_echo = $ua->request($req_echo);
-
- print "$host$echo_url$echo\n";
-
- my @new;
- $db->begin();
- if ( $res_echo->is_success ) {
- print "Request $echo is success\n";
- my @mes = split /\n/, $res_echo->content();
- while (<@mes>) {
- if ( $_ =~ /.{20}/ ) {
- if ( $db->check_hash( $_, $echo ) == 0 ) {
- my $echo_hash = {
- echo => $echo,
- hash => $_,
- };
- my %e_write = (
- echo => $echo,
- hash => $_,
- );
-
- # Write new echo message
- $db->write_echo(%e_write);
- $msgs .= $_ . "\n";
- push( @new, $echo_hash );
- }
- }
- }
- }
- else {
- print $res_echo->status_line, "\n";
- }
- $db->commit();
-
- # Get messages and populate hash
- my $count = 0;
- while ( $count < @new ) {
- my $new_messages_url = "$host$msg_url" . $new[$count]->{hash};
- print "URI: $new_messages_url\n";
- my $req_msg = HTTP::Request->new( GET => $new_messages_url );
- my $res_msg = $ua->request($req_msg);
-
- if ( $res_msg->is_success() ) {
- my ( $hash, $m ) = split(':', $res_msg->content());
- push( @messages_hash, { hash => $hash, base64 => $m } );
- }
- else {
- print $res_msg->status_line, "\n";
- }
- $count++;
- }
-
- print "Sleep for 5 seconds\n";
- sleep(5);
- }
-
- my $new_messages
- = "Новые сообщения
\n";
- if ( defined($msgs) ) {
-
- # Begin transaction
- print localtime() . ": writing messages\n";
- $db->begin();
-
- my $c = 0;
- while ( $c < @messages_hash ) {
- my $mes_hash = $messages_hash[$c]->{hash};
- my $text = II::Enc->decrypt( $messages_hash[$c]->{base64} );
-
- open my $m, "<", \$text
- or die "Cannot open message: $!\n";
-
- my @mes;
- while (<$m>) {
- push( @mes, $_ );
- }
- close $m;
-
- my $count = 7;
- my $post;
- while ( $count < @mes ) {
- $post .= $mes[$count];
- $count++;
- }
-
- chomp( $mes[2] );
- chomp( $mes[1] );
- chomp( $mes[3] );
- chomp( $mes[5] );
- chomp( $mes[6] );
-
- # Make data
- my %data = (
- hash => $mes_hash,
- time => $mes[2],
- echo => $mes[1],
- from_user => $mes[3],
- to_user => $mes[5],
- subg => $mes[6],
- post => "$post",
- read => 0,
- );
-
- # Write message to DB
- $db->write(%data);
-
- push(@messages, {
- hash => $mes_hash,
- time => $mes[2],
- echo => $mes[1],
- from_user => $mes[3],
- to_user => $mes[5],
- subg => $mes[6],
- post => "$post",
- read => 0,
- }
- );
- $c++;
- }
-
- # Commit transaction
- $db->commit();
- print localtime() . ": messages writen to DB!\n";
-
- }
- return @messages;
-}
-
-1;
diff --git a/config.ini b/config.ini
deleted file mode 100644
index 2fb21f8..0000000
--- a/config.ini
+++ /dev/null
@@ -1,15 +0,0 @@
-[auth]
-key =
-nick =
-
-[node]
-host = http://ii-net.tk/ii/ii-point.php?q=/
-list = yes
-;host = http://spline.rooker.ru/i/ii-point.php?q=/
-echoareas = ii.14,pipe.2032,lit.14,linux.14,lor-opennet.15,ru.humor.14,music.14,bone.14,lenta.dark.14,mlp.15,habra.16,ifiction.15,game.rogue.14,vk-news.14,lenta.rss.15.4,piratemedia.rss.15,younglinux.info.14,iing.15,python.15,develop.16
-;echoareas = python.15
-
-[elastic]
-host = 127.0.0.1:9200
-index = iinet4
-
diff --git a/fetch.pl b/fetch.pl
deleted file mode 100755
index 1c469af..0000000
--- a/fetch.pl
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env perl
-# (c) 2015-2015 Difrex
-
-use strict;
-use warnings;
-
-use Search::Elasticsearch;
-
-use II::Config;
-use II::Get;
-use Encode qw(decode encode);
-
-my $config = II::Config->new()->load();
-my $get = II::Get->new($config);
-my @data = $get->get_echo();
-
-
-# Connect to localhost:9200:
-my $e = Search::Elasticsearch->new(
- nodes => [$config->{elastic_host}]
-);
-
-foreach my $message (@data) {
- if ($message) {
- my $body = {
- post => decode("UTF-8", $message->{post}),
- subg => decode("UTF-8", $message->{subg}),
- message => decode("UTF-8", $message->{post}),
- date => $message->{time},
- author => decode("UTF-8", $message->{from_user}),
- to => decode("UTF-8", $message->{to_user}),
- echo => $message->{echo},
- msgid => $message->{hash}
-
- };
-
- # Index post
- print localtime . ": Indexing message" . $message->{hash} . "\n";
- $e->index(
- index => $config->{elastic_index},
- type => 'post',
- id => $message->{hash},
- body => $body
- );
-
- }
-}
-
-print localtime . " Done\n";
-
diff --git a/i2es/elastic.go b/i2es/elastic.go
index 7be320f..44b14c7 100644
--- a/i2es/elastic.go
+++ b/i2es/elastic.go
@@ -12,7 +12,7 @@ import (
// ESDoc Elasticsearch document structure
type ESDoc struct {
Echo string `json:"echo"`
- Subg string `json:"subj"`
+ Subg string `json:"subg"`
To string `json:"to"`
Author string `json:"author"`
Message string `json:"message"`
diff --git a/main.go b/main.go
index f23402b..ba3f210 100644
--- a/main.go
+++ b/main.go
@@ -108,7 +108,7 @@ func main() {
log.Print("Check message ", m.ID)
_, err = elastic.CheckID(m.ID)
if err != nil {
- log.Print("Alredy in index ", err)
+ log.Print(err)
continue
}
err = elastic.PutToIndex(esd)