#!/usr/bin/perl -w # ipstat.pl # (c) 2003 Kim Holburn # (with some additions to the CIDR parser by Matt Gray) # # Download from: # http://www.holburn.net/software/ipstat.perl.txt # # This script analyses logfile from the net accounting daemon net-acctd. # # released under the GPL v2 http://www.gnu.org/copyleft/gpl.html # based on and including parts of (with permission): # # nacctstats.pl v0.9 by Thomas Prokosch # and ## Analyse.pl ## Made by Vincent HANQUEZ ## # Author = $Author: kimh $ # Date = $Date: 2006/05/31 03:45:51 $ # Id = $Id: ipstat.pl,v 1.28 2006/05/31 03:45:51 kimh Exp $ # Revision = $Revision: 1.28 $ # State = $State: Exp $ # Log = $Log: ipstat.pl,v $ # Log = Revision 1.28 2006/05/31 03:45:51 kimh # Log = added upload and download counts # Log = # Log = Revision 1.27 2006/05/17 07:23:10 kimh # Log = fixed some comments in -f mode # Log = # Log = Revision 1.26 2006/05/10 06:13:12 kimh # Log = more flow count stuff fixinf headers # Log = # Log = Revision 1.25 2006/05/09 12:49:28 kimh # Log = more flow count stuff # Log = # Log = Revision 1.24 2006/05/09 05:11:10 kimh # Log = more flow count stuff # Log = # Log = Revision 1.23 2006/05/07 03:46:06 kimh # Log = added flow count esp. to ipq stuff # Log = # Log = Revision 1.22 2006/04/22 08:33:14 kimh # Log = fixed time issues # Log = # Log = Revision 1.21 2006/04/21 05:06:05 kimh # Log = minor fixes and doco # Log = # Log = Revision 1.20 2006/04/21 04:33:40 kimh # Log = converted option parsing to getopt # Log = # Log = Revision 1.19 2006/04/21 00:11:49 kimh # Log = for netflows undo a work around for net-acct ports # Log = # Log = Revision 1.18 2006/03/21 01:03:07 kimh # Log = added turn off exclusions # Log = # Log = Revision 1.17 2006/03/13 01:11:48 kimh # Log = help info # Log = # Log = Revision 1.16 2006/03/12 23:20:55 kimh # Log = fixed formatting # Log = # Log = Revision 1.15 2006/03/09 23:15:20 kimh # Log = added ability to handle netflow # Log = # Log = Revision 1.14 2005/08/18 04:57:36 kimh # Log = fixing rel time bug # Log = # Log = Revision 1.13 2005/08/18 04:56:09 kimh # Log = fixinf rel time bug # Log = # Log = Revision 1.12 2005/08/18 04:41:51 kimh # Log = fixed relative time bug # Log = # Log = Revision 1.11 2004/11/24 06:16:20 kimh # Log = fixed proto totals # Log = # Log = Revision 1.10 2004/11/19 11:01:09 kimh # Log = added other protocols # Log = # Log = Revision 1.9 2004/04/22 04:44:32 kimh # Log = added any port -q # Log = # Log = Revision 1.8 2004/03/29 00:19:37 kimh # Log = added download URL, added port comment # Log = # Log = Revision 1.7 2003/12/16 10:22:22 kimh # Log = added quota format, various options, relative time periods # Log = # Log = Revision 1.6 2003/10/21 23:11:29 kimh # Log = typo # Log = # Log = Revision 1.2 2003/10/20 23:29:24 kimh # Log = Added exclusion list # Log = # Log = Revision 1.1 2003/10/17 09:18:03 kimh # Log = ipstat - net-acct stats # Log = # # use strict; my $version = '$Id: ipstat.pl,v 1.28 2006/05/31 03:45:51 kimh Exp $'; #$version =~ s/\$//g; my $me = $0; #$me =~ s#^.*/?([^/]+)$#$1#; $me =~ s#^.*/##; # these are my defaults: # this is my default for hosts, change to your requirements # or use a conf file see below at end of defaults #my $opt_h = '126\.126\.(?:12[167]|20[23])\..*'; my $opt_p = ""; # destination port pattern my $opt_P = 0; # not destination port pattern my $opt_q = ""; # any port pattern my $opt_Q = 0; # any port pattern my $opt_c = 19; # number of hosts to show my $opt_v = 1; # verbose my $opt_f = ""; # output format my $opt_l = 1; # log format my $opt_w = 0; # count net flows my $opt_s = 0; # stats my $opt_S = 0; # port stats my $opt_g = 0; # show hosts with greater than my $opt_r = 0; # reverse and look at hosts that talk to my $opt_b = 0; # human readable numbers my $opt_d = 1; # download or upload my $opt_D = 0; # download and upload my $opt_e = ""; # exclusions file my $opt_i = 0; # exclusions or inclusions my $opt_to = 0; # date/time to my $opt_from = 0; # date/time from my $opt_rto = 0; # date/time relative to my $opt_rfrom = 0; # date/time relative from my $opt_dnsresolve = 1; # resolve dns my $opt_conf = "" ; # defaults can go in a conf file # we look for ipstat.conf.pl in the same directory # as the script # actually we use name-of-script.conf.pl # there are no checks so be careful # currently this is executed before options are read # even if there is a -C in the options # I want to do this before the options so the # command-line options override defaults # I can't be bothered right now to have # a proper config file and properly # parse and check it. sub read_config { my $file = shift; my $text; my @confargs = (); # warn "debug file=\"$file\"\n"; if (!open (CONF, $file)) { return @confargs; } local $/; $text = ; close CONF; $text =~ s/^\s*//s; my $last = ""; while ($text =~ m{ # the first part groups the phrase inside the quotes. # see explanation of this pattern in MRE # handle double quoted strings ("[^\"\\]*(?:\\.[^\"\\]*)*") | # handle single quoted strings ('[^']+') | #' # ([^\s\#]+) | (\#[^\n]*)\n | (\s+) }scgx ) { my $space=0; my $skip=0; my $bit = $+; # remove quotes, handle escaped \ and " if ($bit =~ /^"/) { $bit =~ s/^"|"$//g; $bit =~ s/\\(["\\])/$1/g; } elsif ($bit =~ /^'/) { $bit =~ s/^'|'$//g; } elsif ($bit =~ /^\s/) { $bit = ""; $space = 1; } elsif ($bit =~ /^#/ || $bit =~ /^$/) { $skip = 1; } if ($skip) { next; } if ($space) { push @confargs, $last; $last = ""; } else { $last .= $bit; } } if ($last) { push @confargs, $last; } #warn "debug confargs:\n"; #my $count=0; #for (@confargs) { $count++; warn "debug ($count)($_)\n"; } return (@confargs); } my @autoconfargs; my $conf = $0; $conf =~ s/$/.conf/; if ( $conf ne $0 && -r $conf ) { parse_opts ('autoconfig',@autoconfargs = read_config ($conf)); } my $pattern_ip = "[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}"; my @dayspm = (31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31); sub fail_usage { my $me = 'ipstat.pl'; my $mess = <] [-c ] ... report on a net-acct or netflow log file. Version $version for detailed help run : $me --help EOM my $messe = <] [-c ] ... report on a net-acct or netflow log file. Version $version (all patterns use perl regular expression format) Options: -h default = '$opt_h' -H example: -H 126.126.126.44 (-d|--download)|-u|--upload (sort by download (default) or upload) (-D|--Download)|-U|--Upload (report sorted by download and upload) -r|--reverse (reverse - look at hosts hostpatt talks to) -S|--show-protocols (show protocol data) -p|dports (only look at these destination ports) -P|--not-dports (don't look at these destination ports) -q|--ports (only look at these ports) -Q|--not-ports (don't look at these ports) -n|--no-dns (no dns resolve) -c|--count (show only hosts default=20, -1 for all) -C|--conf (read configuration from file) Configuration file format: a series of command line style options. strings quoted with ' and " allowed everything after a # is treated as a comment. -l|--log-format (log format: 1=net-acct, 2=netflow) -g|--gt (show hosts with greater than traffic) ( can be bytes, K M G T eg: 10, 10G 30M etc.) -i|--include -e|--exclude (exclude (only include) hosts, subnets, nets, ... in this file format is text CIDR, one CIDR IP addess per line everything after the IP address and any line without an IP address ignored) -E|--no-exclude|-I|--no-include|-e -|-i - no (cancel) exclusions or inclusions --previous-hour --previous-day --day --month --from --from - --from-s --to --to + --to now --to-s -v|--verbose (more verbose) -V|--quiet|--noverbose (more quiet) -f|--format (special output format) -w|--net-flows (count net flows - not bytes) -b|--bytes (bytes not Human readable K,M,G) -s|--stats (show stats) $me --help show this screen $me --version show version Examples: $me -h '123\.45\.(?:67|69|70)\.[[:digit:]]{1,3}' /var/log/nacct/nacct.log $me -c 10 -g 500M logfile dump $me --day 2003-03-03 logfile $me --from 2003-02-02 --to 2003-02-04 log dump $me -H 126.126.127.44 -r -S logfile.gz (show what hosts this host talks to and add port stats) EOM1 # -B|--nobytes|--human (Human readable K,M,G) if (scalar @_ > 0 && $_[0] eq "") { shift; } if (scalar @_ > 0 && $_[0] eq "version") { print $mess; exit 0; } elsif (scalar @_ > 0 && ($_[0] eq "config" || $_[0] eq "autoconfig")) { shift; print "$0: error: \n"; map { print " $_\n"; } @_ ; print "\n"; print $messe; } elsif (scalar @_) { print "$0: error: \n"; map { print " $_\n"; } @_ ; print "\n"; print $mess; } else { print $messe; } exit (scalar @_) ; } my $tcpp = getprotobyname('tcp'); my $udpp = getprotobyname('udp'); my $args = join " ", @ARGV; # B b d D E e f g H h I i l n P p Q q r S s u U V v sub parse_opts { my $cfm = shift; local @ARGV = @_; use Getopt::Long; Getopt::Long::Configure qw(bundling bundling_override no_ignore_case); my $result = GetOptions ( 'config|conf|C=s' => sub { my $myarg = shift; my $ARGV = shift; if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"$myarg needs an argument"); } if (! -e $ARGV) { fail_usage ($cfm,"$myarg file \"$ARGV\" does not exist"); } if (! -r $ARGV) { fail_usage ($cfm,"cannot read $myarg file \"$ARGV\" "); } if ($ARGV eq $conf) { fail_usage ($cfm,"$myarg file \"$ARGV\" is auto-config"); } if ( $ARGV eq $0) { fail_usage ($cfm,"$myarg file \"$ARGV\" is this script"); } $opt_conf = $ARGV; }, 'Host|H=s' => sub { my $myarg = shift; my $ARGV = shift; if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"$myarg needs an argument"); } if ($ARGV !~ /^$pattern_ip$/) { fail_usage ($cfm,"-H hostip is not a valid IP address"); } $opt_h = $ARGV; $opt_h =~ s/\./\\./g; }, 'host-pattern|h=s' => \$opt_h, 'include|include-file|i=s' => sub { my $myarg = shift; my $ARGV = shift; $opt_i = 1; if ($ARGV eq '-' || $ARGV eq '--') { $opt_e = ""; } else { $opt_e = $ARGV; if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"$myarg needs an argument"); } if (! -r $opt_e) { fail_usage ($cfm,"Can't read inclusions file ($opt_e)"); } } }, 'exclude|exclude-file|e=s' => sub { my $myarg = shift; my $ARGV = shift; $opt_i = 0; if ($ARGV eq '-' || $ARGV eq '--') { $opt_e = ""; } else { $opt_e = $ARGV; if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"$myarg needs an argument"); } if (! -r $opt_e) { fail_usage ($cfm,"Can't read exclusions file ($opt_e)"); } } }, 'E|no-exclude' => sub { $opt_e = ""; $opt_i = 0; }, 'I|no-include' => sub { $opt_e = ""; $opt_i = 1; }, 'count|c=i' => sub { my $myarg = shift; my $ARGV = shift; if (!$ARGV || $ARGV < 0) { $opt_c = 0; } else { $opt_c = $ARGV - 1; } }, 'log-format|l=i' => sub { my $myarg = shift; my $ARGV = shift; if ($ARGV eq "0") { $opt_l = 1; } elsif ($ARGV eq "1" || $ARGV eq "2") { $opt_l = $ARGV; } else { fail_usage ( $cfm, " format can only be:", " 1: net-acct", " 2: netflow" ); } }, 'dports|dest-ports|p=s' => sub { my $myarg = shift; my $ARGV = shift; $opt_P = 0; if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"$myarg needs an argument"); } if ($opt_p) { fail_usage ($cfm,"-p already specified ($opt_p)"); } if ($opt_q) { fail_usage ($cfm,"-q already specified ($opt_q)"); } $opt_p = $ARGV; }, 'not-dport|not-dports|P=s' => sub { my $myarg = shift; my $ARGV = shift; $opt_P = 1; if ($opt_p) { fail_usage ($cfm,"-p already specified ($opt_p)"); } if ($opt_q) { fail_usage ($cfm,"-q already specified ($opt_q)"); } if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"$myarg needs an argument"); } $opt_p = $ARGV; }, 'all-ports|ports|q=s' => sub { my $myarg = shift; my $ARGV = shift; $opt_Q = 0; if ($opt_q) { fail_usage ($cfm,"-q already specified ($opt_q)"); } if ($opt_p) { fail_usage ($cfm,"-p already specified ($opt_p)"); } if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"$myarg needs an argument"); } $opt_q = $ARGV; }, 'not-ports|np-ports|noq|Q=s' => sub { my $myarg = shift; my $ARGV = shift; $opt_Q = 1; if ($opt_q) { fail_usage ($cfm,"-q already specified ($opt_q)"); } if ($opt_p) { fail_usage ($cfm,"-p already specified ($opt_p)"); } if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"$myarg needs an argument"); } $opt_q = $ARGV; }, 'gt|greater-than|g=s' => sub { my $myarg = shift; my $ARGV = shift; if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"--gt needs an argument"); } if ($ARGV !~ /^[[:digit:]]+[KMGT]?B?$/i) { fail_usage ($cfm,"-g must be followed by a number"); } $opt_g = $ARGV; $opt_g =~ s/B$//i; if ($opt_g =~ /[KMGT]$/i) { $opt_g =~ s/([KMGT])$//i; if ($1 =~ /K/i) { $opt_g *= 1024; } elsif ($1 =~ /M/i) { $opt_g *= 1024*1024; } elsif ($1 =~ /G/i) { $opt_g *= 1024*1024*1024; } else { $opt_g *= 1024*1024*1024*1024; } } }, 'day=s' => sub { my $myarg = shift; my $ARGV = shift; if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"--day needs an argument"); } ($opt_from,$opt_to) = parseday ($ARGV); }, 'month=s' => sub { my $myarg = shift; my $ARGV = shift; if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"--month needs an argument"); } ($opt_from,$opt_to) = parsemonth ($ARGV); }, # 'from|from.s|from-s=s' 'from|froms|from-s=s' => sub { my $myarg = shift; my $ARGV = shift; my $epoch = ($myarg =~ /^from[\.-]+s$/)?1:0; if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"$myarg needs an argument"); } if ($ARGV =~ /^-[^\d]/) { fail_usage ($cfm,"$myarg must be followed by an argument"); } if ($ARGV =~ /^\+/) { fail_usage ($cfm,"--from must be before --to"); } if ($opt_from || $opt_rfrom) { fail_usage ($cfm,"--from may only be specified once"); } my $rel = $ARGV =~ /^-/; if ($epoch) { $opt_from = $ARGV; } elsif ($rel) { $opt_rfrom = parsertime ($ARGV); } else { $opt_from = parsetime ($ARGV); } if ($opt_from && $opt_to && $opt_from > $opt_to) { fail_usage ($cfm,"--from ($opt_from) greater than --to ($opt_to) "); } }, # 'to|to.s|to-s=s' 'to|tos|to-s=s' => sub { my $myarg = shift; my $ARGV = shift; my $epoch = ($myarg =~ /^to[\.-]+s$/)?1:0; if ($opt_to) { fail_usage $cfm,"--to may only be specified once"; } if (!defined($ARGV) || !$ARGV) { fail_usage ($cfm,"--to needs an argument"); } if ($ARGV =~ /^-[^\d]/) { fail_usage ($cfm,"--to must be followed by an argument"); } # if ($ARGV =~ /^-/) { fail_usage ($cfm,"--to must be after --from"); } if ($opt_to || $opt_rto) { fail_usage $cfm,"--to may only be specified once"; } my $rel = $ARGV =~ /^\[+-]/; if ($ARGV =~ /^now$/i) { $opt_to = time; } elsif ($epoch) { $opt_to = $ARGV; } elsif ($rel) { $opt_rto = parsertime ($ARGV); } else { $opt_to = parsetime ($ARGV); } if ($opt_from && $opt_to && $opt_from > $opt_to) { fail_usage ($cfm,"--to ($opt_to) less than --from ($opt_from) "); } }, 'previous-hour' => sub { my $myarg = shift; my $ARGV = shift; if ($opt_to) { fail_usage $cfm,"--to may only be specified once"; } if ($opt_from) { fail_usage $cfm,"--from may only be specified once"; } $opt_to = time; $opt_from = $opt_to - 3600; }, 'previous-day' => sub { my $myarg = shift; my $ARGV = shift; if ($opt_to) { fail_usage $cfm,"--to may only be specified once"; } if ($opt_from) { fail_usage $cfm,"--from may only be specified once"; } $opt_to = time; $opt_from = $opt_to - 86400; }, 'net-flows|w' => sub { $opt_w = 1; $opt_b = 2; }, 'help|?' => sub { fail_usage; }, 'version' => sub { fail_usage "version"; }, 'show-stats|stats|s' => \$opt_s, 'Show-ports|Show-protocol|S' => \$opt_S, 'format|script|f' => sub { $opt_f = "#"; }, 'reverse|r' => \$opt_r, 'bytes|b' => sub { if ($opt_b<2) { $opt_b = 1; } }, # 'bytes|b' => \$opt_b, # 'nobytes|human|B' => sub { $opt_b=0; }, 'upload|u' => sub { $opt_d = 0; }, 'download|d' => \$opt_d, 'Upload|U' => sub { $opt_d = 0; $opt_D = 1; }, 'Download|D' => sub { $opt_d = 1; $opt_D = 1; }, 'n|no-dns' => sub { $opt_dnsresolve = 0; }, 'verbose|v+' => \$opt_v, 'quiet|V|noverbose' => sub { $opt_v = $opt_v>0?$opt_v-1:0; }, ); return @ARGV; } my @saveargs = @ARGV; my @confargs; @ARGV = parse_opts ("", @ARGV); if ( $opt_conf && -r $opt_conf && $opt_conf ne $0 && $opt_conf ne $conf) { parse_opts ('config',@confargs = read_config ($opt_conf)); } if ($opt_rto && $opt_rfrom) { fail_usage " both --to and --from are relative "; } if ($opt_to && $opt_rfrom) { $opt_from = $opt_to + $opt_rfrom; } if ($opt_rto && $opt_from) { $opt_to = $opt_from + $opt_rto; } if ($opt_v > 1) { print " args were : "; print " -h=\"$opt_h\"\n"; print " -c=\"$opt_c\""; print " -l=\"$opt_l\""; print " -C(auto)=\"$conf\""; print " -C=\"$opt_conf\""; print " -r=\"$opt_r\""; print " -v=\"$opt_v\""; print " -g=\"$opt_g\" \n"; print " -s=\"$opt_s\""; print " -S=\"$opt_S\""; print " -f=\"$opt_f\""; print " -b=\"$opt_b\""; print " -d(1)|-u(0)=\"$opt_d\""; print " -n=\"$opt_dnsresolve\"\n"; print " -p=\"$opt_p\" "; print " -P=\"$opt_P\" "; print " -q=\"$opt_q\" "; print " -Q=\"$opt_Q\" "; print " --to=\"$opt_to\" "; print " --from=\"$opt_from\"\n"; if ($opt_rto || $opt_rfrom) { print " --rto=\"$opt_rto\" "; print " --rfrom=\"$opt_rfrom\"\n"; } print " opt_i=($opt_i) "; if ($opt_i) { print " -i"; } else { print " -e"; } print "=\"$opt_e\"\n"; print "-w=\"$opt_w\"\n"; print "additional arguments were: ", join (':', @ARGV), "\n"; print "\n"; if ($opt_v > 2) { my $count=0; if (scalar @autoconfargs) { warn "debug autoconfargs:\n"; for (@autoconfargs) { $count++; warn "debug ($count)($_)\n"; } } else { warn "debug no autoconfargs\n"; } if (scalar @saveargs) { warn "debug args:\n"; $count=0; for (@saveargs) { $count++; warn "debug ($count)($_)\n"; } } else { warn "debug no saveargs\n"; } if (scalar @confargs) { warn "debug confargs:\n"; $count=0; for (@confargs) { $count++; warn "debug ($count)($_)\n"; } } else { warn "debug no confargs\n"; } } } if ($opt_v) { print "${opt_f}$0 $args\n"; print "${opt_f} "; if ($opt_conf && -r $opt_conf) { print " -C \"$opt_conf\""; } if ($opt_e && -r $opt_e) { if ($opt_i) { print " -i"; } else { print " -e"; } print " \"$opt_e\""; } print "\n"; } print "${opt_f}$me - network stats\n"; print "${opt_f}Version $version\n"; if (!$opt_r) { print "${opt_f}Examining host(s) /$opt_h/\n"; } else { print "${opt_f}Examining hosts talking to /$opt_h/\n"; } if ($opt_p) { if ($opt_P) { print "${opt_f}Not destination port "; } else { print "${opt_f}Only destination port "; } if ($opt_p =~ /^\^(\d+)\$$/) { print "$1\n"; } else { print "/$opt_p/\n"; } } if ($opt_q) { if ($opt_Q) { print "${opt_f}Not port "; } else { print "${opt_f}Only port "; } if ($opt_q =~ /^\^(\d+)\$$/) { print "$1\n"; } else { print "/$opt_q/\n"; } } { my $format = $opt_l == 1? "net-acct": "netflow"; print "${opt_f}Log files in $format format\n"; if ($opt_w) { print "${opt_f}netflow count only\n"; } } if ($opt_v > 3) { warn "debug mode exiting...\n"; exit; } #if (!$opt_h) { fail_usage("No host"); } my @netarrays; sub exclusions { if (!open (NETS, $opt_e)) { die "Couldn't open file ($opt_e)"; } while () { my ($ip, $rest) = split(" ", $_, 2); if ($ip =~ /^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\/(\d{1,2})$/) { my @ips = split(/\./,$1); my $subnetn = $2; my $major = int ($subnetn / 8); my $minor = $2 % 8; # print "$major, $minor\n"; # if ($major > 4 || ($major == 4 && $minor > 0) ) if (32 < $subnetn) { print STDERR "CIDR value out of range ($ip)\n"; } if ($minor == 0) { # divisible by 8 - hoorah my $key = join ".", @ips[0..$major-1]; if ($major != 4) { $key .= "."; } $netarrays[$major]{"$key"} = 1; } else { # minor is not 0, hence mask not multiple of 8...expand it my $pp2 = 2**(8 - $minor); my $pp2m1 = $pp2 - 1; my $subnet = $ips[$major] & (~$pp2m1); my $s; for $s ($subnet..($subnet+$pp2m1)) { my $key = join ".", @ips[0..$major-1]; $key .= ".$s"; if ($major + 1 != 4) {$key .= ".";} $netarrays[$major + 1]{"$key"} = 1; } } } elsif ($ip =~ /^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/) { $netarrays[4]{$1} = 1; } } close NETS; } if ($opt_e) { exclusions(); } sub excluded { my $ip = shift; my $return = $opt_i; if ($opt_e) { my @ip = split(/\./,$ip); if (defined ($netarrays[2]{"$ip[0].$ip[1]."}) || defined ($netarrays[3]{"$ip[0].$ip[1].$ip[2]."}) || defined ($netarrays[4]{"$ip[0].$ip[1].$ip[2].$ip[3]"})) { $return = !$opt_i; } } $return; } use Time::Local; # date must be in this format: # yyyy-mm-dd yyyy/mm/dd sub parsemonth { my $day = shift; my ($start, $end) = (0,0); if ($day =~ m#^([12]\d\d\d)[-/](\d|[01]\d)$#) { my $year = $1; my $month = $2; if ($year < 1998 || 3000 < $year) { fail_usage "bad year"; } if ($month < 1 || 12 < $month) { fail_usage "bad month"; } $year -= 1900; $month -= 1; $start = timelocal (0,0,0,1, $month, $year); # end is more complicated. $end = $start + 24*60*60*$dayspm[$month] - 1; if ($month == 1) { # check it's not a leap year the easy way my ($s,$m,$h,$mday,$mon,$y,$wday,$yday,$isdst) = localtime($end); if ($mon != $month) { $end -= 24*60*60; } } } ($start, $end) ; } sub parseday { my $day = shift; my ($start, $end) = (0,0); if ($day =~ m#^([12]\d\d\d)[-/](\d|[01]\d)[-/](\d|[0123]\d)$#) { my $year = $1; my $month = $2; my $mday = $3; if ($year < 1998 || 3000 < $year) { fail_usage "bad year"; } if ($month < 1 || 12 < $month) { fail_usage "bad month"; } if ($mday < 1 || 31 < $mday) { fail_usage "bad day of month"; } if ($dayspm[$month-1] < $mday) { fail_usage "day not in month"; } $year -= 1900; $month -= 1; # $mday -= 1; $start = timelocal (0,0,0,$mday, $month, $year); $end = $start + 24*60*60 - 1; } ($start, $end) ; } sub parsetime { my $date = shift; my $start = 0; if ($date =~ m#^([12]\d\d\d)[-/](\d|[01]\d)[-/](\d|[0123]\d)$#) { return (parseday($date))[0]; } elsif ($date =~ m#^((?:[12]\d\d\d)[-/](?:\d|[01]\d)[-/](?:\d|[0123]\d))(?:(?:\s+|T)([\d:]+))?$#) { my $day = $1; my $time = $2; $start = (parseday($day))[0]; my ($hours, $minutes, $seconds) = (0,0,0); if ($time =~ /^(\d\d?):(\d\d?):(\d\d?)\s*$/) { $hours = $1; $minutes = $2; $seconds = $3; } elsif ($time =~ /^(\d\d?):(\d\d?)\s*$/) { $hours = $1; $minutes = $2; } elsif ($time =~ /^(\d\d?)\s*$/) { $hours = $1; } elsif (!$time || $time =~ /[^\s]/) { fail_usage "bad time"; } if (23 < $hours) { fail_usage "bad hours"; } if (59 < $minutes) { fail_usage "bad minutes"; } if (59 < $seconds) { fail_usage "bad seconds"; } $start += $hours*60*60 + $minutes*60 + $seconds; } $start ; } sub parserday { my $day = shift; my $start = 0; if ($day =~ m#^([12]\d\d\d|\d\d|\d)[-/](\d|[01]\d)[-/](\d|[0123]\d)$#) { my $year = $1; my $month = $2; my $mday = $3; $start = ($year * 365 + $month*30 + $mday) * 60*60*24; } $start ; } sub parsertime { my $date = shift; my $start = 0; my $day = 0; my $time = 0; my $factor = 1; if ($date =~ /^\-/) { $factor = -1; } $date =~ s/^[\+-]//; if ($date =~ m#^([12]\d\d\d|\d\d|\d)[-/](\d|[01]\d)[-/](\d|[0123]\d)$#) { return parserday($date); } if ($date =~ m#^((?:[12]\d\d\d|\d\d|\d)[-/](?:\d|[01]\d)[-/](?:\d|[0123]\d))(?:(?:\s+|T)([\d:]+))?$#) { $day = $1; $time = $2; } elsif ($date =~ m#^[\d:]+$#) { $time = $date; } if ($day) { $start = parserday($day); } if ($time) { my ($hours, $minutes, $seconds) = (0,0,0); if ($time =~ /^(\d\d?):(\d\d?):(\d\d?)\s*$/) { $hours = $1; $minutes = $2; $seconds = $3; } elsif ($time =~ /^(\d\d?):(\d\d?)\s*$/) { $hours = $1; $minutes = $2; } elsif ($time =~ /^(\d\d?)\s*$/) { $hours = $1; } elsif (!$time || $time =~ /[^\s]/) { fail_usage "bad rtime ($time)"; } $start += $hours*60*60 + $minutes*60 + $seconds; } $start *= $factor; } # ip_to_name: return the name of of the given ip address sub ip_to_name { my ($ip) = @_; if (!$ip) { return (" [no ip]"); } my $myip=sprintf("%-16s", $ip); if (!($ip =~ /^$pattern_ip$/)) { return ("$myip [bad ip]"); } if ($opt_dnsresolve) { my ($host_name, $aliases, $addrtype, $length) ; my @addrs; my $ipaddr = pack("C4", split(/\./, $ip)); if (($host_name, $aliases, $addrtype, $length, @addrs) = gethostbyaddr($ipaddr, 2)) { return ("$myip $host_name"); } else { return ("$myip [lookup failed]"); } } return ($myip); } use POSIX qw(floor); #use Sys::Hostname; my %toprecord; my @protocol; my @days; my @hours = (0) x 24; my $runningtime; # calculates the beginning of a day, in epoch format # $_[0]: the time to process, in seconds sub epochday { my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=gmtime($_[0]); ($_[0]-(($hour*60+$min)*60+$sec))/(24*3600); } # converts the unix epoch time format into human readable form # $_[0]: the time to represent, in seconds sub printdatetime { my $tag = defined($_[1])?$_[1]:""; if ($tag eq "date") { strftime ("%Y-%b-%d", localtime($_[0])); } elsif ($tag eq "short") { strftime ("%b-%d", localtime($_[0])); } else { strftime ("%Y-%b-%d %H:%M:%S", localtime($_[0])); } } # converts bytes into K, M, G, T bytes # $val: the number of bytes # $len: the length of the resulting string, $len>=1 sub printbytes { my ($val, $len, $pre, $suff)= @_; my $suffix = "B" ; if ($opt_b>1) { $suffix = ""; } $val=defined($val)?$val:0; $pre=defined($pre)?$pre:""; $suff=defined($suff)?$suff:""; if ($len) { $len += length($pre) + length($suff); } if (!$opt_b) { if ($val>1024) { $val/=1024; $suffix="K"; } if ($val>1024) { $val/=1024; $suffix="M"; } if ($val>1024) { $val/=1024; $suffix="G"; } if ($val>1024) { $val/=1024; $suffix="T"; } my $val2 = sprintf("%.1f", $val); sprintf("%$len"."s", "$pre$val2$suffix$suff"); } else { ##### sprintf("%$len"."s", "$pre$val$suffix$suff"); } } # prints percentages # $val: the value to represent # $overall: the value representing 100% sub printpercent { my ($val, $overall)=@_; if (!defined($overall)||$overall==0) { return(" undef"); } $val=$val*100/$overall; if ($val<1&&$val!=0) { " < 1%"; } else { sprintf("%5.1f%%", $val); } } # prints a bar # $val: the value which should be visualized # $most: the maximum value which represents 100% # $len: the length of the bar at 100% sub printbar { my ($val, $most, $len)=@_; $len-=2; if ($most==0) { "|".(" "x$len)."|"; } else { my $asterisk=POSIX::floor($val*$len/$most+.5); "|".("*"x$asterisk).(" "x($len-$asterisk))."|"; } } # print some general statistics sub overallstats { print("\n--- Overall statistics ----------------".("-"x40)."\n\n"); my $numdays=($toprecord{'last'}-$toprecord{'first'})/(24*3600); printf("Time evaluated: %s to %s (%0.1f days)\n", printdatetime($toprecord{'first'}), printdatetime($toprecord{'last'}), $numdays); printf("Program running time: %d min %02d sec\n", (time-$runningtime)/60, (time-$runningtime)%60); print("Number of corrupted logfile entries: $toprecord{'corrupted'}\n"); print("Number of parsed logfile entries: $toprecord{'parsed'}\n"); print("Number of parsed lines per second: ". ($toprecord{'parsed'}/((time-$runningtime) or 1))."\n"); print("Overall data transferred: ". printbytes($toprecord{'overall'}, 0)."\n"); print("Average traffic per day/week/month: ". printbytes($toprecord{'overall'}/$numdays, 0)." / ". printbytes($toprecord{'overall'}*7/$numdays, 0)." / ". printbytes($toprecord{'overall'}*30/$numdays, 0)."\n") if ($numdays); printf("Highest traffic - hour: %d\n", $toprecord{'hour'}); printf((" "x16)."- day: %s\n", printdatetime($toprecord{'day'}*24*3600,"date")); } # print statistics on protocols sub protocolstats { my $icmpp = getprotobyname("icmp"); my $igmpp = getprotobyname("igmp"); print("\n\n--- Protocol statistics ---------------".("-"x40)."\n\n"); print("TCP: ".printbytes($protocol[$tcpp], 8)."\n"); print("UDP: ".printbytes($protocol[$udpp ], 8)."\n"); print("ICMP: ".printbytes($protocol[$icmpp], 8)."\n"); print("IGMP: ".printbytes($protocol[$igmpp], 8)."\n"); print("Other protocols: \n"); my $count = 0; for (@protocol) { if ($protocol[$count] && $count != $tcpp && $count != $udpp && $count != $icmpp && $count != $igmpp) { my $proto = getprotobynumber($count); print("$proto: ".printbytes($protocol[$count], 8)."\n"); } $count++; } } # show an hourly diagram sub hourlystats { print("\n\n--- Hourly statistics -----------------".("-"x40)."\n\n"); for (my $i=0; $i<24; $i++) { printf("%2d-%2d: %s %s %s\n", $i, $i+1, printbytes($hours[$i], 7), printpercent($hours[$i], $toprecord{'overall'}), printbar($hours[$i], $hours[$toprecord{'hour'}], 58)); } } # monthly/weekly/daily grid sub gridstats { my ($day, $pday, $pmon); # current and previous processed day/month my ($fday, $lday)=(0,0); # first and last day of current week my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst); my @msum ; my $wsum; # monthly and weekly sums my $line = sub { ("-"x70)."+".("-"x8)."\n"; }; my $weekly_start = sub { $fday=$day-$wday+1; $pday=$fday-1; $lday=$day-$wday+7; $wsum=0; print(printdatetime(24*3600*$fday, "short").":". printdatetime(24*3600*$lday, "short")); }; my $weekly_end = sub { print(" "x(8*($lday-$pday))." |".printbytes($wsum, 8)."\n") if (defined($pday)); }; my $monthly_start = sub { @msum=0; $wsum=0; }; my $monthly_end = sub { my @monids=("January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"); print(&$line()); printf("%13s", $monids[$pmon]); for (my $day=1; $day<=7; $day++) { print(printbytes($msum[$day], 8)); } print(" |".printbytes($msum[0], 8)."\n".&$line()); }; print("\n\n--- Statistical grid ------------------".("-"x40)."\n\n"); print(" week Mon Tue Wed ". "Thu Fri Sat Sun | Sum\n".&$line()); &$monthly_start(); DAY: for ($day=0; $day$lday) { &$weekly_end; &$weekly_start; } print(" "x(8*($day-$pday-1))); print(printbytes($days[$day], 8)); $msum[0]+=$days[$day]; $msum[$wday]+=$days[$day]; # update monthly sum (by day) $wsum+=$days[$day]; # update weekly sum $pday=$day; # we are done, set this day as the previous day $pmon=$mon; # ... and this month as the month previously worked on } &$weekly_end; &$monthly_end; } # a statistic of every day encountered sub dailystats { print("\n\n--- Daily statistics ------------------".("-"x40)."\n\n"); for (my $i=0; $i) { # net-acct #epoch proto src-ip src-port dst-ip dst-port len dev ? #1044905358 6 210.11.144.156 4394 126.126.126.44 110 364 eth0 unknown # netfow #epoch duration source-ip dest-ip source-p dest-p proto packets octets net source-asn dest-asn source-mask dest-mask source-if-index dest-if-index TCP-FLAG # 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 #ddddd dd.dd IP IP dd dd dd dd dd aaaa dd dd dd dd dd dd dd dd # 1 2 3 4 5 6 7 8 9 # /^(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)$/ $count++; my $input=0; if ($_!~ /$netpatt/) { $corrupted++; if ($opt_v>1) { print "corrupt($count)$_"; } next; } # split into individual fields, process time stamp my ($time, $proto, $srcip, $srcpt, $destip, $destpt, $len, $iface, $junk); if ($opt_l==1) { ($time, $proto, $srcip, $srcpt, $destip, $destpt, $len, $iface)= ($1, $2, $3, $4, $5, $6, $7, $8); } else { ($time, $junk, $srcip, $destip, $srcpt, $destpt, $proto, $junk, $len)= ($1, $2, $3, $4, $5, $6, $7, $8, $9); } if ($srcip !~ /^$pattern_ip$/ || $destip !~ /^$pattern_ip$/) { $corrupted++; if ($opt_v>1) { print "corruptip($count)$_"; } next; } $parsed++; if ($start && $time < $start) { $offtime++; if ($opt_v>1) { print "offtime($offtime)\n"; } next; } if ($end && $end < $time) { $offtime++; if ($opt_v>1) { print "offtime($offtime)\n"; } next; } if ($opt_p) { if ($opt_P) { if ($destpt =~ /$opt_p/) { $notfound++; next; } } else { if ($destpt !~ /$opt_p/) { $notfound++; next; } } } elsif ($opt_q) { if ($opt_Q) { if ($destpt =~ /$opt_q/ || $srcpt =~ /$opt_q/) { $notfound++; next; } } else { if ($destpt !~ /$opt_q/ && $srcpt !~ /$opt_q/) { $notfound++; next; } } } if ($srcip =~ /^$opt_h$/) { if ($opt_e && excluded($destip)) { $excluded++; next; } $out+=$len; $outf++; $out{$opt_r?$destip:$srcip}+=$len; $outf{$opt_r?$destip:$srcip}++; $found++; } elsif ($destip =~ /^$opt_h$/) { if ($opt_e && excluded($srcip)) { $excluded++; next; } $in+=$len; $inf++; $in{$opt_r?$srcip:$destip}+=$len; $inf{$opt_r?$srcip:$destip}++; $found++; $input=1; } else { $notfound++; if ($opt_v>1) { print "notfound($notfound)$_"; } next; } if ($time < $mindate || $mindate == 0) { $mindate = $time; } if ($maxdate < $time) { $maxdate = $time; } if ($opt_w) { $len = 1; } if ($opt_S) { if ($opt_l == 1) { if ($input) { if ($proto==$tcpp) { $tcppin{$destpt}+=$len; $tcppout{$srcpt}+=$len; } elsif ($proto==$udpp) { $udppin{$destpt}+=$len; $udppout{$srcpt}+=$len; } else { $protoin {$proto} += $len; } } else { if ($proto==$tcpp) { $tcppout{$destpt}+=$len; $tcppin{$srcpt}+=$len; } elsif ($proto==$udpp) { $udppout{$destpt}+=$len; $udppin{$srcpt}+=$len; } else { $protoout {$proto} += $len; } } } else { if ($input) { if ($proto==$tcpp) { $tcppin{$destpt}+=$len; } elsif ($proto==$udpp) { $udppin{$destpt}+=$len; } else { $protoin {$proto} += $len; } } else { if ($proto==$tcpp) { $tcppout{$destpt}+=$len; } elsif ($proto==$udpp) { $udppout{$destpt}+=$len; } else { $protoout {$proto} += $len; } } } } if ($opt_s) { if (($opt_d != 0) == ($input != 0)) { my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)= localtime($time); my $eday=epochday($time); # update current time frame $toprecord{'first'}=$time if ($toprecord{'first'}==0); $toprecord{'first'}=$time if ($time<$toprecord{'first'}); $toprecord{'last'}=$time if ($time>$toprecord{'last'}); # doing some stats $toprecord{'overall'}+=$len; $protocol[$proto]+=$len; $hours[$hour]+=$len; $toprecord{'hour'}=$hour if ($hours[$hour]>$hours[$toprecord{'hour'}]); $days[$eday]+=$len; $toprecord{'day'}=$eday if ($toprecord{'day'}==0 || \ $days[$eday]>$days[$toprecord{'day'}]); } } } } print "${opt_f}parsed=($parsed) corrupted=($corrupted) \n"; print "${opt_f}found($found) notfound($notfound) "; if ($opt_e) { print $opt_i?"in":"ex"."cluded($excluded) "; } print "offtime($offtime)\n"; #print "days=(",scalar (@days), ") t=($toprecord{'day'}) \n" ; # date or not ? #if ($opt_period) { use POSIX qw(strftime); print "${opt_f}\n"; if ($mindate) { print "${opt_f}Period from "; print strftime("%Y-%m-%d %H:%M:%S", localtime($mindate)); print "\n"; } if ($maxdate) { if ($mindate) { print "${opt_f} to "; } else { print "${opt_f}Period to "; } print strftime("%Y-%m-%d %H:%M:%S", localtime($maxdate)); print "\n"; } my $conns = "bytes"; if ($opt_w) { $conns = "connections"; } if ($opt_d) { print "${opt_f}Sorted by download $conns\n${opt_f}\n"; } else { print "${opt_f}Sorted by upload $conns\n${opt_f}\n"; } #} #print "in=($in) out=($out) \n"; if ($opt_f) { print "${opt_f}IP:in(bytes):in(count):out(count)\n"; } elsif ($opt_b) { print "$opt_f In (Out)\n"; } else { print "$opt_f In (Out)\n"; } if ($opt_f) { print "${opt_f}total:$in:$inf:$outf\n"; } elsif ($opt_w) { print $opt_f, printbytes($inf,$opt_b?16:8), " ", printbytes($outf,$opt_b?16:8,"(",")")," total \n"; } else { print $opt_f, printbytes($in,$opt_b?16:8), " ", printbytes($out,$opt_b?16:8,"(",")")," total \n"; } my $maxin = (keys %in)[0]; if (!defined ($maxin)) { print "${opt_f}nothing found\n"; exit; } { my $keys ; my $keys2 ; my @keys_in ; my @keys_out ; my $in1 = \%in; my $out1 = \%out; if ($opt_w) { $in1 = \%inf; $out1 = \%outf } @keys_in = (sort { $$in1{$b} <=> $$in1{$a} } keys %$in1) ; @keys_out = (sort { $$out1{$b} <=> $$out1{$a} } keys %$out1) ; if ($opt_d) { $keys = \@keys_in; $keys2 = \@keys_out; } else { $keys = \@keys_out; $keys2 = \@keys_in; } my $number = $#$keys; #print "debug c($opt_c) n($number) ki(", scalar (keys %in), ") ko(", scalar (keys %out), ")\n"; if (0 < $opt_c && $opt_c < $number) { $number = $opt_c ; } my $counter=0; if ($opt_f) { for my $key (@$keys[0..$number]) { #print "l($counter)"; if ($opt_g) { if ($opt_d) { if ($$in1{$key} < $opt_g ) { last; } } else { if ($$out1{$key} < $opt_g ) { last; } } } if (defined($in{$key})) { my $minf = (defined ($inf{$key})) ? $inf{$key}:0; my $moutf = (defined ($outf{$key})) ? $outf{$key}:0; print "$key:$in{$key}:$minf:$moutf\n" } $counter++; } } else { for my $key (@$keys[0..$number]) { #print "l($counter)"; my $inkey = $$in1{$key}; my $outkey = $$out1{$key}; if ($opt_g) { if ($opt_d) { if ($inkey < $opt_g ) { last; } } else { if ($outkey < $opt_g ) { last; } } } if (defined($inkey)) { print printbytes($inkey,$opt_b?16:8) ; } else { print $opt_b?" "x16:" "x8; } print " " ; if (defined($outkey)) { print printbytes($outkey,$opt_b?16:8,"(", ")"); } else { print $opt_b?" "x18:" "x10; } print " "; print ip_to_name($key), " \n"; $counter++; } if ($opt_D) { print "\n\n"; if ($opt_d) { print "${opt_f}Sorted by upload $conns\n${opt_f}\n"; } else { print "${opt_f}Sorted by download $conns\n${opt_f}\n"; } for my $key (@$keys2[0..$number]) { #print "l($counter)"; my $inkey = $$in1{$key}; my $outkey = $$out1{$key}; if ($opt_g) { if ($opt_d) { if ($inkey < $opt_g ) { last; } } else { if ($outkey < $opt_g ) { last; } } } if (defined($inkey)) { print printbytes($inkey,$opt_b?16:8) ; } else { print $opt_b?" "x16:" "x8; } print " " ; if (defined($outkey)) { print printbytes($outkey,$opt_b?16:8,"(", ")"); } else { print $opt_b?" "x18:" "x10; } print " "; print ip_to_name($key), " \n"; $counter++; } } } } undef %out; undef %in; undef %outf; undef %inf; sub printhash { my $protoc = shift; my %list = @_; my @keys = (sort { $list{$b} <=> $list{$a} } keys %list) ; my $num = $#keys; if (0 < $opt_c && $opt_c < $num) { $num = $opt_c ; } for my $key (@keys[0..$num]) { my $kn ; if ($opt_g && $list{$key} < $opt_g) { last; } if ($key< 10000) { $kn = getservbyport ($key, $protoc); } $kn = defined($kn)?$kn:""; printf "%s %8.d %s\n", printbytes($list{$key},$opt_b?16:8), $key, $kn ; } } sub printpro { my %protos; for (keys %protoin, keys %protoout) { $protos{$_} = 1; } if (scalar keys %protos) { if ($opt_b) { print "other protocols: In Out \n"; } else { print "other protocols: In Out \n"; } for my $proto (sort keys %protos) { my $name = getprotobynumber ($proto); if (!$name) { $name = "($proto)"; } else { $name = "\"$name\" ($proto)"; } my $inb = 0; if (defined($protoin{$proto})) { $inb = $protoin{$proto}; } my $outb = 0; if (defined($protoout{$proto})) { $outb = $protoout{$proto}; } printf "%13s %s %s \n", $name, printbytes ($inb, $opt_b?16:8), printbytes ($outb, $opt_b?16:8) ; } } } if ($opt_S) { print "Destination TCP ports in:\n"; printhash('tcp', %tcppin); print "Destination TCP ports out:\n"; printhash('tcp', %tcppout); print "Destination UDP ports in:\n"; printhash('udp', %udppin); print "Destination UDP ports out:\n"; printhash('udp', %udppout); printpro; } if ($opt_s) { printf("\nTraffic Report for \"%s\"\n\n", $opt_h); overallstats; protocolstats; hourlystats; gridstats; dailystats; }