forked from OSchip/llvm-project
298 lines
9.8 KiB
Python
298 lines
9.8 KiB
Python
#!/usr/bin/env python
|
|
|
|
""" This runs netstat on a local or remote server. It calculates some simple
|
|
statistical information on the number of external inet connections. It groups
|
|
by IP address. This can be used to detect if one IP address is taking up an
|
|
excessive number of connections. It can also send an email alert if a given IP
|
|
address exceeds a threshold between runs of the script. This script can be used
|
|
as a drop-in Munin plugin or it can be used stand-alone from cron. I used this
|
|
on a busy web server that would sometimes get hit with denial of service
|
|
attacks. This made it easy to see if a script was opening many multiple
|
|
connections. A typical browser would open fewer than 10 connections at once. A
|
|
script might open over 100 simultaneous connections.
|
|
|
|
./topip.py [-s server_hostname] [-u username] [-p password] {-a from_addr,to_addr} {-n N} {-v} {--ipv6}
|
|
|
|
-s : hostname of the remote server to login to.
|
|
-u : username to user for login.
|
|
-p : password to user for login.
|
|
-n : print stddev for the the number of the top 'N' ipaddresses.
|
|
-v : verbose - print stats and list of top ipaddresses.
|
|
-a : send alert if stddev goes over 20.
|
|
-l : to log message to /var/log/topip.log
|
|
--ipv6 : this parses netstat output that includes ipv6 format.
|
|
Note that this actually only works with ipv4 addresses, but for versions of
|
|
netstat that print in ipv6 format.
|
|
--stdev=N : Where N is an integer. This sets the trigger point for alerts and logs.
|
|
Default is to trigger if max value is above 5 standard deviations.
|
|
|
|
Example:
|
|
|
|
This will print stats for the top IP addresses connected to the given host:
|
|
|
|
./topip.py -s www.example.com -u mylogin -p mypassword -n 10 -v
|
|
|
|
This will send an alert email if the maxip goes over the stddev trigger value and
|
|
the the current top ip is the same as the last top ip (/tmp/topip.last):
|
|
|
|
./topip.py -s www.example.com -u mylogin -p mypassword -n 10 -v -a alert@example.com,user@example.com
|
|
|
|
This will print the connection stats for the localhost in Munin format:
|
|
|
|
./topip.py
|
|
|
|
Noah Spurrier
|
|
|
|
$Id: topip.py 489 2007-11-28 23:40:34Z noah $
|
|
"""
|
|
|
|
import pexpect
|
|
import pxssh # See http://pexpect.sourceforge.net/
|
|
import os
|
|
import sys
|
|
import time
|
|
import re
|
|
import getopt
|
|
import pickle
|
|
import getpass
|
|
import smtplib
|
|
import traceback
|
|
from pprint import pprint
|
|
|
|
TOPIP_LOG_FILE = '/var/log/topip.log'
|
|
TOPIP_LAST_RUN_STATS = '/var/run/topip.last'
|
|
|
|
|
|
def exit_with_usage():
|
|
|
|
print globals()['__doc__']
|
|
os._exit(1)
|
|
|
|
|
|
def stats(r):
|
|
"""This returns a dict of the median, average, standard deviation, min and max of the given sequence.
|
|
|
|
>>> from topip import stats
|
|
>>> print stats([5,6,8,9])
|
|
{'med': 8, 'max': 9, 'avg': 7.0, 'stddev': 1.5811388300841898, 'min': 5}
|
|
>>> print stats([1000,1006,1008,1014])
|
|
{'med': 1008, 'max': 1014, 'avg': 1007.0, 'stddev': 5.0, 'min': 1000}
|
|
>>> print stats([1,3,4,5,18,16,4,3,3,5,13])
|
|
{'med': 4, 'max': 18, 'avg': 6.8181818181818183, 'stddev': 5.6216817577237475, 'min': 1}
|
|
>>> print stats([1,3,4,5,18,16,4,3,3,5,13,14,5,6,7,8,7,6,6,7,5,6,4,14,7])
|
|
{'med': 6, 'max': 18, 'avg': 7.0800000000000001, 'stddev': 4.3259218670706474, 'min': 1}
|
|
"""
|
|
|
|
total = sum(r)
|
|
avg = float(total) / float(len(r))
|
|
sdsq = sum([(i - avg)**2 for i in r])
|
|
s = sorted(r)
|
|
return dict(zip(['med', 'avg', 'stddev', 'min', 'max'],
|
|
(s[len(s) // 2], avg, (sdsq / len(r))**.5, min(r), max(r))))
|
|
|
|
|
|
def send_alert(message, subject, addr_from, addr_to, smtp_server='localhost'):
|
|
"""This sends an email alert.
|
|
"""
|
|
|
|
message = 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n' % (
|
|
addr_from, addr_to, subject) + message
|
|
server = smtplib.SMTP(smtp_server)
|
|
server.sendmail(addr_from, addr_to, message)
|
|
server.quit()
|
|
|
|
|
|
def main():
|
|
|
|
######################################################################
|
|
# Parse the options, arguments, etc.
|
|
######################################################################
|
|
try:
|
|
optlist, args = getopt.getopt(
|
|
sys.argv[
|
|
1:], 'h?valqs:u:p:n:', [
|
|
'help', 'h', '?', 'ipv6', 'stddev='])
|
|
except Exception as e:
|
|
print str(e)
|
|
exit_with_usage()
|
|
options = dict(optlist)
|
|
|
|
munin_flag = False
|
|
if len(args) > 0:
|
|
if args[0] == 'config':
|
|
print 'graph_title Netstat Connections per IP'
|
|
print 'graph_vlabel Socket connections per IP'
|
|
print 'connections_max.label max'
|
|
print 'connections_max.info Maximum number of connections per IP'
|
|
print 'connections_avg.label avg'
|
|
print 'connections_avg.info Average number of connections per IP'
|
|
print 'connections_stddev.label stddev'
|
|
print 'connections_stddev.info Standard deviation'
|
|
return 0
|
|
elif args[0] != '':
|
|
print args, len(args)
|
|
return 0
|
|
exit_with_usage()
|
|
if [elem for elem in options if elem in [
|
|
'-h', '--h', '-?', '--?', '--help']]:
|
|
print 'Help:'
|
|
exit_with_usage()
|
|
if '-s' in options:
|
|
hostname = options['-s']
|
|
else:
|
|
# if host was not specified then assume localhost munin plugin.
|
|
munin_flag = True
|
|
hostname = 'localhost'
|
|
# If localhost then don't ask for username/password.
|
|
if hostname != 'localhost' and hostname != '127.0.0.1':
|
|
if '-u' in options:
|
|
username = options['-u']
|
|
else:
|
|
username = raw_input('username: ')
|
|
if '-p' in options:
|
|
password = options['-p']
|
|
else:
|
|
password = getpass.getpass('password: ')
|
|
else:
|
|
use_localhost = True
|
|
|
|
if '-l' in options:
|
|
log_flag = True
|
|
else:
|
|
log_flag = False
|
|
if '-n' in options:
|
|
average_n = int(options['-n'])
|
|
else:
|
|
average_n = None
|
|
if '-v' in options:
|
|
verbose = True
|
|
else:
|
|
verbose = False
|
|
if '-a' in options:
|
|
alert_flag = True
|
|
(alert_addr_from, alert_addr_to) = tuple(options['-a'].split(','))
|
|
else:
|
|
alert_flag = False
|
|
if '--ipv6' in options:
|
|
ipv6_flag = True
|
|
else:
|
|
ipv6_flag = False
|
|
if '--stddev' in options:
|
|
stddev_trigger = float(options['--stddev'])
|
|
else:
|
|
stddev_trigger = 5
|
|
|
|
if ipv6_flag:
|
|
netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+::ffff:(\S+):(\S+)\s+.*?\r'
|
|
else:
|
|
netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(?:::ffff:)*(\S+):(\S+)\s+.*?\r'
|
|
#netstat_pattern = '(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+):(\S+)\s+.*?\r'
|
|
|
|
# run netstat (either locally or via SSH).
|
|
if use_localhost:
|
|
p = pexpect.spawn('netstat -n -t')
|
|
PROMPT = pexpect.TIMEOUT
|
|
else:
|
|
p = pxssh.pxssh()
|
|
p.login(hostname, username, password)
|
|
p.sendline('netstat -n -t')
|
|
PROMPT = p.PROMPT
|
|
|
|
# loop through each matching netstat_pattern and put the ip address in the
|
|
# list.
|
|
ip_list = {}
|
|
try:
|
|
while True:
|
|
i = p.expect([PROMPT, netstat_pattern])
|
|
if i == 0:
|
|
break
|
|
k = p.match.groups()[4]
|
|
if k in ip_list:
|
|
ip_list[k] = ip_list[k] + 1
|
|
else:
|
|
ip_list[k] = 1
|
|
except:
|
|
pass
|
|
|
|
# remove a few common, uninteresting addresses from the dictionary.
|
|
ip_list = dict([(key, value)
|
|
for key, value in ip_list.items() if '192.168.' not in key])
|
|
ip_list = dict([(key, value)
|
|
for key, value in ip_list.items() if '127.0.0.1' not in key])
|
|
|
|
# sort dict by value (count)
|
|
#ip_list = sorted(ip_list.iteritems(),lambda x,y:cmp(x[1], y[1]),reverse=True)
|
|
ip_list = ip_list.items()
|
|
if len(ip_list) < 1:
|
|
if verbose:
|
|
print 'Warning: no networks connections worth looking at.'
|
|
return 0
|
|
ip_list.sort(lambda x, y: cmp(y[1], x[1]))
|
|
|
|
# generate some stats for the ip addresses found.
|
|
if average_n <= 1:
|
|
average_n = None
|
|
# The * unary operator treats the list elements as arguments
|
|
s = stats(zip(*ip_list[0:average_n])[1])
|
|
s['maxip'] = ip_list[0]
|
|
|
|
# print munin-style or verbose results for the stats.
|
|
if munin_flag:
|
|
print 'connections_max.value', s['max']
|
|
print 'connections_avg.value', s['avg']
|
|
print 'connections_stddev.value', s['stddev']
|
|
return 0
|
|
if verbose:
|
|
pprint(s)
|
|
print
|
|
pprint(ip_list[0:average_n])
|
|
|
|
# load the stats from the last run.
|
|
try:
|
|
last_stats = pickle.load(file(TOPIP_LAST_RUN_STATS))
|
|
except:
|
|
last_stats = {'maxip': None}
|
|
|
|
if s['maxip'][1] > (
|
|
s['stddev'] *
|
|
stddev_trigger) and s['maxip'] == last_stats['maxip']:
|
|
if verbose:
|
|
print 'The maxip has been above trigger for two consecutive samples.'
|
|
if alert_flag:
|
|
if verbose:
|
|
print 'SENDING ALERT EMAIL'
|
|
send_alert(
|
|
str(s),
|
|
'ALERT on %s' %
|
|
hostname,
|
|
alert_addr_from,
|
|
alert_addr_to)
|
|
if log_flag:
|
|
if verbose:
|
|
print 'LOGGING THIS EVENT'
|
|
fout = file(TOPIP_LOG_FILE, 'a')
|
|
#dts = time.strftime('%Y:%m:%d:%H:%M:%S', time.localtime())
|
|
dts = time.asctime()
|
|
fout.write('%s - %d connections from %s\n' %
|
|
(dts, s['maxip'][1], str(s['maxip'][0])))
|
|
fout.close()
|
|
|
|
# save state to TOPIP_LAST_RUN_STATS
|
|
try:
|
|
pickle.dump(s, file(TOPIP_LAST_RUN_STATS, 'w'))
|
|
os.chmod(TOPIP_LAST_RUN_STATS, 0o664)
|
|
except:
|
|
pass
|
|
# p.logout()
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
sys.exit(0)
|
|
except SystemExit as e:
|
|
raise e
|
|
except Exception as e:
|
|
print str(e)
|
|
traceback.print_exc()
|
|
os._exit(1)
|