Re: [linux-security] Re: Re: Re: Lightning fast attacks?

[A Mennucc1 <sysman Tonelli sns it>]

On Tue, May 12, 1998 at 10:39:00AM -0500, Kevin Buhr wrote:
>[...]
> 
> It's major drawback is that "tcplogd" logs *all* SYN packets, and
> "icmplogd" logs *all* ICMP packets (except echo replies), so your logs
> tend to fill up with legitimate connections you have to wade through
> to find the "bad guys".  It's great for a "client" machine, however,
> that doesn't receive connections from many external sources.

this is a major problem here indeed... 
almost everything can be logged, but the more you log,
the harder it is to find the valuable informations...

For this reasons I have done two scripts, that I include;
the first one, called watch_logs, is run nightly:
it scans the syslogs 
and extracts those of certain daemons (like, tcplogd);
to do so it uses 
the second, watch_anomaly, that keeps track of what
has happened recently and only shows what has not happened before;
after a few days of "learning", the output decreases a lot,
and it is easy to see if some kind of strange thing
has happened.
The scripts need some improvements (if you improve them, please
e-mail me the new version), but I have been happily using them
for years.


a.m.

#!/bin/sh


### watch  by A.Mennucci 1997   msm tonelli sns it
#
# a script to watch the logs
#
# Copyright: GPL


umask 077
set -e 

TMPDIR=/var/tmp

LOGDIR=/var/log

PATH=${PATH}:/usr/local/sbin

export PATH

# ouput filter to cut away unwanted unprintable chars
# Note that the insertion of that ISO character will 
# crash a terminal if this file is just viewed with 'cat'

filter () { tr  --complement "\n[:print:]" ø ; }


#this will delete the filename given by grep and the initial date in syslogs

nodate () {  cut -d: -f3- | cut -c3- ; }

#this will destry pids in syslogs

nopid () { sed -e 's \[[0-9]*\]: \[NOPID\]: ' ; }

# If the logging to the syslog logfiles has a different format,
# all this must be changed


#this will list all files in syslog.conf , only once


syslogs=` grep -v '^#' /etc/syslog.conf |\
 grep -v '|' | grep / | cut -d/ -f2- | cut -d' ' -f1 | sort -u |\
 awk '{ printf("/%s\n",$0) }' `

list_syslogs () { for i in $syslogs ; do [ -r $i -a -f $i ] && cat $i ; [ -r ${i}.0 -a -f ${i}.0 ] && cat  ${i}.0 ; done ; }

#list_syslogs | nodate | nopid 



#cat /bin/ls | filter | head
#exit

cd $TMPDIR

#echo RUN======================= $0 ====================

############## mail

#the first sed line removes the ORIG-ID
# 2nd removes the SIZE
# 3rd removes the  filenames=ids that smail uses for stored mails

(list_syslogs | grep mail  | nodate | nopid
if [ -r $LOGDIR/smail ] ; then
 cat $LOGDIR/smail/{paniclog,logfile}{,.0} | cut -d: -f4- |\
 sed \
 -e 's ORIG-ID:< * \(.*\)> ORIG-ID:<DEL-ID DEL-IP> ' \
 -e 's SIZE:[0-9]* SIZE:DEL-SIZE ' \
 -e 's 0y[a-zA-Z0-9][a-zA-Z0-9][a-zA-Z0-9][a-zA-Z0-9]-0000[a-zA-Z0-9][a-zA-Z0-9][a-zA-Z0-9] DEL-ID '
fi
)| watch_anomaly mail | filter


########## last, that is , connections
#
# brutal but reasonably working
# it outputs:  USER site SITE day DAY 
# 
last |  awk -F " " -- \
 '/ttyp/{ print  $1 " site " $3 " day "  $4 };\
/tty[0-9]/{print $1 " site localhost day " $3}' |\
watch_anomaly last | filter


################# ftp
#we filter out the lines where tcplogd talks about ftp
#

(list_syslogs | grep ftp  |\
   grep -v "tcplogd: ftp connection attempt"  | nodate | nopid 
cut -d " " -f9-13 /var/log/xferlog | awk "{printf \"xferlog \" ;print  $* }"
)|\
watch_anomaly ftp | filter


############# named

list_syslogs | grep named | nodate | nopid |\
grep -E -v ': NSTAT|: USAGE|: XSTAT' |\
watch_anomaly named | filter 

############# icmplogd

list_syslogs | grep icmplogd  | nodate | nopid |\
watch_anomaly icmplogd | filter 

############# tcplogd
# but not ftp 

list_syslogs | grep tcplogd  | grep -v ftp | nodate | nopid |\
watch_anomaly tcplogd | filter


#echo ------------------- MD5SUM of files in your system -------------

(
cd /
for i in  /var/lib/dpkg/info/*.*sum* ; do 
 md5sum -c $i 2>&1
done
)|\
watch_anomaly md5sums --changed | filter 


#echo END======================= $0 ====================

#!/bin/sh

#see help below


# this is the prefix used for logs of this programs
w=wa


TMPDIR=/var/tmp

LOGDIR=/var/log

# this is the number of days of memory: anything older than
# this is deleted from the history
#
# it is useless to put it below 7 since many log files
#  arent rotated that often

MEMORY=30



#end of user configurable part

###################################### 
p=`basename $0`

if [ "$1" = '-h' -o "$1" = '--help' ] ; then
cat >&2 << __EOF__
Usage:
 $p name [ --changed ]

 $p : a program to screen the logs. By A.Mennucci 97   
            msm tonelli sns it  Copyright: GPL

 The user should pipe in $p all logs about name;
  then $p will show what it has found new (or changed,
  if the option is given); $p has a memory of $MEMORY days.
  The output should be piped thru 'sed -n l'
  'tr --complement [:print:] _ 'if unprintable chars may be found.
  $p will print nothing if there are no  changes.
  The logs that are given to $p should only contain interesting infos
  and so they should be preprocessed to 
  cut away variable infos like: date , pid, ttys, strange ids, ecc
  See /etc/cron.daily/watch_logs for examples.
__EOF__
exit
fi




if [ "$1" = "" ] ; then 
 echo $p: needs an argument. Please read $p -h
 exit 1
fi 


umask 077
cd $TMPDIR

#this rotated logs but now I delete them 

if [ -r  $LOGDIR/$w.$1   ] ; then
sleep 3
if [ -r  $LOGDIR/$w.$1   ] ; then
sleep 3
if [ -r  $LOGDIR/$w.$1   ] ; then
sleep 3
if [ -r  $LOGDIR/$w.$1   ] ; then
 echo "$p: another $p may be runnning on $1" 1>&2
 echo "$p:    otherwise please delete $LOGDIR/$w.$1" 1>&2
 exit 1
fi
fi
fi
fi
#  if [ -r  $LOGDIR/$w.$1.0.gz ] ; then
#     rm $LOGDIR/$w.$1.0.gz
#   fi  
#   mv $LOGDIR/$w.$1.gz $LOGDIR/$w.$1.0.gz



#echo Please pipe in this program any log concerning $1

#TODO: if the log contains a date, this should be used to give the seconds
secs=`date +%s`

oldsecs=`expr $secs - $MEMORY  \* 30 \* 3600 \* 24 `

# for debug
#echo $p for $1 : now is $secs I will forget what is older than $oldsecs 

#in this case the sort -u we use to receive the logs should be changed

# this receives the new logs  #and creates our "lock file"
sort -u  |  awk "{printf \"${secs} \" ;print  }" > $LOGDIR/$w.$1



#echo Thanks



if [ -r  $LOGDIR/$w.$1.history.gz ] ; then
 gunzip  $LOGDIR/$w.$1.history.gz 
fi

if [ -r  $LOGDIR/$w.$1.history ] ; then


  #this forgets old stuff

  awk  '{ if ( $1 > '$oldsecs' ) {print }}' $LOGDIR/$w.$1.history > $LOGDIR/$w.$1.history~newer
  mv $LOGDIR/$w.$1.history~newer $LOGDIR/$w.$1.history
   
  ## prepare differences 

  if [  "$2" = "--changed" ] ; then
   cut -c10- $LOGDIR/$w.$1.history > $LOGDIR/$w.$1.historycut
   cut -c10- $LOGDIR/$w.$1 > $LOGDIR/$w.$1.cut

   diff -c0 $LOGDIR/$w.$1.historycut $LOGDIR/$w.$1.cut > $LOGDIR/$w.$1.diff
   mv $LOGDIR/$w.$1 $LOGDIR/$w.$1.history
   rm $LOGDIR/$w.$1.historycut $LOGDIR/$w.$1.cut
  else
   mv $LOGDIR/$w.$1.history  $LOGDIR/$w.$1.history~  
   sort -t " " -k 2,12 +0rn    $LOGDIR/$w.$1 $LOGDIR/$w.$1.history~  | uniq -s 10 > $LOGDIR/$w.$1.history
   cut -c10- $LOGDIR/$w.$1.history > $LOGDIR/$w.$1.historycut
   cut -c10- $LOGDIR/$w.$1.history~ > $LOGDIR/$w.$1.history~cut
   diff -c0     $LOGDIR/$w.$1.history~cut $LOGDIR/$w.$1.historycut > $LOGDIR/$w.$1.diff
   rm $LOGDIR/$w.$1.history~   
   rm $LOGDIR/$w.$1.historycut $LOGDIR/$w.$1.history~cut
  fi
  
  if [ -s $LOGDIR/$w.$1.diff ] ; then
   if [ ! "$2" = "--changed" ] ;  then
     echo "--- $1 ---This is everything new in the logs about $1 ------"
    else  
     echo "--- $1 ---This is everything changed in the logs about $1 ------"
   fi 
   cat $LOGDIR/$w.$1.diff | grep  "^. "
  fi
  rm   $LOGDIR/$w.$1.diff
  
 else
  echo "-- $1 --What you piped in is new and will be considered history ----"
  mv $LOGDIR/$w.$1 $LOGDIR/$w.$1.history
  cat $LOGDIR/$w.$1.history | cut -c9-
fi

#leaves all clean and gzipped

gzip $LOGDIR/$w.$1.history


#this is our "lock file"
if [ -r  $LOGDIR/$w.$1 ] ; then
    rm $LOGDIR/$w.$1
fi