Awstats 配置详解

[root@test src]# wget http://prdownloads.sourceforge.net/awstats/awstats-6.95.tar.gz [root@test src]# tar zxvf awstats-6.95.tar.gz [root@test src]# mv awstats-6.95 /usr/local/awstats [root@test src]# mkdir /var/lib/awstats [root@test awsta

[root@test src]# wget http://prdownloads.sourceforge.net/awstats/awstats-6.95.tar.gz

[root@test src]# tar zxvf awstats-6.95.tar.gz

[root@test src]# mv awstats-6.95 /usr/local/awstats

[root@test src]# mkdir /var/lib/awstats

[root@test awstats]# mkdir -m 755 /usr/local/awstats/data/

[root@test src]# cd /usr/local/awstats/

[root@test awstat]# perl /usr/local/awstats/tools/awstats_configure.pl

只填写域名，其它选项全部y或默认

[root@test awstats]# vi /etc/awstats/awstats.www.123.com.conf

修改日志文件路径：

LogFile="/usr/local/awstats/tools/logresolvemerge.pl /home/tool/access.20100817.11*.log |"

此为用awstats脚本自动合并分析所有/home/tool/access.20100817.11*.log日志

如果日志使用tar -zcvf 压缩，logresolvemerge.pl会自动调用zcat

也可以同时列出多个日志文件，如以下格式：

LogFile="/usr/local/awstats/tools/logresolvemerge.pl 1.log 2.log |"

合并日志也可用以下方式：

cat a.log b.log c.log | sort -m -t " " –k 4 > log

       将三个日志以空格为分隔符，对第四列进行排序，结果保存在log

 

添加GeoIP插件使awstats可以显示用户地理位置

cpan install Geo::IP

[root@test awstats]# cd /usr/local/src/

[root@test src]# wget http://geolite.maxmind.com/download/geoip/api/c/GeoIP.tar.gz

[root@test src]# tar zxvf GeoIP.tar.gz

[root@test src]# cd GeoIP-1.4.6/

[root@test GeoIP-1.4.6]# ./configure && make && make install

[root@test src]# wget http://geolite.maxmind.com/download/geoip/api/perl/Geo-IP-1.38.tar.gz

[root@test src]# tar zxvf Geo-IP-1.38.tar.gz

[root@test src]#cd Geo-IP-1.38

[root@test Geo-IP-1.38]# perl Makefile.PL && make && make install

[root@test src]# wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz

[root@test src]# gunzip GeoLiteCity.dat.gz

[root@test src]# cp GeoLiteCity.dat /usr/local/share/GeoIP/

[root@test src]# vi /etc/awstats/awstats.www.dayu360.com.conf

启用或添加以下选项：

LoadPlugin="tooltips"

LoadPlugin="decodeutfkeys"

LoadPlugin="geoip GEOIP_STANDARD /usr/local/share/GeoIP/GeoIP.dat"

LoadPlugin="geoip_city_maxmind GEOIP_STANDARD /usr/local/share/GeoIP/GeoLiteCity.dat"

 

[root@test awstats]# perl /usr/local/awstats/wwwroot/cgi-bin/awstats.pl -config=www.dayu360.com –update

更新awstats数据库（即web页面显示的内容）

 

 

当登陆到awstats的web界面时出现提示：

Error: Access to statistics is only allowed from an authenticated session to authenticated users.

则需把/etc/awstats/awstats.www.123.com.conf中：

              AllowAccessFromWebToAuthenticatedUsersOnly=0

 

访问web页：

http://192.168.0.80/awstats/awstats.pl?config=www.123.com

 

以上完成了基本设置，下面进行一些高级设置：

（1）不同的日志格式对应不同的日志格式的设置：

一、若对运行在apache上的网站产生的日志（combined格式）进行分析则在/etc/awstats/awstats.www.123.com.conf文件中配置：

LogType=W

LogFormat=1

二、若要对运行在nginx上的网站产生的日志进行分析，首先需要在nginx的配置文件nginx.conf中更改产生日志的格式：

log_format  main  '$remote_addr - $remote_user [$time_local] $request '

'"$status" $body_bytes_sent "$http_referer" '

'"$http_user_agent" $http_x_forwarded_for';

access_log  logs/access.log  main;

然后在/etc/awstats/awstats.www.123.com.conf文件中配置：

LogType=W

LogFormat=1

三、若nginx日志格式为：

log_format  main  '$remote_addr - $remote_user [$time_local] $request '

'"$status" $body_bytes_sent "$http_referer" '

'"$http_user_agent"';

access_log  logs/access.log  main;

则在/etc/awstats/awstats.www.123.com.conf文件中配置：

LogType=W

LogFormat=%host %other %logname %time1 %methodurl %code %bytesd %refererquot %uaquot"

注：网上某些文章的nginx.conf中log_format后面加了$http_x_forwarded_for（来源ip），然而其在/etc/awstats/awstats.www.123.com.conf文件中配置还是：

LogFormat=%host %other %logname %time1 %methodurl %code %bytesd %refererquot %uaquot"，则因为缺少对应关系而使得awstats报错。

可以通过修改nginx的日志格式（把$http_x_forwarded_for去掉）即可。

最后注意：通过测试发现%methodurl字段必须对应GET的日志，若有POST的日志或“-”则会报错。

 

 

 

 

Sftp自动上传日志：

Apache服务器：

useradd -u 1000 httplog -d /home/httplog/

passwd httplog

awstat 服务器：

创建get日志的脚本

vi /usr/local/shell/get_weblog.sh

#!/bin/sh

export WEB1=172.16.0.115;

export WEB2=172.16.0.116;

export WEB3=172.16.0.117;

export USER=httplog;

export PASS="bdGhS3EI4dSH86QBXjV9";

export CPATH="/home/httpdlog/`date +%Y`/`date --date='1 days ago' +%m`/"

export web1="web1.access.`date --date='1 days ago' +%Y%m%d`.log"

export web2="web2.access.`date --date='1 days ago' +%Y%m%d`.log"

export web3="web3.access.`date --date='1 days ago' +%Y%m%d`.log"

export SPATH="/opt/weblogs/`date +%Y`/`date --date='1 days ago' +%m`/"

echo "sftp start";

 

if [ ! -d $SPATH ]; then

mkdir -p $SPATH; fi

 

lftp -u ${USER},${PASS} sftp://${WEB1}

lcd $SPATH

get $CPATH/$web1

open sftp://${USER}:${PASS}@$WEB2

get $CPATH/$web2

open sftp://${USER}:${PASS}@$WEB3

get $CPATH/$web3

bye

EOF

echo "GET WEB LOG DONE!!!";

 

创建awstat更新脚本

vi /usr/local/shell/awstat-update.sh

#!/bin/bash

configfiles="www"

#configfiles="www bbs buy"

domain=123.com

command=/usr/local/awstats/wwwroot/cgi-bin/awstats.pl

for conf in ${configfiles};

do

        ${command} -update -config=${conf}.${domain} -Databasebreak=month;

        ${command} -update -config=${conf}.${domain} -Databasebreak=day;

done

 

日历显示cgi

vi /usr/local/awstat/wwwroot/cgi-bin/calendar.pl

#!/usr/bin/perl

 

@domain_allow=("www");

#@domain_allow=("www","prom","buy","tag","blog","member");

$site=$ENV{"QUERY_STRING"};

if ($site eq ""){

        $site="www";

}else{

        $reject=1;

        foreach $sub_domain (@domain_allow){

                if ($sub_domain eq $site){

                        $reject=0;

                }

        }

}

# The awstats config file name

$CONFIG=$site.".123.com";

 

### Begin program ###

 

@now=localtime(time);

$today_day=$now[3];

$today_month=$now[4]+1;

$today_year=$now[5]+1900;

 

@yesterday=localtime(time-3600*24);

$ytd_day=$yesterday[3];

$ytd_month=$yesterday[4]+1;

$ytd_year=$yesterday[5]+1900;

 

$lastmonth=$today_month-1;

$lastmonth_year=$today_year;

if($lastmonth

{

        $lastmonth=1;

        $lastmonth_year=$today_year-1;

}

$lastyear=$today_year-1;

 

print "Content-type: text/html\n\n";

print "\n";

if ($reject==1){

        print "subdomain:'$site' is not accepted, please contact admin";

}else{

        print "Choose Subdomain:";

        foreach $subdomain (@domain_allow){

                if($subdomain eq $site) {

                        $selected="selected='selected'";

                }else{

                        $selected="";

                }

                print "$subdomain";

        }

 

        print "
";

        print "Today ";

        print "Yesterday ";

        print "ThisMonth ";

        print "LastMonth ";

        print "ThisYear ";

        print "LastYear ";

        print "\n

---

\n";

 

        printCal($today_year, $today_month);

        print "\n
\n";

        printCal($lastmonth_year, $lastmonth);

}

print "\n

---

\n";

 

##### Methods ######

 

sub getLink

{

        my($year, $month, $day)=@_;

        $query="";

        if($day)

        {

                $query="DatabaseBreak=day&day=${day}&month=${month}&year=${year}";

        }

        elsif($month)

        {

                $query="month=${month}&year=${year}";

        }

        elsif($year)

        {

                $query="year=${year}&month=all";

        }

        return "awstats.pl?config=${CONFIG}&$query";

}

 

sub printCal

{

my($y, $m)=@_;

open(CAL, "cal $m $y |");

@days = ;

close(CAL);

 

$month = $days[0];

$month=~ s/\s\s\s*//g;

$mbg="";

if($m==$today_month && $y==$today_year)

{

        $mbg="bgcolor='#ffaaaa'";

}

print "

\n";

foreach $dy (split(/ /, $days[1]))

{

        print "

";

}

print "

\n";

shift(@days);

shift(@days);

foreach $line (@days)

{

        chomp $line;

        $line =~ s/^\s+//;

        $line =~ s/\s+$//;

        print "

";

        foreach $d (split(/\s+/, $line))

        {

                $bg="";

                if($d==$today_day && $m==$today_month && $y==$today_year)

                {

                        $bg="bgcolor='#ffaaaa'";

                }

                print "

";

        }

        print"

\n";

}

print "

$month

$dy

$d

\n";

}

chmod +x /usr/local/awstat/wwwroot/cgi-bin/calendar.pl

Awstats 6.9.5默认即为GBK编码，改为UTF-8后部分搜索词乱码按照网上所述方式未能解决，GBK状态即无乱码

修改awstats pagecode为GBK

cd /usr/local/awstats/wwwroot/cgi-bin/lang

cp awstats-cn.txt awstats-cn-gbk.txt

vi awstats-cn-gbk.txt

PageCode=utf-8

修改为

PageCode=GBK

iconv -f UTF-8 -t GBK awstats-cn-gbk.txt > awstats-cn.txt

 

添加QQ IP库

wget http://blog.pcware.cn/downloads/ip_geo_qqwry.zip

wget http://www.ieasy.org/download/qqwry.pl

wget http://www.ieasy.org/download/qqhostinfo.pm

ip_geo_qqwry.zip 和 qqwry.pl 任选其一

unzip ip_geo_qqwry.zip

chmod 755 ip_geo_qqwry.pl

mv ip_geo_qqwry.pl /usr/local/awstats/wwwroot/cgi-bin/

wget http://update.cz88.net/soft/qqwry.rar

wget http://packages.sw.be/unrar/unrar-3.9.10-1.el5.rf.x86_64.rpm

rpm -ivh unrar-3.9.10-1.el5.rf.x86_64.rpm

mkdir qqwry && cd qqwry

unrar e ../qqwry.rar

mkdir /usr/local/share/ip_geo/

mv QQWry.Dat /usr/local/share/ip_geo/

cd ..

mv qqhostinfo.pm /usr/local/awstats/wwwroot/cgi-bin/plugins/

vi /usr/local/awstats/wwwroot/cgi-bin/plugins/qqhostinfo.pm

require "${DIR}/plugins/qqwry.pl";

修改为：

require "/usr/local/awstats/wwwroot/cgi-bin/ip_geo_qqwry.pl";

解决Awstats GBK页面IP 物理地址乱码：

vi /usr/local/awstats/wwwroot/cgi-bin/ip_geo_qqwry.pl

注释掉以下两行

#use Encode;

#    $ip_geo_addr=decode("gbk",$ip_geo_addr);

附上ip_geo_qqwry.pl源码，以防下载链接失效

#/* Copyright (C) 2002-2009 Proguru.

#   changuoqiang[at]gmail[dot]com

#   http://blog.pcware.cn

#

#   The Source Code is free; you can redistribute it and/or

#   modify it under the terms of the GNU Lesser General Public

#   License as published by the Free Software Foundation; either

#   version 2.1 of the License, or (at your option) any later version.

#

#   The Source Code is distributed in the hope that it will be useful,

#   but WITHOUT ANY WARRANTY; without even the implied warranty of

#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

#   Lesser General Public License for more details.

#

#   You should have received a copy of the GNU Lesser General Public

#   License along with the KWinUI; if not, write to the Free

#   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA

#   02111-1307 USA.  */

#

#/*

# *  Jun 14,2009   

# */

 

#use Encode;

 

sub ipwhere{

        my $ip      = shift;

        my @ip      = split(/\./, $ip);

        my $ip_num  = $ip[0]*256**3 + $ip[1]*256**2 + $ip[2]*256 + $ip[3];

 

    #my $qqwry_dat="${DIR}/plugins/QQWry.Dat";

        my $qqwry_dat = "/usr/local/share/ip_geo/QQWry.Dat";

        open(INFILE, "$qqwry_dat");

        binmode(INFILE);

 

        my $first_index_of_begin_ip, $last_index_of_begin_ip;

        sysread(INFILE, $first_index_of_begin_ip, 4);

        sysread(INFILE, $last_index_of_begin_ip, 4);

 

        $first_index_of_begin_ip    = unpack("L",$first_index_of_begin_ip);

        $last_index_of_begin_ip     = unpack("L",$last_index_of_begin_ip);

        my $total_index_of_begin_ip = ($last_index_of_begin_ip - $first_index_of_begin_ip)/7 + 1;

 

    #binary search the begin ip

    my $begin_index, $end_index = $total_index_of_begin_ip;

    my $middle_index, $middle_ip, $middle_ip_num;

 

#    while(1){

#        if($begin_index >= $end_index-1){

#            last;

#        }

#        $middle_index = int(($end_index + $begin_index)/2);

#        seek(INFILE, $first_index_of_begin_ip + $middle_index*7, 0);

#        read(INFILE, $middle_ip, 4);

#        $middle_ip_num = unpack("L", $middle_ip);

#        if($ip_num

#            $end_index = $middle_index ;

#        } else {

#            $begin_index = $middle_index ;

#        }

#    }

 

    while($begin_index

 

        $middle_index = int (($end_index + $begin_index)/2);

        seek(INFILE, $first_index_of_begin_ip + 7*$middle_index, 0);

        read(INFILE, $middle_ip, 4);

        $middle_ip_num = unpack("L", $middle_ip);

 

        if($ip_num == $middle_ip_num){

            $begin_index = $middle_index;

            last;

        } elsif ($ip_num

            $end_index = $middle_index;

        } else {

            $begin_index = $middle_index;

        }

    }

 

    my $end_ip_index_offset, $end_ip, $end_ip_num, $end_ip_offset;

    $end_ip_index_offset = $first_index_of_begin_ip + 7*($begin_index) + 4;

    seek(INFILE, $end_ip_index_offset, 0);

    read(INFILE, $end_ip_offset, 3);

   

    $end_ip_offset = unpack("L", $end_ip_offset."\0");

    seek(INFILE, $end_ip_offset, 0);

    read(INFILE, $end_ip, 4);

    $end_ip_num = unpack("L", $end_ip);

 

    if($ip_num

        my $offset, $position_mode, $geo_country_mode_2_pos=0;

 

        $/="\0";

        read(INFILE,$position_mode,1);

 

        #position mode 1  

        if ($position_mode eq "\1") {

            read(INFILE,$offset,3);

            $offset = unpack("L",$offset."\0");

            seek(INFILE,$offset,0);

            read(INFILE,$position_mode,1);

        }

        #position mode 2

        if ($position_mode eq "\2") {

            read(INFILE,$offset,3);

            $geo_country_mode_2_pos = tell(INFILE);

            $offset = unpack("L",$offset."\0");

            seek(INFILE,$offset,0);

        } else {

            seek(INFILE,-1,1);

        }

        $ip_geo_country=;

 

        if($geo_country_mode_2_pos != 0){

            seek(INFILE, $geo_country_mode_2_pos, 0);

        }

 

        #geo local, geo local only position mode 2

        read(INFILE,$position_mode,1);

        if($position_mode eq "\2") {

            read(INFILE,$offset,3);

            $offset = unpack("L",$offset."\0");

            seek(INFILE,$offset,0);

        } else {

            seek(INFILE,-1,1);

        }

        $ip_geo_local=;

    } else{

        $ip_geo_country = "未知数据";

    }

 

        chomp($ip_geo_country, $ip_geo_local);

        $/="\n";

        close(INFILE);

 

        my $ip_geo_addr="$ip_geo_country $ip_geo_local";

        $ip_geo_addr =~ s/CZ88\.NET//isg;

#    $ip_geo_addr=decode("gbk",$ip_geo_addr);

 

        return $ip_geo_addr;

}

 

1;

 

更新IP库脚本：

# Task file of update IP database in [crontab]

TMP_PATH="/tmp/ipdata"

Geo_PATH="/usr/local/share/GeoIP/"

Qip_PATH="/usr/local/share/ip_geo/"

 

#Last file stat

stat /usr/local/share/GeoIP/GeoLiteCity.dat     >>      /var/log/ipdata.log

stat /usr/local/share/GeoIP/GeoIP.dat           >>      /var/log/ipdata.log

stat /usr/local/share/ip_geo/QQWry.Dat          >>      /var/log/ipdata.log

 

 

mkdir $TMP_PATH

cd $TMP_PATH

#

/usr/bin/wget http://geolite.maxmind.com/download/geoip/api/c/GeoIP.tar.gz

/usr/bin/wget http://www.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz

/usr/bin/wget http://update.cz88.net/soft/qqwry.rar

#--> end of download

/bin/tar zxf GeoIP.tar.gz

/usr/bin/gzip -df GeoLiteCity.dat.gz

/usr/bin/unrar e qqwry.rar

mv -f GeoIP-*/data/GeoIP.dat $Geo_PATH

mv -f Geo*.dat $Geo_PATH

mv -f QQWry.Dat $Qip_PATH

cd ..

rm -rf $TMP_PATH

#New file stat

stat /usr/local/share/GeoIP/GeoLiteCity.dat     >>      /var/log/ipdata.log

stat /usr/local/share/GeoIP/GeoIP.dat           >>      /var/log/ipdata.log

stat /usr/local/share/ip_geo/QQWry.Dat          >>      /var/log/ipdata.log

 

 

 

 

 

改进：

Apache Server使用如下脚本处理日志

策略： crontab每天执行此脚本，自动备份前一天日志，并删除一周前的未压缩日志，30天前的压缩日志

#!/bin/sh

HOST=web1

LOG_PATH="/home/httpdlog/`date +%Y`/`date --date='1 days ago' +%m`/"

BAK_PATH="/opt/backup/weblog/`date +%Y`/`date --date='1 days ago' +%m`/"

YESTERDAY=`date --date='1 days ago' +%Y%m%d`

BAK_FILE=$HOST-$YESTERDAY.tgz

LOG_FILE=$HOST.access.$YESTERDAY.log

 

if [ ! -d $BAK_PATH ]; then

mkdir -p $BAK_PATH; fi

 

tar zcfP $BAK_PATH/$BAK_FILE $LOG_PATH/$LOG_FILE

find /opt/backup/ -mtime +30 -name "*.log" -exec rm -rf {} \;

find /opt/backup/ -depth  -empty -type d -exec rmdir -p {} \;

 

find /home/httpdlog/ -mtime +7 -name "*.log" -exec rm -rf {} \;

find /home/httpdlog/ -depth -empty -type d -exec rmdir -p {} \;

 

echo "done!"

 

Awstats Server：

cat get_weblog_tgz.sh

#!/bin/sh

export WEB1=172.16.0.115;

export WEB2=172.16.0.116;

export WEB3=172.16.0.117;

export USER=httplog;

export PASS="bdGhS3EI4dSH86QBXjV9";

export CPATH="/opt/backup/weblog/`date +%Y`/`date --date='1 days ago' +%m`/"

export web1="web1-`date --date='1 days ago' +%Y%m%d`.tgz"

export web2="web2-`date --date='1 days ago' +%Y%m%d`.tgz"

export web3="web3-`date --date='1 days ago' +%Y%m%d`.tgz"

export SPATH="/opt/weblogs/`date +%Y`/`date --date='1 days ago' +%m`/"

echo "sftp start";

 

if [ ! -d $SPATH ]; then

mkdir -p $SPATH; fi

 

lftp -u ${USER},${PASS} sftp://${WEB1}

lcd $SPATH

get $CPATH/$web1

open sftp://${USER}:${PASS}@172.16.0.116

get $CPATH/$web2

open sftp://${USER}:${PASS}@172.16.0.117

get $CPATH/$web3

bye

EOF

echo "GET WEB LOG DONE!!!";