python 日志增量抓取实现方法

下面为大家分享一篇python 日志增量抓取实现方法，具有很好的参考价值，希望对大家有所帮助。一起过来看看吧

实例如下所示：

import time
import pickle
import os
import re
class LogIncScaner(object):
  def __init__(self,log_file, reg_ex,seek_file='/tmp/log-inc-scan.seek.temp'):
    self.log_file = log_file
    self.reg_ex = reg_ex
    self.seek_file = seek_file
  def scan(self):
    seek = self._get_seek()
    file_mtime = os.path.getmtime(self.log_file)
    if file_mtime <= seek[&#39;time&#39;]:
      print &#39;file mtime not change since last scan&#39;
      seek[&#39;time&#39;] = file_mtime
      self._dump_seek(seek)
      return []
    file_size = os.path.getsize(self.log_file)
    if file_size <= seek[&#39;position&#39;]:
      print &#39;file size not change since last scan&#39;
      seek[&#39;position&#39;] = file_size
      self._dump_seek(seek)
      return []
    print &#39;file changed,start to scan&#39;
    matchs = []
    with open(self.log_file, &#39;rb&#39;) as logfd:
      logfd.seek(seek[&#39;position&#39;],os.SEEK_SET)
      for match in re.finditer(self.reg_ex, logfd.read()):
        matchs.append(match)
      seek = {&#39;time&#39;:time.time(),&#39;position&#39;: logfd.tell()}
      print seek
      self._dump_seek(seek)
    return matchs
  def _get_seek(self):
    seek = {&#39;time&#39;:time.time(),&#39;position&#39;:0}
    if os.path.exists(self.seek_file):
      with open(self.seek_file,&#39;rb&#39;) as seekfd:
          try:
            seek = pickle.load(seekfd)
          except:
            pass
    print seek
    return seek
  def _dump_seek(self, seek):
    with open(self.seek_file,&#39;wb&#39;) as seekfd:
      pickle.dump(seek,seekfd)
  def reset_seek(self):
    self._dump_seek({&#39;time&#39;:time.time(),&#39;position&#39;:0})
if __name__ == "__main__":
  scaner = LogIncScaner(&#39;/var/log/messages&#39;,r&#39;(\w+ \d+ \d+:\d+:\d+) .+?exception&#39;)
  scaner.reset_seek()
  while True:
    matchs = scaner.scan()
    for match in matchs:
      print &#39;fond at:&#39; + match.group(1) + &#39; content:&#39; + match.group(0)
    time.sleep(5)

相关推荐：

浅谈python日志的配置文件路径问题

详解python日志打印和写入并发实现代码

Atas ialah kandungan terperinci python 日志增量抓取实现方法. Untuk maklumat lanjut, sila ikut artikel berkaitan lain di laman web China PHP!