Gemeinschaft Lernen Tools-Bibliothek Freizeit
suchen
Deutsch
Heim > Backend-Entwicklung > PHP-Tutorial > Hauptteil

php读取大文件示例分享(文件操作类)_php实例

WBOY
Freigeben： 2016-06-07 17:20:38
Original
898 Leute haben es durchsucht

Lib_File2.php

复制代码 代码如下:

 class Lib_File2
 {
  //文件目录
  private $root = '/data/wwwroot/kkpromo/data/';

  //文件后缀
  private $suffix = '.log';

  //文件句柄
  private $handle=null;

  //一次读取文件的最大记录数
  private $limit=40000;

  //每行读取的字节长度
  private $length=1024;

  //开始时间
  private  $startTime=0;

  //内存使用基准点
  private static $startMemory=0;

  //
  private $conn=null;

  //
  private static  $init=null;

  public static function instance()
  {
   self::$startMemory = memory_get_usage(true);

   if(self::$init && is_object(self::$init))
   {
    return self::$init;
   }

   self::$init = new self();

   return self::$init;
  }

  private function __construct(){}

  public  function setRoot($root)  
  {
   if(!is_dir($root)) die($root.' ROOT DOES NOT EXIST');
   $this->root = $root;
  }

  public function setSuffix($suffix)
  {
   $this->suffix = $suffix;
  }

  public function setLimit($limit)
  {
   if(!is_numeric($limit)) die($limit.' SHOULD BE NUMBERIC');
   if(intval($limit) > 1000000) die($limit.' SHOULD BE LOWER THAN 1000000');
   $this->limit = intval($limit);
  }

  public function _getFile($date , $appid , $op)
  {
   $filename = rtrim($this->root , '/').DIRECTORY_SEPARATOR.$date.DIRECTORY_SEPARATOR.$appid.'.'.$op.$this->suffix;
   if(!file_exists($filename))
   {
    die($filename.' FILE DOES NOT EXISTS!');
   }

   if(!is_file($filename))
   {
    die($filename.' FILE DOES NOT EXIST!');
   }

   if(!is_readable($filename))
   {
    die($filename.'  FILE ACCESS DENY!');
   }

   return $filename;
  }

  
  public function closeFile($date=null , $appid=null , $op=null)
  {
   if($op && $date && $appid)
   {
    if(is_object($this->handle[$date.'_'.$appid.'_'.$op]) || $this->conn[$date.'_'.$appid.'_'.$op])
    {
     unset($this->handle[$date.'_'.$appid.'_'.$op]);
     $this->handle[$date.'_'.$appid.'_'.$op]=null;
    }

    $this->conn[$date.'_'.$appid.'_'.$op]=null;
    $this->handle[$date.'_'.$appid.'_'.$op]=null;
    unset($this->handle[$date.'_'.$appid.'_'.$op]);
   }
   else {
    if(is_array($this->handle) && $this->handle)
    {
     foreach ($this->handle as $key=>$val){
      unset($this->handle[$key]);
      $this->conn[$key]=null;
      $this->handle[$key]=null;
     }
    }
   } 

   return true;
  }

  
  private function _openFile($date , $appid , $op)
  {   
   $this->startTime = microtime(true);
   if(isset($this->conn[$date.'_'.$appid.'_'.$op])  && $this->conn[$date.'_'.$appid.'_'.$op])
   {
    return $this->handle[$date.'_'.$appid.'_'.$op];
   }

   $filename = self::_getFile($date , $appid , $op);
   if(($this->handle[$date.'_'.$appid.'_'.$op] = new SplFileObject($filename , 'r'))!=null)
   {
    $this->conn[$date.'_'.$appid.'_'.$op] = true;
    return $this->handle[$date.'_'.$appid.'_'.$op];
   }
   else {
    die('FILE OPEN FAILED!');
   }
  }

  
  /**
   * 功能：解析数据
   * 格式：  array('timestamp','mid','data');
   * @param string $data
   * @return boolean|array
   */
  private  function _parseData($data , $jsonFlag=true)
  {
   if(empty($data) || !is_string($data)) return false;
   $result = array(
     'timestamp'=>0,
     'mid'=>0,
     'data'=>array(),
   );

   $data = explode('|', $data);
   if(count($data)    $result['timestamp'] = $data[0];
   $result['mid'] = $data[1];
   if($jsonFlag)
   {
    $result['data'] = @json_decode($data[2] , true);
    unset($result['mid']);
   }
   if(empty($result['timestamp']) || empty($result['mid'])) return false;

   unset($data);
   return $result;
  }

  
  /**
   * TODO:读取单一文件
   * @param string $date： 如（20140327）
   * @param int  $appid: 如（1000,9000）
   * @param string $op：如(show，login , index)
   * @param number $startNum 默认从第一行开始
   * @param number $length 默认到$this->limit 读取的行数
   * @param array $condition:array('mid'=>arrray() , 'ip'=>array() , ...)  过滤条件
   * @param bool $jsonFlag：默认为true， 则保留jsondata字段；设为false，则去掉false字段
   * @return array(count , diffTime , memory , data)
   */
  public  function readFile($date , $appid , $op , $startNum=0 , $length=0 , $jsonFlag=false ,  $condition=array())
  {
   $data['data'] = "";
   $data['count'] = 0;
   $index = $startNum;
   $startNum = empty($startNum) ? 0 : $startNum;
   $length = empty($length) ? $this->limit : $length;

   $handle = self::_openFile($date , $appid , $op );
   $line_number=0;

   if($handle)
   { 
    $handle->seek($startNum);
    $handle->setMaxLineLen($this->length);
    while (intval($line_number) - intval($startNum)     {
     $tmp = $handle->current();
     if(empty($tmp)) continue;     
     $tmp = self::_parseData($tmp , $jsonFlag);
     $line_number = $handle->key();
     !$jsonFlag && $condition= array();        
     if(isset($condition) && $condition)
     {
      $key = array_keys($condition);
      if(in_array($tmp['data'][$key[0]], $condition[$key[0]]))
      {
       $data['count']++;
       $data['data'][$line_number] = $tmp;
      }
     }
     else
     {
      $data['data'][$line_number] = $tmp;
      $data['count']++;
     }

     if(intval($line_number) - intval($startNum) >= intval($length)-1) break;
     unset($tmp);
     $handle->next();
    }
    unset($tmp , $length , $line_number , $condition);
   }

   $data['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
   $data['memory'] = doubleval((doubleval(memory_get_usage(true)) -  doubleval(self::$startMemory))/1024/1024) . ' M';

   return $data;
  }
  

  /**
   * TODO:命令行下获取文件总记录数*
   * @param string $date
   * @param int $appid
   * @param string $op
   * @return array
   */
  public  function total_lineFile($date, $appid, $op)
  {
   $this->_openFile($date, $appid, $op);
   $file = escapeshellarg($this->_getFile($date, $appid, $op)); // 对命令行参数进行安全转义
   $line = `wc -l  $file`;
   if(preg_match("/(\d{1,})/", $line , $ret)){
    $data['count']=$ret[1];
   }else{
    $data['count']=0;
   }
   $data['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
   $data['memory'] = doubleval((doubleval(memory_get_usage(true)) -  doubleval(self::$startMemory))/1024/1024) . ' M';
   return $data;
  }

  
  /**
   * TODO:统计{$data}.{$op}.log记录数
   * @param string $date
   * @param int $appid
   * @param string $op
   * @param array $condition
   * @return array
   */
  public function countFile($date , $appid , $op ,$condition=array())
  {
   $data['count'] = 0;
   $handle = self::_openFile($date , $appid , $op );  
   if($handle)
   {
    $handle->setMaxLineLen($this->length);
    while (!$handle->eof())
    {
     $tmp = $handle->current();
     if(empty($tmp)) continue;
     $tmp = self::_parseData($tmp);
     if($condition && is_array($condition) )
     {      
      $key = array_keys($condition);
      if(isset($tmp['data'][$key[0]]) && $tmp['data'][$key[0]] && in_array($tmp['data'][$key[0]], $condition[$key[0]]))
      {
       $data['count']++;
      }
     }
     else
     {
      $data['count']++;      
     }

     unset($tmp);     
     $handle->next();
    }
   }
   unset($handle , $condition , $tmp , $key , $val);
   self::closeFile($date , $appid , $op );

   $data['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
   $data['memory'] = doubleval((doubleval(memory_get_usage(true)) -  doubleval(self::$startMemory))/1024/1024) . ' M';

   return $data;
  }  

  

  /**
   * TODO:统计用户数
   * @param string $date
   * @param int $appid
   * @param string $op
   * @param bool $midflag ：默认为false 则 mid返回空数组；如设为true，则mid数组不为空
   * * @param bool $jsonFlag：默认为true， 则保留jsondata字段；设为false，则去掉jsondata字段
   * @param array $condition
   * @return : array：形如（{"mid":[],"count":2181,"diffTime":0.0397667884827,"memory":"3.75 M"}）
   */
  public function countFileMID($date , $appid , $op  ,  $midflag=false , $jsonFlag=false,  $condition=array())
  {
   //$count = self::total_lineFile($date , $appid , $op );
   $count = self::countFile($date , $appid , $op );
   $index = ceil($count['count'] / $this->limit);
   $result = array('mid'=>array() , 'count'=>0 , 'diffTime'=>0 , 'memory'=>0);

   for ($i=0 ; $i   {
    $startNum = $this->limit*$i;
    $endNum = $this->limit;
    $data = self::readFile($date , $appid , $op  ,  $startNum , $endNum , $jsonFlag);
    var_dump($data);exit();

    if($data['data'] && is_array($data['data']))
    {
     foreach ($data['data'] as $arr)
     {
      if($condition && is_array($condition))
      {
       $key = array_keys($condition);
       if(isset($arr['data'][$key[0]])  && (in_array($arr['data'][$key[0]] , $condition[$key]) || empty($condition[$key[0]])))
       {
        $result['mid'][$arr["mid"]] =1;
        $result['count']++;
       }
      }
      else
      {
        $result['mid'][$arr["mid"]] =1; 
        $result['count']++;
      }
      unset($data);
     }
    }
   }
   unset($index , $count , $condition , $data  , $arr);
   self::closeFile($date , $appid , $op);

   $result['mid'] = array_keys($result['mid']);  
   if(empty($midflag)) unset($result['mid']); 

   $result['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
   $result['memory'] = (memory_get_usage(true) - self::$startMemory)/1024/1024 . ' M';
   return $result;
  }

  
  /**
   * TODO:跨时间段 统计参加$op用户数据
   * @param string $date
   * @param int $appid
   * @param string $op
   * @param number $day
   * @param bool  $midflag ：默认为false 则 mid返回空数组；如设为true，则mid数组不为空
   * @return array 形如（"20140326":{"mid":[],"count":4571,"diffTime":0.0806441307068,"memory":"3.75 M"},
  *          "20140325":{"mid":[],"count":2181,"diffTime":0.0397667884827,"memory":"3.75 M"}）
   */
  public function getReturnUser($date , $appid , $op , $day=1 , $midflag=false)
  {
   $date_i=0;
   for ($i =0; $i    $date_i = date('Ymd' , strtotime($date)-$i*86400);
    $result[$date_i] = self::countFileMID($date_i , $appid , $op , $midflag); 
   }
   unset($date , $date_i , $appid , $op  , $day);

   return $result;
  }
 }
?>

Lib_File1.php

复制代码 代码如下:

 class Lib_File1
 {
  //文件目录
  private $root = '/data/wwwroot/kkpromo/data/';

  //文件后缀
  private $suffix = '.log';

  //文件句柄
  private $hander=null;

  //一次读取文件的最大记录数
  private $limit=40000;

  //每行读取的字节长度
  private $length=1024;

  //开始时间
  private  $startTime=0;

  //内存使用基准点
  private static $startMemory=0;

  //
  private $conn=null;

  //
  private static  $init=null;

  public static function instance()
  {
   self::$startMemory = memory_get_usage(true);

   if(self::$init && is_object(self::$init))
   {
    return self::$init;
   }

   self::$init = new self();

   return self::$init;
  }

  private function __construct(){}

  public  function setRoot($root)  
  {
   if(!is_dir($root)) die($root.' ROOT DOES NOT EXIST');
   $this->root = $root;
  }

  public function setSuffix($suffix)
  {
   $this->suffix = $suffix;
  }

  public function setLimit($limit)
  {
   if(!is_numeric($limit)) die($limit.' SHOULD BE NUMBERIC');
   if(intval($limit) > 1000000) die($limit.' SHOULD BE LOWER THAN 1000000');
   $this->limit = intval($limit);
  }

  private function _getFile($date , $appid , $op)
  {
   $filename = rtrim($this->root , '/').DIRECTORY_SEPARATOR.$date.DIRECTORY_SEPARATOR.$appid.'.'.$op.$this->suffix;
   if(!file_exists($filename))
   {
    die($filename.' FILE DOES NOT EXISTS!');
   }

   if(!is_file($filename))
   {
    die($filename.' FILE DOES NOT EXIST!');
   }

   if(!is_readable($filename))
   {
    die($filename.'  FILE ACCESS DENY!');
   }

   return $filename;
  }

  
  public function closeFile($date=null , $appid=null , $op=null)
  {
   if($op && $date && $appid)
   {
    if(is_object($this->hander[$date.'_'.$appid.'_'.$op]) || $this->conn[$date.'_'.$appid.'_'.$op])
    {
     fclose($this->hander[$date.'_'.$appid.'_'.$op]);
    }

    $this->conn[$date.'_'.$appid.'_'.$op]=null;
    $this->hander[$date.'_'.$appid.'_'.$op]=null;
   }
   else {
    if(is_array($this->hander) && $this->hander)
    {
     foreach ($this->hander as $key=>$val){
      fclose($this->hander[$key]);
      $this->conn[$key]=null;
      $this->hander[$key]=null;
     }
    }
   } 

   return true;
  }

  
  private function _openFile($date , $appid , $op)
  {   
   $this->startTime = microtime(true);
   if(isset($this->conn[$date.'_'.$appid.'_'.$op])  && $this->conn[$date.'_'.$appid.'_'.$op])
   {
    return $this->hander[$date.'_'.$appid.'_'.$op];
   }

   $filename = self::_getFile($date , $appid , $op);
   if(($this->hander[$date.'_'.$appid.'_'.$op] = fopen($filename, 'r'))!=null)
   {
    $this->conn[$date.'_'.$appid.'_'.$op] = true;
    return $this->hander[$date.'_'.$appid.'_'.$op];
   }
   else {
    die('FILE OPEN FAILED!');
   }
  }

  
  /**
   * 功能：解析数据
   * 格式：  array('timestamp','mid','data');
   * @param string $data
   * @return boolean|array
   */
  private  function _parseData($data)
  {
   if(empty($data) || !is_string($data)) return false;
   $result = array(
     'timestamp'=>0,
     'mid'=>0,
     'data'=>array(),
   );

   $data = explode('|', $data);
   if(count($data)    $result['timestamp'] = $data[0];
   $result['mid'] = $data[1];
   $result['data'] = @json_decode($data[2] , true);
   if(empty($result['timestamp']) || empty($result['mid'])) return false;

   unset($data);
   return $result;
  }

  
  /**
   * TODO:读取单一文件
   * @param string $date： 如（20140327）
   * @param int  $appid: 如（1000,9000）
   * @param string $op：如(show，login , index)
   * @param number $startNum 默认从第一行开始
   * @param number $endNum 默认到$this->limit结束
   * @param array $condition:array('mid'=>arrray() , 'ip'=>array() , ...)  过滤条件
   * @param bool $jsonFlag：默认为true， 则保留jsondata字段；设为false，则去掉false字段
   * @return array(count , diffTime , memory , data)
   */
  public  function readFile($date , $appid , $op ,$startNum=0 , $endNum=0 , $jsonFlag=false ,  $condition=array())
  {
   $data['data'] = "";
   $data['count'] = 0;
   $index = $startNum;
   $startNum = empty($startNum) ? 0 : $startNum;
   $endNum = empty($endNum) ? $this->limit : $endNum;

   $hander = self::_openFile($date , $appid , $op );
   $tmpindex=0;

   if($hander)
   {
    //!feof($hander)
    while ($tmpindex     {
     $tmp = fgets($hander , $this->length);
     if(empty($tmp)) continue;
     if($tmpindex =$startNum)
     {
      $tmp = self::_parseData($tmp);    
      if(empty($tmp)) continue;
      //去掉jsondata
       if(!$jsonFlag) { unset($tmp[2]);  $condition= array();  }
      //条件过滤
      if($condition && is_array($condition) )
      {
       foreach ($condition as $key=>$val){
        if(in_array($tmp['data'][$key], $condition[$key]))
         unset($key , $val); 
         $data['count']++;
         $data['data'][$index] = $tmp;
         $index++;
        }
      }
      else{    
       $data['data'][$index] = $tmp;
       $index++;
       $data['count']++;
      }
     }
     if($tmpindex >= $endNum) break;
     $tmpindex++;
     unset($tmp);
    }
    fseek($hander ,  SEEK_END);
   }

   $data['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
   $data['memory'] = doubleval((doubleval(memory_get_usage(true)) -  doubleval(self::$startMemory))/1024/1024) . ' M';

   return $data;
  }
  

  /**
   * TODO:命令行下获取文件总记录数*
   * @param string $date
   * @param int $appid
   * @param string $op
   * @return array
   */
  public  function total_lineFile($date, $appid, $op)
  {
   $this->_openFile($date, $appid, $op);
   $file = escapeshellarg($this->_getFile($date, $appid, $op)); // 对命令行参数进行安全转义
   $line = `wc -l  $file`;
   if(preg_match("/(\d{1,})/", $line , $ret)){
    $data['count']=$ret[1];
   }else{
    $data['count']=0;
   }
   $data['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
   $data['memory'] = doubleval((doubleval(memory_get_usage(true)) -  doubleval(self::$startMemory))/1024/1024) . ' M';
   return $data;
  }

  
  /**
   * TODO:统计{$data}.{$op}.log记录数
   * @param string $date
   * @param int $appid
   * @param string $op
   * @param array $condition
   * @return array
   */
  public function countFile($date , $appid , $op ,$condition=array())
  {
   $data['count'] = 0;
   $hander = self::_openFile($date , $appid , $op );  
   if($hander)
   {
    while (!feof($hander))
    {
     $tmp = fgets($hander , $this->length);
     $tmp = self::_parseData($tmp);
     if(empty($tmp)) continue;
     if($condition && is_array($condition) )
     {
      foreach ($condition as $key=>$val){
       if(isset($tmp['data'][$key]) && $tmp['data'][$key] && in_array($tmp['data'][$key], $condition[$key])){
        unset($key , $val); 
        $data['count']++;
       } 
      }
     }
     else
      $data['count']++;      
     unset($tmp);
    }
    fseek($hander , SEEK_END);
   }
   $data['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
   $data['memory'] = doubleval((doubleval(memory_get_usage(true)) -  doubleval(self::$startMemory))/1024/1024) . ' M';

   return $data;
  }  

  

  /**
   * TODO:统计用户数
   * @param string $date
   * @param int $appid
   * @param string $op
   * @param bool $midflag ：默认为false 则 mid返回空数组；如设为true，则mid数组不为空
   * * @param bool $jsonFlag：默认为true， 则保留jsondata字段；设为false，则去掉jsondata字段
   * @param array $condition
   * @return : array：形如（{"mid":[],"count":2181,"diffTime":0.0397667884827,"memory":"3.75 M"}）
   */
  public function countFileMID($date , $appid , $op  ,  $midflag=false , $jsonFlag=false,  $condition=array())
  {
   $count = self::total_lineFile($date , $appid , $op );
   $index = ceil($count['count'] / $this->limit);
   $result = array('mid'=>array() , 'count'=>0 , 'diffTime'=>0 , 'memory'=>0);

   for ($i=0 ; $i   {
    $startNum = $this->limit*$i;
    $endNum = $this->limit*($i+1);
    $data = self::readFile($date , $appid , $op  ,  $startNum , $endNum , $jsonFlag);
    if($data['data'] && is_array($data['data']))
    {
     foreach ($data['data'] as $arr)
     {
      if($condition && is_array($condition)){
       foreach ($condition as $key=>$val){ 
        if(isset($arr['data'][$key])  && (in_array($arr['data'][$key] , $condition[$key]) || empty($condition[$key]))){
         if(!isset($result['mid'][$arr['mid']]))  { $result['mid'][$arr["mid"]] =1; $result['count']++; }
        }
       }
      }
      else {
       if(!isset($result['mid'][$arr['mid']]))  { $result['mid'][$arr["mid"]] =1; $result['count']++;   }
      }
     }
    }
    unset($data['data'] , $data);
   }
   unset($index , $count , $condition , $data  , $arr);
   self::closeFile($date , $appid , $op);

    $result['mid'] = array_keys($result['mid']);
    //$result['count'] = count($result['mid']);   
    if(empty($midflag)) unset($result['mid']); 

   $result['diffTime'] = doubleval(microtime(true)) - doubleval($this->startTime);
   $result['memory'] = (memory_get_usage(true) - self::$startMemory)/1024/1024 . ' M';
   return $result;
  }

  
  /**
   * TODO:跨时间段 统计参加$op用户数据
   * @param string $date
   * @param int $appid
   * @param string $op
   * @param number $day
   * @param bool  $midflag ：默认为false 则 mid返回空数组；如设为true，则mid数组不为空
   * @return array 形如（"20140326":{"mid":[],"count":4571,"diffTime":0.0806441307068,"memory":"3.75 M"},
  *          "20140325":{"mid":[],"count":2181,"diffTime":0.0397667884827,"memory":"3.75 M"}）
   */
  public function getReturnUser($date , $appid , $op , $day=1 , $midflag=false)
  {
   $date_i=0;
   for ($i =0; $i    $date_i = date('Ymd' , strtotime($date)-$i*86400);
    $result[$date_i] = self::countFileMID($date_i , $appid , $op , $midflag); 
   }
   unset($date , $date_i , $appid , $op  , $day);

   return $result;
  }
 }
?>

Verwandte Etiketten：
php读取大文件
Quelle：php.cn
Vorheriger Artikel：纯PHP生成的一个树叶图片画图例子_php实例 Nächster Artikel：PHP5中实现多态的两种方法实例分享_php实例
Erklärung dieser Website
Der Inhalt dieses Artikels wird freiwillig von Internetnutzern beigesteuert und das Urheberrecht liegt beim ursprünglichen Autor. Diese Website übernimmt keine entsprechende rechtliche Verantwortung. Wenn Sie Inhalte finden, bei denen der Verdacht eines Plagiats oder einer Rechtsverletzung besteht, wenden Sie sich bitte an admin@php.cn
Neueste Artikel des Autors
Aktuelle Ausgaben
Was sind die Best Practices für die Anzeige von Versionsinformationen in Webanwendungen? Ich entwickle eine Webanwendung. Was sind die Best Practices für die Anzeige von Versionsi...
Aus 2024-04-06 19:13:16
0
2
476
Herausforderungen bei der Verwendung von window.location.href zum Weiterleiten auf eine neue Seite Ich habe Probleme mit der Umleitung zu einer lokalen Domänen-PHP-Datei. Ich habe sie für d...
Aus 2024-04-04 12:57:06
0
1
446
Wie finde ich JSON-Daten basierend auf anderen Inhalten in einem JSON-Objekt, wenn Duplikate vorhanden sind? Ich bin gerade dabei, einige Datendiagramme auf einer Website zu implementieren, und ich h...
Aus 2024-04-04 10:57:45
0
1
413
PHP erkennt von Ajax gesendete Inhalte nicht Ich kann PHP nicht dazu bringen, über Ajax gesendeten Text zu lesen. index.html<head>...
Aus 2024-04-02 21:25:42
0
1
341
PHP AJAX wird nicht dynamisch aktualisiert, obwohl Consolelog den richtigen Wert hat Ich versuche, $_POST['category'] durch Klicken auf eine Schaltfläche zu aktualisieren. Ich...
Aus 2024-04-02 17:50:59
0
1
396
verwandte Themen
Mehr>
Beliebte Empfehlungen
Beliebte Tutorials
Mehr>
Verwandte Tutorials
Beliebte Empfehlungen
Aktuelle Kurse
Neueste Downloads
Mehr>
Web-Effekte
Quellcode der Website
Website-Materialien
Frontend-Vorlage