[Sommaire] [Imprimer]

   1  <?php
   2  /*
   3  * Filename.......: class_http.php
   4  * Author.........: Troy Wolf [troy@troywolf.com]
   5  * Last Modified..: Date: 2005/06/21 10:30:00
   6  * Description....: Screen-scraping class with caching. Includes image_cache.php
   7                     companion script. Includes static methods to extract data
   8                     out of HTML tables into arrays or XML.
   9  */
  10  
  11  class http {
  12    var $log;
  13    var $dir;
  14    var $name;
  15    var $filename;
  16    var $url;
  17    var $port;
  18    var $status;
  19    var $header;
  20    var $body;
  21    var $ttl;
  22    var $headers;
  23    var $postvars;
  24    var $connect_timeout;
  25    var $data_ts;
  26     
  27    /*
  28    The class constructor. Configure defaults.
  29    */ 
  30    function http() {
  31      $this->log = "New http() object instantiated.<br />\n";
  32      $this->headers = array();
  33      $this->postvars = array();
  34      $this->ttl = 0;
  35      
  36      /*
  37      Seconds to attempt socket connection before giving up.
  38      */
  39      $this->connect_timeout = 30; 
  40  
  41      /*
  42      Set the 'dir' property to the directory where you want to store the cached
  43      content. End this value with a "/".
  44      */
  45      $this->dir = realpath("./")."/"; //Default to current dir.
  46      
  47      /*
  48      Try to use user agent of the user making this request. If not available,
  49      default to IE6.0 on WinXP, SP1.
  50      */
  51      if (isset($_SERVER['HTTP_USER_AGENT'])) {
  52        $this->headers["User-Agent"] = $_SERVER['HTTP_USER_AGENT'];
  53      } else {
  54        $this->headers["User-Agent"] = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)";
  55      }
  56  
  57      /*
  58      Set referrer to the current script since in essence, it is the referring
  59      page.
  60      */
  61      if (substr($_SERVER['SERVER_PROTOCOL'],0,5) == "HTTPS") {
  62        $this->headers["Referer"] = "https://".$_SERVER['HTTP_HOST'].$_SERVER['REQUEST_URI'];
  63      } else {
  64        $this->headers["Referer"] = "http://".$_SERVER['HTTP_HOST'].$_SERVER['REQUEST_URI'];
  65      }
  66      return true;
  67    }
  68    
  69    /*
  70    fetch() method to get the content. fetch() will use 'ttl' property to
  71    determine whether to get the content from the url or the cache.
  72    */
  73    function fetch($url="", $ttl=0, $name="", $user="", $pwd="") {
  74      $this->log .= "--------------------------------<br />fetch() called<br />\n";
  75      $this->log .= "url: ".$url."<br />\n";
  76          $this->status = "";
  77      $this->header = "";
  78          $this->body = "";
  79          if (!$url) {
  80        $this->log .= "OOPS: You need to pass a URL!<br />";
  81        return false;
  82      }
  83      $this->url = $url;
  84      $this->ttl = $ttl;
  85      $this->name = $name;
  86      $need_to_save = false;
  87      if ($this->ttl == "0") {
  88        if (!$fh = $this->getFromUrl($url, $user, $pwd)) {
  89          return false;
  90        }
  91      } else {
  92        if (strlen(trim($this->name)) == 0) { $this->name = MD5($url); }
  93        $this->filename = $this->dir."http_".$this->name;
  94        $this->log .= "Filename: ".$this->filename."<br />";
  95        $this->getFile_ts();
  96        if ($this->ttl == "daily") {
  97          if (date('Y-m-d',$this->data_ts) != date('Y-m-d',time())) {
  98            $this->log .= "cache has expired<br />";
  99            if (!$fh = $this->getFromUrl($url, $user, $pwd)) {
 100              return false;
 101            }
 102            $need_to_save = true;
 103            if ($this->getFromUrl()) { return $this->saveToCache(); }
 104          } else {
 105            if (!$fh = $this->getFromCache()) {
 106              return false;
 107            }
 108          }
 109        } else {
 110          if ((time() - $this->data_ts) >= $this->ttl) {
 111            $this->log .= "cache has expired<br />";
 112            if (!$fh = $this->getFromUrl($url, $user, $pwd)) {
 113              return false;
 114            }
 115            $need_to_save = true;
 116          } else {
 117            if (!$fh = $this->getFromCache()) {
 118              return false;
 119            }
 120          }
 121        }
 122      }
 123  
 124      /*
 125      Get response header.
 126      */
 127      $this->header = fgets($fh, 1024);
 128          $this->status = substr($this->header,9,3);
 129          while ((trim($line = fgets($fh, 1024)) != "") && (!feof($fh))) {
 130              $this->header .= $line;
 131              if ($this->status=="401" and strpos($line,"WWW-Authenticate: Basic realm=\"")===0) {
 132                  fclose($fh);
 133                  $this->log .= "Could not authenticate<br />\n";
 134          return FALSE;
 135              }
 136          }
 137          
 138      /*
 139      Get response body.
 140      */
 141          while (!feof($fh)) {
 142        $this->body .= fgets($fh, 1024);
 143          }
 144        fclose($fh);
 145          if ($need_to_save) { $this->saveToCache(); }
 146      return $this->status;
 147    }
 148   
 149    /*
 150    PRIVATE getFromUrl() method to scrape content from url.
 151    */
 152    function getFromUrl($url, $user="", $pwd="") {
 153        global $adb;
 154      $this->log .= "getFromUrl() called<br />";
 155      preg_match("~([a-z]*://)?([^:^/]*)(:([0-9]{1,5}))?(/.*)?~i", $url, $parts);
 156        $protocol = $parts[1];
 157          $server = $parts[2];
 158          $port = $parts[4];
 159          $path = $parts[5];
 160      if ($port == "") {
 161        if (strtolower($protocol) == "https://") {
 162          $port = "443";
 163        } else {
 164          $port = "80";
 165        }
 166      }
 167      $query = "select * from vtiger_systems where server_type='proxy'";
 168      $result = $adb->query($query);
 169      $noofrows = $adb->num_rows($result);
 170      if($noofrows != 0)
 171      {
 172          $server = $adb->query_result($result,0,"server");
 173          $port = $adb->query_result($result,0,"server_port");
 174          $user = $adb->query_result($result,0,"server_username"); // added
 175          $pwd = $adb->query_result($result,0,"server_password");    // added
 176      }
 177      
 178        if ($path == "") { $path = "/"; }
 179  
 180          if (!$sock = @fsockopen($server, $port, $errno, $errstr, $this->connect_timeout)) {
 181        $this->log .= "Could not open connection. Error "
 182          .$errno.": ".$errstr."<br />\n";
 183      echo '<b><font color="red">Kindly set proxy server settings under Settings tab</font></b>';
 184      die;
 185        return false;
 186      }
 187    if (!$sock)    {return false;}
 188     fputs($sock, "GET $url HTTP/1.0\r\nHost: $server\r\n");
 189     fputs($sock, "Proxy-Authorization: Basic " . base64_encode ("$user:$pwd") . "\r\n\r\n");   
 190  
 191  
 192          $this->headers["Host"] = $server;
 193          //echo 'server is >>>>>>>>>' .$server;    
 194          if ($user != "" && $pwd != "") {
 195            $this->log .= "Authentication will be attempted<br />\n";
 196              $this->headers["Authorization"] = "Basic ".base64_encode($user.":".$pwd);
 197          }
 198          
 199          if (count($this->postvars) > 0) {
 200              $this->log .= "Variables will be POSTed<br />\n";
 201        $request = "POST ".$path." HTTP/1.0\r\n";
 202              $post_string = "";
 203        foreach ($this->postvars as $key=>$value) {
 204                  $post_string .= "&".$key."=".$value;
 205              }
 206              $post_string = substr($post_string,1);
 207              $this->headers["Content-Length"] = strlen($post_string);
 208          } else {
 209              $request = "GET ".$path." HTTP/1.0\r\n";
 210          }
 211          
 212      if (fwrite($sock, $request) === FALSE) {
 213        fclose($sock);
 214               $this->log .= "Error writing request type to socket<br />\n";
 215        return false;
 216      }
 217          
 218        foreach ($this->headers as $key=>$value) {
 219          if (fwrite($sock, $key.": ".$value."\r\n") === FALSE) {
 220          fclose($sock);
 221          $this->log .= "Error writing headers to socket<br />\n";
 222          return false;
 223        }
 224          }
 225  
 226          if (fwrite($sock, "\r\n") === FALSE) {
 227        fclose($sock);
 228        $this->log .= "Error writing end-of-line to socket<br />\n";
 229        return false;
 230      }
 231      
 232      if (count($this->postvars) > 0) {
 233              if (fwrite($sock, $post_string."\r\n") === FALSE) {
 234          fclose($sock);
 235          $this->log .= "Error writing POST string to socket<br />\n";
 236          return false;
 237        }
 238          }
 239          return $sock;
 240    }
 241    
 242    /*
 243    PRIVATE getFromCache() method to retrieve content from cache file.
 244    */
 245    function getFromCache() {
 246      $this->log .= "getFromCache() called<br />";
 247      //create file pointer
 248      if (!$fp=@fopen($this->filename,"r")) {
 249        $this->log .= "Could not open ".$this->filename."<br />";
 250        return false;
 251      }
 252      return $fp;
 253    }
 254  
 255    /*
 256    PRIVATE saveToCache() method to save content to cache file.
 257    */
 258    function saveToCache() {
 259      $this->log .= "saveToCache() called<br />";
 260  
 261      //create file pointer
 262      if (!$fp=@fopen($this->filename,"w")) {
 263        $this->log .= "Could not open ".$this->filename."<br />";
 264        return false;
 265      }
 266      //write to file
 267      if (!@fwrite($fp,$this->header."\r\n".$this->body)) {
 268        $this->log .= "Could not write to ".$this->filename."<br />";
 269        fclose($fp);
 270        return false;
 271      }
 272      //close file pointer
 273      fclose($fp);
 274      return true;
 275    }
 276  
 277    /*
 278    PRIVATE getFile_ts() method to get cache file modified date.
 279    */
 280    function getFile_ts() {
 281      $this->log .= "getFile_ts() called<br />";
 282      if (!file_exists($this->filename)) {
 283        $this->data_ts = 0;
 284        $this->log .= $this->filename." does not exist<br />";
 285        return false;
 286      }
 287      $this->data_ts = filemtime($this->filename);
 288      return true;
 289    }
 290  
 291    /*
 292    Static method table_into_array()
 293    Generic function to return data array from HTML table data
 294    rawHTML: the page source
 295    needle: optional string to start parsing source from
 296    needle_within: 0 = needle is BEFORE table, 1 = needle is within table
 297    allowed_tags: list of tags to NOT strip from data, e.g. "<a><b>"
 298    */
 299    function table_into_array($rawHTML,$needle="",$needle_within=0,$allowed_tags="") {
 300        $upperHTML = strtoupper($rawHTML);
 301        $idx = 0;
 302        if (strlen($needle) > 0) {
 303            $needle = strtoupper($needle);
 304            $idx = strpos($upperHTML,$needle);
 305            if ($idx === false) { return false; }
 306            if ($needle_within == 1) {
 307                $cnt = 0;
 308                while(($cnt < 100) && (substr($upperHTML,$idx,6) != "<TABLE")) {
 309                    $idx = strrpos(substr($upperHTML,0,$idx-1),"<");
 310                    $cnt++;
 311                }
 312            }
 313        }
 314        $aryData = array();
 315        $rowIdx = 0;
 316        /*    If this table has a header row, it may use TD or TH, so 
 317            check special for this first row. */
 318        $tmp = strpos($upperHTML,"<TR",$idx);
 319        if ($tmp === false) { return false; }
 320        $tmp2 = strpos($upperHTML,"</TR>",$tmp);
 321        if ($tmp2 === false) { return false; }
 322        $row = substr($rawHTML,$tmp,$tmp2-$tmp);
 323        $pattern = "/<TH>|<TH\ |<TD>|<TD\ /";
 324        preg_match($pattern,strtoupper($row),$matches);
 325        $hdrTag = $matches[0];
 326        
 327        while ($tmp = strpos(strtoupper($row),$hdrTag) !== false) {
 328            $tmp = strpos(strtoupper($row),">",$tmp);
 329            if ($tmp === false) { return false; }
 330            $tmp++;
 331            $tmp2 = strpos(strtoupper($row),"</T");
 332            $aryData[$rowIdx][] = trim(strip_tags(substr($row,$tmp,$tmp2-$tmp),$allowed_tags));
 333            $row = substr($row,$tmp2+5);
 334            preg_match($pattern,strtoupper($row),$matches);
 335            $hdrTag = $matches[0];
 336        }
 337        $idx = strpos($upperHTML,"</TR>",$idx)+5;
 338        $rowIdx++;
 339    
 340        /* Now parse the rest of the rows. */
 341        $tmp = strpos($upperHTML,"<TR",$idx);
 342        if ($tmp === false) { return false; }
 343        $tmp2 = strpos($upperHTML,"</TABLE>",$idx);
 344        if ($tmp2 === false) { return false; }
 345        $table = substr($rawHTML,$tmp,$tmp2-$tmp);
 346    
 347        while ($tmp = strpos(strtoupper($table),"<TR") !== false) {
 348            $tmp2 = strpos(strtoupper($table),"</TR");
 349            if ($tmp2 === false) { return false; }
 350            $row = substr($table,$tmp,$tmp2-$tmp);
 351    
 352            while ($tmp = strpos(strtoupper($row),"<TD") !== false) {
 353                $tmp = strpos(strtoupper($row),">",$tmp);
 354                if ($tmp === false) { return false; }
 355                $tmp++;
 356                $tmp2 = strpos(strtoupper($row),"</TD");
 357                $aryData[$rowIdx][] = trim(strip_tags(substr($row,$tmp,$tmp2-$tmp),$allowed_tags));
 358                $row = substr($row,$tmp2+5);
 359            }
 360            $table = substr($table,strpos(strtoupper($table),"</TR>")+5);
 361            $rowIdx++;
 362        }
 363        return $aryData;
 364    }
 365  
 366    /*
 367    Static method table_into_xml()
 368    Generic function to return xml dataset from HTML table data
 369    rawHTML: the page source
 370    needle: optional string to start parsing source from
 371    allowedTags: list of tags to NOT strip from data, e.g. "<a><b>"
 372    */
 373    function table_into_xml($rawHTML,$needle="",$needle_within=0,$allowedTags="") {
 374        if (!$aryTable = http::table_into_array($rawHTML,$needle,$needle_within,$allowedTags)) { return false; }
 375        $xml = "<?xml version=\"1.0\" standalone=\"yes\" ?>\n";
 376        $xml .= "<TABLE>\n";
 377        $rowIdx = 0;
 378        foreach ($aryTable as $row) {
 379            $xml .= "\t<ROW id=\"".$rowIdx."\">\n";
 380            $colIdx = 0;
 381            foreach ($row as $col) {
 382                $xml .= "\t\t<COL id=\"".$colIdx."\">".trim(utf8_encode(htmlspecialchars($col)))."</COL>\n";
 383                $colIdx++;
 384            }
 385            $xml .= "\t</ROW>\n";
 386            $rowIdx++;
 387        }
 388        $xml .= "</TABLE>";
 389        return $xml;
 390    }
 391  }
 392  
 393  ?>