[Sommaire] [Imprimer]

   1  <?php
   2  /**
   3   * Zend Framework
   4   *
   5   * LICENSE
   6   *
   7   * This source file is subject to version 1.0 of the Zend Framework
   8   * license, that is bundled with this package in the file LICENSE, and
   9   * is available through the world-wide-web at the following URL:
  10   * http://www.zend.com/license/framework/1_0.txt. If you did not receive
  11   * a copy of the Zend Framework license and are unable to obtain it
  12   * through the world-wide-web, please send a note to license@zend.com
  13   * so we can mail you a copy immediately.
  14   *
  15   * @package    Zend_Search_Lucene
  16   * @subpackage Analysis
  17   * @copyright  Copyright (c) 2005-2006 Zend Technologies USA Inc. (http://www.zend.com)
  18   * @license    http://www.zend.com/license/framework/1_0.txt Zend Framework License version 1.0
  19   */
  20  
  21  
  22  /** Zend_Search_Lucene_Analysis_Analyzer_Common */
  23  require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php';
  24  
  25  
  26  /**
  27   * @package    Zend_Search_Lucene
  28   * @subpackage Analysis
  29   * @copyright  Copyright (c) 2005-2006 Zend Technologies USA Inc. (http://www.zend.com)
  30   * @license    http://www.zend.com/license/framework/1_0.txt Zend Framework License version 1.0
  31   */
  32  
  33  class Zend_Search_Lucene_Analysis_Analyzer_Common_Text extends Zend_Search_Lucene_Analysis_Analyzer_Common
  34  {
  35      /**
  36       * Tokenize text to a terms
  37       * Returns array of Zend_Search_Lucene_Analysis_Token objects
  38       *
  39       * @param string $data
  40       * @return array
  41       */
  42      public function tokenize($data)
  43      {
  44          $tokenStream = array();
  45  
  46          $position = 0;
  47          while ($position < strlen($data)) {
  48              // skip white space
  49              while ($position < strlen($data) && !ctype_alpha( $data{$position} )) {
  50                  $position++;
  51              }
  52  
  53              $termStartPosition = $position;
  54  
  55              // read token
  56              while ($position < strlen($data) && ctype_alpha( $data{$position} )) {
  57                  $position++;
  58              }
  59  
  60              // Empty token, end of stream.
  61              if ($position == $termStartPosition) {
  62                  break;
  63              }
  64  
  65              $token = new Zend_Search_Lucene_Analysis_Token(substr($data,
  66                                               $termStartPosition,
  67                                               $position-$termStartPosition),
  68                                        $termStartPosition,
  69                                        $position);
  70              $tokenStream[] = $this->normalize($token);
  71          }
  72  
  73          return $tokenStream;
  74      }
  75  }
  76