git.roojs.org Git - web.mtrack/blob - Zend/Search/Lucene/Index/Writer.php

   1 <?php
   2 /**
   3  * Zend Framework
   4  *
   5  * LICENSE
   6  *
   7  * This source file is subject to the new BSD license that is bundled
   8  * with this package in the file LICENSE.txt.
   9  * It is also available through the world-wide-web at this URL:
  10  * http://framework.zend.com/license/new-bsd
  11  * If you did not receive a copy of the license and are unable to
  12  * obtain it through the world-wide-web, please send an email
  13  * to license@zend.com so we can send you a copy immediately.
  14  *
  15  * @category   Zend
  16  * @package    Zend_Search_Lucene
  17  * @subpackage Index
  18  * @copyright  Copyright (c) 2005-2009 Zend Technologies USA Inc. (http://www.zend.com)
  19  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  20  * @version    $Id: Writer.php 16541 2009-07-07 06:59:03Z bkarwin $
  21  */
  22
  23
  24 /** Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter */
  25 require_once 'Zend/Search/Lucene/Index/SegmentWriter/DocumentWriter.php';
  26
  27 /** Zend_Search_Lucene_Index_SegmentInfo */
  28 require_once 'Zend/Search/Lucene/Index/SegmentInfo.php';
  29
  30 /** Zend_Search_Lucene_Index_SegmentMerger */
  31 require_once 'Zend/Search/Lucene/Index/SegmentMerger.php';
  32
  33 /** Zend_Search_Lucene_LockManager */
  34 require_once 'Zend/Search/Lucene/LockManager.php';
  35
  36
  37
  38 /**
  39  * @category   Zend
  40  * @package    Zend_Search_Lucene
  41  * @subpackage Index
  42  * @copyright  Copyright (c) 2005-2009 Zend Technologies USA Inc. (http://www.zend.com)
  43  * @license    http://framework.zend.com/license/new-bsd     New BSD License
  44  */
  45 class Zend_Search_Lucene_Index_Writer
  46 {
  47     /**
  48      * @todo Implement Analyzer substitution
  49      * @todo Implement Zend_Search_Lucene_Storage_DirectoryRAM and Zend_Search_Lucene_Storage_FileRAM to use it for
  50      *       temporary index files
  51      * @todo Directory lock processing
  52      */
  53
  54     /**
  55      * Number of documents required before the buffered in-memory
  56      * documents are written into a new Segment
  57      *
  58      * Default value is 10
  59      *
  60      * @var integer
  61      */
  62     public $maxBufferedDocs = 10;
  63
  64     /**
  65      * Largest number of documents ever merged by addDocument().
  66      * Small values (e.g., less than 10,000) are best for interactive indexing,
  67      * as this limits the length of pauses while indexing to a few seconds.
  68      * Larger values are best for batched indexing and speedier searches.
  69      *
  70      * Default value is PHP_INT_MAX
  71      *
  72      * @var integer
  73      */
  74     public $maxMergeDocs = PHP_INT_MAX;
  75
  76     /**
  77      * Determines how often segment indices are merged by addDocument().
  78      *
  79      * With smaller values, less RAM is used while indexing,
  80      * and searches on unoptimized indices are faster,
  81      * but indexing speed is slower.
  82      *
  83      * With larger values, more RAM is used during indexing,
  84      * and while searches on unoptimized indices are slower,
  85      * indexing is faster.
  86      *
  87      * Thus larger values (> 10) are best for batch index creation,
  88      * and smaller values (< 10) for indices that are interactively maintained.
  89      *
  90      * Default value is 10
  91      *
  92      * @var integer
  93      */
  94     public $mergeFactor = 10;
  95
  96     /**
  97      * File system adapter.
  98      *
  99      * @var Zend_Search_Lucene_Storage_Directory
 100      */
 101     private $_directory = null;
 102
 103
 104     /**
 105      * Changes counter.
 106      *
 107      * @var integer
 108      */
 109     private $_versionUpdate = 0;
 110
 111     /**
 112      * List of the segments, created by index writer
 113      * Array of Zend_Search_Lucene_Index_SegmentInfo objects
 114      *
 115      * @var array
 116      */
 117     private $_newSegments = array();
 118
 119     /**
 120      * List of segments to be deleted on commit
 121      *
 122      * @var array
 123      */
 124     private $_segmentsToDelete = array();
 125
 126     /**
 127      * Current segment to add documents
 128      *
 129      * @var Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter
 130      */
 131     private $_currentSegment = null;
 132
 133     /**
 134      * Array of Zend_Search_Lucene_Index_SegmentInfo objects for this index.
 135      *
 136      * It's a reference to the corresponding Zend_Search_Lucene::$_segmentInfos array
 137      *
 138      * @var array Zend_Search_Lucene_Index_SegmentInfo
 139      */
 140     private $_segmentInfos;
 141
 142     /**
 143      * Index target format version
 144      *
 145      * @var integer
 146      */
 147     private $_targetFormatVersion;
 148
 149     /**
 150      * List of indexfiles extensions
 151      *
 152      * @var array
 153      */
 154     private static $_indexExtensions = array('.cfs' => '.cfs',
 155                                              '.cfx' => '.cfx',
 156                                              '.fnm' => '.fnm',
 157                                              '.fdx' => '.fdx',
 158                                              '.fdt' => '.fdt',
 159                                              '.tis' => '.tis',
 160                                              '.tii' => '.tii',
 161                                              '.frq' => '.frq',
 162                                              '.prx' => '.prx',
 163                                              '.tvx' => '.tvx',
 164                                              '.tvd' => '.tvd',
 165                                              '.tvf' => '.tvf',
 166                                              '.del' => '.del',
 167                                              '.sti' => '.sti' );
 168
 169
 170     /**
 171      * Create empty index
 172      *
 173      * @param Zend_Search_Lucene_Storage_Directory $directory
 174      * @param integer $generation
 175      * @param integer $nameCount
 176      */
 177     public static function createIndex(Zend_Search_Lucene_Storage_Directory $directory, $generation, $nameCount)
 178     {
 179         if ($generation == 0) {
 180             // Create index in pre-2.1 mode
 181             foreach ($directory->fileList() as $file) {
 182                 if ($file == 'deletable' ||
 183                     $file == 'segments'  ||
 184                     isset(self::$_indexExtensions[ substr($file, strlen($file)-4)]) ||
 185                     preg_match('/\.f\d+$/i', $file) /* matches <segment_name>.f<decimal_nmber> file names */) {
 186                         $directory->deleteFile($file);
 187                     }
 188             }
 189
 190             $segmentsFile = $directory->createFile('segments');
 191             $segmentsFile->writeInt((int)0xFFFFFFFF);
 192
 193             // write version (initialized by current time)
 194             $segmentsFile->writeLong(round(microtime(true)));
 195
 196             // write name counter
 197             $segmentsFile->writeInt($nameCount);
 198             // write segment counter
 199             $segmentsFile->writeInt(0);
 200
 201             $deletableFile = $directory->createFile('deletable');
 202             // write counter
 203             $deletableFile->writeInt(0);
 204         } else {
 205             $genFile = $directory->createFile('segments.gen');
 206
 207             $genFile->writeInt((int)0xFFFFFFFE);
 208             // Write generation two times
 209             $genFile->writeLong($generation);
 210             $genFile->writeLong($generation);
 211
 212             $segmentsFile = $directory->createFile(Zend_Search_Lucene::getSegmentFileName($generation));
 213             $segmentsFile->writeInt((int)0xFFFFFFFD);
 214
 215             // write version (initialized by current time)
 216             $segmentsFile->writeLong(round(microtime(true)));
 217
 218             // write name counter
 219             $segmentsFile->writeInt($nameCount);
 220             // write segment counter
 221             $segmentsFile->writeInt(0);
 222         }
 223     }
 224
 225     /**
 226      * Open the index for writing
 227      *
 228      * @param Zend_Search_Lucene_Storage_Directory $directory
 229      * @param array $segmentInfos
 230      * @param integer $targetFormatVersion
 231      * @param Zend_Search_Lucene_Storage_File $cleanUpLock
 232      */
 233     public function __construct(Zend_Search_Lucene_Storage_Directory $directory, &$segmentInfos, $targetFormatVersion)
 234     {
 235         $this->_directory           = $directory;
 236         $this->_segmentInfos        = &$segmentInfos;
 237         $this->_targetFormatVersion = $targetFormatVersion;
 238     }
 239
 240     /**
 241      * Adds a document to this index.
 242      *
 243      * @param Zend_Search_Lucene_Document $document
 244      */
 245     public function addDocument(Zend_Search_Lucene_Document $document)
 246     {
 247         if ($this->_currentSegment === null) {
 248             $this->_currentSegment =
 249                 new Zend_Search_Lucene_Index_SegmentWriter_DocumentWriter($this->_directory, $this->_newSegmentName());
 250         }
 251         $this->_currentSegment->addDocument($document);
 252
 253         if ($this->_currentSegment->count() >= $this->maxBufferedDocs) {
 254             $this->commit();
 255         }
 256
 257         $this->_maybeMergeSegments();
 258
 259         $this->_versionUpdate++;
 260     }
 261
 262
 263     /**
 264      * Check if we have anything to merge
 265      *
 266      * @return boolean
 267      */
 268     private function _hasAnythingToMerge()
 269     {
 270         $segmentSizes = array();
 271         foreach ($this->_segmentInfos as $segName => $segmentInfo) {
 272             $segmentSizes[$segName] = $segmentInfo->count();
 273         }
 274
 275         $mergePool   = array();
 276         $poolSize    = 0;
 277         $sizeToMerge = $this->maxBufferedDocs;
 278         asort($segmentSizes, SORT_NUMERIC);
 279         foreach ($segmentSizes as $segName => $size) {
 280             // Check, if segment comes into a new merging block
 281             while ($size >= $sizeToMerge) {
 282                 // Merge previous block if it's large enough
 283                 if ($poolSize >= $sizeToMerge) {
 284                     return true;
 285                 }
 286                 $mergePool   = array();
 287                 $poolSize    = 0;
 288
 289                 $sizeToMerge *= $this->mergeFactor;
 290
 291                 if ($sizeToMerge > $this->maxMergeDocs) {
 292                     return false;
 293                 }
 294             }
 295
 296             $mergePool[] = $this->_segmentInfos[$segName];
 297             $poolSize += $size;
 298         }
 299
 300         if ($poolSize >= $sizeToMerge) {
 301             return true;
 302         }
 303
 304         return false;
 305     }
 306
 307     /**
 308      * Merge segments if necessary
 309      */
 310     private function _maybeMergeSegments()
 311     {
 312         if (Zend_Search_Lucene_LockManager::obtainOptimizationLock($this->_directory) === false) {
 313             return;
 314         }
 315
 316         if (!$this->_hasAnythingToMerge()) {
 317             Zend_Search_Lucene_LockManager::releaseOptimizationLock($this->_directory);
 318             return;
 319         }
 320
 321         // Update segments list to be sure all segments are not merged yet by another process
 322         //
 323         // Segment merging functionality is concentrated in this class and surrounded
 324         // by optimization lock obtaining/releasing.
 325         // _updateSegments() refreshes segments list from the latest index generation.
 326         // So only new segments can be added to the index while we are merging some already existing
 327         // segments.
 328         // Newly added segments will be also included into the index by the _updateSegments() call
 329         // either by another process or by the current process with the commit() call at the end of _mergeSegments() method.
 330         // That's guaranteed by the serialisation of _updateSegments() execution using exclusive locks.
 331         $this->_updateSegments();
 332
 333         // Perform standard auto-optimization procedure
 334         $segmentSizes = array();
 335         foreach ($this->_segmentInfos as $segName => $segmentInfo) {
 336             $segmentSizes[$segName] = $segmentInfo->count();
 337         }
 338
 339         $mergePool   = array();
 340         $poolSize    = 0;
 341         $sizeToMerge = $this->maxBufferedDocs;
 342         asort($segmentSizes, SORT_NUMERIC);
 343         foreach ($segmentSizes as $segName => $size) {
 344             // Check, if segment comes into a new merging block
 345             while ($size >= $sizeToMerge) {
 346                 // Merge previous block if it's large enough
 347                 if ($poolSize >= $sizeToMerge) {
 348                     $this->_mergeSegments($mergePool);
 349                 }
 350                 $mergePool   = array();
 351                 $poolSize    = 0;
 352
 353                 $sizeToMerge *= $this->mergeFactor;
 354
 355                 if ($sizeToMerge > $this->maxMergeDocs) {
 356                     Zend_Search_Lucene_LockManager::releaseOptimizationLock($this->_directory);
 357                     return;
 358                 }
 359             }
 360
 361             $mergePool[] = $this->_segmentInfos[$segName];
 362             $poolSize += $size;
 363         }
 364
 365         if ($poolSize >= $sizeToMerge) {
 366             $this->_mergeSegments($mergePool);
 367         }
 368
 369         Zend_Search_Lucene_LockManager::releaseOptimizationLock($this->_directory);
 370     }
 371
 372     /**
 373      * Merge specified segments
 374      *
 375      * $segments is an array of SegmentInfo objects
 376      *
 377      * @param array $segments
 378      */
 379     private function _mergeSegments($segments)
 380     {
 381         $newName = $this->_newSegmentName();
 382         $merger = new Zend_Search_Lucene_Index_SegmentMerger($this->_directory,
 383                                                              $newName);
 384         foreach ($segments as $segmentInfo) {
 385             $merger->addSource($segmentInfo);
 386             $this->_segmentsToDelete[$segmentInfo->getName()] = $segmentInfo->getName();
 387         }
 388
 389         $newSegment = $merger->merge();
 390         if ($newSegment !== null) {
 391             $this->_newSegments[$newSegment->getName()] = $newSegment;
 392         }
 393
 394         $this->commit();
 395     }
 396
 397     /**
 398      * Update segments file by adding current segment to a list
 399      *
 400      * @throws Zend_Search_Lucene_Exception
 401      */
 402     private function _updateSegments()
 403     {
 404         // Get an exclusive index lock
 405         Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory);
 406
 407         // Write down changes for the segments
 408         foreach ($this->_segmentInfos as $segInfo) {
 409             $segInfo->writeChanges();
 410         }
 411
 412
 413         $generation = Zend_Search_Lucene::getActualGeneration($this->_directory);
 414         $segmentsFile   = $this->_directory->getFileObject(Zend_Search_Lucene::getSegmentFileName($generation), false);
 415         $newSegmentFile = $this->_directory->createFile(Zend_Search_Lucene::getSegmentFileName(++$generation), false);
 416
 417         try {
 418             $genFile = $this->_directory->getFileObject('segments.gen', false);
 419         } catch (Zend_Search_Lucene_Exception $e) {
 420             if (strpos($e->getMessage(), 'is not readable') !== false) {
 421                 $genFile = $this->_directory->createFile('segments.gen');
 422             } else {
 423                 throw $e;
 424             }
 425         }
 426
 427         $genFile->writeInt((int)0xFFFFFFFE);
 428         // Write generation (first copy)
 429         $genFile->writeLong($generation);
 430
 431         try {
 432             // Write format marker
 433             if ($this->_targetFormatVersion == Zend_Search_Lucene::FORMAT_2_1) {
 434                 $newSegmentFile->writeInt((int)0xFFFFFFFD);
 435             } else if ($this->_targetFormatVersion == Zend_Search_Lucene::FORMAT_2_3) {
 436                 $newSegmentFile->writeInt((int)0xFFFFFFFC);
 437             }
 438
 439             // Read src file format identifier
 440             $format = $segmentsFile->readInt();
 441             if ($format == (int)0xFFFFFFFF) {
 442                 $srcFormat = Zend_Search_Lucene::FORMAT_PRE_2_1;
 443             } else if ($format == (int)0xFFFFFFFD) {
 444                 $srcFormat = Zend_Search_Lucene::FORMAT_2_1;
 445             } else if ($format == (int)0xFFFFFFFC) {
 446                 $srcFormat = Zend_Search_Lucene::FORMAT_2_3;
 447             } else {
 448                 throw new Zend_Search_Lucene_Exception('Unsupported segments file format');
 449             }
 450
 451             $version = $segmentsFile->readLong() + $this->_versionUpdate;
 452             $this->_versionUpdate = 0;
 453             $newSegmentFile->writeLong($version);
 454
 455             // Write segment name counter
 456             $newSegmentFile->writeInt($segmentsFile->readInt());
 457
 458             // Get number of segments offset
 459             $numOfSegmentsOffset = $newSegmentFile->tell();
 460             // Write dummy data (segment counter)
 461             $newSegmentFile->writeInt(0);
 462
 463             // Read number of segemnts
 464             $segmentsCount = $segmentsFile->readInt();
 465
 466             $segments = array();
 467             for ($count = 0; $count < $segmentsCount; $count++) {
 468                 $segName = $segmentsFile->readString();
 469                 $segSize = $segmentsFile->readInt();
 470
 471                 if ($srcFormat == Zend_Search_Lucene::FORMAT_PRE_2_1) {
 472                     // pre-2.1 index format
 473                     $delGen            = 0;
 474                     $hasSingleNormFile = false;
 475                     $numField          = (int)0xFFFFFFFF;
 476                     $isCompoundByte    = 0;
 477                     $docStoreOptions   = null;
 478                 } else {
 479                     $delGen = $segmentsFile->readLong();
 480
 481                     if ($srcFormat == Zend_Search_Lucene::FORMAT_2_3) {
 482                         $docStoreOffset = $segmentsFile->readInt();
 483
 484                         if ($docStoreOffset != (int)0xFFFFFFFF) {
 485                             $docStoreSegment        = $segmentsFile->readString();
 486                             $docStoreIsCompoundFile = $segmentsFile->readByte();
 487
 488                             $docStoreOptions = array('offset'     => $docStoreOffset,
 489                                                      'segment'    => $docStoreSegment,
 490                                                      'isCompound' => ($docStoreIsCompoundFile == 1));
 491                         } else {
 492                             $docStoreOptions = null;
 493                         }
 494                     } else {
 495                         $docStoreOptions = null;
 496                     }
 497
 498                     $hasSingleNormFile = $segmentsFile->readByte();
 499                     $numField          = $segmentsFile->readInt();
 500
 501                     $normGens = array();
 502                     if ($numField != (int)0xFFFFFFFF) {
 503                         for ($count1 = 0; $count1 < $numField; $count1++) {
 504                             $normGens[] = $segmentsFile->readLong();
 505                         }
 506                     }
 507                     $isCompoundByte    = $segmentsFile->readByte();
 508                 }
 509
 510                 if (!in_array($segName, $this->_segmentsToDelete)) {
 511                     // Load segment if necessary
 512                     if (!isset($this->_segmentInfos[$segName])) {
 513                         if ($isCompoundByte == 0xFF) {
 514                             // The segment is not a compound file
 515                             $isCompound = false;
 516                         } else if ($isCompoundByte == 0x00) {
 517                             // The status is unknown
 518                             $isCompound = null;
 519                         } else if ($isCompoundByte == 0x01) {
 520                             // The segment is a compound file
 521                             $isCompound = true;
 522                         }
 523
 524                         $this->_segmentInfos[$segName] =
 525                                     new Zend_Search_Lucene_Index_SegmentInfo($this->_directory,
 526                                                                              $segName,
 527                                                                              $segSize,
 528                                                                              $delGen,
 529                                                                              $docStoreOptions,
 530                                                                              $hasSingleNormFile,
 531                                                                              $isCompound);
 532                     } else {
 533                         // Retrieve actual deletions file generation number
 534                         $delGen = $this->_segmentInfos[$segName]->getDelGen();
 535                     }
 536
 537                     $newSegmentFile->writeString($segName);
 538                     $newSegmentFile->writeInt($segSize);
 539                     $newSegmentFile->writeLong($delGen);
 540                     if ($this->_targetFormatVersion == Zend_Search_Lucene::FORMAT_2_3) {
 541                         if ($docStoreOptions !== null) {
 542                             $newSegmentFile->writeInt($docStoreOffset);
 543                             $newSegmentFile->writeString($docStoreSegment);
 544                             $newSegmentFile->writeByte($docStoreIsCompoundFile);
 545                         } else {
 546                             // Set DocStoreOffset to -1
 547                             $newSegmentFile->writeInt((int)0xFFFFFFFF);
 548                         }
 549                     } else if ($docStoreOptions !== null) {
 550                         // Release index write lock
 551                         Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory);
 552
 553                         throw new Zend_Search_Lucene_Exception('Index conversion to lower format version is not supported.');
 554                     }
 555
 556                     $newSegmentFile->writeByte($hasSingleNormFile);
 557                     $newSegmentFile->writeInt($numField);
 558                     if ($numField != (int)0xFFFFFFFF) {
 559                         foreach ($normGens as $normGen) {
 560                             $newSegmentFile->writeLong($normGen);
 561                         }
 562                     }
 563                     $newSegmentFile->writeByte($isCompoundByte);
 564
 565                     $segments[$segName] = $segSize;
 566                 }
 567             }
 568             $segmentsFile->close();
 569
 570             $segmentsCount = count($segments) + count($this->_newSegments);
 571
 572             foreach ($this->_newSegments as $segName => $segmentInfo) {
 573                 $newSegmentFile->writeString($segName);
 574                 $newSegmentFile->writeInt($segmentInfo->count());
 575
 576                 // delete file generation: -1 (there is no delete file yet)
 577                 $newSegmentFile->writeInt((int)0xFFFFFFFF);$newSegmentFile->writeInt((int)0xFFFFFFFF);
 578                 if ($this->_targetFormatVersion == Zend_Search_Lucene::FORMAT_2_3) {
 579                     // docStoreOffset: -1 (segment doesn't use shared doc store)
 580                     $newSegmentFile->writeInt((int)0xFFFFFFFF);
 581                 }
 582                 // HasSingleNormFile
 583                 $newSegmentFile->writeByte($segmentInfo->hasSingleNormFile());
 584                 // NumField
 585                 $newSegmentFile->writeInt((int)0xFFFFFFFF);
 586                 // IsCompoundFile
 587                 $newSegmentFile->writeByte($segmentInfo->isCompound() ? 1 : -1);
 588
 589                 $segments[$segmentInfo->getName()] = $segmentInfo->count();
 590                 $this->_segmentInfos[$segName] = $segmentInfo;
 591             }
 592             $this->_newSegments = array();
 593
 594             $newSegmentFile->seek($numOfSegmentsOffset);
 595             $newSegmentFile->writeInt($segmentsCount);  // Update segments count
 596             $newSegmentFile->close();
 597         } catch (Exception $e) {
 598             /** Restore previous index generation */
 599             $generation--;
 600             $genFile->seek(4, SEEK_SET);
 601             // Write generation number twice
 602             $genFile->writeLong($generation); $genFile->writeLong($generation);
 603
 604             // Release index write lock
 605             Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory);
 606
 607             // Throw the exception
 608             throw $e;
 609         }
 610
 611         // Write generation (second copy)
 612         $genFile->writeLong($generation);
 613
 614
 615         // Check if another update or read process is not running now
 616         // If yes, skip clean-up procedure
 617         if (Zend_Search_Lucene_LockManager::escalateReadLock($this->_directory)) {
 618             /**
 619              * Clean-up directory
 620              */
 621             $filesToDelete = array();
 622             $filesTypes    = array();
 623             $filesNumbers  = array();
 624
 625             // list of .del files of currently used segments
 626             // each segment can have several generations of .del files
 627             // only last should not be deleted
 628             $delFiles = array();
 629
 630             foreach ($this->_directory->fileList() as $file) {
 631                 if ($file == 'deletable') {
 632                     // 'deletable' file
 633                     $filesToDelete[] = $file;
 634                     $filesTypes[]    = 0; // delete this file first, since it's not used starting from Lucene v2.1
 635                     $filesNumbers[]  = 0;
 636                 } else if ($file == 'segments') {
 637                     // 'segments' file
 638                     $filesToDelete[] = $file;
 639                     $filesTypes[]    = 1; // second file to be deleted "zero" version of segments file (Lucene pre-2.1)
 640                     $filesNumbers[]  = 0;
 641                 } else if (preg_match('/^segments_[a-zA-Z0-9]+$/i', $file)) {
 642                     // 'segments_xxx' file
 643                     // Check if it's not a just created generation file
 644                     if ($file != Zend_Search_Lucene::getSegmentFileName($generation)) {
 645                         $filesToDelete[] = $file;
 646                         $filesTypes[]    = 2; // first group of files for deletions
 647                         $filesNumbers[]  = (int)base_convert(substr($file, 9), 36, 10); // ordered by segment generation numbers
 648                     }
 649                 } else if (preg_match('/(^_([a-zA-Z0-9]+))\.f\d+$/i', $file, $matches)) {
 650                     // one of per segment files ('<segment_name>.f<decimal_number>')
 651                     // Check if it's not one of the segments in the current segments set
 652                     if (!isset($segments[$matches[1]])) {
 653                         $filesToDelete[] = $file;
 654                         $filesTypes[]    = 3; // second group of files for deletions
 655                         $filesNumbers[]  = (int)base_convert($matches[2], 36, 10); // order by segment number
 656                     }
 657                 } else if (preg_match('/(^_([a-zA-Z0-9]+))(_([a-zA-Z0-9]+))\.del$/i', $file, $matches)) {
 658                     // one of per segment files ('<segment_name>_<del_generation>.del' where <segment_name> is '_<segment_number>')
 659                     // Check if it's not one of the segments in the current segments set
 660                     if (!isset($segments[$matches[1]])) {
 661                         $filesToDelete[] = $file;
 662                         $filesTypes[]    = 3; // second group of files for deletions
 663                         $filesNumbers[]  = (int)base_convert($matches[2], 36, 10); // order by segment number
 664                     } else {
 665                         $segmentNumber = (int)base_convert($matches[2], 36, 10);
 666                         $delGeneration = (int)base_convert($matches[4], 36, 10);
 667                         if (!isset($delFiles[$segmentNumber])) {
 668                             $delFiles[$segmentNumber] = array();
 669                         }
 670                         $delFiles[$segmentNumber][$delGeneration] = $file;
 671                     }
 672                 } else if (isset(self::$_indexExtensions[substr($file, strlen($file)-4)])) {
 673                     // one of per segment files ('<segment_name>.<ext>')
 674                     $segmentName = substr($file, 0, strlen($file) - 4);
 675                     // Check if it's not one of the segments in the current segments set
 676                     if (!isset($segments[$segmentName])  &&
 677                         ($this->_currentSegment === null  ||  $this->_currentSegment->getName() != $segmentName)) {
 678                         $filesToDelete[] = $file;
 679                         $filesTypes[]    = 3; // second group of files for deletions
 680                         $filesNumbers[]  = (int)base_convert(substr($file, 1 /* skip '_' */, strlen($file)-5), 36, 10); // order by segment number
 681                     }
 682                 }
 683             }
 684
 685             $maxGenNumber = 0;
 686             // process .del files of currently used segments
 687             foreach ($delFiles as $segmentNumber => $segmentDelFiles) {
 688                 ksort($delFiles[$segmentNumber], SORT_NUMERIC);
 689                 array_pop($delFiles[$segmentNumber]); // remove last delete file generation from candidates for deleting
 690
 691                 end($delFiles[$segmentNumber]);
 692                 $lastGenNumber = key($delFiles[$segmentNumber]);
 693                 if ($lastGenNumber > $maxGenNumber) {
 694                     $maxGenNumber = $lastGenNumber;
 695                 }
 696             }
 697             foreach ($delFiles as $segmentNumber => $segmentDelFiles) {
 698                 foreach ($segmentDelFiles as $delGeneration => $file) {
 699                         $filesToDelete[] = $file;
 700                         $filesTypes[]    = 4; // third group of files for deletions
 701                         $filesNumbers[]  = $segmentNumber*$maxGenNumber + $delGeneration; // order by <segment_number>,<del_generation> pair
 702                 }
 703             }
 704
 705             // Reorder files for deleting
 706             array_multisort($filesTypes,    SORT_ASC, SORT_NUMERIC,
 707                             $filesNumbers,  SORT_ASC, SORT_NUMERIC,
 708                             $filesToDelete, SORT_ASC, SORT_STRING);
 709
 710             foreach ($filesToDelete as $file) {
 711                 try {
 712                     /** Skip shared docstore segments deleting */
 713                     /** @todo Process '.cfx' files to check if them are already unused */
 714                     if (substr($file, strlen($file)-4) != '.cfx') {
 715                         $this->_directory->deleteFile($file);
 716                     }
 717                 } catch (Zend_Search_Lucene_Exception $e) {
 718                     if (strpos($e->getMessage(), 'Can\'t delete file') === false) {
 719                         // That's not "file is under processing or already deleted" exception
 720                         // Pass it through
 721                         throw $e;
 722                     }
 723                 }
 724             }
 725
 726             // Return read lock into the previous state
 727             Zend_Search_Lucene_LockManager::deEscalateReadLock($this->_directory);
 728         } else {
 729             // Only release resources if another index reader is running now
 730             foreach ($this->_segmentsToDelete as $segName) {
 731                 foreach (self::$_indexExtensions as $ext) {
 732                     $this->_directory->purgeFile($segName . $ext);
 733                 }
 734             }
 735         }
 736
 737         // Clean-up _segmentsToDelete container
 738         $this->_segmentsToDelete = array();
 739
 740
 741         // Release index write lock
 742         Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory);
 743
 744         // Remove unused segments from segments list
 745         foreach ($this->_segmentInfos as $segName => $segmentInfo) {
 746             if (!isset($segments[$segName])) {
 747                 unset($this->_segmentInfos[$segName]);
 748             }
 749         }
 750     }
 751
 752     /**
 753      * Commit current changes
 754      */
 755     public function commit()
 756     {
 757         if ($this->_currentSegment !== null) {
 758             $newSegment = $this->_currentSegment->close();
 759             if ($newSegment !== null) {
 760                 $this->_newSegments[$newSegment->getName()] = $newSegment;
 761             }
 762             $this->_currentSegment = null;
 763         }
 764
 765         $this->_updateSegments();
 766     }
 767
 768
 769     /**
 770      * Merges the provided indexes into this index.
 771      *
 772      * @param array $readers
 773      * @return void
 774      */
 775     public function addIndexes($readers)
 776     {
 777         /**
 778          * @todo implementation
 779          */
 780     }
 781
 782     /**
 783      * Merges all segments together into new one
 784      *
 785      * Returns true on success and false if another optimization or auto-optimization process
 786      * is running now
 787      *
 788      * @return boolean
 789      */
 790     public function optimize()
 791     {
 792         if (Zend_Search_Lucene_LockManager::obtainOptimizationLock($this->_directory) === false) {
 793             return false;
 794         }
 795
 796         // Update segments list to be sure all segments are not merged yet by another process
 797         //
 798         // Segment merging functionality is concentrated in this class and surrounded
 799         // by optimization lock obtaining/releasing.
 800         // _updateSegments() refreshes segments list from the latest index generation.
 801         // So only new segments can be added to the index while we are merging some already existing
 802         // segments.
 803         // Newly added segments will be also included into the index by the _updateSegments() call
 804         // either by another process or by the current process with the commit() call at the end of _mergeSegments() method.
 805         // That's guaranteed by the serialisation of _updateSegments() execution using exclusive locks.
 806         $this->_updateSegments();
 807
 808         $this->_mergeSegments($this->_segmentInfos);
 809
 810         Zend_Search_Lucene_LockManager::releaseOptimizationLock($this->_directory);
 811
 812         return true;
 813     }
 814
 815     /**
 816      * Get name for new segment
 817      *
 818      * @return string
 819      */
 820     private function _newSegmentName()
 821     {
 822         Zend_Search_Lucene_LockManager::obtainWriteLock($this->_directory);
 823
 824         $generation = Zend_Search_Lucene::getActualGeneration($this->_directory);
 825         $segmentsFile = $this->_directory->getFileObject(Zend_Search_Lucene::getSegmentFileName($generation), false);
 826
 827         $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version)
 828         $segmentNameCounter = $segmentsFile->readInt();
 829
 830         $segmentsFile->seek(12); // 12 = 4 (int, file format marker) + 8 (long, index version)
 831         $segmentsFile->writeInt($segmentNameCounter + 1);
 832
 833         // Flash output to guarantee that wrong value will not be loaded between unlock and
 834         // return (which calls $segmentsFile destructor)
 835         $segmentsFile->flush();
 836
 837         Zend_Search_Lucene_LockManager::releaseWriteLock($this->_directory);
 838
 839         return '_' . base_convert($segmentNameCounter, 10, 36);
 840     }
 841
 842 }