7 * This source file is subject to the new BSD license that is bundled
8 * with this package in the file LICENSE.txt.
9 * It is also available through the world-wide-web at this URL:
10 * http://framework.zend.com/license/new-bsd
11 * If you did not receive a copy of the license and are unable to
12 * obtain it through the world-wide-web, please send an email
13 * to license@zend.com so we can send you a copy immediately.
16 * @package Zend_Search_Lucene
17 * @subpackage Document
18 * @copyright Copyright (c) 2005-2009 Zend Technologies USA Inc. (http://www.zend.com)
19 * @license http://framework.zend.com/license/new-bsd New BSD License
20 * @version $Id: Html.php 16971 2009-07-22 18:05:45Z mikaelkael $
24 /** Zend_Search_Lucene_Document */
25 require_once 'Zend/Search/Lucene/Document.php';
32 * @package Zend_Search_Lucene
33 * @subpackage Document
34 * @copyright Copyright (c) 2005-2009 Zend Technologies USA Inc. (http://www.zend.com)
35 * @license http://framework.zend.com/license/new-bsd New BSD License
37 class Zend_Search_Lucene_Document_Html extends Zend_Search_Lucene_Document
40 * List of document links
44 private $_links = array();
47 * List of document header links
51 private $_headerLinks = array();
54 * Stored DOM representation
61 * Exclud nofollow links flag
63 * If true then links with rel='nofollow' attribute are not included into
68 private static $_excludeNoFollowLinks = false;
73 * @param string $data HTML string (may be HTML fragment, )
74 * @param boolean $isFile
75 * @param boolean $storeContent
76 * @param string $defaultEncoding HTML encoding, is used if it's not specified using Content-type HTTP-EQUIV meta tag.
78 private function __construct($data, $isFile, $storeContent, $defaultEncoding = '')
80 $this->_doc = new DOMDocument();
81 $this->_doc->substituteEntities = true;
84 $htmlData = file_get_contents($data);
88 @$this->_doc->loadHTML($htmlData);
90 if ($this->_doc->encoding === null) {
91 // Document encoding is not recognized
93 /** @todo improve HTML vs HTML fragment recognition */
94 if (preg_match('/<html>/i', $htmlData, $matches, PREG_OFFSET_CAPTURE)) {
95 // It's an HTML document
96 // Add additional HEAD section and recognize document
97 $htmlTagOffset = $matches[0][1] + strlen($matches[0][1]);
99 @$this->_doc->loadHTML(iconv($defaultEncoding, 'UTF-8//IGNORE', substr($htmlData, 0, $htmlTagOffset))
100 . '<head><META HTTP-EQUIV="Content-type" CONTENT="text/html; charset=UTF-8"/></head>'
101 . iconv($defaultEncoding, 'UTF-8//IGNORE', substr($htmlData, $htmlTagOffset)));
103 // Remove additional HEAD section
104 $xpath = new DOMXPath($this->_doc);
105 $head = $xpath->query('/html/head')->item(0);
106 if (!$head || !$head->parentNode) {
107 throw new Exception("could not find html/head in this doc");
109 $head->parentNode->removeChild($head);
111 // It's an HTML fragment
112 @$this->_doc->loadHTML('<html><head><META HTTP-EQUIV="Content-type" CONTENT="text/html; charset=UTF-8"/></head><body>'
113 . iconv($defaultEncoding, 'UTF-8//IGNORE', $htmlData)
118 /** @todo Add correction of wrong HTML encoding recognition processing
120 * Content-type HTTP-EQUIV meta tag is presented, but ISO-8859-5 encoding is actually used,
121 * even $this->_doc->encoding demonstrates another recognized encoding
124 $xpath = new DOMXPath($this->_doc);
127 $titleNodes = $xpath->query('/html/head/title');
128 foreach ($titleNodes as $titleNode) {
129 // title should always have only one entry, but we process all nodeset entries
130 $docTitle .= $titleNode->nodeValue . ' ';
132 $this->addField(Zend_Search_Lucene_Field::Text('title', $docTitle, 'UTF-8'));
134 $metaNodes = $xpath->query('/html/head/meta[@name]');
135 foreach ($metaNodes as $metaNode) {
136 $this->addField(Zend_Search_Lucene_Field::Text($metaNode->getAttribute('name'),
137 $metaNode->getAttribute('content'),
142 $bodyNodes = $xpath->query('/html/body');
143 foreach ($bodyNodes as $bodyNode) {
144 // body should always have only one entry, but we process all nodeset entries
145 $this->_retrieveNodeText($bodyNode, $docBody);
148 $this->addField(Zend_Search_Lucene_Field::Text('body', $docBody, 'UTF-8'));
150 $this->addField(Zend_Search_Lucene_Field::UnStored('body', $docBody, 'UTF-8'));
153 $linkNodes = $this->_doc->getElementsByTagName('a');
154 foreach ($linkNodes as $linkNode) {
155 if (($href = $linkNode->getAttribute('href')) != '' &&
156 (!self::$_excludeNoFollowLinks || strtolower($linkNode->getAttribute('rel')) != 'nofollow' )
158 $this->_links[] = $href;
161 $this->_links = array_unique($this->_links);
163 $linkNodes = $xpath->query('/html/head/link');
164 foreach ($linkNodes as $linkNode) {
165 if (($href = $linkNode->getAttribute('href')) != '') {
166 $this->_headerLinks[] = $href;
169 $this->_headerLinks = array_unique($this->_headerLinks);
173 * Set exclude nofollow links flag
175 * @param boolean $newValue
177 public static function setExcludeNoFollowLinks($newValue)
179 self::$_excludeNoFollowLinks = $newValue;
183 * Get exclude nofollow links flag
187 public static function getExcludeNoFollowLinks()
189 return self::$_excludeNoFollowLinks;
195 * We should exclude scripts, which may be not included into comment tags, CDATA sections,
197 * @param DOMNode $node
198 * @param string &$text
200 private function _retrieveNodeText(DOMNode $node, &$text)
202 if ($node->nodeType == XML_TEXT_NODE) {
203 $text .= $node->nodeValue ;
205 } else if ($node->nodeType == XML_ELEMENT_NODE && $node->nodeName != 'script') {
206 foreach ($node->childNodes as $childNode) {
207 $this->_retrieveNodeText($childNode, $text);
213 * Get document HREF links
217 public function getLinks()
219 return $this->_links;
223 * Get document header links
227 public function getHeaderLinks()
229 return $this->_headerLinks;
233 * Load HTML document from a string
235 * @param string $data
236 * @param boolean $storeContent
237 * @param string $defaultEncoding HTML encoding, is used if it's not specified using Content-type HTTP-EQUIV meta tag.
238 * @return Zend_Search_Lucene_Document_Html
240 public static function loadHTML($data, $storeContent = false, $defaultEncoding = '')
242 return new Zend_Search_Lucene_Document_Html($data, false, $storeContent, $defaultEncoding);
246 * Load HTML document from a file
248 * @param string $file
249 * @param boolean $storeContent
250 * @param string $defaultEncoding HTML encoding, is used if it's not specified using Content-type HTTP-EQUIV meta tag.
251 * @return Zend_Search_Lucene_Document_Html
253 public static function loadHTMLFile($file, $storeContent = false, $defaultEncoding = '')
255 return new Zend_Search_Lucene_Document_Html($file, true, $storeContent, $defaultEncoding);
260 * Highlight text in text node
262 * @param DOMText $node
263 * @param array $wordsToHighlight
264 * @param callback $callback Callback method, used to transform (highlighting) text.
265 * @param array $params Array of additionall callback parameters (first non-optional parameter is a text to transform)
266 * @throws Zend_Search_Lucene_Exception
268 protected function _highlightTextNode(DOMText $node, $wordsToHighlight, $callback, $params)
270 $analyzer = Zend_Search_Lucene_Analysis_Analyzer::getDefault();
271 $analyzer->setInput($node->nodeValue, 'UTF-8');
273 $matchedTokens = array();
275 while (($token = $analyzer->nextToken()) !== null) {
276 if (isset($wordsToHighlight[$token->getTermText()])) {
277 $matchedTokens[] = $token;
281 if (count($matchedTokens) == 0) {
285 $matchedTokens = array_reverse($matchedTokens);
287 foreach ($matchedTokens as $token) {
288 // Cut text after matched token
289 $node->splitText($token->getEndOffset());
292 $matchedWordNode = $node->splitText($token->getStartOffset());
294 // Retrieve HTML string representation for highlihted word
295 $fullCallbackparamsList = $params;
296 array_unshift($fullCallbackparamsList, $matchedWordNode->nodeValue);
297 $highlightedWordNodeSetHtml = call_user_func_array($callback, $fullCallbackparamsList);
299 // Transform HTML string to a DOM representation and automatically transform retrieved string
300 // into valid XHTML (It's automatically done by loadHTML() method)
301 $highlightedWordNodeSetDomDocument = new DOMDocument('1.0', 'UTF-8');
302 $success = @$highlightedWordNodeSetDomDocument->
303 loadHTML('<html><head><meta http-equiv="Content-type" content="text/html; charset=UTF-8"/></head><body>'
304 . $highlightedWordNodeSetHtml
307 require_once 'Zend/Search/Lucene/Exception.php';
308 throw new Zend_Search_Lucene_Exception("Error occured while loading highlighted text fragment: '$highlightedNodeHtml'.");
310 $highlightedWordNodeSetXpath = new DOMXPath($highlightedWordNodeSetDomDocument);
311 $highlightedWordNodeSet = $highlightedWordNodeSetXpath->query('/html/body')->item(0)->childNodes;
313 for ($count = 0; $count < $highlightedWordNodeSet->length; $count++) {
314 $nodeToImport = $highlightedWordNodeSet->item($count);
315 $node->parentNode->insertBefore($this->_doc->importNode($nodeToImport, true /* deep copy */),
319 $node->parentNode->removeChild($matchedWordNode);
325 * highlight words in content of the specified node
327 * @param DOMNode $contextNode
328 * @param array $wordsToHighlight
329 * @param callback $callback Callback method, used to transform (highlighting) text.
330 * @param array $params Array of additionall callback parameters (first non-optional parameter is a text to transform)
332 protected function _highlightNodeRecursive(DOMNode $contextNode, $wordsToHighlight, $callback, $params)
334 $textNodes = array();
336 if (!$contextNode->hasChildNodes()) {
340 foreach ($contextNode->childNodes as $childNode) {
341 if ($childNode->nodeType == XML_TEXT_NODE) {
342 // process node later to leave childNodes structure untouched
343 $textNodes[] = $childNode;
345 // Process node if it's not a script node
346 if ($childNode->nodeName != 'script') {
347 $this->_highlightNodeRecursive($childNode, $wordsToHighlight, $callback, $params);
352 foreach ($textNodes as $textNode) {
353 $this->_highlightTextNode($textNode, $wordsToHighlight, $callback, $params);
358 * Standard callback method used to highlight words.
360 * @param string $stringToHighlight
364 public function applyColour($stringToHighlight, $colour)
366 return '<b style="color:black;background-color:' . $colour . '">' . $stringToHighlight . '</b>';
370 * Highlight text with specified color
372 * @param string|array $words
373 * @param string $colour
376 public function highlight($words, $colour = '#66ffff')
378 return $this->highlightExtended($words, array($this, 'applyColour'), array($colour));
384 * Highlight text using specified View helper or callback function.
386 * @param string|array $words Words to highlight. Words could be organized using the array or string.
387 * @param callback $callback Callback method, used to transform (highlighting) text.
388 * @param array $params Array of additionall callback parameters passed through into it
389 * (first non-optional parameter is an HTML fragment for highlighting)
391 * @throws Zend_Search_Lucene_Exception
393 public function highlightExtended($words, $callback, $params = array())
395 if (!is_array($words)) {
396 $words = array($words);
399 $wordsToHighlightList = array();
400 $analyzer = Zend_Search_Lucene_Analysis_Analyzer::getDefault();
401 foreach ($words as $wordString) {
402 $wordsToHighlightList[] = $analyzer->tokenize($wordString);
404 $wordsToHighlight = call_user_func_array('array_merge', $wordsToHighlightList);
406 if (count($wordsToHighlight) == 0) {
407 return $this->_doc->saveHTML();
410 $wordsToHighlightFlipped = array();
411 foreach ($wordsToHighlight as $id => $token) {
412 $wordsToHighlightFlipped[$token->getTermText()] = $id;
415 if (!is_callable($callback)) {
416 require_once 'Zend/Search/Lucene/Exception.php';
417 throw new Zend_Search_Lucene_Exception('$viewHelper parameter mast be a View Helper name, View Helper object or callback.');
420 $xpath = new DOMXPath($this->_doc);
422 $matchedNodes = $xpath->query("/html/body");
423 foreach ($matchedNodes as $matchedNode) {
424 $this->_highlightNodeRecursive($matchedNode, $wordsToHighlightFlipped, $callback, $params);
434 public function getHTML()
436 return $this->_doc->saveHTML();
444 public function getHtmlBody()
446 $xpath = new DOMXPath($this->_doc);
447 $bodyNodes = $xpath->query('/html/body')->item(0)->childNodes;
449 $outputFragments = array();
450 for ($count = 0; $count < $bodyNodes->length; $count++) {
451 $outputFragments[] = $this->_doc->saveXML($bodyNodes->item($count));
454 return implode($outputFragments);