git.roojs.org Git - pear/blob - HTML/WordDiff.php

   1 <?php
   2
   3 /**
   4  * Description of WordDiff
   5  *
   6  *  require_once 'HTML/WordDiff.php';
   7  *       $init = array(
   8  *           'lang' => 'en',
   9  *           'file' => '/home/press/rss/2014/03/31/3952.html'
  10  *       );
  11  *       $wd = new HTML_WordDiff($init);
  12  *        $percent = $wd->compare('/home/press/rss/2014/03/31/3954.html');
  13  *
  14  *
  15  *
  16  * @author chris
  17  */
  18 //
  19 //require_once 'PEAR.php';
  20 //require_once 'DB/DataObject.php';
  21
  22 class HTML_WordDiff
  23 {
  24     //put your code here
  25
  26     var $lang = 'en'; // the press release language
  27     var $original = array(); // original html words
  28     var $target = array(); // diff target html words
  29     var $countTotal = 0; // Total words count form original html
  30     var $targetTotal = 0; // Total words count form target html
  31     var $wordMax = -1;
  32     //word type classification
  33     var $nonSinoTibetan = array(//non Sino-Tibetan languages
  34         'aa',
  35         'ab',
  36         'en',
  37         'pt',
  38         'ar',
  39         'de',
  40         'fr',
  41         'es',
  42         'vi',
  43         'id',
  44     );
  45     var $sinoTibetan = array(//Sino-Tibetan languages
  46         'my',
  47         'th',
  48         'ko',
  49         'zh_HK',
  50         'ja',
  51         'zh_TW',
  52         'zh_CN',
  53     );
  54
  55     var $alternatives = array(
  56         '.',
  57         ',',
  58         '--'
  59     );
  60
  61     var $htmlDom = false; // HTML Dom elements
  62     var $debug_on = false;
  63     /**
  64      * Constructor
  65      *
  66      *
  67      * @param Array $config
  68      * lang = language of article
  69      * file = name of file...
  70      * string = string contents
  71      *
  72      * @return type
  73      *
  74      */
  75     function __construct($config = false)
  76     {
  77         //print_r($config);
  78
  79         if(!$config){
  80             return;
  81         }
  82
  83         if(!is_array($config)){
  84             trigger_error("Word Diff got error, the argument IS NOT array");
  85             return;
  86         }
  87
  88         if(empty($config['lang'])){
  89             trigger_error("the language is missing.");
  90             return;
  91         }
  92         if(empty($config['file']) && !isset($config['string'])){
  93             trigger_error("File is missing");
  94             return;
  95         }
  96         if (isset($config['debug_on'])) {
  97             $this->debug_on = $config['debug_on'];
  98         }
  99
 100
 101         // not in used now??
 102         if(!in_array($this->lang, $this->nonSinoTibetan)){
 103             if(!in_array($this->lang, $this->sinoTibetan)){
 104                 trigger_error("This ({$this->lang}) language is not on our word type classification");
 105             }
 106             return;
 107         }
 108
 109
 110         $this->htmlDom = isset($config['string']) ? $config['string'] : '';
 111
 112
 113         if(isset($config['file']) && file_exists($config['file'])){
 114             $this->htmlDom = file_get_contents($config['file']);
 115         }
 116
 117         $this->lang = $config['lang'];
 118
 119
 120         $m = 'buildWords';// default run sino-tibetan
 121
 122         if(!method_exists($this, $m)){
 123             trigger_error("Method not found ($m)");
 124             return;
 125         }
 126         $this->$m();
 127     }
 128
 129     function isSino()
 130     {
 131         return in_array($this->lang, $this->sinoTibetan);
 132     }
 133
 134     /**
 135      * set the words array
 136      *
 137      * for non Sino-Tibetan languages etc. English, French
 138      *
 139      *
 140      * @param $String $target for the array index
 141      *
 142      */
 143     function buildWords($target = 'original')
 144     {
 145         static $cache= array();
 146
 147         if (isset($cache[md5($this->htmlDom)])) {
 148
 149             $this->$target = $cache[md5($this->htmlDom)];
 150
 151             if ($this->wordMax < 0) {
 152                 $this->wordMax = array_sum(array_values($this->$target)) * 10 ;
 153             }
 154
 155             if($target == 'original'){
 156                 $this->countTotal = array_sum(array_values($this->$target));
 157             }else{
 158                 $this->targetTotal= array_sum(array_values($this->$target));
 159             }
 160
 161             return;
 162         }
 163
 164         $words = $this->DomToStrings();
 165
 166         if ($this->wordMax < 0) {
 167             $this->wordMax = 10 * count($words);
 168         }
 169
 170         if($this->debug_on){
 171             var_Dump("domstrings"); print_r($words);
 172         }
 173
 174         $ret = array();
 175         $last_w = false;
 176
 177         foreach($words as $str){
 178
 179             if(empty($str) || !trim(strlen($str))) {
 180                 continue;
 181             }
 182
 183             if ($last_w !== false) {
 184
 185                 if(!isset($ret[$last_w.'|'.$str])){
 186                     $ret[$last_w.'|'.$str] = 1;
 187                 } else {
 188                     $ret[$last_w.'|'.$str] += 1;
 189                 }
 190             }
 191
 192             $last_w = $str;
 193
 194         }
 195
 196         if($target == 'original'){
 197             $this->countTotal = array_sum(array_values($ret));
 198         }else{
 199             $this->targetTotal= array_sum(array_values($ret));
 200         }
 201
 202         $this->$target = $ret;
 203
 204         $cache[md5($this->htmlDom)] = $ret;
 205
 206     }
 207
 208     function DomToStrings($target = '')
 209     {
 210         $charset = 'UTF-8';
 211
 212         $pageDom = new DomDocument('1.0', $charset);
 213         $pageDom->formatOutput = true;
 214
 215         $searchPage = preg_replace('#charset=([^"]+)#', '', $this->htmlDom);
 216
 217         @$pageDom->loadHTML(($charset == 'UTF-8' ? '<?xml version="1.0" encoding="UTF-8"?>' : ''). $searchPage);
 218
 219         $sentence = $this->domExtractWords($pageDom->documentElement, array(), $charset);
 220
 221
 222
 223
 224         $content = implode('', $sentence);
 225
 226         $content = preg_replace('/\n+/', ' ', $content);
 227
 228         $content = preg_replace('/\s+/', ' ', $content);
 229
 230         if ($charset != 'auto') {
 231             if (($this->lang == 'zh_HK' || $this->lang == 'zh_TW') && $charset == 'gb2312') {
 232                 $content = mb_convert_encoding($content, $charset,  "UTF-8");
 233                 $content = mb_convert_encoding($content, "BIG5",$charset);
 234                 $content = mb_convert_encoding($content, "UTF-8",  "BIG5");
 235             } else {
 236                 $content = mb_convert_encoding($content, "UTF-8",  $charset);
 237             }
 238         }
 239
 240         $words = "";
 241
 242         for ($i = 0; $i < mb_strlen($content); $i++){
 243
 244             $word = mb_substr($content, $i, 1);
 245
 246             if(preg_match('/'.$this->cjkpreg().'/u', $word)){
 247                 $words .= " {$word} ";
 248                 continue;
 249             }
 250
 251             if (preg_match('/[^\w]+/u', $word)) {
 252                 $words .= ' ';
 253                 continue;
 254             }
 255
 256             $words .= $word;
 257         }
 258
 259         $words = preg_split('/\s+/', trim($words));
 260          //var_dump($words);exit;
 261         return $words;
 262     }
 263
 264     function domExtractWords($node, $sentence, $charset)
 265     {
 266         if (empty($node)) {
 267             return $sentence;
 268         }
 269
 270         if ($node->nodeType == XML_TEXT_NODE) {
 271             $sentence[] = $node->textContent;
 272         }
 273
 274         if (!$node->hasChildNodes()) {
 275             return $sentence;
 276         }
 277
 278         for($i = 0; $i < $node->childNodes->length; $i++) {
 279
 280             $n = $node->childNodes->item($i);
 281
 282             $sentence = $this->domExtractWords($n, $sentence, $charset);
 283         }
 284
 285         return $sentence;
 286     }
 287
 288     function cjkpreg() {
 289
 290         static $ret = false;
 291         if ($ret !== false) {
 292             return $ret;
 293         }
 294
 295
 296         $ret = '['.implode('', array(
 297                     "\x{0E00}-\x{0E7F}", // thai ??
 298                     "\x{2E80}-\x{2EFF}",      # CJK Radicals Supplement
 299                     "\x{2F00}-\x{2FDF}",      # Kangxi Radicals
 300                     "\x{2FF0}-\x{2FFF}",      # Ideographic Description Characters
 301 //                    "\x{3000}-\x{303F}",      # CJK Symbols and Punctuation
 302                     "\x{3040}-\x{309F}",      # Hiragana
 303                     "\x{30A0}-\x{30FF}",      # Katakana
 304                     "\x{3100}-\x{312F}",      # Bopomofo
 305                     "\x{3130}-\x{318F}",      # Hangul Compatibility Jamo
 306                     "\x{3190}-\x{319F}",      # Kanbun
 307                     "\x{31A0}-\x{31BF}",      # Bopomofo Extended
 308                     "\x{31F0}-\x{31FF}",      # Katakana Phonetic Extensions
 309                     "\x{3200}-\x{32FF}",      # Enclosed CJK Letters and Months
 310                     "\x{3300}-\x{33FF}",      # CJK Compatibility
 311                     "\x{3400}-\x{4DBF}",      # CJK Unified Ideographs Extension A
 312                     "\x{4DC0}-\x{4DFF}",      # Yijing Hexagram Symbols
 313                     "\x{4E00}-\x{9FFF}",      # CJK Unified Ideographs
 314                     "\x{A000}-\x{A48F}",      # Yi Syllables
 315                     "\x{A490}-\x{A4CF}",      # Yi Radicals
 316                     "\x{AC00}-\x{D7AF}",      # Hangul Syllables
 317                     "\x{F900}-\x{FAFF}",      # CJK Compatibility Ideographs
 318                     "\x{FE30}-\x{FE4F}",      # CJK Compatibility Forms
 319                     "\x{1D300}-\x{1D35F}",    # Tai Xuan Jing Symbols
 320                     "\x{20000}-\x{2A6DF}",    # CJK Unified Ideographs Extension B
 321                     "\x{2F800}-\x{2FA1F}"     # CJK Compatibility Ideographs Supplement
 322         )). ']';
 323
 324 //        print_R($ret);
 325         return $ret;
 326     }
 327
 328     /**
 329      *
 330      *
 331      *
 332      *
 333      * @param (array|string) $file either file path or array('string'=>'....')
 334      *
 335      * @return int $percent percentage of match
 336      *
 337      */
 338     public function compare($file)
 339     {
 340
 341         if (is_array($file)) {
 342             $this->htmlDom = $file['string'];
 343         }
 344
 345         if(is_string($file) && file_exists($file)){
 346             $this->htmlDom = file_get_contents($file);
 347         }
 348
 349         $m = 'buildWords';
 350
 351         if(!method_exists($this, $m)){
 352             trigger_error("Method not found ($m)");
 353             return;
 354         }
 355
 356         $this->$m('target');
 357
 358         $matchs = 0;
 359
 360         foreach($this->original as $k => $t){
 361
 362             if(!isset($this->target[$k])){
 363                 continue;
 364             }
 365
 366             if($this->original[$k] == $this->target[$k]){
 367                 $matchs += $this->original[$k];
 368                 continue;
 369             }
 370
 371             if($this->original[$k] > $this->target[$k]){
 372                 $matchs += $this->target[$k];
 373                 continue;
 374             }
 375
 376             $matchs += $this->original[$k];
 377
 378         }
 379
 380         $percent = ( $matchs / ($this->countTotal) * 100);
 381
 382         return (int)$percent;
 383
 384     }
 385
 386 }