git.roojs.org Git - roojspacker/blob - src/jsdoc/Packer.vala

   1
   2 /**
   3  * @namespace JSDOC
   4  * @class  Packer
   5  * Create a new packer
   6  *
   7  * Use with pack.js
   8  *
   9  *
  10  * Usage:
  11  * <code>
  12  *
  13
  14 var x = new  JSON.Packer(target, debugTarget);
  15
  16 x.files = an array of files
  17 x.srcfiles = array of files (that list other files...) << not supported?
  18 x.target = "output.pathname.js"
  19 x.debugTarget = "output.pathname.debug.js"
  20
  21
  22
  23 x.pack();  // writes files  etc..
  24
  25  *</code>
  26  *
  27  * Notes for improving compacting:
  28  *  if you add a jsdoc comment
  29  * <code>
  30  * /**
  31  *   eval:var:avarname
  32  *   eval:var:bvarname
  33  *   ....
  34  * </code>
  35  * directly before an eval statement, it will compress all the code around the eval,
  36  * and not rename the variables 'avarname'
  37  *
  38  * Dont try running this on a merged uncompressed large file - it's used to be horrifically slow. not sure about now..
  39  * Best to use lot's of small classes, and use it to merge, as it will cache the compaction
  40  *
  41  *
  42  *
  43  * Notes for translation
  44  *  - translation relies on you using double quotes for strings if they need translating
  45  *  - single quoted strings are ignored.
  46  *
  47  * Generation of indexFiles
  48  *   - translateIndex = the indexfile
  49  *
  50  *
  51  *
  52  *
  53
  54  */
  55 namespace JSDOC
  56 {
  57         public errordomain PackerError {
  58             ArgumentError
  59     }
  60
  61
  62         public class Packer : Object
  63         {
  64                 /**
  65                 * @cfg {String} target to write files to - must be full path.
  66                 */
  67                 string target = "";
  68                 GLib.FileOutputStream targetStream = null;
  69                 /**
  70                  * @cfg {String} debugTarget target to write files debug version to (uncompacted)- must be full path.
  71                  */
  72                 string targetDebug = "";
  73
  74
  75                 GLib.FileOutputStream targetDebugStream  = null;
  76                 /**
  77                  * @cfg {String} tmpDir  (optional) where to put the temporary files.
  78                  *      if you set this, then files will not be cleaned up
  79                  *
  80                  *  at present we need tmpfiles - as we compile multiple files into one.
  81                  *  we could do this in memory now, as I suspect vala will not be as bad as javascript for leakage...
  82                  *
  83                  */
  84                 //public string tmpDir = "/tmp";  // FIXME??? in ctor?
  85
  86
  87
  88                 // list of files to compile...
  89                 public Gee.ArrayList<string> files;
  90
  91                 /**
  92                 * @cfg activeFile ??? used???
  93                 */
  94
  95                 public string activeFile = "";
  96
  97
  98
  99                 public  string outstr = ""; // if no target is specified - then this will contain the result
 100
 101
 102
 103
 104                 public Packer()
 105                 {
 106 #if HAVE_JSON_GLIB
 107                         this.result = new Json.Object();
 108 #else
 109                         this.result_count = new  Gee.HashMap <string,int>();
 110
 111                         this.result =  new Gee.HashMap<
 112                                 string /* errtype*/ , Gee.HashMap<string /*fn*/,     Gee.HashMap<int /*line*/, Gee.ArrayList<string>>>
 113                         >();
 114
 115 #endif
 116                         this.files = new Gee.ArrayList<string>();
 117
 118                         new Lang_Class(); ///initilizaze lang..
 119
 120                         //this.tmp = Glib.get_tmp_dir(); // do we have to delete this?
 121
 122
 123                 }
 124
 125
 126                 // this could be another class really..
 127
 128                 public enum ResultType {
 129                         err ,
 130                         warn;
 131                         public string to_string() {
 132                                 switch(this) {
 133                                         case err: return "ERR";
 134                                         case warn: return "WARN";
 135                                         default: assert_not_reached();
 136                                 }
 137
 138                           }
 139                   }
 140                 /**
 141                 *  result of complication - a JSON object containing warnings / errors etc..
 142                 *  FORMAT:
 143                 *     warn-TOTAL : X  (number of warnings.
 144                 *     err-TOTAL: X  (number of errors) << this indicates failure...
 145                 *     warn : {
 146                 *            FILENAME : {
 147                 *                  line : [ Errors,Errors,.... ]
 148                 *     err : {
 149                 *           .. sane format..
 150                 *
 151                 */
 152
 153 #if HAVE_JSON_GLIB
 154
 155                 public Json.Object result;   // output - what's the complication result
 156
 157                 public void  logError(ResultType type, string filename, int line, string message) {
 158
 159                          if (!this.result.has_member(type.to_string()+"-TOTAL")) {
 160                                  this.result.set_int_member(type.to_string()+"-TOTAL", 1);
 161                          } else {
 162                                 this.result.set_int_member(type.to_string()+"-TOTAL",
 163                                         this.result.get_int_member(type.to_string()+"-TOTAL") +1
 164                                 );
 165                          }
 166
 167
 168                          if (!this.result.has_member(type.to_string())) {
 169                                  this.result.set_object_member(type.to_string(), new Json.Object());
 170                          }
 171                          var t = this.result.get_object_member(type.to_string());
 172                          if (!t.has_member(filename)) {
 173                                  t.set_object_member(filename, new Json.Object());
 174                          }
 175                          var tt = t.get_object_member(filename);
 176                          if (!tt.has_member(line.to_string())) {
 177                                  tt.set_array_member(line.to_string(), new Json.Array());
 178                          }
 179                          var tl = tt.get_array_member(line.to_string());
 180                          tl.add_string_element(message);
 181
 182                 }
 183
 184                 public bool hasErrors(string fn)
 185                 {
 186                          if (!this.result.has_member(ResultType.err.to_string())) {
 187                                  return false;
 188                          }
 189
 190                          if (fn.length < 1) {
 191                                 return true;
 192                          }
 193                          var t = this.result.get_object_member(ResultType.err.to_string());
 194
 195                          if (t.has_member(fn)) {
 196                                  return true;
 197                          }
 198                          return false;
 199                 }
 200                 public void dumpErrors(ResultType type)
 201                 {
 202                          if (!this.result.has_member(type.to_string())) {
 203                                  return;
 204                          }
 205                         var t = this.result.get_object_member(type.to_string());
 206                         t.foreach_member((obj, filename, node) => {
 207                                         var linelist = node.dup_object();
 208                                         linelist.foreach_member((linelistobj, linestr, nodear) => {
 209                                                 var errors=  nodear.dup_array();
 210                                                 errors.foreach_element((errorar, ignore, nodestr) => {
 211                                                         print("%s: %s:%s %s\n", type.to_string(), filename, linestr, nodestr.get_string());
 212                                                 });
 213                                         });
 214
 215                         });
 216                 }
 217 #else
 218                 public Gee.HashMap <string,int> result_count;   // output - what's the complication result
 219
 220                 public Gee.HashMap<
 221                                 string /* errtype*/ , Gee.HashMap<string /*fn*/,     Gee.HashMap<int /*line*/, Gee.ArrayList<string>>>
 222                 > result;
 223
 224                 public void  logError(ResultType type, string filename, int line, string message) {
 225
 226
 227                          if (!this.result_count.has_key(type.to_string()+"-TOTAL")) {
 228                                  this.result_count.set(type.to_string()+"-TOTAL", 1);
 229                          } else {
 230                                 this.result_count.set(type.to_string()+"-TOTAL",
 231                                         this.result_count.get(type.to_string()+"-TOTAL") +1
 232                                 );
 233                          }
 234
 235
 236
 237                          if (!this.result.has_key(type.to_string())) {
 238                                  this.result.set(type.to_string(),
 239                                          new Gee.HashMap<string /*fn*/,     Gee.HashMap<int /*line*/, Gee.ArrayList<string>>>()
 240                                  );
 241                          }
 242                          var t = this.result.get(type.to_string());
 243                          if (!t.has_key(filename)) {
 244                                  t.set(filename, new  Gee.HashMap<int /*line*/, Gee.ArrayList<string>>());
 245                          }
 246                          var tt = t.get(filename);
 247                          if (!tt.has_key(line)) {
 248                                  tt.set(line, new Gee.ArrayList<string>());
 249                          }
 250                          var tl = tt.get(line);
 251                          tl.add(message);
 252
 253                 }
 254
 255                 public bool hasErrors(string fn)
 256                 {
 257                          if (!this.result.has_key(ResultType.err.to_string())) {
 258                                  return false;
 259                          }
 260
 261                          if (fn.length < 1) {
 262                                 return true;
 263                          }
 264                          var t = this.result.get(ResultType.err.to_string());
 265
 266                          if (t.has_key(fn)) {
 267                                  return true;
 268                          }
 269                          return false;
 270                 }
 271                 public void dumpErrors(ResultType type)
 272                 {
 273                          if (!this.result.has_key(type.to_string())) {
 274                                  return;
 275                          }
 276                         var t = this.result.get(type.to_string());
 277                         foreach(string filename in t.keys) {
 278                                 var node = t.get(filename);
 279                                 foreach(int line in node.keys) {
 280                                         var errors = node.get(line);
 281                                         foreach(string errstr in errors) {
 282                                                         print("%s: %s:%d %s\n", type.to_string(), filename, line, errstr);
 283                                         }
 284                                 }
 285
 286                         }
 287                 }
 288
 289
 290 #endif
 291
 292
 293
 294                 public void loadSourceIndexes(Gee.ArrayList<string> indexes)
 295                 {
 296                         foreach(var f in indexes) {
 297                                 this.loadSourceIndex(f);
 298                         }
 299                 }
 300
 301                 public void loadFiles(string[] fs)
 302                 {
 303                         // fixme -- prefix baseDir?
 304                         foreach(var f in fs) {
 305                             GLib.debug("add File: %s", f);
 306                                 this.files.add(f); //?? easier way?
 307                         }
 308                 }
 309                 public void loadFile(string f)
 310                 {
 311                     // fixme -- prefix baseDir?
 312                     GLib.debug("add File: %s", f);
 313                         this.files.add(f);
 314                         GLib.debug("FILE LEN: %d", this.files.size);
 315                 }
 316
 317
 318                 public string pack(string target, string targetDebug = "") throws PackerError
 319                 {
 320                     this.target = target;
 321                         this.targetDebug  = targetDebug;
 322
 323                     if (this.files.size < 1) {
 324                                 throw new PackerError.ArgumentError("No Files loaded before pack() called");
 325                         }
 326                         if (this.target.length > 0 ) {
 327                                 this.targetStream = File.new_for_path(this.target).replace(null, false,FileCreateFlags.NONE);
 328                         }
 329                         if (this.targetDebug.length > 0 ) {
 330                                 this.targetDebugStream = File.new_for_path(this.targetDebug).replace(null, false,FileCreateFlags.NONE);
 331                         }
 332                         return this.packAll();
 333                 }
 334
 335
 336
 337
 338
 339                 /**
 340                  * load a dependancy list -f option
 341                  * @param {String} srcfile sourcefile to parse
 342                  *
 343                  */
 344
 345                 public void loadSourceIndex(string in_srcfile)
 346                 {
 347
 348                     var srcfile = in_srcfile;
 349                     if (srcfile[0] != '/') {
 350                         //      srcfile = PackerRun.opt_real_basedir + in_srcfile;
 351                         }
 352                     string str;
 353                     FileUtils.get_contents(srcfile,out str);
 354
 355                     var lines = str.split("\n");
 356                     for(var i =0; i < lines.length;i++) {
 357
 358                             var f = lines[i].strip();
 359                         if (f.length < 1 ||
 360                                 Regex.match_simple ("^/", f) ||
 361                                 !Regex.match_simple ("[a-zA-Z]+", f)
 362                         ){
 363                                 continue; // blank comment or not starting with a-z
 364                         }
 365
 366                         if (Regex.match_simple ("\\.js$", f)) {
 367                             this.files.add( f);
 368                             // js file..
 369                             continue;
 370                         }
 371
 372                                 // this maps Roo.bootstrap.XXX to Roo/bootstrap/xxx.js
 373                                 // should we prefix? =- or should this be done elsewhere?
 374
 375                         var add = f.replace(".", "/") + ".js";
 376
 377                         if (add[0] != '/') {
 378                         //              add = PackerRun.opt_real_basedir + add;
 379                                 }
 380
 381                         if (this.files.contains(add)) {
 382                             continue;
 383                         }
 384
 385
 386
 387                         this.files.add( add );
 388
 389                     }
 390                 }
 391
 392
 393                 private string packAll()   // do the packing (run from constructor)
 394                 {
 395
 396                     //this.transOrigFile= bpath + '/../lang.en.js'; // needs better naming...
 397                     //File.write(this.transfile, "");
 398                     if (this.target.length > 0) {
 399                         this.targetStream.write("".data);
 400                     }
 401
 402                     if (this.targetDebugStream != null) {
 403                             this.targetDebugStream.write("".data);
 404                     }
 405
 406
 407                     var tmpDir = GLib.DirUtils.make_tmp("roojspacker_XXXXXX");
 408
 409                     foreach(var file in this.files) {
 410
 411                         print("reading %s\n",file );
 412
 413                         if (!FileUtils.test (file, FileTest.EXISTS) || FileUtils.test (file, FileTest.IS_DIR)) {
 414                             print("SKIP (is not a file) %s\n ", file);
 415                             continue;
 416                         }
 417
 418                                 var loaded_string = false;
 419                                 string file_contents = "";
 420                         // debug Target
 421
 422                         if (this.targetDebugStream !=null) {
 423
 424                                 FileUtils.get_contents(file,out file_contents);
 425                             this.targetDebugStream.write(file_contents.data);
 426                             loaded_string = false;
 427                         }
 428                         // it's a good idea to check with 0 compression to see if the code can parse!!
 429
 430                         // debug file..
 431                         //File.append(dout, str +"\n");
 432
 433
 434
 435                         var minfile = tmpDir + "/" + file.replace("/", ".");
 436
 437
 438                         // let's see if we have a min file already?
 439                         // this might happen if tmpDir is set ..
 440
 441
 442                         if ( FileUtils.test (minfile, FileTest.EXISTS)) {
 443
 444                                 var otv = File.new_for_path(file).query_info (FileAttribute.TIME_MODIFIED, 0).get_modification_time();
 445                                 var mtv = File.new_for_path(minfile).query_info (FileAttribute.TIME_MODIFIED, 0).get_modification_time();
 446
 447
 448                            // print("%s : compare : Cache file  %s to Orignal Time %s\n", file, mtv.to_iso8601(), otv.to_iso8601());
 449                             if (mtv.tv_usec > otv.tv_usec) {
 450                                 continue; // file is newer or the same time..
 451
 452                             }
 453
 454                         }
 455
 456                         print("COMPRESSING to %s\n", minfile);
 457                         //var codeComp = pack(str, 10, 0, 0);
 458                        // if (PackerRun.opt_clean_cache && FileUtils.test (minfile, FileTest.EXISTS)) {
 459                        //     FileUtils.remove(minfile);
 460                        // }
 461                         if (!loaded_string) {
 462                                 FileUtils.get_contents(file,out file_contents);
 463                         }
 464
 465                          this.packFile(file_contents, file, minfile);
 466
 467
 468                     }
 469
 470                     // at this point if we have errors, we should stop..
 471
 472
 473                         this.dumpErrors(ResultType.warn);
 474                         this.dumpErrors(ResultType.err); // since they are fatal - display them last...
 475
 476
 477
 478
 479                 //      if (PackerRun.opt_dump_tokens || this.hasErrors("")) {
 480
 481                 //              GLib.Process.exit(0);
 482                 //      }
 483                     print("MERGING SOURCE\n");
 484
 485                     for(var i=0; i < this.files.size; i++)  {
 486                         var file = this.files[i];
 487                         var minfile = tmpDir + "/" + file.replace("/", ".");
 488
 489
 490                         if ( !FileUtils.test(minfile, FileTest.EXISTS)) {
 491                                 print("skipping source %s - does not exist\n", minfile);
 492                             continue;
 493                         }
 494                         string str;
 495                         FileUtils.get_contents(minfile, out str);
 496                         print("using MIN FILE  %s\n", minfile);
 497                         if (str.length > 0) {
 498                             if (this.targetStream != null) {
 499                                         this.targetStream.write(("// " +
 500                                                 ( (file.length > PackerRun.opt_real_basedir.length) ? file.substring(PackerRun.opt_real_basedir.length)  : file ) +
 501                                                 "\n").data);
 502                                         this.targetStream.write(("// " + file ) + "\n").data);                                  this.targetStream.write((str + "\n").data);
 503
 504                             } else {
 505                                 this.outstr += "//" +
 506                                         ( (file.length > PackerRun.opt_real_basedir.length) ? file.substring(PackerRun.opt_real_basedir.length)  : file ) +  "\n";
 507                                 this.outstr += str + "\n";
 508                             }
 509
 510                         }
 511                         if (PackerRun.opt_clean_cache) {
 512                             FileUtils.remove(minfile);
 513                         }
 514
 515                     }
 516                     if (PackerRun.opt_clean_cache) {
 517                                 FileUtils.remove(tmpDir);
 518                         }
 519
 520                     if (this.target.length > 0 ) {
 521                             print("Output file: " + this.target);
 522                     }
 523                     if (this.targetDebug.length > 0) {
 524                                  print("Output debug file: %s\n" , this.targetDebug);
 525                         }
 526
 527
 528
 529                         // OUTPUT should be handled by PackerRun (so that this can be used as a library...)
 530                         if (this.outstr.length > 0 ) {
 531                 return this.outstr;
 532                         //      stdout.printf ("%s", this.outstr);
 533                         }
 534                     return "";
 535
 536
 537                 }
 538                 /**
 539                  * Core packing routine  for a file
 540                  *
 541                  * @param str - str source text..
 542                  * @param fn - filename (for reference?)
 543                  * @param minfile - min file location...
 544                  *
 545                  */
 546
 547                 public  string packFile  (string str,string fn, string minfile)
 548                 {
 549
 550                         var tr = new  TokenReader(this);
 551                         tr.keepDocs =true;
 552                         tr.keepWhite = true;
 553                         tr.keepComments = true;
 554                         tr.sepIdents = true;
 555                         tr.collapseWhite = false;
 556                         tr.filename = fn;
 557
 558                         // we can load translation map here...
 559
 560                         TokenArray toks = tr.tokenize(new TextStream(str)); // dont merge xxx + . + yyyy etc.
 561
 562                         if (PackerRun.opt_dump_tokens) {
 563                                 toks.dump();
 564                                 return "";
 565                                 //GLib.Process.exit(0);
 566                         }
 567
 568                         this.activeFile = fn;
 569
 570                         // and replace if we are generating a different language..
 571
 572
 573                         //var ts = new TokenStream(toks);
 574                         //print(JSON.stringify(toks, null,4 )); Seed.quit();
 575                         var ts = new Collapse(toks.tokens, this, fn);
 576
 577                         //ts.dumpAll("");                       print("Done collaps"); Process.exit(1);
 578
 579                    // print(JSON.stringify(ts.tokens, null,4 )); Seed.quit();
 580                         //return;//
 581                         if (!PackerRun.opt_skip_scope) {
 582                                 var sp = new ScopeParser(ts, this, fn);
 583
 584                                 //sp.packer = this;
 585                                 sp.buildSymbolTree();
 586                                 sp.mungeSymboltree();
 587
 588
 589                                 sp.printWarnings();
 590                         }
 591
 592
 593                         //print(sp.warnings.join("\n"));
 594                         //(new TokenStream(toks.tokens)).dumpAll(""); GLib.Process.exit(1);
 595                         // compress works on the original array - in theory the replacements have already been done by now
 596                         var outf = CompressWhite(new TokenStream(toks.tokens), this, PackerRun.opt_keep_whitespace); // do not kill whitespace..
 597
 598
 599                         debug("RESULT: \n %s\n", outf);
 600
 601
 602
 603                         if (outf.length > 0 && minfile.length > 0 && !this.hasErrors(fn)) {
 604                                 FileUtils.set_contents(minfile, outf);
 605
 606                         }
 607
 608
 609                         return outf;
 610
 611
 612
 613                 }
 614
 615
 616                 public string md5(string str)
 617                 {
 618
 619                         return GLib.Checksum.compute_for_string(GLib.ChecksumType.MD5, str);
 620
 621                 }
 622
 623          //stringHandler : function(tok) -- not used...
 624     }
 625
 626 }