git.roojs.org Git - Pman.Xtuple/blob - Migrate.php

   1 <?php
   2 /**
   3  *
   4  * migration script runner..
   5  * xx.php Xtuple/Migrate
   6  *
   7  * This is the controller that runs all the table migrations.
   8  *
   9  * it' calls it'self.. to try and reduce memrory
  10  *
  11  * Caching is a bit of a nightmare..
  12  *  - Usage scenarios
  13  *    a) run the whole thing   Xtuple/Migrate
  14  *    b) run a specific year/table combo    Xtuple/Migrate -t Netsuite_XXX -y 2010
  15  *
  16  *
  17  *
  18  *
  19  *
  20  */
  21 require_once 'Pman.php';
  22
  23 class Pman_Xtuple_Migrate extends Pman
  24 {
  25     static $cli_desc = "Migrate Netsuite json files to xtuple";
  26
  27     static $cli_opts = array(
  28         'debug' => array(
  29             'desc' => 'Turn on debugging (see DataObjects debugLevel )',
  30             'default' => 0,
  31             'short' => 'v',
  32             'min' => 1,
  33             'max' => 1,
  34
  35         ),
  36
  37         'source' => array(
  38             'desc' => 'Source directory for json files.',
  39             'short' => 'f',
  40             'default' => '',
  41             'min' => 1,
  42             'max' => 1,
  43         ),
  44          'no-cache' => array(
  45             'desc' => 'Do not cache.',
  46             'short' => 'n',
  47             'default' => 0,
  48             'min' => 1,
  49             'max' => 1,
  50         ),
  51         'list-order' => array(
  52             'desc' => 'List Order.',
  53             'short' => 'l',
  54             'default' => 0,
  55             'min' => 1,
  56             'max' => 1,
  57         ),
  58          'table' => array(
  59             'desc' => 'Only a specific table, eg. -t Netsuite_SalesOrder',
  60             'short' => 't',
  61
  62             'max' => 1,
  63         ),
  64          'year' => array(
  65             'desc' => 'Only a specific year, eg. -y 2009',
  66             'short' => 'y',
  67             'default' => 2009,
  68             'min' => 1,
  69             'max' => 1,
  70         ),
  71          'mapped' => array(
  72             'desc' => "Get a mapped value or 'ALL' to show everything",
  73             'short' => 'm',
  74             'default' => 0,
  75             'min' => 1,
  76             'max' => 1,
  77         ),
  78
  79          'where' => array(
  80             'desc' => "Import only items matching this condition, eg. --where='ItemFulfillment_id=123'",
  81             'short' => 'w',
  82             'default' => '',
  83             'min' => 1,
  84             'max' => 1,
  85         ),
  86
  87         'print' => array(
  88             'desc' => "print_r the data for import (matching -w)",
  89             'short' => 'p',
  90             'default' => 0,
  91             'min' => 0,
  92             'max' => 1,
  93         ),
  94
  95
  96     );
  97
  98
  99     function getAuth()
 100     {
 101         $ff = HTML_FlexyFramework::get();
 102         if (!$ff->cli) {
 103             die("run form cli only");
 104         }
 105
 106     }
 107     var $map = array();
 108
 109     var $tables = array();
 110
 111
 112     var $opts = array();
 113
 114     function initOpts($opts)
 115     {
 116         $this->opts = $opts;
 117
 118         //print_R($opts);
 119
 120         if (empty($this->opts['source'])) {
 121             $this->opts['source'] = __DIR__.'/data';
 122         }
 123
 124
 125         $sd = strtoupper(substr( HTML_FlexyFramework::get()->database, -2));
 126         $this->department = $sd;
 127         $os = $this->opts['source'] ;
 128         $this->opts['source'] .= '/'. $sd;
 129
 130         //print_R($this->opts);exit;
 131
 132         if (!is_dir($this->opts['source'])) {
 133             if (file_exists($os)) {
 134                 $this->opts['source']  = $os;
 135             } else {
 136
 137
 138                 die("--source must be a directory {$this->opts['source']}\n");
 139             }
 140         }
 141
 142
 143
 144     }
 145     var $department = 'UNKNOWN';
 146     function cachedir()
 147     {
 148         $uinfo = posix_getpwuid( posix_getuid () );
 149         $user = $uinfo['name'];
 150
 151         $cd = ini_get('session.save_path') ."/xtuple{$this->department}-{$user}";
 152         if (!file_exists($cd)) {
 153             mkdir($cd, 0700, true);
 154         }
 155         return $cd;
 156
 157
 158     }
 159
 160
 161     function get($path, $opts)
 162     {
 163         ini_set('memory_limit', -1); // unlimited!
 164
 165         PEAR::setErrorHandling(PEAR_ERROR_CALLBACK, array($this, 'onPearError'));
 166
 167
 168         $this->initOpts($opts);
 169         $cd = $this->cachedir();
 170
 171         if (!empty($opts['mapped']) && !empty($opts['table'])) {
 172             require_once 'Pman/Xtuple/Migration/Base.php';
 173             $b = new Pman_Xtuple_Migration_Base($this);
 174             echo "Mapped value: ";
 175             $this->map(preg_replace('/^Netsuite_/i', '',$opts['table']));
 176             print_r(
 177                     $opts['mapped'] == 'ALL' ?
 178                         $this->map[preg_replace('/^Netsuite_/i', '',$opts['table'])] :
 179                         $b->mapped(
 180                             preg_replace('/^Netsuite_/i', '',$opts['table']),
 181                             $opts['mapped']
 182                         )
 183             );
 184             echo "\n";
 185             exit;
 186         }
 187
 188         $this->buildDeps();
 189
 190         $nocache = $this->opts['no-cache'];
 191        // print_R($this->deps); exit;
 192         // push salesorderitem up the list..
 193         $this->deps['Netsuite_Invoice'][] = 'Netsuite_SalesOrderItem';
 194         //$this->deps['Netsuite_Invoice'][] = 'Netsuite_SalesOrder';
 195         $this->deps['Netsuite_Rates'] = array('Netsuite_Currency');
 196         $this->deps['Netsuite_VendorBill'] = array('Netsuite_Rates');
 197         $this->deps['Netsuite_PurchaseOrder'] = array('Netsuite_Rates');
 198         $this->deps['Netsuite_ItemReceipt'] = array('Netsuite_Rates');
 199         $this->deps['Netsuite_JournalEntry'] = array('Netsuite_Rates');
 200         $this->deps['Netsuite_VendorPayment'][]  = 'Netsuite_VendorBill';
 201         $this->deps['Netsuite_CustomerPayment'][]  = 'Netsuite_CustomerRefund';
 202         $this->deps['Netsuite_CustomerRefund'][]  = 'Netsuite_CreditMemo';
 203
 204         // delay inventory transfer?
 205         $this->deps['Netsuite_InventoryTransfer'][]  = 'Netsuite_InventoryItemLocations';
 206       //   $this->deps['Netsuite_InventoryTransfer'][]  = 'Netsuite_VendorBill';
 207         // bump up customer payment
 208         //$this->deps['Netsuite_VendorBill'][]  = 'Netsuite_CustomerPayment';
 209         $this->deps['Netsuite_Check'][]  = 'Netsuite_VendorBill';
 210         $this->deps['Netsuite_VendorReturnAuthorization'][]  = 'Netsuite_VendorBill';
 211         $this->deps['Netsuite_Check'][]  = 'Netsuite_VendorBill';
 212         $this->deps['Netsuite_VendorCredit'][]  = 'Netsuite_VendorBill';
 213
 214         $this->deps['Netsuite_Transfer'] = array( 'Netsuite_JournalEntry');
 215         $this->tables[] = 'Netsuite_Transfer';
 216
 217         $this->deps['Netsuite_Deposit'] = array( 'Netsuite_JournalEntry');
 218         $this->tables[] = 'Netsuite_Deposit';
 219
 220
 221         $this->deps['Netsuite_InventoryTransfer'][] = 'Netsuite_ItemReceipt';
 222
 223          $this->deps['Netsuite_InventoryAdjustment'][] = 'Netsuite_ItemReceipt';
 224
 225         // locations have to have customers.
 226         $this->deps['Netsuite_Location'][]  = 'Netsuite_Customer';
 227         $this->deps['Netsuite_InventoryItem'][]  = 'Netsuite_Location';
 228         $this->deps['Netsuite_ItemReceipt'][]  =  'Netsuite_InventoryItem';
 229         $this->deps['Netsuite_PurchaseOrder'][]  = 'Netsuite_InventoryItem';
 230        $this->deps['Netsuite_VendorBill'][]  = 'Netsuite_InventoryItem';
 231
 232
 233
 234         //$this->deps['Netsuite_VendorBill'][]  = 'Netsuite_Location';
 235
 236         // remove me later...- when elliot has finished it.
 237         //$this->deps['Netsuite_ExpenseReport'][]  = 'Netsuite_CustomerPayment';
 238
 239         //$this->deps['Netsuite_CreditMemo'] = array('Netsuite_InventoryItem');
 240
 241         array_unshift($this->tables, 'Netsuite_Rates');
 242         $this->tables = $this->sortDeps($this->tables);
 243
 244         /// append Netsuite_ if doesn't already exist
 245         if (empty($this->opts['table'])) {
 246             // nothing
 247         } else  if(is_string($this->opts['table'])) {
 248             $this->opts['table'] = (strrpos($this->opts['table'], 'Netsuite_') !== false) ?
 249                                     $this->opts['table'] : 'Netsuite_'.$this->opts['table'];
 250         } else if (is_array($this->opts['table'])) {
 251             foreach($this->opts['table'] as &$t) {
 252                 $t = (strrpos($t, 'Netsuite_') !== false) ? $t : 'Netsuite_'.$t;
 253             }
 254         }
 255
 256         if (!empty($this->opts['table']) && !in_array( $this->opts['table'], $this->tables)) {
 257                 die("Invalid table ! - {$this->opts['table']}\n");
 258
 259         }
 260
 261
 262         //print_R($this->tables);
 263
 264         if ($this->opts['list-order']) {
 265             echo implode("\n", $this->tables) ."\n";exit;
 266         }
 267
 268
 269         /// make sure all our accounting periods are open...
 270
 271         $per = DB_DataObject::Factory('period');
 272         $per->query('UPDATE period set period_closed=false');
 273
 274
 275         // remove invalid taxzones..
 276
 277         $per->query("delete from taxass where taxass_taxzone_id = (
 278                     SELECT taxzone_id FROM taxzone where taxzone_descrip='State Sales Tax Authority'
 279             )");
 280
 281         $per->query("delete from taxzone where taxzone_descrip='State Sales Tax Authority'");
 282
 283         $wkey = false;
 284         if (!empty($this->opts['where'])) {
 285
 286             if (empty($this->opts['table'])) {
 287                 die("you must specify table if using --where\n");
 288             }
 289
 290             list($wkey, $wval) = explode('=', $this->opts['where']);
 291         }
 292
 293         $y = $this->opts['year'];
 294
 295
 296         $done = array();
 297         foreach($this->tables as $tbl) {
 298
 299             $n = preg_replace('/^Netsuite_/i', '', $tbl);
 300
 301             // skip these tables
 302             if ($n == 'TransactionApply') {
 303                 // we actually use it for all the transaction based tables,
 304                 // and our dump code now merges it into the parent transactions.
 305                 continue;
 306             }
 307
 308
 309
 310
 311
 312             // should we use map() for this?
 313             $f =  $this->opts['source'] ."/{$tbl}.sql.json";
 314             $all = file_exists($f.'.all') ;
 315             $f = $all ? $f.'.all' : $f.'.'.$y;
 316             $yy = $all ? 'all' : $y;
 317             echo "Migrating $tbl from $f\n";
 318
 319
 320             if (empty($this->opts['table'])) {
 321
 322
 323
 324
 325                 $export_run = false;
 326
 327
 328                 foreach (array( 2008,2009,2010,2011,2012) as $yr) {
 329                     $f =  $this->opts['source'] ."/{$tbl}.sql.json";
 330                     $cyr = $all ? 'all' : $yr;
 331                     $f = $all ? $f.'.all' : $f.'.'.$yr;
 332                     $cache = $cd ."/" .md5($f) . "-{$tbl}-{$cyr}.sql.cache";
 333
 334                     $sourcetime = file_exists($f) ? filemtime($f)  : 0;
 335
 336                     $is_other = false;
 337                     if ($n == 'Transfer' || $n == 'Deposit') {
 338                        // no checks done..
 339                     } else {
 340
 341
 342                         if (!$sourcetime && !$is_other) {
 343                             echo "SKIP FILE - does not exists : $f\n";
 344                             continue;
 345
 346                         }
 347
 348                         $migtime = file_exists(dirname(__FILE__) ."/Migration/{$n}.php") ?
 349                                 filemtime(dirname(__FILE__) ."/Migration/{$n}.php") : 0;
 350                         $cachetime = file_exists($cache) && filesize($cache) ? filemtime($cache) : 0;
 351
 352                         print_R(array( $f=> $sourcetime, 'PHP'=>$migtime,$cache => $cachetime));
 353                         if ($cachetime >= $sourcetime && $cachetime  >= $migtime) {
 354                             echo "SKIP FILE UP TO DATE? $tbl : $yr \n";
 355                             continue;
 356                         }
 357                         if (!$migtime) {
 358                             echo "SKIP TABLE - No convertor exists $tbl : $yr \n";
 359                             continue;
 360                         }
 361
 362
 363                         if (!$export_run) {
 364
 365                             $this->runExport($tbl);
 366
 367                             $export_run = true;
 368
 369                         }
 370                     }
 371
 372                     echo "RUNNING AS CACHE MISSING / OUT OF DATE: $cache \n";
 373                     $cmd = implode(' ', array(
 374                         '/usr/bin/php', // php
 375                         $_SERVER['SCRIPT_NAME'], // hk.php
 376                         'Xtuple/Migrate',
 377                         '-t ' . $tbl,
 378                         '-y ' . $yr,
 379                         '-f ' . $this->opts['source']
 380                     ));
 381                     echo "\n\n-------------------------\n$cmd\n-----------------\n\n";
 382                     $ret = 0;
 383                     passthru( $cmd , $ret );
 384                     $ret = (int) $ret;
 385                     if ($ret !== 4  ) {
 386                         var_dump($ret);
 387                         die("FAILED ");
 388
 389                     }
 390                     if ($all) {
 391                         break;
 392                     }
 393                 }
 394                 continue;
 395
 396             }
 397
 398
 399
 400
 401             $cache = $cd ."/" .md5($f) . "-{$tbl}-{$yy}.sql.cache";
 402
 403
 404             if ($n == 'Transfer' || $n == 'Deposit') {
 405                 if (!empty($this->opts['table']) &&  $this->opts['table'] != $tbl) {
 406                     continue;
 407
 408                 }
 409                 if (file_exists($cache)) {
 410                 //    continue;
 411                 }
 412                 //DB_DataObject::DebugLevel(1);
 413                 $cl = $this->factory($n);
 414                 $cl->postMigrate();
 415                 touch($cache);
 416
 417                 echo  "Completed ". $this->opts['table'] . "\n";
 418                 exit(4);
 419                 continue;
 420
 421
 422
 423             }
 424
 425
 426
 427             if (!file_exists($f)) {
 428                 $this->updateReport($n, "Skipped -  no dump exists ");
 429                 echo "Skip $n - no dump exists $f \n";
 430                 echo "creating empty cache $cache\n";
 431                 $done[] = $tbl;
 432                 file_put_contents($cache, serialize(array()));
 433
 434                  if (!empty($this->opts['table']) && $this->opts['table'] == $tbl) {
 435                     echo "Completed ". $this->opts['table'] . "\n";
 436                     exit(4);
 437                 }
 438
 439                 continue;
 440             }
 441              //var_dump($this->opts);
 442
 443             if ($this->opts['debug']) {
 444
 445                 DB_DataObject::debugLevel( $this->opts['debug'] );
 446             }
 447
 448             $cl = $this->factory($n);
 449
 450             if (!$cl) {
 451                 $this->updateReport($n, "Skipped -  no migration tool exists");
 452                 if (!empty($this->opts['table']) && $this->opts['table'] == $tbl) {
 453                     echo "Skipped -  no migration tool exists ". $this->opts['table'] . "\n";
 454                     exit(4);
 455                 }
 456                 continue;
 457             }
 458
 459             $doimport = empty($this->opts['table']) || $this->opts['table'] == $tbl;
 460
 461             $usecache = !$nocache;
 462             if (!empty($this->opts['table'])) {
 463                 if ($this->opts['table'] == $tbl) {
 464                     $usecache = false;
 465                 }
 466             }
 467
 468
 469
 470             $sourcetime = filemtime($f);
 471             $migtime = filemtime(dirname(__FILE__) ."/Migration/{$n}.php");
 472             $cachetime = file_exists($cache) && filesize($cache) ? filemtime($cache) : 0;
 473
 474
 475             if ($usecache && $cachetime >= $sourcetime && $cachetime  >= $migtime) {
 476                 // use cache..
 477
 478                 $this->map($n);
 479                 //$doimport ?  $cl->validateUsed((array)json_decode($data[1])) : false;
 480                 $done[] = $tbl;
 481
 482                 $this->updateReport($n, false);
 483
 484
 485                 echo "Migrating $tbl - using cache : $cache \n";
 486                 continue;
 487             }
 488
 489
 490             if ($nocache &&  $cachetime < $sourcetime) {
 491                 echo "Migrating $tbl - cache to old, or does not exist\n";
 492             }
 493             if ($nocache && $cachetime  < $migtime) {
 494                 echo "Migrating $tbl - migration code was updated\n";
 495             }
 496
 497
 498             // finally do the migration..
 499
 500
 501             $cl->validateOutputSupport();
 502
 503             $data = file($f);
 504
 505             $missing_dep  = false;
 506             foreach( $this->deps[$tbl] as $dep) {
 507                 if (in_array($dep, $done)) {
 508                     continue;
 509                 }
 510                 echo "Checking for dep $dep\n";
 511                 if ($this->map(preg_replace('/^Netsuite_/', '', $dep))) {
 512                     continue;
 513                 }
 514                 $missing_dep = $dep;
 515                 break;
 516
 517             }
 518
 519
 520
 521
 522             if ($missing_dep !== false) {
 523                 echo "Skip $n - dependant table $missing_dep has not been done yet.\n";
 524                 $this->updateReport($n, "Skip dependant table $missing_dep has not been done yet.");
 525
 526                 if (!empty($this->opts['table']) && $this->opts['table'] == $tbl) {
 527                     echo "Failed  ". $this->opts['table'] . "\n";
 528                     exit(3); // error condtion..
 529                 }
 530                 exit(3);
 531                 continue;
 532             }
 533             if (!$doimport ) {
 534                 echo "Skip $n - no specified by --table .\n";
 535                 $this->updateReport($n, false);
 536                 if (!empty($this->opts['table']) && $this->opts['table'] == $tbl) {
 537                     echo  "Completed ". $this->opts['table'] . "\n";
 538                     exit(4);
 539                 }
 540
 541                 continue;
 542             }
 543
 544
 545
 546
 547             $cl->validateUsed((array)json_decode($data[1]));
 548             $def = false;
 549             $this->map[$n] = array();
 550             $total = count($data);
 551             echo "Migrating $total Records\n";
 552             $last = 0;
 553             foreach($data as $i=> $l) {
 554
 555
 556                 if (empty($cl->def)) { // first line is default.s
 557                     $cl->def = $l;
 558                     continue;
 559                 }
 560                 // print out some progress.
 561                 if ($i % 500 == 0) {
 562                     echo floor(($i/$total) * 100)."%\n";
 563                 }
 564
 565                 $row = (array)json_decode($l);
 566
 567                 // testing a single entry..
 568                 if ($wkey) {
 569                     if (!isset($row[$wkey])) {
 570                         die("--where condition is invalid, key '$wkey' does not exist\n");
 571                     }
 572                     if ($row[$wkey] != $wval) {
 573                         continue;
 574                     }
 575                     if (!empty($opts['print'])) {
 576                         print_R($row);
 577                     }
 578                     //DB_DataObject::debugLevel(1);
 579
 580                 }
 581
 582
 583
 584                 $add = $cl->migrate($row);
 585                 if ($wkey) {
 586                     echo "Migrate return\n";
 587                     var_dump($add);
 588                 }
 589                 // rates does not have an id col.
 590                 if ($add !== false )  {
 591                     //echo "SAVEMAP  {$row['id']} :  $add \n";
 592                     $cl->saveMap(isset($row['id']) ? $row['id'] : $i, $add);
 593                 }
 594
 595                 // free up some memory..
 596                 $GLOBALS['_DB_DATAOBJECT']['RESULTFIELDS'] = array();
 597                 $GLOBALS['_DB_DATAOBJECT']['RESULTS'] = array();
 598
 599
 600
 601             }
 602
 603
 604             //var_Dump($n);
 605             // works ok here..
 606             //print_R(array_keys($this->map));
 607             echo "Migrating $tbl - ". count(array_keys($this->map[$n])) . " Records added\n";
 608
 609
 610
 611             $cl->postMigrate();
 612
 613             if ($wkey) {
 614                 die("ONLY RUNNING IN TESTING MODE with
 615                     --where condition exiting before any damage is done..\n");
 616             }
 617
 618
 619             $str = $cl->resultsToString();
 620
 621             $this->updateReport($n, $cl->resultsToString());
 622
 623
 624
 625             if ($str !== false) {
 626                 file_put_contents($cd ."/{$tbl}-{$yy}.results.txt", $str);
 627                 echo "RESULTS IN {$cd}/{$tbl}-{$yy}.results.txt\n";
 628             }
 629             // sorting is pretty irrelivant, and screws things up horriably anyway.
 630             //ksort($this->map[$n],SORT_STRING);
 631             //var_dump($this->map[$n][3263]);
 632
 633
 634             file_put_contents($cache, serialize($this->map[$n]));
 635             echo "CACHED DATA IN php -r 'print_r(unserialize(file_get_contents(\"{$cache}\")));'\n";
 636             touch($cache, max($sourcetime, $migtime, filemtime($cache)));
 637             $done[] = $tbl;
 638
 639             if (!empty($this->opts['table']) && $this->opts['table'] == $tbl) {
 640
 641                 //print_r($this->map[$n]);
 642
 643                 //var_dump($this->map[$n][3263]);
 644
 645                 echo  "Completed ". $this->opts['table'] . "\n";
 646                 exit(4);
 647             }
 648             die("\nRun again - to import more\n");
 649         }
 650         die("DONE");
 651         //die("DONE\n");
 652
 653     }
 654
 655     function updateReport($n, $res)
 656     {
 657         $cd = $this->cachedir();
 658
 659         $f = $this->opts['source'] ."/Netsuite_{$n}.sql.json";
 660
 661         // we need to update individual reports, and a 'merged one'..
 662
 663         if ($res !== false) {
 664             file_put_contents( "{$cd}/" .md5($f) . "-Netsuite_{$n}.results.txt", $res);
 665         } else {
 666             $res = file_exists("{$cd}/" .md5($f) . "-Netsuite_{$n}.results.txt") ?
 667                 file_get_contents("{$cd}/" .md5($f) . "-Netsuite_{$n}.results.txt") :
 668                 "No results available";
 669         }
 670
 671         static $latest = false;
 672         if (!$latest) {
 673             $latest = "{$cd}/latest.results.txt";
 674             file_put_contents($latest, "Import Report - " . date("Y-m-d H:i:s"). "\n\n");
 675         }
 676         $fh = fopen($latest, 'a');
 677
 678         fwrite($fh, "RESULTS FOR: $n:\n");
 679         fwrite($fh, "$res\n\n");
 680         fclose($fh);
 681
 682
 683
 684
 685     }
 686
 687     function buildDeps()
 688     {
 689         $deps = array();
 690
 691
 692
 693         $ini = parse_ini_file(dirname(__FILE__).'/Migration/netsuite.links.ini', true);
 694
 695         foreach($ini as $tbl => $maps) {
 696              if (!in_array($tbl, $this->tables)) {
 697                 $this->tables[] = $tbl;
 698                 $deps[$tbl] = array();
 699             }
 700
 701             foreach($maps as $col=>$to) {
 702                 $kv = explode(':', $to);
 703
 704                 if (!in_array($kv[0], $this->tables)) {
 705                     $this->tables[] = $kv[0];
 706                     $deps[$kv[0]] = array();
 707                 }
 708
 709                 if (in_array($kv[0], $deps[$tbl])) {
 710                     continue;
 711                 }
 712                 $deps[$tbl][] = $kv[0];
 713             }
 714
 715         }
 716
 717
 718         $this->deps = $deps;
 719
 720
 721     }
 722     /**
 723      *
 724      * has the map been loaded..
 725      */
 726     var $loaded = array();
 727
 728     function map($n)
 729     {
 730
 731         if (isset($this->loaded[$n])) {
 732             return true;
 733         }
 734         $this->loaded[$n] = true;
 735
 736         echo "RESET MAP FOR $n\n";
 737         $this->map[$n] = array();
 738         $cd = $this->cachedir();
 739
 740         $years = array('all', 2008,2009,2010,2011,2012);
 741
 742         $tbl = 'Netsuite_'. $n;
 743         // make sure teh map is loaded for a class..
 744         $ret = false;
 745         foreach($years as $yy) {
 746
 747
 748             $f =  $this->opts['source'] ."/{$tbl}.sql.json.". $yy;
 749
 750             $cache = $cd ."/" .md5($f) . "-{$tbl}-{$yy}.sql.cache";
 751
 752             if (!file_exists($cache) || !filesize($cache)) {
 753                 echo "NO CACHE $cache\n";
 754                 continue;
 755             }
 756
 757             $ret = true;
 758             echo "LOADED CACHE $cache\n";
 759             $this->map[$n] +=   unserialize(file_get_contents($cache));
 760             if ($yy == 'all') {
 761                 break;
 762             }
 763         }
 764         if (!$ret) {
 765             unset($this->map[$n]);
 766         }
 767         return $ret;
 768     }
 769
 770     function sortDeps($ar)
 771     {
 772         // this could be done by usort, but we need to build a reverse dependancy list as well.
 773
 774         // loop through an array,
 775         // if we can add an item - add it, otherwise push it to the end..
 776
 777         $ret = array();
 778         $done = array();
 779         for($i = 0  ; $i < count($ar); $i++ )
 780         {
 781             $ltest = array();
 782             while (true)
 783             {
 784                 $t = $ar[$i];
 785                 if (isset($ltest[$t])) {
 786                     echo "DEPENDANCIE FAILED FOR $t";
 787                     exit;
 788
 789                 }
 790                 // no dependancies.. it can be added..
 791                 if (empty($this->deps[$t])) {
 792                     $done[$t] = true;
 793                     break;
 794                 }
 795
 796                 // have all the dependancies been added..
 797                 $missing = false;
 798                 foreach($this->deps[$t] as $dep) {
 799                     if (!isset($done[$dep])) {
 800                         $missing = true;
 801                         break;
 802                     }
 803                 }
 804                 if (!$missing) {
 805                     $done[$t] = true;
 806                     break;
 807                 }
 808                 // a dependancy is missing..
 809                 // move this element to the end..
 810
 811                 unset($ar[$i]);
 812                 $ar = array_values($ar); // renumber...
 813                 $ar[] = $t;
 814                 $ltest[$t] = true;
 815                 // try ataing..
 816             }
 817
 818
 819
 820         }
 821         return $ar;
 822
 823     }
 824
 825
 826     function factory($n)
 827     {
 828         $parts = explode('/', $n);
 829         $n = $parts[0];
 830         $cf = dirname(__FILE__) ."/Migration/{$n}.php";
 831         $cn = "Pman_Xtuple_Migration_{$n}";
 832
 833         if (!file_exists($cf)) {
 834             echo "Skip $n - no file '$cf' exists yet\n";
 835             return false;
 836         }
 837         require_once $cf;
 838         if (!class_exists($cn)) {
 839             echo "SKIP class does not exist in file: $cn\n";
 840             return false;
 841         }
 842         $cl = new $cn($this, isset($parts[1]) ? $parts[1] : false);
 843
 844         return $cl;
 845
 846     }
 847
 848     function onPearError($err)
 849     {
 850
 851         $out = $err->toString();
 852
 853
 854         //print_R($bt); exit;
 855         $ret = array();
 856         foreach($err->backtrace as $b) {
 857             $ret[] = $b['file'] . '(' . $b['line'] . ')@' .   @$bt['class'] . '::' . @$bt['function'];
 858         }
 859         //convert the huge backtrace into something that is readable..
 860         $out .= "\n" . implode("\n",  $ret);
 861
 862
 863         echo $out;
 864
 865         echo "\n-- FAILED --\n";
 866         exit(3);
 867
 868
 869
 870     }
 871     function runExport($tbl)
 872     {
 873         echo "RUNNING EXPORT to ensure data is up-to-date: $tbl \n";
 874         $country = strtoupper($this->department);
 875         // this is in Pman.Xtuple.Migrate directory now..
 876         $cmd = implode(' ', array(
 877             '/usr/bin/php', // php
 878             __DIR__  . '/../web.Netsuite/index.php',
 879             'Netsuite/Dump',
 880             '-t ' . $tbl,
 881             '-c ' . $country
 882         ));
 883         echo "\n\n-------------------------\n$cmd\n-----------------\n\n";
 884         //exit;
 885         //$ret = 0;
 886         passthru( $cmd , $ret );
 887
 888     }
 889
 890
 891
 892 }