Import/Core_geoip.php

[Pman.Core] / Import / Core_geoip.php

diff --git a/Import/Core_geoip.php b/Import/Core_geoip.php
index 103b802..567e000 100644 (file)
--- a/Import/Core_geoip.php
+++ b/Import/Core_geoip.php
@@ -9,7 +9,7 @@ class Pman_Core_Import_Core_geoip extends Pman_Roo
     static $cli_opts = array();
     
     var $id_mapping = array();
-    
+
     function getAuth()
     {
         $ff = HTML_FlexyFramework::get();
@@ -25,6 +25,10 @@ class Pman_Core_Import_Core_geoip extends Pman_Roo
         $this->get();
     }
     
+    var $processed = 0;
+    var $total = 0;
+    var $echo = '';
+    
     function get()
     {
         
@@ -37,16 +41,25 @@ class Pman_Core_Import_Core_geoip extends Pman_Roo
             $this->jerr('GeoLite2-City-Locations.csv OR GeoLite2-City-Blocks.csv does not exists?!');
         }
         
+        $this->log("Insert location data start");
+        
         $this->insertLocation($location);
         
+        $this->log("Insert Block data start");
+        
         $this->insertBlock($block);
+        
+        $this->jok("DONE");
     }
     
     function insertLocation($csv)
     {
         ini_set("auto_detect_line_endings", true);
         
+        
+        
         $fh = fopen($csv, 'r');
+        
         if (!$fh) {
             $this->jerr("invalid location file");
         }
@@ -60,6 +73,10 @@ class Pman_Core_Import_Core_geoip extends Pman_Roo
         
         $cols = false;
         
+        $this->processed = 0;
+        $this->total = count(file($csv));
+        $this->echo = '';
+        
         while(false !== ($n = fgetcsv($fh,10000, ',', '"'))) {
             if(!array_filter($n)){ // empty row
                 continue;
@@ -90,12 +107,68 @@ class Pman_Core_Import_Core_geoip extends Pman_Roo
             }
             
             $this->processLocation($row);
+            
+            $this->processStatus();
         }
         
     }
     
-    function processLocation($row)
+    function insertBlock($csv)
     {
+        ini_set("auto_detect_line_endings", true);
+        
+        $fh = fopen($csv, 'r');
+        if (!$fh) {
+            $this->jerr("invalid location file");
+        }
+        
+        $req = array(
+            'NETWORK_START_IP', 'NETWORK_MASK_LENGTH', 'GEONAME_ID',
+            'REGISTERED_COUNTRY_GEONAME_ID', 'REPRESENTED_COUNTRY_GEONAME_ID', 'POSTAL_CODE',
+            'LATITUDE', 'LONGITUDE', 'IS_ANONYMOUS_PROXY',
+            'IS_SATELLITE_PROVIDER'
+        );
+        
+        $cols = false;
+        
+        while(false !== ($n = fgetcsv($fh,10000, ',', '"'))) {
+            if(!array_filter($n)){ // empty row
+                continue;
+            }
+            
+            if (!$cols) {
+                $cols = array();
+                foreach($n as $k) {
+                    $cols[] = strtoupper(trim($k));
+                }
+                
+                if (empty($cols)) {
+                    continue;
+                }
+                foreach($req as $r) {
+                    if (!in_array($r,$cols)) {
+                        $cols = false;
+                        break;
+                    }
+                }
+                continue;
+            }
+            
+            $row = array();
+            
+            foreach($cols as $i=>$k) {
+                $row[$k] = trim($n[$i]);
+            }
+            
+            $this->processBlock($row);
+        }
+        
+    }
+    
+    
+    
+    function processLocation($row)
+    {   
         $continent = $this->processContinent($row['CONTINENT_CODE'], $row['CONTINENT_NAME']);
         
         $country = $this->processCountry($row['COUNTRY_ISO_CODE'], $row['COUNTRY_NAME'], $continent);
@@ -193,10 +266,69 @@ class Pman_Core_Import_Core_geoip extends Pman_Roo
         
     }
     
+    function processBlock($row)
+    {
+        if(empty($this->id_mapping[$row['GEONAME_ID']])){
+            $this->log("Missing mapping for {$row['GEONAME_ID']}");
+            $this->log("IP : {$row['NETWORK_START_IP']}");
+            return;
+        }
+        
+        $network_mapping = DB_DataObject::factory('core_geoip_network_mapping');
+        
+        $start_ip = array_pop(explode(":", $row['NETWORK_START_IP']));
+        
+        $network_mapping->setFrom(array(
+            'start_ip' => $start_ip,
+            'mask_length' => pow(2, (128 - $row['NETWORK_MASK_LENGTH'])),
+            'city_id' => $this->id_mapping[$row['GEONAME_ID']]
+        ));
+        
+        if(!$network_mapping->find(true)){
+            $network_mapping->insert();
+        }
+        
+        $location = DB_DataObject::factory('core_geoip_location');
+        if(!$location->get('city_id', $network_mapping->city_id)){
+            $location->setFrom(array(
+                'latitude' => $row['LATITUDE'],
+                'longitude' => $row['LONGITUDE'],
+                'city_id' => $network_mapping->city_id
+            ));
+        }
+        
+        
+        if(!empty($row['POSTAL_CODE'])){
+            $city = DB_DataObject::factory('core_geoip_city');
+            if($city->get($network_mapping->city_id)){
+                return;
+            }
+
+            $oc = clone($city);
+            $city->postal_code = $row['POSTAL_CODE'];
+            
+            $city->update($oc);
+        }
+        
+    }
     
-    
-    function log($str)
+    function processStatus()
     {
-        echo "$str \n";
+        echo "\033[K"; // Erase to end of line:
+            
+        if (strlen($this->echo)) {
+            echo "\033[".strlen($this->echo)."D";    // Move $length characters backward
+        }
+
+        $this->echo = str_pad(ROUND(($this->processed / $this->total),2) * 100, 3, ' ', STR_PAD_LEFT) .
+            " % (" . str_pad(($this->processed), strlen($this->total), ' ', STR_PAD_LEFT) .
+            " / {$this->total})";
+
+
+        echo $this->echo;
+
+        if($this->processed == $this->total){
+            echo "\n";
+        }
     }
 }