git.roojs.org Git - Pman.Core/blob - Import/Core_geoip.php

   1 <?php
   2
   3 require_once 'Pman/Roo.php';
   4
   5 class Pman_Core_Import_Core_geoip extends Pman_Roo
   6 {
   7     static $cli_desc = "Insert the geoip database";
   8
   9     static $cli_opts = array();
  10
  11     var $id_mapping = array();
  12
  13     function getAuth()
  14     {
  15         $ff = HTML_FlexyFramework::get();
  16         if (!$ff->cli) {
  17             die("access denied");
  18         }
  19         HTML_FlexyFramework::ensureSingle(__FILE__, $this);
  20         return true;
  21     }
  22
  23     function post()
  24     {
  25         $this->get();
  26     }
  27
  28     var $processed = 0;
  29     var $total = 0;
  30     var $echo = '';
  31
  32     function get()
  33     {
  34
  35         PEAR::setErrorHandling(PEAR_ERROR_CALLBACK, array($this, 'onPearError'));
  36
  37         $location = '/tmp/GeoLite2-City-Locations.csv';
  38         $block = '/tmp/GeoLite2-City-Blocks.csv';
  39
  40         if(!file_exists($location) || !file_exists($block)){
  41             $this->jerr('GeoLite2-City-Locations.csv OR GeoLite2-City-Blocks.csv does not exists?!');
  42         }
  43
  44         $this->log("Insert location data start");
  45
  46         $this->insertLocation($location);
  47
  48         $this->log("Insert Block data start");
  49
  50         $this->insertBlock($block);
  51
  52         $this->jok("DONE");
  53     }
  54
  55     function insertLocation($csv)
  56     {
  57         ini_set("auto_detect_line_endings", true);
  58
  59         $linecount = count(file($csv));
  60
  61         $fh = fopen($csv, 'r');
  62
  63         if (!$fh) {
  64             $this->jerr("invalid location file");
  65         }
  66
  67         $req = array(
  68             'GEONAME_ID', 'CONTINENT_CODE', 'CONTINENT_NAME',
  69             'COUNTRY_ISO_CODE', 'COUNTRY_NAME', 'SUBDIVISION_ISO_CODE',
  70             'SUBDIVISION_NAME', 'CITY_NAME', 'METRO_CODE',
  71             'TIME_ZONE'
  72         );
  73
  74         $cols = false;
  75
  76         $processed = 0
  77
  78         while(false !== ($n = fgetcsv($fh,10000, ',', '"'))) {
  79             if(!array_filter($n)){ // empty row
  80                 continue;
  81             }
  82
  83             if (!$cols) {
  84                 $cols = array();
  85                 foreach($n as $k) {
  86                     $cols[] = strtoupper(trim($k));
  87                 }
  88
  89                 if (empty($cols)) {
  90                     continue;
  91                 }
  92                 foreach($req as $r) {
  93                     if (!in_array($r,$cols)) {
  94                         $cols = false;
  95                         break;
  96                     }
  97                 }
  98                 continue;
  99             }
 100
 101             $row = array();
 102
 103             foreach($cols as $i=>$k) {
 104                 $row[$k] = trim($n[$i]);
 105             }
 106
 107             $this->processLocation($row);
 108
 109             echo "\033[K"; // Erase to end of line:
 110
 111             $echo = '';
 112
 113             if (strlen($echo)) {
 114                 echo "\033[".strlen($echo)."D";    // Move $length characters backward
 115             }
 116
 117             $echo = str_pad(ROUND(($this->processed / $this->total),2) * 100, 3, ' ', STR_PAD_LEFT) .
 118                 " % (" . str_pad(($this->processed), strlen($this->total), ' ', STR_PAD_LEFT) .
 119                 " / {$this->total}) - out {$this->out_count}";
 120
 121
 122             echo $this->echo;
 123
 124             if($this->processed == $this->total){
 125                 echo "\n";
 126             }
 127         }
 128
 129     }
 130
 131     function insertBlock($csv)
 132     {
 133         ini_set("auto_detect_line_endings", true);
 134
 135         $fh = fopen($csv, 'r');
 136         if (!$fh) {
 137             $this->jerr("invalid location file");
 138         }
 139
 140         $req = array(
 141             'NETWORK_START_IP', 'NETWORK_MASK_LENGTH', 'GEONAME_ID',
 142             'REGISTERED_COUNTRY_GEONAME_ID', 'REPRESENTED_COUNTRY_GEONAME_ID', 'POSTAL_CODE',
 143             'LATITUDE', 'LONGITUDE', 'IS_ANONYMOUS_PROXY',
 144             'IS_SATELLITE_PROVIDER'
 145         );
 146
 147         $cols = false;
 148
 149         while(false !== ($n = fgetcsv($fh,10000, ',', '"'))) {
 150             if(!array_filter($n)){ // empty row
 151                 continue;
 152             }
 153
 154             if (!$cols) {
 155                 $cols = array();
 156                 foreach($n as $k) {
 157                     $cols[] = strtoupper(trim($k));
 158                 }
 159
 160                 if (empty($cols)) {
 161                     continue;
 162                 }
 163                 foreach($req as $r) {
 164                     if (!in_array($r,$cols)) {
 165                         $cols = false;
 166                         break;
 167                     }
 168                 }
 169                 continue;
 170             }
 171
 172             $row = array();
 173
 174             foreach($cols as $i=>$k) {
 175                 $row[$k] = trim($n[$i]);
 176             }
 177
 178             $this->processBlock($row);
 179         }
 180
 181     }
 182
 183
 184
 185     function processLocation($row)
 186     {
 187         $continent = $this->processContinent($row['CONTINENT_CODE'], $row['CONTINENT_NAME']);
 188
 189         $country = $this->processCountry($row['COUNTRY_ISO_CODE'], $row['COUNTRY_NAME'], $continent);
 190
 191         $division = $this->processDivision($row['SUBDIVISION_ISO_CODE'], $row['SUBDIVISION_NAME']);
 192
 193         $city = $this->processCity($row['CITY_NAME'], $row['METRO_CODE'], $row['TIME_ZONE'], $country, $division);
 194
 195         if(!empty($city) && !empty($city->id)){
 196             $this->id_mapping[$row['GEONAME_ID']] = $city->id;
 197         }
 198     }
 199
 200     function processContinent($code, $name)
 201     {
 202         if(empty($code)){
 203             return false;
 204         }
 205
 206         $continent = DB_DataObject::factory('core_geoip_continent');
 207         if(!$continent->get('code', $code)){
 208             $continent->setFrom(array(
 209                 'code' => $code,
 210                 'name' => (!empty($name)) ? $name : $code
 211             ));
 212
 213             $continent->insert();
 214         }
 215
 216         return $continent;
 217     }
 218
 219     function processCountry($code, $name, $continent)
 220     {
 221         if(empty($code)){
 222             return false;
 223         }
 224
 225         $country = DB_DataObject::factory('core_geoip_country');
 226         if(!$country->get('code', $code)){
 227             $country->setFrom(array(
 228                 'code' => $code,
 229                 'name' => (!empty($name)) ? $name : $code,
 230                 'continent_id' => (!empty($continent) && !empty($continent->id)) ? $continent->id : 0
 231             ));
 232
 233             $country->insert();
 234         }
 235
 236         return $country;
 237     }
 238
 239     function processDivision($code, $name)
 240     {
 241         if(empty($code)){
 242             return false;
 243         }
 244
 245         $division = DB_DataObject::factory('core_geoip_division');
 246         if(!$division->get('code', $code)){
 247             $division->setFrom(array(
 248                 'code' => $code,
 249                 'name' => (!empty($name)) ? $name : $code
 250             ));
 251
 252             $division->insert();
 253         }
 254
 255         return $division;
 256     }
 257
 258     function processCity($name, $metro_code, $time_zone, $country, $division)
 259     {
 260         if(empty($name)){
 261             return false;
 262         }
 263
 264         $city = DB_DataObject::factory('core_geoip_city');
 265
 266         if($city->get('name', $name)){
 267             return $city;
 268         }
 269
 270         $city->setFrom(array(
 271             'name' => $name,
 272             'metro_code' => $metro_code,
 273             'time_zone' => $time_zone,
 274             'country_id' => (!empty($country) && !empty($country->id)) ? $country->id : 0,
 275             'division_id' => (!empty($division) && !empty($division->id)) ? $division->id : 0
 276         ));
 277
 278         $city->insert();
 279
 280         return $city;
 281
 282     }
 283
 284     function processBlock($row)
 285     {
 286         if(empty($this->id_mapping[$row['GEONAME_ID']])){
 287             $this->log("Missing mapping for {$row['GEONAME_ID']}");
 288             $this->log("IP : {$row['NETWORK_START_IP']}");
 289             return;
 290         }
 291
 292         $network_mapping = DB_DataObject::factory('core_geoip_network_mapping');
 293
 294         $start_ip = array_pop(explode(":", $row['NETWORK_START_IP']));
 295
 296         $network_mapping->setFrom(array(
 297             'start_ip' => $start_ip,
 298             'mask_length' => pow(2, (128 - $row['NETWORK_MASK_LENGTH'])),
 299             'city_id' => $this->id_mapping[$row['GEONAME_ID']]
 300         ));
 301
 302         if(!$network_mapping->find(true)){
 303             $network_mapping->insert();
 304         }
 305
 306         $location = DB_DataObject::factory('core_geoip_location');
 307         if(!$location->get('city_id', $network_mapping->city_id)){
 308             $location->setFrom(array(
 309                 'latitude' => $row['LATITUDE'],
 310                 'longitude' => $row['LONGITUDE'],
 311                 'city_id' => $network_mapping->city_id
 312             ));
 313         }
 314
 315
 316         if(!empty($row['POSTAL_CODE'])){
 317             $city = DB_DataObject::factory('core_geoip_city');
 318             if($city->get($network_mapping->city_id)){
 319                 return;
 320             }
 321
 322             $oc = clone($city);
 323             $city->postal_code = $row['POSTAL_CODE'];
 324
 325             $city->update($oc);
 326         }
 327
 328     }
 329
 330     function log($str)
 331     {
 332         echo "$str \n";
 333     }
 334 }