git.roojs.org Git - Pman.Core/blob - Import/Core_geoip.php

   1 <?php
   2
   3 require_once 'Pman/Roo.php';
   4
   5 class Pman_Core_Import_Core_geoip extends Pman_Roo
   6 {
   7     static $cli_desc = "Insert the geoip database";
   8
   9     static $cli_opts = array();
  10
  11     var $id_mapping = array();
  12
  13     function getAuth()
  14     {
  15         $ff = HTML_FlexyFramework::get();
  16         if (!$ff->cli) {
  17             die("access denied");
  18         }
  19         HTML_FlexyFramework::ensureSingle(__FILE__, $this);
  20         return true;
  21     }
  22
  23     function post()
  24     {
  25         $this->get();
  26     }
  27
  28     function get()
  29     {
  30
  31         PEAR::setErrorHandling(PEAR_ERROR_CALLBACK, array($this, 'onPearError'));
  32
  33         $location = '/tmp/GeoLite2-City-Locations.csv';
  34         $block = '/tmp/GeoLite2-City-Blocks.csv';
  35
  36         if(!file_exists($location) || !file_exists($block)){
  37             $this->jerr('GeoLite2-City-Locations.csv OR GeoLite2-City-Blocks.csv does not exists?!');
  38         }
  39
  40         $this->insertLocation($location);
  41
  42         $this->insertBlock($block);
  43
  44         $this->jok("DONE");
  45     }
  46
  47     function insertLocation($csv)
  48     {
  49         ini_set("auto_detect_line_endings", true);
  50
  51         $fh = fopen($csv, 'r');
  52         if (!$fh) {
  53             $this->jerr("invalid location file");
  54         }
  55
  56         $req = array(
  57             'GEONAME_ID', 'CONTINENT_CODE', 'CONTINENT_NAME',
  58             'COUNTRY_ISO_CODE', 'COUNTRY_NAME', 'SUBDIVISION_ISO_CODE',
  59             'SUBDIVISION_NAME', 'CITY_NAME', 'METRO_CODE',
  60             'TIME_ZONE'
  61         );
  62
  63         $cols = false;
  64
  65         while(false !== ($n = fgetcsv($fh,10000, ',', '"'))) {
  66             if(!array_filter($n)){ // empty row
  67                 continue;
  68             }
  69
  70             if (!$cols) {
  71                 $cols = array();
  72                 foreach($n as $k) {
  73                     $cols[] = strtoupper(trim($k));
  74                 }
  75
  76                 if (empty($cols)) {
  77                     continue;
  78                 }
  79                 foreach($req as $r) {
  80                     if (!in_array($r,$cols)) {
  81                         $cols = false;
  82                         break;
  83                     }
  84                 }
  85                 continue;
  86             }
  87
  88             $row = array();
  89
  90             foreach($cols as $i=>$k) {
  91                 $row[$k] = trim($n[$i]);
  92             }
  93
  94             $this->processLocation($row);
  95         }
  96
  97     }
  98
  99     function insertBlock($csv)
 100     {
 101         ini_set("auto_detect_line_endings", true);
 102
 103         $fh = fopen($csv, 'r');
 104         if (!$fh) {
 105             $this->jerr("invalid location file");
 106         }
 107
 108         $req = array(
 109             'NETWORK_START_IP', 'NETWORK_MASK_LENGTH', 'GEONAME_ID',
 110             'REGISTERED_COUNTRY_GEONAME_ID', 'REPRESENTED_COUNTRY_GEONAME_ID', 'POSTAL_CODE',
 111             'LATITUDE', 'LONGITUDE', 'IS_ANONYMOUS_PROXY',
 112             'IS_SATELLITE_PROVIDER'
 113         );
 114
 115         $cols = false;
 116
 117         while(false !== ($n = fgetcsv($fh,10000, ',', '"'))) {
 118             if(!array_filter($n)){ // empty row
 119                 continue;
 120             }
 121
 122             if (!$cols) {
 123                 $cols = array();
 124                 foreach($n as $k) {
 125                     $cols[] = strtoupper(trim($k));
 126                 }
 127
 128                 if (empty($cols)) {
 129                     continue;
 130                 }
 131                 foreach($req as $r) {
 132                     if (!in_array($r,$cols)) {
 133                         $cols = false;
 134                         break;
 135                     }
 136                 }
 137                 continue;
 138             }
 139
 140             $row = array();
 141
 142             foreach($cols as $i=>$k) {
 143                 $row[$k] = trim($n[$i]);
 144             }
 145
 146             $this->processBlock($row);
 147         }
 148
 149     }
 150
 151
 152
 153     function processLocation($row)
 154     {
 155         $continent = $this->processContinent($row['CONTINENT_CODE'], $row['CONTINENT_NAME']);
 156
 157         $country = $this->processCountry($row['COUNTRY_ISO_CODE'], $row['COUNTRY_NAME'], $continent);
 158
 159         $division = $this->processDivision($row['SUBDIVISION_ISO_CODE'], $row['SUBDIVISION_NAME']);
 160
 161         $city = $this->processCity($row['CITY_NAME'], $row['METRO_CODE'], $row['TIME_ZONE'], $country, $division);
 162
 163         if(!empty($city) && !empty($city->id)){
 164             $this->id_mapping[$row['GEONAME_ID']] = $city->id;
 165         }
 166     }
 167
 168     function processContinent($code, $name)
 169     {
 170         if(empty($code)){
 171             return false;
 172         }
 173
 174         $continent = DB_DataObject::factory('core_geoip_continent');
 175         if(!$continent->get('code', $code)){
 176             $continent->setFrom(array(
 177                 'code' => $code,
 178                 'name' => (!empty($name)) ? $name : $code
 179             ));
 180
 181             $continent->insert();
 182         }
 183
 184         return $continent;
 185     }
 186
 187     function processCountry($code, $name, $continent)
 188     {
 189         if(empty($code)){
 190             return false;
 191         }
 192
 193         $country = DB_DataObject::factory('core_geoip_country');
 194         if(!$country->get('code', $code)){
 195             $country->setFrom(array(
 196                 'code' => $code,
 197                 'name' => (!empty($name)) ? $name : $code,
 198                 'continent_id' => (!empty($continent) && !empty($continent->id)) ? $continent->id : 0
 199             ));
 200
 201             $country->insert();
 202         }
 203
 204         return $country;
 205     }
 206
 207     function processDivision($code, $name)
 208     {
 209         if(empty($code)){
 210             return false;
 211         }
 212
 213         $division = DB_DataObject::factory('core_geoip_division');
 214         if(!$division->get('code', $code)){
 215             $division->setFrom(array(
 216                 'code' => $code,
 217                 'name' => (!empty($name)) ? $name : $code
 218             ));
 219
 220             $division->insert();
 221         }
 222
 223         return $division;
 224     }
 225
 226     function processCity($name, $metro_code, $time_zone, $country, $division)
 227     {
 228         if(empty($name)){
 229             return false;
 230         }
 231
 232         $city = DB_DataObject::factory('core_geoip_city');
 233
 234         if($city->get('name', $name)){
 235             return $city;
 236         }
 237
 238         $city->setFrom(array(
 239             'name' => $name,
 240             'metro_code' => $metro_code,
 241             'time_zone' => $time_zone,
 242             'country_id' => (!empty($country) && !empty($country->id)) ? $country->id : 0,
 243             'division_id' => (!empty($division) && !empty($division->id)) ? $division->id : 0
 244         ));
 245
 246         $city->insert();
 247
 248         return $city;
 249
 250     }
 251
 252     function processBlock($row)
 253     {
 254         if(empty($this->id_mapping[$row['GEONAME_ID']])){
 255             $this->log("Missing mapping for {$row['GEONAME_ID']}");
 256             $this->log("IP : {$row['NETWORK_START_IP']}");
 257             return;
 258         }
 259
 260         $network_mapping = DB_DataObject::factory('core_geoip_network_mapping');
 261
 262         $start_ip = array_pop(explode(":", $row['NETWORK_START_IP']));
 263
 264         $network_mapping->setFrom(array(
 265             'start_ip' => $start_ip,
 266             'mask_length' => pow(2, (128 - $row['NETWORK_MASK_LENGTH'])),
 267             'city_id' => $this->id_mapping[$row['GEONAME_ID']]
 268         ));
 269
 270         if(!$network_mapping->find(true)){
 271             $network_mapping->insert();
 272         }
 273
 274         $location = DB_DataObject::factory('core_geoip_location');
 275         if(!$location->get('city_id', $network_mapping->city_id)){
 276             $location->setFrom(array(
 277                 'latitude' => $row['LATITUDE'],
 278                 'longitude' => $row['LONGITUDE'],
 279                 'city_id' => $network_mapping->city_id
 280             ));
 281         }
 282
 283
 284         if(!empty($row['POSTAL_CODE'])){
 285             $city = DB_DataObject::factory('core_geoip_city');
 286             if($city->get($network_mapping->city_id)){
 287                 return;
 288             }
 289
 290             $oc = clone($city);
 291             $city->postal_code = $row['POSTAL_CODE'];
 292
 293             $city->update($oc);
 294         }
 295
 296     }
 297
 298     function log($str)
 299     {
 300         echo "$str \n";
 301     }
 302 }