git.roojs.org Git - Pman.Core/blob - Import/Core_geoip.php

   1 <?php
   2
   3 require_once 'Pman/Roo.php';
   4
   5 class Pman_Core_Import_Core_geoip extends Pman_Roo
   6 {
   7     static $cli_desc = "Insert the geoip database";
   8
   9     static $cli_opts = array();
  10
  11     var $id_mapping = array();
  12
  13     function getAuth()
  14     {
  15         $ff = HTML_FlexyFramework::get();
  16         if (!$ff->cli) {
  17             die("access denied");
  18         }
  19         HTML_FlexyFramework::ensureSingle(__FILE__, $this);
  20         return true;
  21     }
  22
  23     function post()
  24     {
  25         $this->get();
  26     }
  27
  28     var $processed = 0;
  29     var $total = 0;
  30     var $echo = '';
  31
  32     function get()
  33     {
  34
  35         PEAR::setErrorHandling(PEAR_ERROR_CALLBACK, array($this, 'onPearError'));
  36
  37         $location = '/tmp/GeoLite2-City-Locations.csv';
  38         $block = '/tmp/GeoLite2-City-Blocks.csv';
  39
  40         if(!file_exists($location) || !file_exists($block)){
  41             $this->jerr('GeoLite2-City-Locations.csv OR GeoLite2-City-Blocks.csv does not exists?!');
  42         }
  43
  44         $this->log("Insert location data start");
  45
  46         $this->insertLocation($location);
  47
  48         $this->log("Insert Block data start");
  49
  50         $this->insertBlock($block);
  51
  52         $this->jok("DONE");
  53     }
  54
  55     function insertLocation($csv)
  56     {
  57         ini_set("auto_detect_line_endings", true);
  58
  59
  60
  61         $fh = fopen($csv, 'r');
  62
  63         if (!$fh) {
  64             $this->jerr("invalid location file");
  65         }
  66
  67         $req = array(
  68             'GEONAME_ID', 'CONTINENT_CODE', 'CONTINENT_NAME',
  69             'COUNTRY_ISO_CODE', 'COUNTRY_NAME', 'SUBDIVISION_ISO_CODE',
  70             'SUBDIVISION_NAME', 'CITY_NAME', 'METRO_CODE',
  71             'TIME_ZONE'
  72         );
  73
  74         $cols = false;
  75
  76         $this->processed = 0;
  77         $this->total = count(file($csv));
  78         $this->echo = '';
  79
  80         while(false !== ($n = fgetcsv($fh,10000, ',', '"'))) {
  81             if(!array_filter($n)){ // empty row
  82                 continue;
  83             }
  84
  85             if (!$cols) {
  86                 $cols = array();
  87                 foreach($n as $k) {
  88                     $cols[] = strtoupper(trim($k));
  89                 }
  90
  91                 if (empty($cols)) {
  92                     continue;
  93                 }
  94                 foreach($req as $r) {
  95                     if (!in_array($r,$cols)) {
  96                         $cols = false;
  97                         break;
  98                     }
  99                 }
 100                 continue;
 101             }
 102
 103             $row = array();
 104
 105             foreach($cols as $i=>$k) {
 106                 $row[$k] = trim($n[$i]);
 107             }
 108
 109             $this->processLocation($row);
 110
 111             $this->processStatus();
 112         }
 113
 114     }
 115
 116     function insertBlock($csv)
 117     {
 118         ini_set("auto_detect_line_endings", true);
 119
 120         $fh = fopen($csv, 'r');
 121         if (!$fh) {
 122             $this->jerr("invalid location file");
 123         }
 124
 125         $req = array(
 126             'NETWORK_START_IP', 'NETWORK_MASK_LENGTH', 'GEONAME_ID',
 127             'REGISTERED_COUNTRY_GEONAME_ID', 'REPRESENTED_COUNTRY_GEONAME_ID', 'POSTAL_CODE',
 128             'LATITUDE', 'LONGITUDE', 'IS_ANONYMOUS_PROXY',
 129             'IS_SATELLITE_PROVIDER'
 130         );
 131
 132         $cols = false;
 133
 134         while(false !== ($n = fgetcsv($fh,10000, ',', '"'))) {
 135             if(!array_filter($n)){ // empty row
 136                 continue;
 137             }
 138
 139             if (!$cols) {
 140                 $cols = array();
 141                 foreach($n as $k) {
 142                     $cols[] = strtoupper(trim($k));
 143                 }
 144
 145                 if (empty($cols)) {
 146                     continue;
 147                 }
 148                 foreach($req as $r) {
 149                     if (!in_array($r,$cols)) {
 150                         $cols = false;
 151                         break;
 152                     }
 153                 }
 154                 continue;
 155             }
 156
 157             $row = array();
 158
 159             foreach($cols as $i=>$k) {
 160                 $row[$k] = trim($n[$i]);
 161             }
 162
 163             $this->processBlock($row);
 164         }
 165
 166     }
 167
 168
 169
 170     function processLocation($row)
 171     {
 172         $continent = $this->processContinent($row['CONTINENT_CODE'], $row['CONTINENT_NAME']);
 173
 174         $country = $this->processCountry($row['COUNTRY_ISO_CODE'], $row['COUNTRY_NAME'], $continent);
 175
 176         $division = $this->processDivision($row['SUBDIVISION_ISO_CODE'], $row['SUBDIVISION_NAME']);
 177
 178         $city = $this->processCity($row['CITY_NAME'], $row['METRO_CODE'], $row['TIME_ZONE'], $country, $division);
 179
 180         if(!empty($city) && !empty($city->id)){
 181             $this->id_mapping[$row['GEONAME_ID']] = $city->id;
 182         }
 183     }
 184
 185     function processContinent($code, $name)
 186     {
 187         if(empty($code)){
 188             return false;
 189         }
 190
 191         $continent = DB_DataObject::factory('core_geoip_continent');
 192         if(!$continent->get('code', $code)){
 193             $continent->setFrom(array(
 194                 'code' => $code,
 195                 'name' => (!empty($name)) ? $name : $code
 196             ));
 197
 198             $continent->insert();
 199         }
 200
 201         return $continent;
 202     }
 203
 204     function processCountry($code, $name, $continent)
 205     {
 206         if(empty($code)){
 207             return false;
 208         }
 209
 210         $country = DB_DataObject::factory('core_geoip_country');
 211         if(!$country->get('code', $code)){
 212             $country->setFrom(array(
 213                 'code' => $code,
 214                 'name' => (!empty($name)) ? $name : $code,
 215                 'continent_id' => (!empty($continent) && !empty($continent->id)) ? $continent->id : 0
 216             ));
 217
 218             $country->insert();
 219         }
 220
 221         return $country;
 222     }
 223
 224     function processDivision($code, $name)
 225     {
 226         if(empty($code)){
 227             return false;
 228         }
 229
 230         $division = DB_DataObject::factory('core_geoip_division');
 231         if(!$division->get('code', $code)){
 232             $division->setFrom(array(
 233                 'code' => $code,
 234                 'name' => (!empty($name)) ? $name : $code
 235             ));
 236
 237             $division->insert();
 238         }
 239
 240         return $division;
 241     }
 242
 243     function processCity($name, $metro_code, $time_zone, $country, $division)
 244     {
 245         if(empty($name)){
 246             return false;
 247         }
 248
 249         $city = DB_DataObject::factory('core_geoip_city');
 250
 251         if($city->get('name', $name)){
 252             return $city;
 253         }
 254
 255         $city->setFrom(array(
 256             'name' => $name,
 257             'metro_code' => $metro_code,
 258             'time_zone' => $time_zone,
 259             'country_id' => (!empty($country) && !empty($country->id)) ? $country->id : 0,
 260             'division_id' => (!empty($division) && !empty($division->id)) ? $division->id : 0
 261         ));
 262
 263         $city->insert();
 264
 265         return $city;
 266
 267     }
 268
 269     function processBlock($row)
 270     {
 271         if(empty($this->id_mapping[$row['GEONAME_ID']])){
 272             $this->log("Missing mapping for {$row['GEONAME_ID']}");
 273             $this->log("IP : {$row['NETWORK_START_IP']}");
 274             return;
 275         }
 276
 277         $network_mapping = DB_DataObject::factory('core_geoip_network_mapping');
 278
 279         $start_ip = array_pop(explode(":", $row['NETWORK_START_IP']));
 280
 281         $network_mapping->setFrom(array(
 282             'start_ip' => $start_ip,
 283             'mask_length' => pow(2, (128 - $row['NETWORK_MASK_LENGTH'])),
 284             'city_id' => $this->id_mapping[$row['GEONAME_ID']]
 285         ));
 286
 287         if(!$network_mapping->find(true)){
 288             $network_mapping->insert();
 289         }
 290
 291         $location = DB_DataObject::factory('core_geoip_location');
 292         if(!$location->get('city_id', $network_mapping->city_id)){
 293             $location->setFrom(array(
 294                 'latitude' => $row['LATITUDE'],
 295                 'longitude' => $row['LONGITUDE'],
 296                 'city_id' => $network_mapping->city_id
 297             ));
 298         }
 299
 300
 301         if(!empty($row['POSTAL_CODE'])){
 302             $city = DB_DataObject::factory('core_geoip_city');
 303             if($city->get($network_mapping->city_id)){
 304                 return;
 305             }
 306
 307             $oc = clone($city);
 308             $city->postal_code = $row['POSTAL_CODE'];
 309
 310             $city->update($oc);
 311         }
 312
 313     }
 314
 315     function processStatus()
 316     {
 317         echo "$str \n";
 318     }
 319 }