git.roojs.org Git - Pman.Core/blob - Import/Core_geoip.php

   1 <?php
   2
   3 require_once 'Pman/Roo.php';
   4
   5 class Pman_Core_Import_Core_geoip extends Pman_Roo
   6 {
   7     static $cli_desc = "Insert the geoip database";
   8
   9     static $cli_opts = array();
  10
  11     var $id_mapping = array();
  12
  13     function getAuth()
  14     {
  15         $ff = HTML_FlexyFramework::get();
  16         if (!$ff->cli) {
  17             die("access denied");
  18         }
  19         HTML_FlexyFramework::ensureSingle(__FILE__, $this);
  20         return true;
  21     }
  22
  23     function post()
  24     {
  25         $this->get();
  26     }
  27
  28     function get()
  29     {
  30
  31         PEAR::setErrorHandling(PEAR_ERROR_CALLBACK, array($this, 'onPearError'));
  32
  33         $location = '/tmp/GeoLite2-City-Locations.csv';
  34         $block = '/tmp/GeoLite2-City-Blocks.csv';
  35
  36         if(!file_exists($location) || !file_exists($block)){
  37             $this->jerr('GeoLite2-City-Locations.csv OR GeoLite2-City-Blocks.csv does not exists?!');
  38         }
  39
  40         $this->log("Insert location data start");
  41
  42         $this->insertLocation($location);
  43
  44         $this->log("Insert Block data start");
  45
  46         $this->insertBlock($block);
  47
  48         $this->jok("DONE");
  49     }
  50
  51     function insertLocation($csv)
  52     {
  53         ini_set("auto_detect_line_endings", true);
  54
  55         $fh = fopen($csv, 'r');
  56
  57         if (!$fh) {
  58             $this->jerr("invalid location file");
  59         }
  60
  61         $req = array(
  62             'GEONAME_ID', 'CONTINENT_CODE', 'CONTINENT_NAME',
  63             'COUNTRY_ISO_CODE', 'COUNTRY_NAME', 'SUBDIVISION_ISO_CODE',
  64             'SUBDIVISION_NAME', 'CITY_NAME', 'METRO_CODE',
  65             'TIME_ZONE'
  66         );
  67
  68         $cols = false;
  69
  70         while(false !== ($n = fgetcsv($fh,10000, ',', '"'))) {
  71             if(!array_filter($n)){ // empty row
  72                 continue;
  73             }
  74
  75             if (!$cols) {
  76                 $cols = array();
  77                 foreach($n as $k) {
  78                     $cols[] = strtoupper(trim($k));
  79                 }
  80
  81                 if (empty($cols)) {
  82                     continue;
  83                 }
  84                 foreach($req as $r) {
  85                     if (!in_array($r,$cols)) {
  86                         $cols = false;
  87                         break;
  88                     }
  89                 }
  90                 continue;
  91             }
  92
  93             $row = array();
  94
  95             foreach($cols as $i=>$k) {
  96                 $row[$k] = trim($n[$i]);
  97             }
  98
  99             $this->processLocation($row);
 100         }
 101
 102     }
 103
 104     function insertBlock($csv)
 105     {
 106         ini_set("auto_detect_line_endings", true);
 107
 108         $fh = fopen($csv, 'r');
 109         if (!$fh) {
 110             $this->jerr("invalid location file");
 111         }
 112
 113         $req = array(
 114             'NETWORK_START_IP', 'NETWORK_MASK_LENGTH', 'GEONAME_ID',
 115             'REGISTERED_COUNTRY_GEONAME_ID', 'REPRESENTED_COUNTRY_GEONAME_ID', 'POSTAL_CODE',
 116             'LATITUDE', 'LONGITUDE', 'IS_ANONYMOUS_PROXY',
 117             'IS_SATELLITE_PROVIDER'
 118         );
 119
 120         $cols = false;
 121
 122         while(false !== ($n = fgetcsv($fh,10000, ',', '"'))) {
 123             if(!array_filter($n)){ // empty row
 124                 continue;
 125             }
 126
 127             if (!$cols) {
 128                 $cols = array();
 129                 foreach($n as $k) {
 130                     $cols[] = strtoupper(trim($k));
 131                 }
 132
 133                 if (empty($cols)) {
 134                     continue;
 135                 }
 136                 foreach($req as $r) {
 137                     if (!in_array($r,$cols)) {
 138                         $cols = false;
 139                         break;
 140                     }
 141                 }
 142                 continue;
 143             }
 144
 145             $row = array();
 146
 147             foreach($cols as $i=>$k) {
 148                 $row[$k] = trim($n[$i]);
 149             }
 150
 151             $this->processBlock($row);
 152         }
 153
 154     }
 155
 156
 157
 158     function processLocation($row)
 159     {
 160         $continent = $this->processContinent($row['CONTINENT_CODE'], $row['CONTINENT_NAME']);
 161
 162         $country = $this->processCountry($row['COUNTRY_ISO_CODE'], $row['COUNTRY_NAME'], $continent);
 163
 164         $division = $this->processDivision($row['SUBDIVISION_ISO_CODE'], $row['SUBDIVISION_NAME']);
 165
 166         $city = $this->processCity($row['CITY_NAME'], $row['METRO_CODE'], $row['TIME_ZONE'], $country, $division);
 167
 168         if(!empty($city) && !empty($city->id)){
 169             $this->id_mapping[$row['GEONAME_ID']] = $city->id;
 170         }
 171     }
 172
 173     function processContinent($code, $name)
 174     {
 175         if(empty($code)){
 176             return false;
 177         }
 178
 179         $continent = DB_DataObject::factory('core_geoip_continent');
 180         if(!$continent->get('code', $code)){
 181             $continent->setFrom(array(
 182                 'code' => $code,
 183                 'name' => (!empty($name)) ? $name : $code
 184             ));
 185
 186             $continent->insert();
 187         }
 188
 189         return $continent;
 190     }
 191
 192     function processCountry($code, $name, $continent)
 193     {
 194         if(empty($code)){
 195             return false;
 196         }
 197
 198         $country = DB_DataObject::factory('core_geoip_country');
 199         if(!$country->get('code', $code)){
 200             $country->setFrom(array(
 201                 'code' => $code,
 202                 'name' => (!empty($name)) ? $name : $code,
 203                 'continent_id' => (!empty($continent) && !empty($continent->id)) ? $continent->id : 0
 204             ));
 205
 206             $country->insert();
 207         }
 208
 209         return $country;
 210     }
 211
 212     function processDivision($code, $name)
 213     {
 214         if(empty($code)){
 215             return false;
 216         }
 217
 218         $division = DB_DataObject::factory('core_geoip_division');
 219         if(!$division->get('code', $code)){
 220             $division->setFrom(array(
 221                 'code' => $code,
 222                 'name' => (!empty($name)) ? $name : $code
 223             ));
 224
 225             $division->insert();
 226         }
 227
 228         return $division;
 229     }
 230
 231     function processCity($name, $metro_code, $time_zone, $country, $division)
 232     {
 233         if(empty($name)){
 234             return false;
 235         }
 236
 237         $city = DB_DataObject::factory('core_geoip_city');
 238
 239         if($city->get('name', $name)){
 240             return $city;
 241         }
 242
 243         $city->setFrom(array(
 244             'name' => $name,
 245             'metro_code' => $metro_code,
 246             'time_zone' => $time_zone,
 247             'country_id' => (!empty($country) && !empty($country->id)) ? $country->id : 0,
 248             'division_id' => (!empty($division) && !empty($division->id)) ? $division->id : 0
 249         ));
 250
 251         $city->insert();
 252
 253         return $city;
 254
 255     }
 256
 257     function processBlock($row)
 258     {
 259         if(empty($this->id_mapping[$row['GEONAME_ID']])){
 260             $this->log("Missing mapping for {$row['GEONAME_ID']}");
 261             $this->log("IP : {$row['NETWORK_START_IP']}");
 262             return;
 263         }
 264
 265         $network_mapping = DB_DataObject::factory('core_geoip_network_mapping');
 266
 267         $start_ip = array_pop(explode(":", $row['NETWORK_START_IP']));
 268
 269         $network_mapping->setFrom(array(
 270             'start_ip' => $start_ip,
 271             'mask_length' => pow(2, (128 - $row['NETWORK_MASK_LENGTH'])),
 272             'city_id' => $this->id_mapping[$row['GEONAME_ID']]
 273         ));
 274
 275         if(!$network_mapping->find(true)){
 276             $network_mapping->insert();
 277         }
 278
 279         $location = DB_DataObject::factory('core_geoip_location');
 280         if(!$location->get('city_id', $network_mapping->city_id)){
 281             $location->setFrom(array(
 282                 'latitude' => $row['LATITUDE'],
 283                 'longitude' => $row['LONGITUDE'],
 284                 'city_id' => $network_mapping->city_id
 285             ));
 286         }
 287
 288
 289         if(!empty($row['POSTAL_CODE'])){
 290             $city = DB_DataObject::factory('core_geoip_city');
 291             if($city->get($network_mapping->city_id)){
 292                 return;
 293             }
 294
 295             $oc = clone($city);
 296             $city->postal_code = $row['POSTAL_CODE'];
 297
 298             $city->update($oc);
 299         }
 300
 301     }
 302
 303     function log($str)
 304     {
 305         echo "$str \n";
 306     }
 307 }