git.roojs.org Git - Pman.Core/blob - Import/Core_geoip.php

   1 <?php
   2
   3 require_once 'Pman/Roo.php';
   4
   5 class Pman_Core_Import_Core_geoip extends Pman_Roo
   6 {
   7     static $cli_desc = "Insert the geoip database";
   8
   9     static $cli_opts = array();
  10
  11     var $id_mapping = array();
  12
  13     function getAuth()
  14     {
  15         $ff = HTML_FlexyFramework::get();
  16         if (!$ff->cli) {
  17             die("access denied");
  18         }
  19         HTML_FlexyFramework::ensureSingle(__FILE__, $this);
  20         return true;
  21     }
  22
  23     function post()
  24     {
  25         $this->get();
  26     }
  27
  28     function get()
  29     {
  30
  31         PEAR::setErrorHandling(PEAR_ERROR_CALLBACK, array($this, 'onPearError'));
  32
  33         $location = '/tmp/GeoLite2-City-Locations.csv';
  34         $block = '/tmp/GeoLite2-City-Blocks.csv';
  35
  36         if(!file_exists($location) || !file_exists($block)){
  37             $this->jerr('GeoLite2-City-Locations.csv OR GeoLite2-City-Blocks.csv does not exists?!');
  38         }
  39
  40         $this->log("Insert location data start");
  41
  42 //        $this->insertLocation($location);
  43
  44         $this->log("Insert Block data start");
  45
  46         $this->insertBlock($block);
  47
  48         $this->jok("DONE");
  49     }
  50
  51     function insertLocation($csv)
  52     {
  53         ini_set("auto_detect_line_endings", true);
  54
  55         $fh = fopen($csv, 'r');
  56
  57         if (!$fh) {
  58             $this->jerr("invalid location file");
  59         }
  60
  61         $linecount = count(file($csv));
  62
  63         print_r($linecount);exit;
  64
  65         $req = array(
  66             'GEONAME_ID', 'CONTINENT_CODE', 'CONTINENT_NAME',
  67             'COUNTRY_ISO_CODE', 'COUNTRY_NAME', 'SUBDIVISION_ISO_CODE',
  68             'SUBDIVISION_NAME', 'CITY_NAME', 'METRO_CODE',
  69             'TIME_ZONE'
  70         );
  71
  72         $cols = false;
  73
  74         while(false !== ($n = fgetcsv($fh,10000, ',', '"'))) {
  75             if(!array_filter($n)){ // empty row
  76                 continue;
  77             }
  78
  79             if (!$cols) {
  80                 $cols = array();
  81                 foreach($n as $k) {
  82                     $cols[] = strtoupper(trim($k));
  83                 }
  84
  85                 if (empty($cols)) {
  86                     continue;
  87                 }
  88                 foreach($req as $r) {
  89                     if (!in_array($r,$cols)) {
  90                         $cols = false;
  91                         break;
  92                     }
  93                 }
  94                 continue;
  95             }
  96
  97             $row = array();
  98
  99             foreach($cols as $i=>$k) {
 100                 $row[$k] = trim($n[$i]);
 101             }
 102
 103             $this->processLocation($row);
 104         }
 105
 106     }
 107
 108     function insertBlock($csv)
 109     {
 110         ini_set("auto_detect_line_endings", true);
 111
 112         $linecount = count(file($csv));
 113
 114         print_r($linecount);exit;
 115
 116         $fh = fopen($csv, 'r');
 117         if (!$fh) {
 118             $this->jerr("invalid location file");
 119         }
 120
 121         $req = array(
 122             'NETWORK_START_IP', 'NETWORK_MASK_LENGTH', 'GEONAME_ID',
 123             'REGISTERED_COUNTRY_GEONAME_ID', 'REPRESENTED_COUNTRY_GEONAME_ID', 'POSTAL_CODE',
 124             'LATITUDE', 'LONGITUDE', 'IS_ANONYMOUS_PROXY',
 125             'IS_SATELLITE_PROVIDER'
 126         );
 127
 128         $cols = false;
 129
 130         while(false !== ($n = fgetcsv($fh,10000, ',', '"'))) {
 131             if(!array_filter($n)){ // empty row
 132                 continue;
 133             }
 134
 135             if (!$cols) {
 136                 $cols = array();
 137                 foreach($n as $k) {
 138                     $cols[] = strtoupper(trim($k));
 139                 }
 140
 141                 if (empty($cols)) {
 142                     continue;
 143                 }
 144                 foreach($req as $r) {
 145                     if (!in_array($r,$cols)) {
 146                         $cols = false;
 147                         break;
 148                     }
 149                 }
 150                 continue;
 151             }
 152
 153             $row = array();
 154
 155             foreach($cols as $i=>$k) {
 156                 $row[$k] = trim($n[$i]);
 157             }
 158
 159             $this->processBlock($row);
 160         }
 161
 162     }
 163
 164
 165
 166     function processLocation($row)
 167     {
 168         $continent = $this->processContinent($row['CONTINENT_CODE'], $row['CONTINENT_NAME']);
 169
 170         $country = $this->processCountry($row['COUNTRY_ISO_CODE'], $row['COUNTRY_NAME'], $continent);
 171
 172         $division = $this->processDivision($row['SUBDIVISION_ISO_CODE'], $row['SUBDIVISION_NAME']);
 173
 174         $city = $this->processCity($row['CITY_NAME'], $row['METRO_CODE'], $row['TIME_ZONE'], $country, $division);
 175
 176         if(!empty($city) && !empty($city->id)){
 177             $this->id_mapping[$row['GEONAME_ID']] = $city->id;
 178         }
 179     }
 180
 181     function processContinent($code, $name)
 182     {
 183         if(empty($code)){
 184             return false;
 185         }
 186
 187         $continent = DB_DataObject::factory('core_geoip_continent');
 188         if(!$continent->get('code', $code)){
 189             $continent->setFrom(array(
 190                 'code' => $code,
 191                 'name' => (!empty($name)) ? $name : $code
 192             ));
 193
 194             $continent->insert();
 195         }
 196
 197         return $continent;
 198     }
 199
 200     function processCountry($code, $name, $continent)
 201     {
 202         if(empty($code)){
 203             return false;
 204         }
 205
 206         $country = DB_DataObject::factory('core_geoip_country');
 207         if(!$country->get('code', $code)){
 208             $country->setFrom(array(
 209                 'code' => $code,
 210                 'name' => (!empty($name)) ? $name : $code,
 211                 'continent_id' => (!empty($continent) && !empty($continent->id)) ? $continent->id : 0
 212             ));
 213
 214             $country->insert();
 215         }
 216
 217         return $country;
 218     }
 219
 220     function processDivision($code, $name)
 221     {
 222         if(empty($code)){
 223             return false;
 224         }
 225
 226         $division = DB_DataObject::factory('core_geoip_division');
 227         if(!$division->get('code', $code)){
 228             $division->setFrom(array(
 229                 'code' => $code,
 230                 'name' => (!empty($name)) ? $name : $code
 231             ));
 232
 233             $division->insert();
 234         }
 235
 236         return $division;
 237     }
 238
 239     function processCity($name, $metro_code, $time_zone, $country, $division)
 240     {
 241         if(empty($name)){
 242             return false;
 243         }
 244
 245         $city = DB_DataObject::factory('core_geoip_city');
 246
 247         if($city->get('name', $name)){
 248             return $city;
 249         }
 250
 251         $city->setFrom(array(
 252             'name' => $name,
 253             'metro_code' => $metro_code,
 254             'time_zone' => $time_zone,
 255             'country_id' => (!empty($country) && !empty($country->id)) ? $country->id : 0,
 256             'division_id' => (!empty($division) && !empty($division->id)) ? $division->id : 0
 257         ));
 258
 259         $city->insert();
 260
 261         return $city;
 262
 263     }
 264
 265     function processBlock($row)
 266     {
 267         if(empty($this->id_mapping[$row['GEONAME_ID']])){
 268             $this->log("Missing mapping for {$row['GEONAME_ID']}");
 269             $this->log("IP : {$row['NETWORK_START_IP']}");
 270             return;
 271         }
 272
 273         $network_mapping = DB_DataObject::factory('core_geoip_network_mapping');
 274
 275         $start_ip = array_pop(explode(":", $row['NETWORK_START_IP']));
 276
 277         $network_mapping->setFrom(array(
 278             'start_ip' => $start_ip,
 279             'mask_length' => pow(2, (128 - $row['NETWORK_MASK_LENGTH'])),
 280             'city_id' => $this->id_mapping[$row['GEONAME_ID']]
 281         ));
 282
 283         if(!$network_mapping->find(true)){
 284             $network_mapping->insert();
 285         }
 286
 287         $location = DB_DataObject::factory('core_geoip_location');
 288         if(!$location->get('city_id', $network_mapping->city_id)){
 289             $location->setFrom(array(
 290                 'latitude' => $row['LATITUDE'],
 291                 'longitude' => $row['LONGITUDE'],
 292                 'city_id' => $network_mapping->city_id
 293             ));
 294         }
 295
 296
 297         if(!empty($row['POSTAL_CODE'])){
 298             $city = DB_DataObject::factory('core_geoip_city');
 299             if($city->get($network_mapping->city_id)){
 300                 return;
 301             }
 302
 303             $oc = clone($city);
 304             $city->postal_code = $row['POSTAL_CODE'];
 305
 306             $city->update($oc);
 307         }
 308
 309     }
 310
 311     function log($str)
 312     {
 313         echo "$str \n";
 314     }
 315 }