[Python] Geographical locating and traceroute.

[gray-fox]

So what i've been working with is basically geographical locator + geographical traceroute based on ip address or domain name. Propably nothing new or innovative in this, but all this is just for training purposes. For locating i've used maxmind's free database(http://dev.maxmind.com/geoip/legacy/geolite/#Downloads) with pygeoip library and pygmaps(python wrapper for google maps) to draw map of the location/tracerout. For colors i've used ansicolors module.

Code: (python) [Select]

#!/usr/bin/python
# -*- coding:utf-8  -*-

import argparse, pygeoip, webbrowser, sys, os, pygmaps, urllib2, gzip, time
from scapy.all import traceroute
from colors import red, yellow, green

def download_db():
    print "Downloading database.."
    try:
        gz_file = "GeoLiteCity.dat.gz"
        url = "http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz"
        res = urllib2.urlopen(url)
        fh = open(gz_file, "w")
        fh.write(res.read())
        fh.close()
        dat_file = gzip.open(gz_file, 'rb')
        content = dat_file.read()
        dat_file.close()
        outF = file("GeoLiteCity.dat", 'wb')
        outF.write(content)
        outF.close()
        os.remove('GeoLiteCity.dat.gz')
    except:
        print "[!] Error while downloading/updating database, try manually from http://dev.maxmind.com/geoip/legacy/geolite/"
        if os.path.isfile('GeoLiteCity.dat') == True:
            pass                    #if updating fails
        else:
            sys.exit(0)          #if first time download fails
       
     
def db_status():
    try:
        update_time = 60*60*24*30 #ca. month
        db_time = os.path.getmtime('GeoLiteCity.dat')
        curr_time = time.time()
        db_age = curr_time-db_time
        if db_age > update_time:
            print "[!] Database update recommended!"
            print "[!] Updating..."
            download_db()
    except:
        if os.path.isfile('GeoLiteCity.dat') == False:
            print "[!] Database file not found!"
            download_db()
        else:
            print "[!] Error while checking database status"
     
def location(target):
    rawdata = pygeoip.GeoIP('GeoLiteCity.dat')
    try:
        data = rawdata.record_by_addr(target)
    except:
        data = rawdata.record_by_name(target)
    country = data['country_name']
    longi = data['longitude']
    lat = data['latitude']
    mymap = pygmaps.maps(lat, longi, 12)
    mymap.addpoint(lat, longi, "Target is near this area")
    mymap.draw('./target_location.html')
    print yellow("[!] Target: ")+red(str(target))
    print yellow("[x] Target's country: ")+red(str(country))
    print yellow("[x] Longitude: ")+red(str(longi))
    print yellow("[x] Latitude: ")+red(str(lat))
    print yellow('[x] Map of the location >')+red('"target_location.html"')
    while True:
        o_map = raw_input("Try to open target_location.html map in browser(y/ n):\n>").lower()
        if o_map.startswith('y'):
            try:
                new = 2
                webbrowser.open("target_location.html", new=new)
                break
            except:
                print '[!]Could not open map in web browser. Open "geo_trace.html" -file manually'
                break
        elif o_map.startswith('n'):
            break
   
def geo_trace(tracer):
    if os.getuid() != 0:
        print red("[!]Youd must be root for this part to run correctly!") #scapy needs root
    trace, _ = traceroute([tracer] , verbose=0)
    hosts = trace.get_trace().values()[0]
    ips = [hosts[i][0] for i in range(1, len(hosts) + 1)]
    rawdata = pygeoip.GeoIP('GeoLiteCity.dat')
    i = 0
    path = []
    print yellow("Geotrace:")+red(str(tracer))
    while ( i < len(ips)):
        data = rawdata.record_by_addr(ips[i])
        if data == None:
            pass
        else:
            longi = data['longitude']
            lat = data['latitude']
            path.append((lat, longi))
            print yellow("[X] IP")+":"+yellow(str(ips[i]))+":"+ red(str(data['country_name']))
        i += 1
    tracemap = pygmaps.maps(lat ,longi,3)
    tracemap.addpath(path,"#FF0000")
    tracemap.draw('./geo_trace.html')
    while True:
        o_map = raw_input("Try to open geo_trace.html map in browser(y/ n):\n>").lower()
        if o_map.startswith('y'):
            try:
                new = 2
                webbrowser.open("geo_trace.html", new=new)
                break
            except:
                print '[!]Could not open map in web browser. Open "geo_trace.html" -file manually'
                break
        elif o_map.startswith('n'):
            break
             
if __name__ == '__main__':

    parser = argparse.ArgumentParser(description="IP-address geolocator and geotracer")
    parser.add_argument("-t", "--target", type=str, help="Give target's ip-address.")
    parser.add_argument("-gT", "--geoTrace", type=str, help="Give target's ip-address or domain-name")
    parser.add_argument("-d", "--download", type=str, help="Download maxmind geoip database")
   
    args = parser.parse_args()
    target = args.target
    tracer = args.geoTrace
    downl = args.download
   
    if len(sys.argv)==1:
        parser.print_help()
        sys.exit(0)
    os.system('clear')
    db_status()
    if downl:
        download_db()
    if target:
        location(target)
    if tracer:
        geo_trace(tracer)

Like i said i have done this for training purposes and i'm most interested for your opinion of the code itself. Is it in ok-, bad- or "rewrite that shit" state. So constructive hints and tips would be appreciated.

Here is anyway some pics and usage examples of the program. I'm sending these using my phone+tapatalk so if there is something strange in formatting or in pics, i try to fix them when i have chance/time to use my computer.

Basic locating:
(No particular reason why i used nhl's site in examples)

python geotrace.py -t nhl.com

Terminal output:

Map viewed in browser:

Accuracy depends on maxmind's database, so there's not much i can do to improve it. When is tested with my own IP, both country and city were correct.

Geotrace example:

python geotrace.py -gT nhl.com

Terminal output:

Map viewed in browser:



Here's link for the github repo https://github.com/gray-fox/geotrace

[TheWormKill]

[size=xx-small ! important]1[/size][size=xx-small ! important]1[/size]All in all, the code looks good to me.
There are some minor aspects, I'd change, though. These are mostly related to user-friendlyness:

• Try to catch as many wrong inputs and typos as possible ( not just checking, whether a string equals to "yes"), altough this isn't really important.
• Sometimes you don't use python's syntax to the full: use the += notation etc.

You see: not many real issues, just minor quirks. All in all, good work, although I'd try to find a workaround to the issue that you need to be root to use this.

[gray-fox]

Thank you for the tips TheWormKill.
I just want to make sure i understood them right:

Try to catch as many wrong inputs and typos as possible ( not just checking, whether a string equals to "yes"),

You mean doing something like this?:

Code: (python) [Select]

while True:
    open_map = raw_input("Try to open target_location.html map in browser(yes / no):\n>")
    if open_map ==  "yes" or open_map == "no":
        break
   
if open_map == "yes":
    try:
        new = 2
        webbrowser.open("target_location.html", new=new)
....


Sometimes you don't use python's syntax to the full: use the += notation etc.

Do you mean, for example when i use "i = i + 1", what i should do is "i += 1"?

About that running without root, i did thought that myself, but haven't yet found propier way to do it without root-privildges. At least not with using scapy. Afaik problem is sending and receiving raw packets without root. But i have to give this more thought.

[TheWormKill]

[size=xx-small ! important]1[/size]

Thank you for the tips TheWormKill.
I just want to make sure i understood them right:You mean doing something like this?:

Code: (python) [Select]

while True:
    open_map = raw_input("Try to open target_location.html map in browser(yes / no):\n>")
    if open_map ==  "yes" or open_map == "no":
        break
   
if open_map == "yes":
    try:
        new = 2
        webbrowser.open("target_location.html", new=new)
....
Do you mean, for example when i use "i = i + 1", what i should do is "i += 1"?

About that running without root, i did thought that myself, but haven't yet found propier way to do it without root-privildges. At least not with using scapy. Afaik problem is sending and receiving raw packets without root. But i have to give this more thought.

About the first: I'd make the user re-input something if his input wasn't valid, and make as much possibilities as possible valid: 'y', 'Y', 'yes', 'Yes' etc. Sorry about not making it clear enough.

Code: (python) [Select]

input_valid = False
while not input_valid:
    r = raw_input('Input: ').lower()
    if r.startswith('y'): input_valid = 1
    elif r.startswith('n'): input_valid = -1
Or you can interpret everything not starting with 'y' as False.
 About the second: Yes, that's what I meant, it just looks cleaner. On the third point: I agree. Maybe it would be useful to search for alternatives.

[d4rkcat]

Pretty damn cool script.
Awesome work!
+1 

[gray-fox]

Once again, thank you for your notes TheWormKill . When i have more time i will use your tips and go trough my code to see how i'm going to rewrite those parts we have discussed. I have also tryed to find alternatives to make traceroute without root privilidges, but still haven't been able to find anything useful. I thought that using setcap would have allowed to run script as non-root, but it seems that system ignores suid/cap that has been set on the script. Or i just lack skills to do it right. I guess i just have to keep googling.

Pretty damn cool script.
Awesome work!
+1 

Thanks d4rkcat. 

[HTH]

Scapy will always require root because you are using an ICMP traceroute (as per standards) which requires raw sockets to complete. It is, in my opinion, very poor form to run an interpreter with root privileges so you should look into a UDP or TCP traceroute, or even using the systems traceroute tool and just parsing the input (they use udp by default and as such dont require root)

You should also only perform the root check if the person selects traceroute, this would halve the amount of time the script needs to be root, if you dont find a work around.


Either way it is a cool script and +1

[TheWormKill]

Some googling brought up this:
http://synack.me/blog/using-scapy-without-root-privileges
It seems to be a starting point if you consider not using raw packets, by creating a regular socket and "[using] scapy to generate and parse the payloads".

[Matriplex]

As a side note I did something similar to what you're doing just about a week ago. But I instead just pinged the IPs to see if they were up and then plotted their position on an image. I also added in some CPU based additive blending to define hot spots. The original image was 3400x1800 but I can't get a hold of it right now so this one is a little more compressed and filtered looking but it gets the point across. Not posting directly because of it's size.

Edit: check out North Korea.

Le image

[gray-fox]

I now made some small adjustments to code in my OP. Larger changes,mostly concerning traceroute, will maybe come when i have time to see if i can make something that works good enough.

@HTH I actually first consider using systems traceroute and started making version wich did traceroute and ip-address parsing more or less like this:

Code: (python) [Select]

def geo_trace(tracer):
    o_put = os.popen("traceroute -n "+tracer+"| tail -n+2 | awk '{ print $2 }'").read()
    ips = o_put.split() # IP-address list
...But then i decide, that i want to use purely python as much as i can. I'm now trying to see if i can make udp or tcp traceroute using python's socket module. I have done some socket programming in python before, but not sure if i yet have skills for this. But this is definetly good way to learn, so maybe i eventually can make this work.

@TheWormKill Thanks for the link. That seems intresting option how to use scapy. I will look more into it.

@Matriplex Wow, that map looks really cool. Would you be willing to share bit more about how you created your map?

[Matriplex]

It's really simple, exactly like I explained haha.

Here's the full size image, I found it again. http://i.imgur.com/9eSVY2t.png
Here are the two methods that actually do the mapping and additive blending. I used Pillow for the images.

Code: [Select]

def plotPoint(im, draw, x, y):
        final = [0, 0, 0, 0]
        base = (6, 255, 6, 255)
        col = im.getpixel((x, y))
        for i in range(len(base)):
                final[i] = base[i] + col[i]
        draw.line((x, y, x, y), tuple(final), 1)
 
def handleCoords(im, draw, x, y):
        x += 180
        y += 90
        y = 180 - y
        x *= 10 ############
        y *= 10 # scales the image
        plotPoint(im, draw, x, y)


[kenjoe41]

Here are my 2-cents.
We would forego all the asking for a "yes/no" inorder to print open the map in the browser and rather include a commandline arg to it instead. But if you stay with it, remember to rather use one letter; "y/n" to minimise the margin for error and also turn the given letter to one case ; lowercase, also take care of any case reply.

Another thing, since we all might not have the maxmind database on our systems, a download class for the database could do. Check a default location for the database, if empty or non-existent, check if a custom path for the database was provided [this is another flag to specify], if not provided, prompt them for your script to download it or just exit and tell then to include the download flag[another flag to include]. You could also have an update flag, i don't know how often the maxmind database is updated but checking for how old the database is and advising to update it would be nice.

Argparse should have taken care of the printing of the help message incase the args are not enough, in which case either the target flag should be made required.
I notice you only ask for IP addresses, how about someone provides a domain name and you take care of resolving the IP then do your magic. Just remember to tell me which IP of the domain you are using.
In argparse, the type value here wouldn't be needed as they are by default all strings. I would reck my brain about it if i needed an int or bool.
I was also wondering how i will exit the infinite loop after the browser opens, what is going to make the value true for it to exit!
This script doesn't include complex constructs that you actually could practice both; writing portable code for all systems and portable code for both python varieties, 2 and 3.
It would all be nice and dandy if you rolled a class or two for this. Nice project to practice on.

Anyway, thanks for the script. These are the projects  i also thought about when i was first learning python.
Check this out for inspiration on that GUID problem: https://github.com/fiorix/txtraceroute/blob/master/txtraceroute.py

[gray-fox]

Here is some sort of version 2 of the script. I now tried to use classes even if i'm still not sure that i'm using them completely right. I also now added geo-traceroute option where ip addresses are parsed from os's traceroute command. So if script is runned as root it still uses scapy, but as regular user it tryes to use "system's" traceroute.
Every now and then i have been trying to make my own traceroute versions with both tcp and udp, But haven't been able to make anything that works properly without root access and it was kind of driving me crazy, so i have been taking small break from the subject.
 
So here is the code:

Code: (python) [Select]

#!/usr/bin/python
# -*- coding:utf-8  -*-

import argparse, pygeoip, webbrowser, sys, os, pygmaps, urllib2, gzip, time
from scapy.all import traceroute
from colors import red, yellow, green

class maintain(object):
    def download_db(self):
        print "Downloading database.."
        try:
            self.gz_file = "GeoLiteCity.dat.gz"
            self.url = "http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz"
            self.res = urllib2.urlopen(self.url)
            self.fh = open(self.gz_file, "w")
            self.fh.write(self.res.read())
            self.fh.close()
            self.dat_file = gzip.open(self.gz_file, 'rb')
            self.content = self.dat_file.read()
            self.dat_file.close()
            self.outF = file("GeoLiteCity.dat", 'wb')
            self.outF.write(self.content)
            self.outF.close()
            os.remove('GeoLiteCity.dat.gz')
        except:
            print "[!] Error downloading/updating database, try manually from http://dev.maxmind.com/geoip/legacy/geolite/"
            if os.path.isfile('GeoLiteCity.dat') == True:
                pass
            else:
                sys.exit(0)
       
     
    def db_status(self):
        try:
            self.update_time = 60*60*24*30 #ca. month
            self.db_time = os.path.getmtime('GeoLiteCity.dat')
            self.curr_time = time.time()
            self.db_age = self.curr_time-self.db_time
            if self.db_age > self.update_time:
                print "[!] Database update recommended!"
                print "[!] Updating..."
                self.download_db()
        except:
            if os.path.isfile('GeoLiteCity.dat') == False:
                print "[!] Database file not found!"
                self.download_db()
            else:
                print "[!] Error while checking database status"
                sys.exit(0)
               
class trace_route(object): 
    def geo_trace(self, tracer):
        print red("[!] Using scapy's traceroute")
        time.sleep(1)
        self.tracer = tracer
        self.trace, _ = traceroute([self.tracer] , verbose=0)
        self.hosts = self.trace.get_trace().values()[0]
        self.ips = [self.hosts[self.i][0] for self.i in range(1, len(self.hosts) + 1)]
        self.rawdata = pygeoip.GeoIP('GeoLiteCity.dat')
        self.i = 0
        self.path = []
        print yellow("Geotrace:")+red(str(self.tracer))
        while ( self.i < len(self.ips)):
            self.data = self.rawdata.record_by_addr(self.ips[self.i])
            if self.data == None:
                pass
            else:
                self.longi = self.data['longitude']
                self.lat = self.data['latitude']
                self.path.append((self.lat, self.longi))
                print yellow("[X] IP")+":"+yellow(str(self.ips[self.i]))+":"+ red(str(self.data['country_name']))
            self.i += 1
        self.tracemap = pygmaps.maps(self.lat ,self.longi,3)
        self.tracemap.addpath(self.path,"#FF0000")
        self.tracemap.draw('./geo_trace.html')
        while True:
            self.o_map = raw_input("Try to open geo_trace.html map in browser(y/ n):\n>").lower()
            if self.o_map.startswith('y'):
                try:
                    self.new = 2
                    webbrowser.open("geo_trace.html", new=self.new)
                    break
                except:
                    print '[!]Could not open map in web browser. Open "geo_trace.html" -file manually'
                break
            elif self.o_map.startswith('n'):
                break
               
    def alt_traceroute(self,tracer):
        print red("[!] Using systems 'default' traceroute")
        time.sleep(1)
        self.tracer = tracer
        self.o_put = os.popen("traceroute -n "+self.tracer+"| tail -n+2 | awk '{ print $2 }'").read()
        self.raw_ips = self.o_put.split() # IP-address list
        self.ips = [ self.j for self.j in self.raw_ips if not self.j.startswith('*') ]
        self.rawdata = pygeoip.GeoIP('GeoLiteCity.dat')
        self.i = 0
        self.path = []
        print yellow("Geotrace:")+red(str(self.tracer))
        while ( self.i < len(self.ips)):
            self.data = self.rawdata.record_by_addr(self.ips[self.i])
            if self.data == None:
                pass
            else:
                self.longi = self.data['longitude']
                self.lat = self.data['latitude']
                self.path.append((self.lat, self.longi))
                print yellow("[X] IP")+":"+yellow(str(self.ips[self.i]))+":"+ red(str(self.data['country_name']))
            self.i += 1
        self.tracemap = pygmaps.maps(self.lat ,self.longi,3)
        self.tracemap.addpath(self.path,"#FF0000")
        self.tracemap.draw('./geo_trace.html')
        while True:
            self.o_map = raw_input("Tryccc to open geo_trace.html map in browser(y/ n):\n>").lower()
            if self.o_map.startswith('y'):
                try:
                    self.new = 2
                    webbrowser.open("geo_trace.html", new=self.new)
                    break
                except:
                    print '[!]Could not open map in web browser. Open "geo_trace.html" -file manually'
                break
            elif self.o_map.startswith('n'):
                break
       
 
               
class location(object):
    def locate(self, target):
        self.target = target
        self.rawdata = pygeoip.GeoIP('GeoLiteCity.dat')
        try:
            self.data = self.rawdata.record_by_addr(self.target)
        except:
            self.data = self.rawdata.record_by_name(self.target)
        self.country = self.data['country_name']
        self.longi = self.data['longitude']
        self.lat = self.data['latitude']
        self.mymap = pygmaps.maps(self.lat, self.longi, 12)
        self.mymap.addpoint(self.lat, self.longi, "Target is near this area")
        self.mymap.draw('./target_location.html')
        print yellow("[!] Target: ")+red(str(self.target))
        print yellow("[x] Target's country: ")+red(str(self.country))
        print yellow("[x] Longitude: ")+red(str(self.longi))
        print yellow("[x] Latitude: ")+red(str(self.lat))
        print yellow('[x] Map of the location >')+red('"target_location.html"')
        while True:
            self.o_map = raw_input("Try to open target_location.html map in browser(y/ n):\n>").lower()
            if self.o_map.startswith('y'):
                try:
                    self.new = 2
                    webbrowser.open("target_location.html", new=self.new)
                    break
                except:
                     print '[!]Could not open map in web browser. Open "geo_trace.html" -file manually'
                break
            elif self.o_map.startswith('n'):
                break

             
if __name__ == '__main__':

    parser = argparse.ArgumentParser(description="IP-address geolocator and geotracer")
    parser.add_argument("-t", "--target", type=str, help="Give ip-address or domain-name for locating.")
    parser.add_argument("-gT", "--geoTrace", type=str, help="Give ip-address or domain-name to tracing.")
    parser.add_argument("-d", "--download",action='store_true', help="Download/update maxmind geoip database")
   
    args = parser.parse_args()
    target = args.target
    tracer = args.geoTrace
    downl = args.download
    maintain = maintain()
    if not downl and not target and not tracer:
        parser.print_help()
        sys.exit(0)
    os.system('clear')
    maintain.db_status()
    if downl:
        maintain.download_db()
    if target:
        location = location()
        location.locate(target)
    if tracer:
        trace_route = trace_route()
        if os.getuid() == 0:   
            trace_route.geo_trace(tracer)
        elif os.getuid() != 0:
            trace_route.alt_traceroute(tracer)
I'm still intrested if someone is willing to share new tips/opinions about the script,  now especially about class usage in it.

[kenjoe41]

Haven't looked thoroughly through it. But in line 42 and 46, those functions are in the same class, why not just call self.download_db()

[gray-fox]

But in line 42 and 46, those functions are in the same class, why not just call self.download_db()

Honestly, i didn't know i can do it that way. Thanks!