yantisj · Feb 14, 2018 · Feb 14, 2018 · Feb 14, 2018 · Feb 14, 2018 · Feb 14, 2018
Showing with 195 additions and 76 deletions.

+1 −0 .gitignore

+37 −0 README.md

+3 −2 ndcrawl-sample.ini

+16 −12 ndcrawl.py

+4 −3 ndlib/execute.py

+41 −17 ndlib/output.py

+48 −18 ndlib/parse.py

+45 −24 ndlib/topology.py
diff --git a/.gitignore b/.gitignore
@@ -6,3 +6,4 @@ start-env.sh
 output/
 *.csv
 ndcrawl.ini
+.idea/
diff --git a/README.md b/README.md
@@ -1,5 +1,15 @@
 # CDP/LLDP Network Discovery Crawler for Cisco networks
 
+## Todo
+- Add Serial Number collection
+- Dedup devices on Serial Number
+- Generate Graphviz Output
+- Collect platform version and ios image
+
+## Fork information
+This has been forked to be able to accomodate the needs of Craig Armstrong
+Commits will be as detailed as possible, code maintenance may die as quickly as this repo was forked
+
 This is experimental at the moment. Uses netmiko and some seed devices to scrape
 your network and output CSV files of all neighbor topology data, as well as a
 device list file. Uses threaded connections at each iteration, moving out from
@@ -17,6 +27,33 @@ can only scrape cisco devices to discover next level devices at the moment.
 
 ```./ndcrawl.py -seed core1.domain.com,core2.domain.com --user yantisj -nei_file nd.csv -dev_file devices.csv --debug 1```
 
+```
+usage: ndcrawl.py [-h] [-seed switch1[,switch2]] [-nei_file file]
+                  [-dev_file file] [-gv_file file] [-ng_file file] [--quiet]
+                  [--seed_os cisco_nxos] [--seed_file file] [--user username]
+                  [--max_crawl int] [--conf file] [--debug DEBUG] [-v]
+                  [--en secret]
+
+Discover Network Topology via CDP/LLDP
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -seed switch1[,switch2]
+                        Seed devices to start crawl
+  -nei_file file        Output Neighbors to File
+  -dev_file file        Output Devices to File
+  -gv_file file         Output GraphViz Topology File
+  -ng_file file         Output NetGrph Topology File
+  --quiet               Quiet output, log to file only
+  --seed_os cisco_nxos  Netmiko OS type for seed devices
+  --seed_file file      Seed devices from a file, one per line
+  --user username       Username to execute as
+  --max_crawl int       Max devices to crawl (default 10000)
+  --conf file           Alternate Config File
+  --debug DEBUG         Set debugging level
+  -v                    Verbose Output
+  --en secret           Activate privilege level 15
+ ```
 ## Config File Notes
 
 Copy the ndcrawl-sample.ini to ndcrawl.ini and edit options. All CLI options can be specified

diff --git a/ndcrawl-sample.ini b/ndcrawl-sample.ini
@@ -6,13 +6,13 @@ log_file = ndcrawl.log
 thread_count = 50
 
 # Ignore any CDP neighbors that match this regex
-ignore_regex = (oobsw|lab)
+ignore_regex = (oobsw|lab|SEP)
 
 # Max devices to crawl
 max_crawl = 10000
 
 # Seed OS type (otherwise discovered)
-seed_os = cisco_nxos
+seed_os = cisco_ios
 
 ## Optional Variables, define to enable
 
@@ -22,6 +22,7 @@ seeds =
 # Stored username and password (define to enable)
 username =
 password =
+secret =
 
 # Neighbor File
 nei_file = neighbors.csv

diff --git a/ndcrawl.py b/ndcrawl.py
@@ -14,21 +14,18 @@
 parser = argparse.ArgumentParser(description='Discover Network Topology via CDP/LLDP')
 parser.add_argument('-seed', metavar="switch1[,switch2]", help="Seed devices to start crawl")
 parser.add_argument('-nei_file', metavar="file", help="Output Neighbors to File", type=str)
-parser.add_argument('-dev_file', metavar="file", help="Output Neighbors to File", type=str)
+parser.add_argument('-dev_file', metavar="file", help="Output Devices to File", type=str)
+parser.add_argument('-gv_file', metavar="file", help="Output GraphViz Topology File", type=str)
 parser.add_argument('-ng_file', metavar="file", help="Output NetGrph Topology File", type=str)
 parser.add_argument('--quiet', help='Quiet output, log to file only', action="store_true")
-parser.add_argument("--seed_os", metavar='cisco_nxos', help="Netmiko OS type for seed devices",
-                    type=str)
-parser.add_argument("--seed_file", metavar='file', help="Seed devices from a file, one per line",
-                    type=str)
-parser.add_argument("--user", metavar='username', help="Username to execute as",
-                    type=str)
-parser.add_argument("--max_crawl", metavar='int', help="Max devices to crawl (default 10000)",
-                    type=int)
-parser.add_argument("--conf", metavar='file', help="Alternate Config File",
-                    type=str)
+parser.add_argument("--seed_os", metavar='cisco_nxos', help="Netmiko OS type for seed devices", type=str)
+parser.add_argument("--seed_file", metavar='file', help="Seed devices from a file, one per line", type=str)
+parser.add_argument("--user", metavar='username', help="Username to execute as", type=str)
+parser.add_argument("--max_crawl", metavar='int', help="Max devices to crawl (default 10000)", type=int)
+parser.add_argument("--conf", metavar='file', help="Alternate Config File", type=str)
 parser.add_argument("--debug", help="Set debugging level", type=int)
 parser.add_argument("-v", help="Verbose Output", action="store_true")
+parser.add_argument("--en", metavar='secret', help="Activate privilege level 15", type=str)
 
 args = parser.parse_args()
 
@@ -92,13 +89,20 @@
     else:
         password = getpass.getpass('Password for ' + args.user + ': ')
 
+    if not args.en:
+        if 'secret' in config['main'] and config['main']['secret']:
+            args.en = config['main']['secret']
+
     # Check for output files from config
     if not args.nei_file:
         if 'nei_file' in config['main'] and config['main']['nei_file']:
             args.nei_file = config['main']['nei_file']
     if not args.dev_file:
         if 'dev_file' in config['main'] and config['main']['dev_file']:
             args.dev_file = config['main']['dev_file']
+    if not args.gv_file:
+        if 'gv_file' in config['main'] and config['main']['gv_file']:
+            args.dev_file = config['main']['gv_file']
 
     if args.seed_file:
         seeds = list()
@@ -113,7 +117,7 @@
     if not args.quiet:
         print('Beginning crawl on:', ', '.join(seeds))
 
-    topology.crawl(seeds, args.user, password, outf=args.nei_file, dout=args.dev_file, ngout=args.ng_file)
+    topology.crawl(seeds, args.user, password, args.nei_file, args.dev_file, args.ng_file, args.en)
 else:
     print('\nError: Must provide -seed devices if not using config file\n')
     parser.print_help()
diff --git a/ndlib/execute.py b/ndlib/execute.py
@@ -6,17 +6,18 @@
 
 logger = logging.getLogger(__name__)
 
-def get_session(host, platform, username, password):
+def get_session(host, platform, username, password, secret):
     """ Get an SSH session on device """
 
     net_connect = ConnectHandler(device_type=platform,
                                  ip=host,
                                  global_delay_factor=0.2,
                                  username=username,
                                  password=password,
+                                 secret=secret,
                                  timeout=20)
-
-    net_connect.enable()
+    if secret:
+        net_connect.enable()
 
     return net_connect
 

diff --git a/ndlib/output.py b/ndlib/output.py
@@ -2,29 +2,38 @@
 import csv
 import logging
 
+
 logger = logging.getLogger(__name__)
 
-def output_files(outf, ngout, dout, neighbors, devices, distances):
+
+def output_files(outf, ngout, dout, gvout, neighbors, devices, distances):
     """ Output files to CSV if requested """
 
     # Output Neighbor CSV File
     if outf:
-        fieldnames = ['local_device_id', 'remote_device_id', 'distance', 'local_int', \
-                      'remote_int', 'ipv4', 'os', 'platform', 'description']
-        f = open(outf, 'w')
+        fieldnames = ['local_device_id', 'local_int',
+                      'remote_device_id', 'remote_int',
+                      'remote_ipv4', 'os', 'platform', 'description']
+        f = open(outf, 'w', newline="\n")
         dw = csv.DictWriter(f, fieldnames=fieldnames)
         dw.writeheader()
         for n in neighbors:
-            nw = n.copy()
-            if 'logged_in' in nw:
-                nw.pop('logged_in')
+            # nw = n.copy()
+            if n['local_device_id'] == "Unknown" or n['local_device_id'] == "Seed":
+                continue
+
+            nw = {'local_device_id': n['local_device_id'], 'local_int': n['local_int'],
+                  'remote_device_id': n['remote_device_id'], 'remote_int': n['remote_int'],
+                  'remote_ipv4': n['ipv4'], 'os': n['os'],
+                  'platform': n['platform'], 'description': n['description']}
+
             dw.writerow(nw)
         f.close()
 
     # Output NetGrph CSV File
     if ngout:
         fieldnames = ['LocalName', 'LocalPort', 'RemoteName', 'RemotePort']
-        f = open(ngout, 'w')
+        f = open(ngout, 'w', newline="\n")
         dw = csv.DictWriter(f, fieldnames=fieldnames)
         dw.writeheader()
         for n in neighbors:
@@ -38,20 +47,35 @@ def output_files(outf, ngout, dout, neighbors, devices, distances):
         f.close()
 
     if dout:
-        fieldnames = ['device_id', 'ipv4', 'platform', 'os', 'distance', 'logged_in']
-        f = open(dout, 'w')
+        fieldnames = ['device_id', 'ipv4', 'platform', 'os', 'version', 'image', 'logged_in']
+        f = open(dout, 'w', newline="\n")
         dw = csv.DictWriter(f, fieldnames=fieldnames)
         dw.writeheader()
         for d in sorted(devices):
-            dist = 100
-            if devices[d]['remote_device_id'] in distances:
-                dist = distances[devices[d]['remote_device_id']]
-
             logged_in = False
             if 'logged_in' in devices[d] and devices[d]['logged_in']:
                 logged_in = True
 
-            dd = {'device_id': devices[d]['remote_device_id'], 'ipv4': devices[d]['ipv4'], \
-                  'platform': devices[d]['platform'], 'os': devices[d]['os'], \
-                  'distance': dist, 'logged_in': logged_in}
+            dd = {'device_id': devices[d]['remote_device_id'], 'ipv4': devices[d]['ipv4'],
+                  'platform': devices[d]['platform'], 'os': devices[d]['os'],
+                  'version' : devices[d]['version'], 'image': devices[d]['image'],
+                  'logged_in': logged_in}
             dw.writerow(dd)
+
+    if gvout:
+        #TODO Remove duplicate lines
+        #TODO Add interface names to lines
+        #TODO Add information to devices
+        dot_graph = "graph G { \n"
+        for d in sorted(devices):
+            dot_graph = dot_graph + devices[d]['remote_device_id'].split('.')[0] + "\n"
+
+        for n in neighbors:
+            if n['local_device_id'] == "Unknown" or n['local_device_id'] == "Seed":
+                continue
+            dot_graph = dot_graph + n['local_device_id'].split('.')[0] + " -- " + n['remote_device_id'].split('.')[0] + "\n"
+
+        dot_graph = dot_graph + "\n }"
+        f = open(gvout, 'w')
+        f.write(dot_graph)
+        f.close()
diff --git a/ndlib/parse.py b/ndlib/parse.py
@@ -6,7 +6,8 @@
 
 config = dict()
 
-def merge_nd(nd_cdp, nd_lldp):
+
+def merge_nd(nd_cdp, nd_lldp, serial=None):
     """ Merge CDP and LLDP data into one structure """
 
     neis = dict()
@@ -21,14 +22,16 @@ def merge_nd(nd_cdp, nd_lldp):
         if (n['local_device_id'], n['remote_device_id'], n['local_int'], n['remote_int']) in n:
             if 'description' in neis[(n['local_device_id'], n['remote_device_id'], n['local_int'], n['remote_int'])]:
                 n['description'] = neis[(n['local_device_id'], n['remote_device_id'], n['local_int'], n['remote_int'])]['description']
+        if serial:
+            n['serial'] = serial
         neis[(n['local_device_id'], n['remote_device_id'], n['local_int'], n['remote_int'])] = n
 
-
     for n in neis:
         nd.append(neis[n])
 
     return nd
 
+
 def parse_cdp(cdp, device):
     'Return nd neighbors for IOS/NXOS CDP output'
 
@@ -43,15 +46,16 @@ def parse_cdp(cdp, device):
         ints = re.search(r'^Interface\:\s([A-Za-z0-9\.\-\_\/]+).*\:\s([A-Za-z0-9\.\-\_\/]+)$', l)
         ipv4 = re.search(r'^\s+IPv4\sAddress\:\s(\d+\.\d+\.\d+\.\d+)', l)
         ip = re.search(r'^\s+IP\saddress\:\s(\d+\.\d+\.\d+\.\d+)', l)
-        nxos = re.search(r'Cisco Nexus', l)
-        ios = re.search(r'Cisco IOS', l)
+        nxos = re.search(r'(Cisco Nexus .+)$', l)
+        ios = re.search(r'(Cisco IOS .+)$', l)
+        version = re.search(r'\, Version ([^\,]+)\,', l)
+
         if devid:
             if current:
                 if not re.search(config['main']['ignore_regex'], current['remote_device_id']):
                     nd.append(current.copy())
                 else:
-                    logger.info('Regex Ignore on %s neighbor from %s', \
-                                    current['remote_device_id'], current['local_device_id'])
+                    logger.info('Regex Ignore on %s neighbor from %s', current['remote_device_id'], current['local_device_id'])
             current = dict()
             rname = devid.group(1)
             current['local_device_id'] = dname
@@ -62,8 +66,10 @@ def parse_cdp(cdp, device):
             current['ipv4'] = 'Unknown'
             current['os'] = 'Unknown'
             current['description'] = ''
+            current['version'] = 'Unknown'
+            current['image'] = 'Unknown'
+
         if ints:
-            #print(l, ints.group(1), ints.group(2))
             current['local_int'] = ints.group(1)
             current['remote_int'] = ints.group(2)
         if ipv4:
@@ -79,13 +85,13 @@ def parse_cdp(cdp, device):
             current['os'] = 'cisco_nxos'
         if ios:
             current['os'] = 'cisco_ios'
+        if version:
+            current['version'] = version.group(1)
 
     if current:
-        if not re.search(config['main']['ignore_regex'], current['remote_device_id']):
-            nd.append(current.copy())
+        if not re.search(config['main']['ignore_regex'], current['remote_device_id']): nd.append(current.copy())
         else:
-            logger.warning('Regex Ignore on %s neighbor from %s', \
-                            current['remote_device_id'], current['local_device_id'])
+            logger.warning('Regex Ignore on %s neighbor from %s', current['remote_device_id'], current['local_device_id'])
     return nd
 
 
@@ -123,8 +129,7 @@ def parse_lldp(lldp_det, lldp_sum, device):
                 if not re.search(config['main']['ignore_regex'], current['remote_device_id']):
                     nd.append(current.copy())
                 else:
-                    logger.info('Regex Ignore on %s neighbor from %s', \
-                                    current['remote_device_id'], current['local_device_id'])
+                    logger.info('Regex Ignore on %s neighbor from %s', current['remote_device_id'], current['local_device_id'])
             current = dict()
             rname = devid.group(1)
             current['local_device_id'] = dname
@@ -135,7 +140,8 @@ def parse_lldp(lldp_det, lldp_sum, device):
             current['ipv4'] = 'Unknown'
             current['os'] = 'Unknown'
             current['description'] = ''
-
+            current['version'] = 'Unknown'
+            current['image'] = 'Unknown'
 
         if sysname:
             if not re.search('advertised', sysname.group(1)):
@@ -146,8 +152,7 @@ def parse_lldp(lldp_det, lldp_sum, device):
             # Try to map interface via summary if Unknown (IOS)
             if device['os'] == 'cisco_ios':
                 if current['remote_int'] in dmap:
-                    logger.debug('Mapping %s local interface %s to chassis id %s', \
-                                dname, dmap[current['remote_int']], current['remote_int'])
+                    logger.debug('Mapping %s local interface %s to chassis id %s', dname, dmap[current['remote_int']], current['remote_int'])
                     current['local_int'] = dmap[current['remote_int']]
                 elif current['remote_device_id'] in dmap:
                     current['local_int'] = dmap[current['remote_device_id']]
@@ -175,6 +180,31 @@ def parse_lldp(lldp_det, lldp_sum, device):
         if not re.search(config['main']['ignore_regex'], current['remote_device_id']):
             nd.append(current.copy())
         else:
-            logger.warning('Regex Ignore on %s neighbor from %s', \
-                            current['remote_device_id'], current['local_device_id'])
+            logger.warning('Regex Ignore on %s neighbor from %s', current['remote_device_id'], current['local_device_id'])
     return nd
+
+def parse_serial(serial_data, device=None):
+    'Return the serial number regex`d from the input'
+    if serial_data and serial_data[0]:
+        serial = re.search(r'^Processor board ID (.+)$', serial_data[0])
+        rtrn = serial.group(1)
+    else:
+        rtrn = None
+
+    return rtrn
+
+
+def dedup_devices(devices):
+    """Depuplicate device list based on IPv4"""
+    devs = dict()
+
+    for d in devices:
+        ip = devices[d]['ipv4']
+        if ip in devs:
+            for sub in devices[d]:
+                if devs[devices[d]['ipv4']][sub] == "Unknown" and devices[d][sub] != "Unknown":
+                    devs[devices[d]['ipv4']][sub] = devices[d][sub]
+        else:
+            devs[devices[d]['ipv4']] = devices[d]
+
+    return devs
-Original file line number
+Diff line change
@@ @@ -6,3 +6,4 @@ start-env.sh @@
     output/
     *.csv
     ndcrawl.ini
+    .idea/