finally tidied up a bit

.gitignore

@@ -0,0 +1 @@
+runs/

FreshGrab.py

@@ -1,5 +1,25 @@
 #!/usr/bin/python
 
+# Copyright (C) 2018 Stephen Farrell, [email protected]
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 import os, sys, argparse, tempfile, gc
 import json, jsonpickle
 import time

GrabIPs.py

@@ -1,5 +1,25 @@
 #!/usr/bin/python
 
+# Copyright (C) 2018 Stephen Farrell, [email protected]
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 # grab just the IPs from a censys file 
 # or any other with one json structure per line and an 'ip' key in that dict
 

GraphKeyReuse3.py

@@ -1,5 +1,25 @@
 #!/usr/bin/python
 
+# Copyright (C) 2018 Stephen Farrell, [email protected]
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 # check who's re-using the same keys 
 # CensysIESMTP.py
 

KeyTypes.py

@@ -1,5 +1,25 @@
 #!/usr/bin/python
 
+# Copyright (C) 2018 Stephen Farrell, [email protected]
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 # check what kind of keys are in use
 # CensysIESMTP.py
 

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017 sftcd
+Copyright (c) 2018 Stephen Farrell, [email protected]
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,6 +1,26 @@
 # Our two populations are servers in IE and EE (according to
 # maxmind) who listen on port 25.
 
+# Copyright (C) 2018 Stephen Farrell, [email protected]
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 SRCDIR=${HOME}/code/surveys
 DATADIR=${HOME}/data/smtp/runs
 

README.md

@@ -1,59 +1,11 @@
 # surveys
 
-Code for various survey-related stuff
+Code for various crypto-survey related stuff. 
 
-Note that this very much in flux as I figure out what's interesting. I 
-promise to put manners on the code soon as I know what I want it do to:-)
-Getting there though...
+The current code collects and collates keys and sees which of those
+are the same. 
 
-Initial work is based on using the [censys.io](https://censys.io/) API
+The ```misc``` directory has various bits and pieces knocked up along
+the way in case they prove handy later.
 
-## Cron job
-
-```CensysSMTP-cron.py``` is set to pull a weekly view of the SMTP speakers,
-in my case from IE and EE (the latter for comparison).
-
-## Analysis
-
-As a first cut, I've analysed the mail speakers into the following classes:
-
-- "good-looking" : Talks SMTPTLS with certs that chain to a browser-trusted root 
-- "medium-looking" : Talks STARTTLS ok but without a cert like the above (maybe a local root)
-- "self-signed" : Talks STARTTLS ok but with a self-signed cert 
-- "bad-sig" : bad signature on certificate somewhere
-- "bad-looking" : doesn't manage to talk STARTTLS for some reason
-- "dunno" : doesn't fit the above
-
-```Classify.py``` takes a filename (produced by CensysSMTP-cron for examplle) as input and splits that file into
-those classes i.e. good, badsig, medium, selfsigned, bad, and dunno, and 
-produces basic counts. Those files are dropped into an "outs" 
-directory below the ```$CWD``` for now.
-
-Later, we can try do some stats etc.
-
-## A run for Sep 29th for Ireland and Estonia
-
-<pre>
-For SMTP speakers apparently in Ireland:
-
-     1852   good
-     2713   badsig
-     2116   medium
-     1010   selfsigned
-     4670   bad
-      135   dunno
-
-    12496   total
-
-For SMTP speakers apparently in Estonia:
-
-      832   good
-     4196   badsig
-     3415   medium
-     2047   selfsigned
-     1921   bad
-       59   dunno
-
-    12470   total
-</pre>
 

SameKeys.py

@@ -1,5 +1,25 @@
 #!/usr/bin/python
 
+# Copyright (C) 2018 Stephen Farrell, [email protected]
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 # check who's re-using the same keys 
 # CensysIESMTP.py
 

SurveyFuncs.py

@@ -1,4 +1,24 @@
 #!/usr/bin/python
+
+# Copyright (C) 2018 Stephen Farrell, [email protected]
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
 import json
 import jsonpickle
 import copy

TwentyTwos.py

@@ -1,5 +1,25 @@
 #!/usr/bin/python
 
+# Copyright (C) 2018 Stephen Farrell, [email protected]
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 # read out the port 22 collisions and verify those using ssh-keyscan
 
 import os, sys, argparse, tempfile, gc

misc/README.md

@@ -0,0 +1,65 @@
+# surveys
+
+This used to be the main README for this repo. This directory just has
+bits and pieces assembled along the way in case they turn out useful
+sometime. You should mostly ignore it all.
+
+# Stuff to ignore below here...
+
+Code for various survey-related stuff
+
+Note that this very much in flux as I figure out what's interesting. I 
+promise to put manners on the code soon as I know what I want it do to:-)
+Getting there though...
+
+Initial work is based on using the [censys.io](https://censys.io/) API
+
+## Cron job
+
+```CensysSMTP-cron.py``` is set to pull a weekly view of the SMTP speakers,
+in my case from IE and EE (the latter for comparison).
+
+## Analysis
+
+As a first cut, I've analysed the mail speakers into the following classes:
+
+- "good-looking" : Talks SMTPTLS with certs that chain to a browser-trusted root 
+- "medium-looking" : Talks STARTTLS ok but without a cert like the above (maybe a local root)
+- "self-signed" : Talks STARTTLS ok but with a self-signed cert 
+- "bad-sig" : bad signature on certificate somewhere
+- "bad-looking" : doesn't manage to talk STARTTLS for some reason
+- "dunno" : doesn't fit the above
+
+```Classify.py``` takes a filename (produced by CensysSMTP-cron for examplle) as input and splits that file into
+those classes i.e. good, badsig, medium, selfsigned, bad, and dunno, and 
+produces basic counts. Those files are dropped into an "outs" 
+directory below the ```$CWD``` for now.
+
+Later, we can try do some stats etc.
+
+## A run for Sep 29th for Ireland and Estonia
+
+<pre>
+For SMTP speakers apparently in Ireland:
+
+     1852   good
+     2713   badsig
+     2116   medium
+     1010   selfsigned
+     4670   bad
+      135   dunno
+
+    12496   total
+
+For SMTP speakers apparently in Estonia:
+
+      832   good
+     4196   badsig
+     3415   medium
+     2047   selfsigned
+     1921   bad
+       59   dunno
+
+    12470   total
+</pre>
+

mm_update.sh

@@ -1,5 +1,25 @@
 #!/bin/bash
 
+# Copyright (C) 2018 Stephen Farrell, [email protected]
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 # grab and explode updated versions of maxmind's free DBs
 
 set -x