From 1615bdc8abb249cd553144e54a2a7aabbec19eab Mon Sep 17 00:00:00 2001 From: Ed Summers Date: Thu, 24 Aug 2017 11:03:56 -0400 Subject: [PATCH 1/6] add python3 support --- utils/wordcloud.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/utils/wordcloud.py b/utils/wordcloud.py index 775acca8..f0864568 100755 --- a/utils/wordcloud.py +++ b/utils/wordcloud.py @@ -2,6 +2,7 @@ from __future__ import print_function import re +import sys import json import fileinput @@ -48,7 +49,7 @@ wordcloud_js = urlopen('https://raw.githubusercontent.com/jasondavies/d3-cloud/master/build/d3.layout.cloud.js').read() -print((""" +output = """ @@ -93,4 +94,6 @@ -""" % (wordcloud_js.decode('utf8'), json.dumps(words, indent=2))).encode('utf8')) +""" % (wordcloud_js.decode('utf8'), json.dumps(words, indent=2)) + +sys.stdout.write(output) From 9aa8af30fd667c7a867d5b0ebf2826d8ab536dc4 Mon Sep 17 00:00:00 2001 From: Ed Summers Date: Thu, 24 Aug 2017 12:56:57 -0400 Subject: [PATCH 2/6] add 2.7 support --- utils/json2csv.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/utils/json2csv.py b/utils/json2csv.py index 89ce678a..589e16d5 100755 --- a/utils/json2csv.py +++ b/utils/json2csv.py @@ -5,13 +5,17 @@ CSV columns. If you'd like it adjusted send a pull request! """ -import csv import sys import json import fileinput if sys.version_info[0] < 3: - sys.exit("Sorry, json2csv.py requires python v3") + try: + import unicodecsv as csv + except ImportError: + sys.exit("unicodecsv is required for python 2") +else: + import csv def main(): sheet = csv.writer(sys.stdout) From d55630e56a009fa9843149e563276289d4caf939 Mon Sep 17 00:00:00 2001 From: Ed Summers Date: Thu, 24 Aug 2017 14:41:03 -0400 Subject: [PATCH 3/6] insure stdout is utf-8 --- utils/json2csv.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/json2csv.py b/utils/json2csv.py index 589e16d5..e684ef68 100755 --- a/utils/json2csv.py +++ b/utils/json2csv.py @@ -7,6 +7,7 @@ import sys import json +import codecs import fileinput if sys.version_info[0] < 3: @@ -17,6 +18,9 @@ else: import csv +if sys.stdout.encoding != 'UTF-8': + sys.stdout = codecs.getwriter('utf8')(sys.stdout, 'strict') + def main(): sheet = csv.writer(sys.stdout) sheet.writerow(get_headings()) From 604e741c8f5ea8681b46a3be21b6bdb15f0af16f Mon Sep 17 00:00:00 2001 From: Ed Summers Date: Thu, 24 Aug 2017 14:57:41 -0400 Subject: [PATCH 4/6] ensure input is utf-8 too? --- utils/json2csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/json2csv.py b/utils/json2csv.py index e684ef68..fbd5dc64 100755 --- a/utils/json2csv.py +++ b/utils/json2csv.py @@ -24,7 +24,7 @@ def main(): sheet = csv.writer(sys.stdout) sheet.writerow(get_headings()) - for line in fileinput.input(): + for line in fileinput.input(openhook=fileinput.hook_encoded("utf-8")): tweet = json.loads(line) sheet.writerow(get_row(tweet)) From 5507dcc5381080120be3695b7504b68de4c32ac0 Mon Sep 17 00:00:00 2001 From: Ed Summers Date: Fri, 25 Aug 2017 15:08:48 +0000 Subject: [PATCH 5/6] added --output option --- utils/json2csv.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/utils/json2csv.py b/utils/json2csv.py index fbd5dc64..e5653e1e 100755 --- a/utils/json2csv.py +++ b/utils/json2csv.py @@ -8,6 +8,7 @@ import sys import json import codecs +import argparse import fileinput if sys.version_info[0] < 3: @@ -18,13 +19,22 @@ else: import csv -if sys.stdout.encoding != 'UTF-8': - sys.stdout = codecs.getwriter('utf8')(sys.stdout, 'strict') - def main(): - sheet = csv.writer(sys.stdout) + parser = argparse.ArgumentParser() + parser.add_argument('--output', '-o', help='write output to file instead of stdout') + parser.add_argument('files', metavar='FILE', nargs='*', help='files to read, if empty, stdin is used') + args = parser.parse_args() + + if args.output: + sheet = csv.writer(open(args.output, "w")) + else: + sheet = csv.writer(sys.stdout) + sheet.writerow(get_headings()) - for line in fileinput.input(openhook=fileinput.hook_encoded("utf-8")): + + files = args.files if len(args.files) > 0 else ('-',) + for line in fileinput.input(files, openhook=fileinput.hook_encoded("utf-8")): + tweet = json.loads(line) sheet.writerow(get_row(tweet)) From 8ea95691bfaa45dc3a82d361eef3f62409a0b792 Mon Sep 17 00:00:00 2001 From: Ed Summers Date: Fri, 25 Aug 2017 15:30:15 +0000 Subject: [PATCH 6/6] set encoding --- utils/json2csv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/json2csv.py b/utils/json2csv.py index e5653e1e..5c261f49 100755 --- a/utils/json2csv.py +++ b/utils/json2csv.py @@ -26,7 +26,7 @@ def main(): args = parser.parse_args() if args.output: - sheet = csv.writer(open(args.output, "w")) + sheet = csv.writer(codecs.open(args.output, 'wb', 'utf-8')) else: sheet = csv.writer(sys.stdout)