Upgrade to py3

.gitignore

@@ -21,3 +21,7 @@
 /web2py/parameters_*.py
 /web2py/welcome.w2p
 /web2py/NEWINSTALL
+/db
+/apps/temp*
+/apps/20newsgroups_*
+**/*.bak

bin/export_corpus.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import print_function
 import sys
 sys.path.append("web2py")
 
@@ -9,7 +10,7 @@
 
 def ExportCorpus(database_path, corpus_filename):
 	database_filename = '{}/corpus.db'.format(database_path)
-	print 'Exporting database [{}] to file [{}]'.format(database_filename, corpus_filename)
+	print('Exporting database [{}] to file [{}]'.format(database_filename, corpus_filename))
 
 	with Corpus_DB(database_path) as corpusDB:
 		corpusDB.ExportToFile(corpus_filename)

bin/export_spreadsheet.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import print_function
 import sys
 sys.path.append("web2py")
 
@@ -12,7 +13,7 @@
 
 def ExportSpreadsheet(database_path, spreadsheet_filename, id_key, content_key, is_csv):
 	database_filename = '{}/corpus.db'.format(database_path)
-	print 'Exporting database [{}] to spreadsheet [{}]'.format(database_filename, spreadsheet_filename)
+	print('Exporting database [{}] to spreadsheet [{}]'.format(database_filename, spreadsheet_filename))
 
 	with Corpus_DB(database_path) as corpus_db:
 		corpus_db.ExportToSpreadsheet(spreadsheet_filename, is_csv = is_csv, id_key = id_key, content_key = content_key)

bin/import_corpus.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import print_function
 import sys
 sys.path.append("web2py")
 
@@ -13,10 +14,10 @@ def ImportCorpus(corpus_filename_or_folder, database_path):
 
 	with Corpus_DB(database_path, isInit=True) as corpus_db:
 		if os.path.isfile(corpus_filename_or_folder):
-			print 'Importing file [{}] into database [{}]'.format(corpus_filename_or_folder, database_filename)
+			print('Importing file [{}] into database [{}]'.format(corpus_filename_or_folder, database_filename))
 			corpus_db.ImportFromFile(corpus_filename_or_folder)
 		else:
-			print 'Importing folder [{}] into database [{}]'.format(corpus_filename_or_folder, database_filename)
+			print('Importing folder [{}] into database [{}]'.format(corpus_filename_or_folder, database_filename))
 			corpus_db.ImportFromFolder(corpus_filename_or_folder)
 
 def main():

bin/import_spreadsheet.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import print_function
 import sys
 sys.path.append("web2py")
 
@@ -12,7 +13,7 @@
 
 def ImportSpreadsheet(spreadsheet_filename, database_path, id_key, content_key, is_csv):
 	database_filename = '{}/corpus.db'.format(database_path)
-	print 'Importing spreadsheet [{}] into database [{}]'.format(spreadsheet_filename, database_filename)
+	print('Importing spreadsheet [{}] into database [{}]'.format(spreadsheet_filename, database_filename))
 
 	with Corpus_DB(database_path, isInit=True) as corpus_db:
 		corpus_db.ImportFromSpreadsheet(spreadsheet_filename, is_csv = is_csv, id_key = id_key, content_key = content_key)

bin/read_treetm.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 import sys
+# This adds web2py to path, so that stuff like gluon can be imported properly 
 sys.path.append("web2py")
 
 import argparse

bin/setup_treetm.sh

@@ -26,7 +26,7 @@ function __setup_treetm__ {
 			mkdir -p $EXTERNALS_PATH
 			mkdir -p $EXTERNALS_SUBPATH
 			echo "    Downloading..."
-			curl --insecure --location http://www.cs.umd.edu/~ynhu/code/tree-TM.zip > $EXTERNALS_SUBPATH/tree-tm.zip
+			curl --insecure --location https://github.com/BYU-NLP-Lab/ITM_Mallet/raw/master/tree-TM.zip > $EXTERNALS_SUBPATH/tree-tm.zip
 			echo "    Extracting README..."
 			unzip $EXTERNALS_SUBPATH/tree-tm.zip tree-TM/readme.txt -d $EXTERNALS_SUBPATH &&\
 				mv $EXTERNALS_SUBPATH/tree-TM/readme.txt $EXTERNALS_SUBPATH/README &&\

bin/unpack_ChinaRuns.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import division
+from past.utils import old_div
 import sys
 sys.path.append("web2py")
 
@@ -49,11 +51,11 @@ def main():
 				"value" : topic_weight
 			})
 	for elem in data:
-		elem['value'] = elem['value'] / max_value
+		elem['value'] = old_div(elem['value'], max_value)
 
 	filename = '{}/meta.json'.format(path)
 	with open(filename, 'w') as f:
-		json.dump(data, f, encoding = 'utf-8', indent = 2, sort_keys = True)
+		json.dump(data, f, indent = 2, sort_keys = True)
 
 if __name__ == '__main__':
 	main()

bin/unpack_PoliblogRuns.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import division
+from past.utils import old_div
 import sys
 sys.path.append("web2py")
 
@@ -51,11 +53,11 @@ def main():
 				"value" : topic_weight
 			})
 	for elem in data:
-		elem['value'] = elem['value'] / max_value
+		elem['value'] = old_div(elem['value'], max_value)
 
 	filename = '{}/meta.json'.format(path)
 	with open(filename, 'w') as f:
-		json.dump(data, f, encoding = 'utf-8', indent = 2, sort_keys = True)
+		json.dump(data, f, indent = 2, sort_keys = True)
 
 if __name__ == '__main__':
 	main()

dataset_src/controllers/upload.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+from builtins import str
 import csv
 import os
 from db.Corpus_DB import Corpus_DB

demo.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import print_function
 import argparse
 import subprocess
 
@@ -13,9 +14,9 @@
 def Shell(command):
 	p = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
 	while p.poll() is None:
-		line = p.stdout.readline().rstrip('\n')
+		line = p.stdout.readline().decode('UTF-8').strip('\n')
 		if len(line) > 0:
-			print line
+			print(line)
 
 def Demonstrate(dataset, model, is_quiet, force_overwrite):
 	database_folder = 'data/demo/{}/corpus'.format(dataset)
@@ -59,15 +60,15 @@ def ImportModel():
 			command.append('--overwrite')
 		Shell(command)
 
-	print '--------------------------------------------------------------------------------'
-	print 'Build a topic model ({}) using a demo dataset ({})'.format(model, dataset)
-	print '  database = {}'.format(database_folder)
-	print '    corpus = {}'.format(corpus_folder)
-	print '     model = {}'.format(model_folder)
-	print '       app = {}'.format(app_name)
-	print '--------------------------------------------------------------------------------'
+	print('--------------------------------------------------------------------------------')
+	print('Build a topic model ({}) using a demo dataset ({})'.format(model, dataset))
+	print('  database = {}'.format(database_folder))
+	print('    corpus = {}'.format(corpus_folder))
+	print('     model = {}'.format(model_folder))
+	print('       app = {}'.format(app_name))
+	print('--------------------------------------------------------------------------------')
 
-	PrepareDataset()
+	# PrepareDataset()
 	PrepareModel()
 	PrepareOthers()
 	TrainModel()

echo_src/controllers/default.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+from builtins import str
 import json
 
 def index():
@@ -9,8 +10,8 @@ def index():
 		value = envObject[ key ]
 		if isinstance( value, dict ) or \
 		   isinstance( value, list ) or isinstance( value, tuple ) or \
-		   isinstance( value, str ) or isinstance( value, unicode ) or \
-		   isinstance( value, int ) or isinstance( value, long ) or isinstance( value, float ) or \
+		   isinstance( value, str ) or isinstance( value, str ) or \
+		   isinstance( value, int ) or isinstance( value, int ) or isinstance( value, float ) or \
 		   value is None or value is True or value is False:
 			envJSON[ key ] = value
 		else:
@@ -30,6 +31,6 @@ def index():
 		'extension' : request.extension,
 		'now' : str( request.now )
 	}
-	dataStr = json.dumps( data, encoding = 'utf-8', indent = 2, sort_keys = True )
+	dataStr = json.dumps(data, indent = 2, sort_keys = True )
 	response.headers['Content-Type'] = 'application/json'
 	return dataStr

server_src/controllers/GroupInBox.py

@@ -24,6 +24,6 @@ def gib():
 	handler.UpdateModel()
 	handler.InspectModel()
 	handler.LoadGIB()
-	dataStr = json.dumps(handler.content, encoding='utf-8', indent=2, sort_keys=True)
+	dataStr = json.dumps(handler.content, indent=2, sort_keys=True)
 	response.headers['Content-Type'] = 'application/json'
 	return dataStr

server_src/controllers/ScatterPlot1.py

@@ -24,6 +24,6 @@ def gib():
 	handler.UpdateModel()
 	handler.InspectModel()
 	handler.LoadGIB()
-	dataStr = json.dumps(handler.content, encoding='utf-8', indent=2, sort_keys=True)
+	dataStr = json.dumps(handler.content, indent=2, sort_keys=True)
 	response.headers['Content-Type'] = 'application/json'
 	return dataStr

server_src/controllers/TermTopicMatrix1.py

@@ -17,7 +17,7 @@ def StateModel():
 		with LDA_DB() as lda_db:
 			handler = TermTopicMatrix1(request, response, bow_db, lda_db)
 			data = handler.GetStateModel()
-	dataStr = json.dumps(data, encoding='utf-8', indent=2, sort_keys=True)
+	dataStr = json.dumps(data, indent=2, sort_keys=True)
 	response.headers['Content-Type'] = 'application/json'
 	return dataStr
 
@@ -26,7 +26,7 @@ def SeriatedTermTopicProbabilityModel():
 		with LDA_DB() as lda_db:
 			handler = TermTopicMatrix1(request, response, bow_db, lda_db)
 			data = handler.GetSeriatedTermTopicProbabilityModel()
-	dataStr = json.dumps(data, encoding='utf-8', indent=2, sort_keys=True)
+	dataStr = json.dumps(data, indent=2, sort_keys=True)
 	response.headers['Content-Type'] = 'application/json'
 	return dataStr
 
@@ -35,7 +35,7 @@ def FilteredTermTopicProbabilityModel():
 		with LDA_DB() as lda_db:
 			handler = TermTopicMatrix1(request, response, bow_db, lda_db)
 	data = handler.GetFilteredTermTopicProbabilityModel()
-	dataStr = json.dumps(data, encoding='utf-8', indent=2, sort_keys=True)
+	dataStr = json.dumps(data, indent=2, sort_keys=True)
 	response.headers['Content-Type'] = 'application/json'
 	return dataStr
 
@@ -44,6 +44,6 @@ def TermFrequencyModel():
 		with LDA_DB() as lda_db:
 			handler = TermTopicMatrix1(request, response, bow_db, lda_db)
 			data = handler.GetTermFrequencyModel()
-	dataStr = json.dumps(data, encoding='utf-8', indent=2, sort_keys=True)
+	dataStr = json.dumps(data, indent=2, sort_keys=True)
 	response.headers['Content-Type'] = 'application/json'
 	return dataStr

server_src/controllers/TermTopicMatrix2.py

@@ -17,6 +17,6 @@ def GetEntry():
 		with LDA_DB() as lda_db:
 			handler = TermTopicMatrix2(request, response, bow_db, lda_db)
 			data = handler.GetEntry()
-	dataStr = json.dumps(data, encoding='utf-8', indent=2, sort_keys=True)
+	dataStr = json.dumps(data, indent=2, sort_keys=True)
 	response.headers['Content-Type'] = 'application/json'
 	return dataStr

server_src/controllers/TermTopicMatrix3.py

@@ -17,7 +17,7 @@ def GetTerms():
 		with LDA_DB() as lda_db:
 			handler = TermTopicMatrix3(request, response, bow_db, lda_db)
 			data = handler.GetTerms()
-	dataStr = json.dumps(data, encoding='utf-8', indent=2, sort_keys=True)
+	dataStr = json.dumps(data, indent=2, sort_keys=True)
 	response.headers['Content-Type'] = 'application/json'
 	return dataStr
 
@@ -26,7 +26,7 @@ def GetTopics():
 		with LDA_DB() as lda_db:
 			handler = TermTopicMatrix3(request, response, bow_db, lda_db)
 			data = handler.GetTopics()
-	dataStr = json.dumps(data, encoding='utf-8', indent=2, sort_keys=True)
+	dataStr = json.dumps(data, indent=2, sort_keys=True)
 	response.headers['Content-Type'] = 'application/json'
 	return dataStr
 
@@ -35,6 +35,6 @@ def GetTermTopicMatrix():
 		with LDA_DB() as lda_db:
 			handler = TermTopicMatrix3(request, response, bow_db, lda_db)
 			data = handler.GetTermTopicMatrix()
-	dataStr = json.dumps(data, encoding='utf-8', indent=2, sort_keys=True)
+	dataStr = json.dumps(data, indent=2, sort_keys=True)
 	response.headers['Content-Type'] = 'application/json'
 	return dataStr

server_src/modules/apps/CreateApp.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from builtins import object
 import logging
 import os
 import random
@@ -29,7 +30,7 @@ def __init__( self, appName, APPS_ROOT = 'apps' ):
 	def RunCommand( self, command ):
 		p = subprocess.Popen( command, stdout = subprocess.PIPE, stderr = subprocess.STDOUT )
 		while p.poll() is None:
-			line = p.stdout.readline().rstrip('\n')
+			line = p.stdout.readline().decode('UTF-8').strip('\n')
 			if len(line) > 0:
 				self.logger.debug( line )
 

server_src/modules/apps/SplitSentences.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from builtins import object
 import logging
 import subprocess
 
-class SplitSentences():
+class SplitSentences(object):
 
 	SENTENCE_SPLITTER = 'utils/corenlp/SentenceSplitter.jar'
 
@@ -16,6 +17,6 @@ def __init__( self, inputCorpusFilename, outputSentenceFilename ):
 	def Shell( self, command ):
 		p = subprocess.Popen( command, stdout = subprocess.PIPE, stderr = subprocess.STDOUT )
 		while p.poll() is None:
-			line = p.stdout.readline().rstrip('\n')
+			line = p.stdout.readline().decode('UTF-8').strip('\n')
 			if len(line) > 0:
 				self.logger.debug( line )