WIP: Python 3 integration branch

openlibrary/catalog/marc/marc_xml.py

@@ -1,8 +1,9 @@
-from lxml import etree
-from marc_base import MarcBase, MarcException
 from unicodedata import normalize
 
 import six
+from lxml import etree
+
+from openlibrary.catalog.marc.marc_base import MarcBase, MarcException
 
 data_tag = '{http://www.loc.gov/MARC21/slim}datafield'
 control_tag = '{http://www.loc.gov/MARC21/slim}controlfield'

openlibrary/catalog/merge/names.py

@@ -1,6 +1,6 @@
 from __future__ import print_function
 import re
-from normalize import normalize
+from openlibrary.catalog.merge.normalize import normalize
 
 re_split_parts = re.compile('(.*?[. ]+)')
 re_marc_name = re.compile('^(.*), (.*)$')

openlibrary/catalog/oca/parse.py

@@ -1,6 +1,6 @@
 from __future__ import print_function
 import sys
-from cStringIO import StringIO
+from six import StringIO
 from xml.parsers.expat import error as xml_error
 from elementtree import ElementTree
 from types import *

openlibrary/core/init.py

@@ -14,7 +14,7 @@
 import sys
 import time
 
-import formats
+from openlibrary.core import formats
 
 class Init:
     """Init process for starting and managing OL services.

openlibrary/core/inlibrary.py

@@ -1,10 +1,8 @@
 """Tools for in-library lending.
 """
 import web
-import cache
-import iprange
-import geo_ip
 from infogami.utils import delegate
+from openlibrary.core import cache, geo_ip, iprange
 
 def _get_libraries(site=None):
     """Returns all the libraries each as a dict."""

openlibrary/core/middleware.py

@@ -1,7 +1,7 @@
 """WSGI middleware used in Open Library.
 """
 import web
-import StringIO
+from six import StringIO
 import gzip
 
 class GZipMiddleware:
@@ -23,7 +23,7 @@ def get_response_header(name, default=None):
             return default
 
         def compress(text, level=9):
-            f = StringIO.StringIO()
+            f = StringIO()
             gz = gzip.GzipFile(None, 'wb', level, fileobj=f)
             gz.write(text)
             gz.close()

openlibrary/core/models.py

@@ -7,21 +7,19 @@
 import iptools
 from infogami.infobase import client
 
-import helpers as h
+from openlibrary.core import helpers as h
 
 #TODO: fix this. openlibrary.core should not import plugins.
 from openlibrary import accounts
-from openlibrary.utils import extract_numeric_id_from_olid
-from openlibrary.plugins.upstream.utils import get_history
-from openlibrary.core.helpers import private_collection_in
+from openlibrary.core import db, cache, iprange, inlibrary, loanstats, waitinglist, lending
 from openlibrary.core.bookshelves import Bookshelves
+from openlibrary.core.helpers import private_collection_in
+from openlibrary.core.lists.model import ListMixin, Seed
 from openlibrary.core.ratings import Ratings
-from openlibrary.utils.isbn import to_isbn_13, isbn_13_to_isbn_10
 from openlibrary.core.vendors import create_edition_from_amazon_metadata
-
-# relative imports
-from lists.model import ListMixin, Seed
-from . import db, cache, iprange, inlibrary, loanstats, waitinglist, lending
+from openlibrary.plugins.upstream.utils import get_history
+from openlibrary.utils import extract_numeric_id_from_olid
+from openlibrary.utils.isbn import to_isbn_13, isbn_13_to_isbn_10
 
 from six.moves import urllib
 

openlibrary/core/olmarkdown.py

@@ -11,7 +11,7 @@
 the javascript markdown editor used in OL.
 """
 
-import helpers as h
+from openlibrary.core import helpers as h
 import re
 from infogami.utils.markdown import markdown
 

openlibrary/core/processors/__init__.py

@@ -1 +1 @@
-from readableurls import ReadableUrlProcessor
+from openlibrary.core.processors.readableurls import ReadableUrlProcessor

openlibrary/coverstore/tests/test_code.py

@@ -1,5 +1,5 @@
 from .. import code
-from cStringIO import StringIO
+from six import StringIO
 import web
 import datetime
 

openlibrary/coverstore/warc.py

@@ -4,18 +4,19 @@
 http://archive-access.sourceforge.net/warc/warc_file_format-0.10.html
 """
 
-import urllib
-import httplib
 import datetime
 
+from six.moves.http_client import HTTPConnection
+from six.moves.urllib.parse import urlparse
+
 WARC_VERSION = "0.10"
 CRLF = "\r\n"
 
 class WARCReader:
     """Reader to read records from a warc file.
 
-    >>> import StringIO
-    >>> f = StringIO.StringIO()
+    >>> from six import StringIO
+    >>> f = StringIO()
     >>> r1 = WARCRecord("resource", "subject_uri", "image/jpeg", {"hello": "world"}, "foo")
     >>> r2 = WARCRecord("resource", "subject_uri", "image/jpeg", {"hello": "world"}, "bar")
     >>> w = WARCWriter(f)
@@ -111,7 +112,7 @@ def _readuntil(self, condition):
 
     def read(self, size):
         protocol, host, port, path = self.urlsplit(self.url)
-        conn = httplib.HTTPConnection(host, port)
+        conn = HTTPConnection(host, port)
         headers = {'Range': 'bytes=%d-%d' % (self.offset, self.offset + size - 1)}
         conn.request('GET', path, None, headers)
         response = conn.getresponse()
@@ -123,13 +124,16 @@ def urlsplit(self, url):
         """Splits url into protocol, host, port and path.
 
             >>> f = HTTPFile('')
+            >>> f.urlsplit("http://www.google.com")
+            ('http', 'www.google.com', None, '')
             >>> f.urlsplit("http://www.google.com/search?q=hello")
             ('http', 'www.google.com', None, '/search?q=hello')
+            >>> f.urlsplit("http://www.google.com:80/search?q=hello")
+            ('http', 'www.google.com', 80, '/search?q=hello')
         """
-        protocol, rest = urllib.splittype(url)
-        hostport, path = urllib.splithost(rest)
-        host, port = urllib.splitport(hostport)
-        return protocol, host, port, path
+        p = urlparse(url)
+        fullpath = "?".join((p.path, p.query)) if (p.path or p.query) else ""
+        return p.scheme, p.hostname, p.port, fullpath
 
 class WARCHeader:
     r"""WARCHeader class represents the header in the WARC file format.
@@ -219,9 +223,9 @@ def __repr__(self):
 class LazyWARCRecord(WARCRecord):
     """Class to create WARCRecord lazily.
 
-    >>> import StringIO
+    >>> from six import StringIO
     >>> r1 = WARCRecord("resource", "subject_uri", "image/jpeg", {"hello": "world"}, "foo bar", creation_date="20080808080808", record_id="record_42")
-    >>> f = StringIO.StringIO(str(r1))
+    >>> f = StringIO(str(r1))
     >>> offset = len(str(r1.get_header()))
     >>> r2 = LazyWARCRecord(f, offset, r1.get_header())
     >>> r1 == r2
@@ -247,8 +251,9 @@ def get_data(self):
 class WARCWriter:
     r"""Writes to write warc records to file.
 
-    >>> import re, StringIO
-    >>> f = StringIO.StringIO()
+    >>> import re
+    >>> from six import StringIO
+    >>> f = StringIO()
     >>> r1 = WARCRecord("resource", "subject_uri", "image/jpeg", {"hello": "world"}, "foo", creation_date="20080808080808", record_id="record_42")
     >>> r2 = WARCRecord("resource", "subject_uri", "image/jpeg", {"hello": "world"}, "bar", creation_date="20080808090909", record_id="record_43")
     >>> w = WARCWriter(f)

openlibrary/plugins/importapi/import_opds.py

@@ -2,7 +2,7 @@
 OL Import API OPDS parser
 """
 
-import import_edition_builder
+from openlibrary.plugins.importapi import import_edition_builder
 
 import six
 

openlibrary/plugins/importapi/import_rdf.py

@@ -2,7 +2,7 @@
 OL Import API RDF parser
 """
 
-import import_edition_builder
+from openlibrary.plugins.importapi import import_edition_builder
 
 import six
 

openlibrary/plugins/importapi/metaxml_to_json.py

@@ -10,7 +10,7 @@
 """
 from __future__ import print_function
 
-from import_edition_builder import import_edition_builder
+from openlibrary.plugins.importapi.import_edition_builder import import_edition_builder
 
 def parse_collection(collection):
     collection_dict = {

openlibrary/plugins/inside/code.py

@@ -1,21 +1,19 @@
-
 from time import time
 import re
 import simplejson
-import httplib
 import logging
 
 import web
 from infogami.utils import delegate
 from infogami.utils.view import render_template
 from openlibrary.core.fulltext import fulltext_search
 
-
 from six.moves import urllib
-
+import six.moves.http_client as httplib
 
 RESULTS_PER_PAGE = 20
 
+
 class search_inside(delegate.page):
 
     path = '/search/inside'

openlibrary/plugins/openlibrary/borrow_home.py

@@ -11,9 +11,9 @@
 
 from openlibrary.core import helpers as h
 from openlibrary.core import inlibrary
+from openlibrary.plugins.openlibrary.libraries import LoanStats
 from openlibrary.plugins.worksearch.subjects import SubjectEngine
 
-from libraries import LoanStats
 
 class borrow(delegate.page):
     path = "/borrow"

openlibrary/plugins/openlibrary/libraries.py

@@ -5,7 +5,7 @@
 import logging
 import datetime
 import itertools
-from cStringIO import StringIO
+from six import StringIO
 import csv
 import simplejson
 

openlibrary/plugins/openlibrary/stats.py

@@ -4,14 +4,13 @@
 import logging
 import traceback
 
-from openlibrary.core import stats as graphite_stats
-
 import web
 from infogami import config
 from infogami.utils import stats
-import openlibrary.core.stats
 
-import filters as stats_filters
+from openlibrary.core import stats as graphite_stats
+import openlibrary.core.stats
+import openlibrary.plugins.openlibrary.filters as stats_filters
 
 l = logging.getLogger("openlibrary.stats")
 

openlibrary/plugins/search/code.py

@@ -1,26 +1,23 @@
 from __future__ import print_function
-import web
-import stopword
-import pdb
-
-from infogami import utils
-from infogami.utils import delegate
-from infogami.infobase.client import Thing
-from infogami.utils import view, template
-from infogami import config
-from infogami.plugins.api.code import jsonapi
 
+import pdb
 import re
-import web
-import solr_client
 import time
-import simplejson
+from collections import defaultdict
 from functools import partial
 from gzip import open as gzopen
-import cPickle
-from collections import defaultdict
 
+import simplejson
 import six
+import web
+from infogami import config, utils
+from infogami.infobase.client import Thing
+from infogami.plugins.api.code import jsonapi
+from infogami.utils import delegate, template, view
+from six.moves import cPickle
+
+from openlibrary.plugins.search import facet_hash, solr_client, stopword
+from openlibrary.plugins.search.collapse import collapse_groups
 
 render = template.render
 
@@ -70,7 +67,6 @@ def lookup_ocaid(ocaid):
     w = web.ctx.site.get(ocat[0]) if ocat else None
     return w
 
-from collapse import collapse_groups
 class fullsearch(delegate.page):
     def POST(self):
         errortext = None
@@ -135,7 +131,6 @@ class Result_nums: pass
 
     GET = POST
 
-import facet_hash
 facet_token = view.public(facet_hash.facet_token)
 
 class Timestamp(object):

openlibrary/plugins/search/solr_client.py

@@ -1,19 +1,20 @@
 #!/usr/bin/python
 from xml.etree.cElementTree import ElementTree
-from cStringIO import StringIO
 import os
 import re
 from collections import defaultdict
 import cgi
 import web
 import simplejson
-from facet_hash import facet_token
 import pdb
 
 import six
+from six import StringIO
 from six.moves.urllib.parse import quote_plus
 from six.moves.urllib.request import urlopen
 
+from openlibrary.plugins.search.facet_hash import facet_token
+
 php_location = "/petabox/setup.inc"
 
 # server_addr = ('pharosdb.us.archive.org', 8983)

openlibrary/plugins/upstream/account.py

@@ -18,19 +18,15 @@
 import infogami.core.code as core
 
 from openlibrary import accounts
-from openlibrary.i18n import gettext as _
-from openlibrary.core import helpers as h, lending
-from openlibrary.core.bookshelves import Bookshelves
-from openlibrary.plugins.recaptcha import recaptcha
-from openlibrary.plugins import openlibrary as olib
 from openlibrary.accounts import (
     audit_accounts, Account, OpenLibraryAccount, InternetArchiveAccount, valid_email)
+from openlibrary.core import helpers as h, lending
+from openlibrary.core.bookshelves import Bookshelves
 from openlibrary.core.sponsorships import get_sponsored_editions
-
-import forms
-import utils
-import borrow
-
+from openlibrary.i18n import gettext as _
+from openlibrary.plugins import openlibrary as olib
+from openlibrary.plugins.recaptcha import recaptcha
+from openlibrary.plugins.upstream import borrow, forms, utils
 
 from six.moves import range
 from six.moves import urllib