From d46bd94ad51d9117b222d9e1b82cb7b59f95c5b2 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 06:43:17 +0100 Subject: [PATCH 01/47] from web.browser import AppBrowser Cherry-picked from #2416 Corresponds to the suggestion at https://github.com/internetarchive/openlibrary/pull/2416#issuecomment-536190367 --- openlibrary/mocks/mock_ol.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openlibrary/mocks/mock_ol.py b/openlibrary/mocks/mock_ol.py index 19a587eafb6..5bee83a9b91 100644 --- a/openlibrary/mocks/mock_ol.py +++ b/openlibrary/mocks/mock_ol.py @@ -2,6 +2,11 @@ import pytest import re import web +try: # newer versions of web.py + from web.browser import AppBrowser + except ImportError: # older versions of web.py + from web import AppBrowser + from infogami import config from infogami.infobase import client from infogami.utils import delegate @@ -34,11 +39,11 @@ def extract_links(self): """Extracts link from the email message.""" return re.findall(r"http://[^\s]*", self.message) -class OLBrowser(web.AppBrowser): +class OLBrowser(AppBrowser): def get_text(self, e=None, name=None, **kw): if name or kw: e = self.get_soup().find(name=name, **kw) - return web.AppBrowser.get_text(self, e) + return AppBrowser.get_text(self, e) class OL: """Mock OL object for all tests. From 6497b0ac5e72fadf4596685f0110ff89cb815e75 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 06:47:00 +0100 Subject: [PATCH 02/47] from web.browser import AppBrowser --- openlibrary/mocks/mock_ol.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openlibrary/mocks/mock_ol.py b/openlibrary/mocks/mock_ol.py index 5bee83a9b91..10375ac834f 100644 --- a/openlibrary/mocks/mock_ol.py +++ b/openlibrary/mocks/mock_ol.py @@ -3,9 +3,9 @@ import re import web try: # newer versions of web.py - from web.browser import AppBrowser - except ImportError: # older versions of web.py - from web import AppBrowser + from web.browser import AppBrowser +except ImportError: # older versions of web.py + from web import AppBrowser from infogami import config from infogami.infobase import client From 07f7dbb3ace822cfcda60a0739bdccd123cd29bd Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 07:29:16 +0100 Subject: [PATCH 03/47] from openlibrary.catalog.marc import mnemonics --- openlibrary/catalog/marc/fast_parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/catalog/marc/fast_parse.py b/openlibrary/catalog/marc/fast_parse.py index e906454e0ae..624f3e15b45 100644 --- a/openlibrary/catalog/marc/fast_parse.py +++ b/openlibrary/catalog/marc/fast_parse.py @@ -3,8 +3,8 @@ import re from pymarc import MARC8ToUnicode -import mnemonics from unicodedata import normalize +from openlibrary.catalog.marc import mnemonics from openlibrary.catalog.utils import tidy_isbn re_real_book = re.compile('(pbk|hardcover|alk[^a-z]paper|cloth)', re.I) From 9279fe6f15b444dffd70dfacad3ab72cc1d88dbf Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 07:32:25 +0100 Subject: [PATCH 04/47] from openlibrary.core import formats --- openlibrary/core/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/core/init.py b/openlibrary/core/init.py index 70d4cb46e5e..4d4c89207d5 100644 --- a/openlibrary/core/init.py +++ b/openlibrary/core/init.py @@ -14,7 +14,7 @@ import sys import time -import formats +from openlibrary.core import formats class Init: """Init process for starting and managing OL services. From 7d37ee3abb69cf3dcc385760ab26c4001f8cdd19 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 07:34:25 +0100 Subject: [PATCH 05/47] from openlibrary.core import helpers --- openlibrary/core/olmarkdown.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/core/olmarkdown.py b/openlibrary/core/olmarkdown.py index 7d26ecdd443..e33503894ec 100644 --- a/openlibrary/core/olmarkdown.py +++ b/openlibrary/core/olmarkdown.py @@ -11,7 +11,7 @@ the javascript markdown editor used in OL. """ -import helpers as h +from openlibrary.core import helpers as h import re from infogami.utils.markdown import markdown From c819b29e595bee79521ae98007e0ab848d7a75ce Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 07:36:04 +0100 Subject: [PATCH 06/47] from openlibrary.core.readableurls import ReadableUrlProcessor --- openlibrary/core/processors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/core/processors/__init__.py b/openlibrary/core/processors/__init__.py index ac7199d807b..455d16a57ec 100644 --- a/openlibrary/core/processors/__init__.py +++ b/openlibrary/core/processors/__init__.py @@ -1 +1 @@ -from readableurls import ReadableUrlProcessor +from openlibrary.core.readableurls import ReadableUrlProcessor From c880526c2509fc2ea0b346601f6d9791acfd2afb Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 07:39:04 +0100 Subject: [PATCH 07/47] from openlibrary.core import cache, geo_ip, iprange --- openlibrary/core/inlibrary.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openlibrary/core/inlibrary.py b/openlibrary/core/inlibrary.py index 5af6b1de7ed..e2e4ad374bb 100644 --- a/openlibrary/core/inlibrary.py +++ b/openlibrary/core/inlibrary.py @@ -1,10 +1,8 @@ """Tools for in-library lending. """ import web -import cache -import iprange -import geo_ip from infogami.utils import delegate +from openlibrary.core import cache, geo_ip, iprange def _get_libraries(site=None): """Returns all the libraries each as a dict.""" From bb0b80fd695cb1dfadede303507d1c836bf674db Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 07:48:30 +0100 Subject: [PATCH 08/47] from openlibrary.core.processors.readableurls import ReadableUrlProcessor --- openlibrary/core/processors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/core/processors/__init__.py b/openlibrary/core/processors/__init__.py index 455d16a57ec..01fd63f75c2 100644 --- a/openlibrary/core/processors/__init__.py +++ b/openlibrary/core/processors/__init__.py @@ -1 +1 @@ -from openlibrary.core.readableurls import ReadableUrlProcessor +from openlibrary.core.processors.readableurls import ReadableUrlProcessor From 98bd644a882d2d877bbd0922f09b194122bf55de Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 08:12:56 +0100 Subject: [PATCH 09/47] from openlibrary.catalog.marc.marc_base import MarcBase, MarcException, BadMARC --- openlibrary/catalog/marc/marc_binary.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openlibrary/catalog/marc/marc_binary.py b/openlibrary/catalog/marc/marc_binary.py index 44bd959530e..b7a1ab8ce95 100644 --- a/openlibrary/catalog/marc/marc_binary.py +++ b/openlibrary/catalog/marc/marc_binary.py @@ -1,9 +1,9 @@ -from openlibrary.catalog.marc import fast_parse -#TODO: Move fast_parse get_tag_lines(), handle_wrapped_lines(), get_all_tag_lines() into this class -from marc_base import MarcBase, MarcException, BadMARC from unicodedata import normalize from pymarc import MARC8ToUnicode -from openlibrary.catalog.marc import mnemonics + +#TODO: Move fast_parse get_tag_lines(), handle_wrapped_lines(), get_all_tag_lines() into this class +from openlibrary.catalog.marc import fast_parse, mnemonics +from openlibrary.catalog.marc.marc_base import MarcBase, MarcException, BadMARC import six From 8e361c9d02017a36eb6d2456505a5d6977ac7781 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 08:15:37 +0100 Subject: [PATCH 10/47] from six import StringIO --- openlibrary/plugins/upstream/addbook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/plugins/upstream/addbook.py b/openlibrary/plugins/upstream/addbook.py index 2d4f1bb8e50..73a385ef55c 100644 --- a/openlibrary/plugins/upstream/addbook.py +++ b/openlibrary/plugins/upstream/addbook.py @@ -3,7 +3,6 @@ import web import simplejson from collections import defaultdict -from StringIO import StringIO import csv import datetime @@ -29,6 +28,7 @@ from . import spamcheck import six +from six import StringIO from six.moves import urllib From 721b4ad996a94b1a3b7f20b9933dc5733ee7fdcd Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 08:24:27 +0100 Subject: [PATCH 11/47] from openlibrary.catalog.marc.get_subjects import subjects_for_work --- openlibrary/catalog/marc/parse.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openlibrary/catalog/marc/parse.py b/openlibrary/catalog/marc/parse.py index 1db516a0cc5..6ffd8b09c99 100644 --- a/openlibrary/catalog/marc/parse.py +++ b/openlibrary/catalog/marc/parse.py @@ -1,8 +1,9 @@ import re -from openlibrary.catalog.utils import pick_first_date, tidy_isbn, flip_name, remove_trailing_dot, remove_trailing_number_dot -from get_subjects import subjects_for_work from collections import defaultdict -from marc_base import BadMARC, NoTitle, MarcException + +from openlibrary.catalog.marc.get_subjects import subjects_for_work +from openlibrary.catalog.marc.marc_base import BadMARC, NoTitle, MarcException +from openlibrary.catalog.utils import pick_first_date, tidy_isbn, flip_name, remove_trailing_dot, remove_trailing_number_dot re_question = re.compile('^\?+$') re_lccn = re.compile('(...\d+).*') From 7375c7c30e1ba6d9a4f35285e0fc7370e23b29be Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 08:26:07 +0100 Subject: [PATCH 12/47] from openlibrary.catalog.marc.marc_base import MarcBase, MarcException --- openlibrary/catalog/marc/marc_xml.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openlibrary/catalog/marc/marc_xml.py b/openlibrary/catalog/marc/marc_xml.py index 1fedc970d02..5d5113be648 100644 --- a/openlibrary/catalog/marc/marc_xml.py +++ b/openlibrary/catalog/marc/marc_xml.py @@ -1,8 +1,9 @@ -from lxml import etree -from marc_base import MarcBase, MarcException from unicodedata import normalize import six +from lxml import etree + +from openlibrary.catalog.marc.marc_base import MarcBase, MarcException data_tag = '{http://www.loc.gov/MARC21/slim}datafield' control_tag = '{http://www.loc.gov/MARC21/slim}controlfield' From a489a1ab66bda86a82e63f69372b1dc13e88ce0d Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 08:27:16 +0100 Subject: [PATCH 13/47] from openlibrary.catalog.merge.normalize import normalize --- openlibrary/catalog/merge/names.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/catalog/merge/names.py b/openlibrary/catalog/merge/names.py index 626974f2377..594e5a66dc7 100644 --- a/openlibrary/catalog/merge/names.py +++ b/openlibrary/catalog/merge/names.py @@ -1,6 +1,6 @@ from __future__ import print_function import re -from normalize import normalize +from openlibrary.catalog.merge.normalize import normalize re_split_parts = re.compile('(.*?[. ]+)') re_marc_name = re.compile('^(.*), (.*)$') From 0d282175a273f3f8a5ac7655f7bc6d2cb6d97b02 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 08:57:34 +0100 Subject: [PATCH 14/47] iptools==0.7.0; python_version >= '3.0' --- requirements_common.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_common.txt b/requirements_common.txt index 6ad1b86e2e4..7f546155381 100644 --- a/requirements_common.txt +++ b/requirements_common.txt @@ -4,6 +4,7 @@ DBUtils==1.3 Genshi==0.7.1 gunicorn==19.9.0 iptools==0.6.1; python_version < '3.0' +iptools==0.7.0; python_version >= '3.0' internetarchive==1.8.5 lxml==4.4.1 Pillow==6.2.1 From 807e960829b4414cf7160ce83d68d7bd8721b1b8 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 09:04:13 +0100 Subject: [PATCH 15/47] from openlibrary.catalog.works.find_works import ... --- openlibrary/catalog/works/test_find_works.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/openlibrary/catalog/works/test_find_works.py b/openlibrary/catalog/works/test_find_works.py index 2815706e42d..b06e8f26099 100644 --- a/openlibrary/catalog/works/test_find_works.py +++ b/openlibrary/catalog/works/test_find_works.py @@ -1,6 +1,11 @@ # -*- coding: utf-8 -*- -from find_works import top_rev_wt, has_dot, freq_dict_top, find_works, get_books, find_works2, build_work_title_map, find_works3, find_work_sort from openlibrary.catalog.merge.normalize import normalize +from openlibrary.catalog.works.find_works import (top_rev_wt, has_dot, + freq_dict_top, find_works, + get_books, find_works2, + build_work_title_map, + find_works3, find_work_sort) + def test_has_dot(): assert has_dot('Magic.') From 8f3475f2ff7c3f1565a270293f46f7f348be914d Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 09:07:01 +0100 Subject: [PATCH 16/47] from scripts import _init_path --- scripts/copydocs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/copydocs.py b/scripts/copydocs.py index 08484bc7fc7..53a953761c7 100755 --- a/scripts/copydocs.py +++ b/scripts/copydocs.py @@ -15,7 +15,6 @@ from collections import namedtuple -import _init_path import sys import os import simplejson @@ -23,6 +22,7 @@ from openlibrary.api import OpenLibrary, marshal, unmarshal from optparse import OptionParser +from scripts import _init_path import six From 625a9b6774b0987999c177c80b0d1110160a8041 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 09:17:44 +0100 Subject: [PATCH 17/47] from openlibrary.core import helpers --- openlibrary/core/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/core/models.py b/openlibrary/core/models.py index 021756f6660..89c23f39de7 100644 --- a/openlibrary/core/models.py +++ b/openlibrary/core/models.py @@ -7,7 +7,7 @@ import iptools from infogami.infobase import client -import helpers as h +from openlibrary.core import helpers as h #TODO: fix this. openlibrary.core should not import plugins. from openlibrary import accounts From 09fab8cb535978d781c2f2a52c9e48a5535cb13b Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 09:24:30 +0100 Subject: [PATCH 18/47] from six.moves.http_client import HTTPConnection --- openlibrary/solr/update_work.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openlibrary/solr/update_work.py b/openlibrary/solr/update_work.py index e062b41966c..eb1e3f86c9f 100644 --- a/openlibrary/solr/update_work.py +++ b/openlibrary/solr/update_work.py @@ -1,6 +1,5 @@ from __future__ import print_function import datetime -import httplib import logging import os import re @@ -12,6 +11,7 @@ import simplejson as json import six from six.moves import urllib +from six.moves.http_client import HTTPConnection import web from lxml.etree import tostring, Element, SubElement @@ -815,7 +815,7 @@ def solr_update(requests, debug=False, commitWithin=60000): :param bool debug: :param int commitWithin: Solr commitWithin, in ms """ - h1 = httplib.HTTPConnection(get_solr()) + h1 = HTTPConnection(get_solr()) url = 'http://%s/solr/update' % get_solr() logger.info("POSTing update to %s", url) From ea05f6108e88530861357733721af90da587d645 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 09:37:20 +0100 Subject: [PATCH 19/47] from openlibrary.solr.data_provider import get_data_provider --- openlibrary/solr/update_work.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/solr/update_work.py b/openlibrary/solr/update_work.py index eb1e3f86c9f..cf9717f0f92 100644 --- a/openlibrary/solr/update_work.py +++ b/openlibrary/solr/update_work.py @@ -15,12 +15,12 @@ import web from lxml.etree import tostring, Element, SubElement -from data_provider import get_data_provider from infogami.infobase.client import ClientException from openlibrary import config from openlibrary.catalog.utils.query import set_query_host, base_url as get_ol_base_url from openlibrary.core import helpers as h from openlibrary.core import ia +from openlibrary.solr.data_provider import get_data_provider from openlibrary.utils.isbn import opposite_isbn logger = logging.getLogger("openlibrary.solr") From afc98081ad8ebd35f91394d7e9d2deb7a7327735 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 09:52:42 +0100 Subject: [PATCH 20/47] from six import StringIO --- openlibrary/tests/solr/test_update_work.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/tests/solr/test_update_work.py b/openlibrary/tests/solr/test_update_work.py index 2a4b3bf5def..275bc4519a9 100644 --- a/openlibrary/tests/solr/test_update_work.py +++ b/openlibrary/tests/solr/test_update_work.py @@ -1,7 +1,7 @@ from openlibrary.solr import update_work from openlibrary.solr.data_provider import DataProvider from openlibrary.solr.update_work import build_data -from StringIO import StringIO +from six import StringIO author_counter = 0 edition_counter = 0 From 2fb9b53678c4794d76aba7617dab76479d834769 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 10:29:15 +0100 Subject: [PATCH 21/47] from infogami.utils.view import public, render_template --- openlibrary/plugins/upstream/borrow.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openlibrary/plugins/upstream/borrow.py b/openlibrary/plugins/upstream/borrow.py index c2b56a992f0..014e4ee9e30 100644 --- a/openlibrary/plugins/upstream/borrow.py +++ b/openlibrary/plugins/upstream/borrow.py @@ -12,11 +12,9 @@ from infogami import config from infogami.utils import delegate -from infogami.utils.view import public +from infogami.utils.view import public, render_template from infogami.infobase.utils import parse_datetime -from utils import render_template - from openlibrary.core import stats from openlibrary.core import msgbroker from openlibrary.core import lending From b0a65bf5490aa10bf0a5999da9dde7c76173fdba Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 11:25:38 +0100 Subject: [PATCH 22/47] isort imports, from six.moves.html_parser import HTMLParser --- openlibrary/plugins/upstream/utils.py | 52 +++++++++++++-------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/openlibrary/plugins/upstream/utils.py b/openlibrary/plugins/upstream/utils.py index ba60efc14e3..03a00fb04a7 100644 --- a/openlibrary/plugins/upstream/utils.py +++ b/openlibrary/plugins/upstream/utils.py @@ -1,32 +1,37 @@ -import web -import simplejson -import babel -import babel.core -import babel.dates -from UserDict import DictMixin -from collections import defaultdict -import re -import random -import xml.etree.ElementTree as etree import datetime -import gzip -import StringIO import logging -from HTMLParser import HTMLParser +import random +import re +import xml.etree.ElementTree as etree +from collections import defaultdict +try: + from UserDict import DictMixin +except ImportError: + from collections.abc import MutableMapping as DictMixin +import babel +import babel.core +import babel.dates +import memcache +import simplejson import six -from six.moves import urllib - +import web from infogami import config -from infogami.utils import view, delegate, stats -from infogami.utils.view import render, get_template, public -from infogami.utils.macro import macro +from infogami.infobase.client import Changeset, Thing, storify +from infogami.utils import delegate, stats, view from infogami.utils.context import context -from infogami.infobase.client import Thing, Changeset, storify - +from infogami.utils.macro import macro +from infogami.utils.view import get_template, public, render +from openlibrary.core import cache from openlibrary.core.helpers import commify, parse_datetime from openlibrary.core.middleware import GZipMiddleware -from openlibrary.core import cache, ab +from openlibrary.core.olmarkdown import OLMarkdown +from openlibrary.plugins.upstream import adapter +from openlibrary.utils import olmemcache +from openlibrary.utils.olcompress import OLCompressor +from six.moves import urllib +from six.moves.html_parser import HTMLParser + class MultiDict(DictMixin): """Ordered Dictionary that can store multiple values. @@ -494,7 +499,6 @@ def _get_edition_config(): roles = thing.roles return web.storage(classifications=classifications, identifiers=identifiers, roles=roles) -from openlibrary.core.olmarkdown import OLMarkdown def get_markdown(text, safe_mode=False): md = OLMarkdown(source=text, safe_mode=safe_mode) view._register_mdx_extensions(md) @@ -519,10 +523,6 @@ def websafe(text): return _websafe(text) -from openlibrary.utils.olcompress import OLCompressor -from openlibrary.utils import olmemcache -import adapter -import memcache class UpstreamMemcacheClient: """Wrapper to memcache Client to handle upstream specific conversion and OL specific compression. From a07b457f677a8743776417e296914c9d180b7aae Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 11:45:48 +0100 Subject: [PATCH 23/47] from six import StringIO --- openlibrary/core/middleware.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openlibrary/core/middleware.py b/openlibrary/core/middleware.py index d4487ae1195..339bbb5d09b 100644 --- a/openlibrary/core/middleware.py +++ b/openlibrary/core/middleware.py @@ -1,7 +1,7 @@ """WSGI middleware used in Open Library. """ import web -import StringIO +from six import StringIO import gzip class GZipMiddleware: @@ -23,7 +23,7 @@ def get_response_header(name, default=None): return default def compress(text, level=9): - f = StringIO.StringIO() + f = StringIO() gz = gzip.GzipFile(None, 'wb', level, fileobj=f) gz.write(text) gz.close() From c0ccae0d6b36a36684edeae283c4ef9070ef6978 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 20:19:00 +0100 Subject: [PATCH 24/47] from six.moves.collections_abc import Mapping --- openlibrary/plugins/upstream/utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/openlibrary/plugins/upstream/utils.py b/openlibrary/plugins/upstream/utils.py index 03a00fb04a7..e6fb1f2483b 100644 --- a/openlibrary/plugins/upstream/utils.py +++ b/openlibrary/plugins/upstream/utils.py @@ -4,10 +4,6 @@ import re import xml.etree.ElementTree as etree from collections import defaultdict -try: - from UserDict import DictMixin -except ImportError: - from collections.abc import MutableMapping as DictMixin import babel import babel.core @@ -30,10 +26,11 @@ from openlibrary.utils import olmemcache from openlibrary.utils.olcompress import OLCompressor from six.moves import urllib +from six.moves.collections_abc import Mapping from six.moves.html_parser import HTMLParser -class MultiDict(DictMixin): +class MultiDict(Mapping): """Ordered Dictionary that can store multiple values. >>> d = MultiDict() From ccdd0e94df29e29f417325deb8aabf7d8a0061c4 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 21:03:48 +0100 Subject: [PATCH 25/47] Remove unused import httplib --- openlibrary/plugins/inside/code.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openlibrary/plugins/inside/code.py b/openlibrary/plugins/inside/code.py index 1833ce6a54d..b91e5e42bf1 100644 --- a/openlibrary/plugins/inside/code.py +++ b/openlibrary/plugins/inside/code.py @@ -2,7 +2,6 @@ from time import time import re import simplejson -import httplib import logging import web From 3dd16c71aae6572231bbd73baff3239ebb38e71c Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 21:20:21 +0100 Subject: [PATCH 26/47] import six.moves.http_client as httplib --- openlibrary/plugins/inside/code.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openlibrary/plugins/inside/code.py b/openlibrary/plugins/inside/code.py index b91e5e42bf1..a06888839a6 100644 --- a/openlibrary/plugins/inside/code.py +++ b/openlibrary/plugins/inside/code.py @@ -1,4 +1,3 @@ - from time import time import re import simplejson @@ -9,12 +8,12 @@ from infogami.utils.view import render_template from openlibrary.core.fulltext import fulltext_search - from six.moves import urllib - +import six.moves.http_client as httplib RESULTS_PER_PAGE = 20 + class search_inside(delegate.page): path = '/search/inside' From 19e374deec16bcc7fddeeeb870f3f9ab10074fae Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 21:23:06 +0100 Subject: [PATCH 27/47] from openlibrary.plugins.upstream import acs4 --- openlibrary/plugins/upstream/borrow.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/openlibrary/plugins/upstream/borrow.py b/openlibrary/plugins/upstream/borrow.py index 014e4ee9e30..662d2c42853 100644 --- a/openlibrary/plugins/upstream/borrow.py +++ b/openlibrary/plugins/upstream/borrow.py @@ -15,19 +15,18 @@ from infogami.utils.view import public, render_template from infogami.infobase.utils import parse_datetime +from openlibrary import accounts +from openlibrary.accounts.model import OpenLibraryAccount from openlibrary.core import stats from openlibrary.core import msgbroker from openlibrary.core import lending from openlibrary.core import waitinglist from openlibrary.core import ab -from openlibrary.accounts.model import OpenLibraryAccount -from openlibrary import accounts +from openlibrary.plugins.upstream import acs4 from openlibrary.utils import dateutil from lxml import etree -import acs4 - from six.moves import urllib From e8599800c533eba6dc088315930b4209a7bc8869 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 21:28:54 +0100 Subject: [PATCH 28/47] from openlibrary.plugins.upstream import utils --- openlibrary/plugins/upstream/addbook.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/openlibrary/plugins/upstream/addbook.py b/openlibrary/plugins/upstream/addbook.py index 73a385ef55c..ac2f380f201 100644 --- a/openlibrary/plugins/upstream/addbook.py +++ b/openlibrary/plugins/upstream/addbook.py @@ -5,6 +5,7 @@ from collections import defaultdict import csv import datetime +import logging from infogami import config from infogami.core import code as core @@ -18,14 +19,11 @@ from openlibrary.utils.solr import Solr from openlibrary.i18n import gettext as _ from openlibrary import accounts -import logging - -import utils -from utils import render_template, fuzzy_find -from account import as_admin from openlibrary.plugins.recaptcha import recaptcha -from . import spamcheck +from openlibrary.plugins.upstream import utils +from openlibrary.plugins.upstream.account import as_admin, spamcheck +from openlibrary.plugins.upstream.utils import render_template, fuzzy_find import six from six import StringIO From dc3b19f8b587a19f3b976ec9ecd2f983dc61677b Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 21:35:37 +0100 Subject: [PATCH 29/47] from openlibrary.plugins.upstream import spamcheck, utils --- openlibrary/plugins/upstream/addbook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openlibrary/plugins/upstream/addbook.py b/openlibrary/plugins/upstream/addbook.py index ac2f380f201..fd5668c85b3 100644 --- a/openlibrary/plugins/upstream/addbook.py +++ b/openlibrary/plugins/upstream/addbook.py @@ -21,8 +21,8 @@ from openlibrary import accounts from openlibrary.plugins.recaptcha import recaptcha -from openlibrary.plugins.upstream import utils -from openlibrary.plugins.upstream.account import as_admin, spamcheck +from openlibrary.plugins.upstream import spamcheck, utils +from openlibrary.plugins.upstream.account import as_admin from openlibrary.plugins.upstream.utils import render_template, fuzzy_find import six From f25541430d633149ae1848934e687e4720211a88 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 22:23:21 +0100 Subject: [PATCH 30/47] import openlibrary.plugins.openlibrary.filters as stats_filters --- openlibrary/plugins/openlibrary/stats.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/openlibrary/plugins/openlibrary/stats.py b/openlibrary/plugins/openlibrary/stats.py index dedaf5c6022..799f5fb805e 100644 --- a/openlibrary/plugins/openlibrary/stats.py +++ b/openlibrary/plugins/openlibrary/stats.py @@ -4,14 +4,13 @@ import logging import traceback -from openlibrary.core import stats as graphite_stats - import web from infogami import config from infogami.utils import stats -import openlibrary.core.stats -import filters as stats_filters +from openlibrary.core import stats as graphite_stats +import openlibrary.core.stats +import openlibrary.plugins.openlibrary.filters as stats_filters l = logging.getLogger("openlibrary.stats") From c51ec0daf33e3a31745c86928f1f8a1bfdbd0747 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 23:15:59 +0100 Subject: [PATCH 31/47] from openlibrary.core.lists.model import ListMixin, Seed --- openlibrary/core/models.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/openlibrary/core/models.py b/openlibrary/core/models.py index 89c23f39de7..60568229ff8 100644 --- a/openlibrary/core/models.py +++ b/openlibrary/core/models.py @@ -11,17 +11,15 @@ #TODO: fix this. openlibrary.core should not import plugins. from openlibrary import accounts -from openlibrary.utils import extract_numeric_id_from_olid -from openlibrary.plugins.upstream.utils import get_history -from openlibrary.core.helpers import private_collection_in +from openlibrary.core import db, cache, iprange, inlibrary, loanstats, waitinglist, lending from openlibrary.core.bookshelves import Bookshelves +from openlibrary.core.helpers import private_collection_in +from openlibrary.core.lists.model import ListMixin, Seed from openlibrary.core.ratings import Ratings -from openlibrary.utils.isbn import to_isbn_13, isbn_13_to_isbn_10 from openlibrary.core.vendors import create_edition_from_amazon_metadata - -# relative imports -from lists.model import ListMixin, Seed -from . import db, cache, iprange, inlibrary, loanstats, waitinglist, lending +from openlibrary.plugins.upstream.utils import get_history +from openlibrary.utils import extract_numeric_id_from_olid +from openlibrary.utils.isbn import to_isbn_13, isbn_13_to_isbn_10 from six.moves import urllib From 5691d761285a89230095f40deb88848383991cc5 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 23:21:20 +0100 Subject: [PATCH 32/47] from openlibrary.plugins.upstream import borrow, forms, utils --- openlibrary/plugins/upstream/account.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/openlibrary/plugins/upstream/account.py b/openlibrary/plugins/upstream/account.py index 6f2be8b6bcc..1390587e78b 100644 --- a/openlibrary/plugins/upstream/account.py +++ b/openlibrary/plugins/upstream/account.py @@ -18,19 +18,15 @@ import infogami.core.code as core from openlibrary import accounts -from openlibrary.i18n import gettext as _ -from openlibrary.core import helpers as h, lending -from openlibrary.core.bookshelves import Bookshelves -from openlibrary.plugins.recaptcha import recaptcha -from openlibrary.plugins import openlibrary as olib from openlibrary.accounts import ( audit_accounts, Account, OpenLibraryAccount, InternetArchiveAccount, valid_email) +from openlibrary.core import helpers as h, lending +from openlibrary.core.bookshelves import Bookshelves from openlibrary.core.sponsorships import get_sponsored_editions - -import forms -import utils -import borrow - +from openlibrary.i18n import gettext as _ +from openlibrary.plugins import openlibrary as olib +from openlibrary.plugins.recaptcha import recaptcha +from openlibrary.plugins.upstream import borrow, forms, utils from six.moves import range from six.moves import urllib From 231989510416254b3102b4f0b0714717ca13a1b9 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 23:24:01 +0100 Subject: [PATCH 33/47] from openlibrary.plugins.worksearch import searchapi, subjects --- openlibrary/plugins/worksearch/code.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openlibrary/plugins/worksearch/code.py b/openlibrary/plugins/worksearch/code.py index 8b7d74dbf1d..b8c3efd96a5 100644 --- a/openlibrary/plugins/worksearch/code.py +++ b/openlibrary/plugins/worksearch/code.py @@ -803,11 +803,9 @@ def GET(self): return delegate.RawText(json.dumps(response, indent=True)) def setup(): - import searchapi + from openlibrary.plugins.worksearch import searchapi, subjects searchapi.setup() - from . import subjects - # subjects module needs read_author_facet and solr_select_url. # Importing this module to access them will result in circular import. # Setting them like this to avoid circular-import. From 48bd7ad1eaa4f140ba4fcb362443299c8ac68499 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 23:45:29 +0100 Subject: [PATCH 34/47] from openlibrary.plugins.search import facet_hash, solr_client, stopword --- openlibrary/plugins/search/code.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/openlibrary/plugins/search/code.py b/openlibrary/plugins/search/code.py index 1ae7b963803..abc8a473997 100644 --- a/openlibrary/plugins/search/code.py +++ b/openlibrary/plugins/search/code.py @@ -1,26 +1,23 @@ from __future__ import print_function -import web -import stopword -import pdb - -from infogami import utils -from infogami.utils import delegate -from infogami.infobase.client import Thing -from infogami.utils import view, template -from infogami import config -from infogami.plugins.api.code import jsonapi +import pdb import re -import web -import solr_client import time -import simplejson +from collections import defaultdict from functools import partial from gzip import open as gzopen -import cPickle -from collections import defaultdict +import simplejson import six +import web +from infogami import config, utils +from infogami.infobase.client import Thing +from infogami.plugins.api.code import jsonapi +from infogami.utils import delegate, template, view +from six.moves import cPickle + +from openlibrary.plugins.search import facet_hash, solr_client, stopword +from openlibrary.plugins.search.collapse import collapse_groups render = template.render @@ -70,7 +67,6 @@ def lookup_ocaid(ocaid): w = web.ctx.site.get(ocat[0]) if ocat else None return w -from collapse import collapse_groups class fullsearch(delegate.page): def POST(self): errortext = None @@ -135,7 +131,6 @@ class Result_nums: pass GET = POST -import facet_hash facet_token = view.public(facet_hash.facet_token) class Timestamp(object): From 750adf4b2d1ec48e8d113696fee2eeabdd9fc3df Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Jan 2020 23:47:54 +0100 Subject: [PATCH 35/47] from openlibrary.plugins.openlibrary.libraries import LoanStats --- openlibrary/plugins/openlibrary/borrow_home.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/plugins/openlibrary/borrow_home.py b/openlibrary/plugins/openlibrary/borrow_home.py index 90ac86fd2ed..3a4359d174d 100644 --- a/openlibrary/plugins/openlibrary/borrow_home.py +++ b/openlibrary/plugins/openlibrary/borrow_home.py @@ -11,9 +11,9 @@ from openlibrary.core import helpers as h from openlibrary.core import inlibrary +from openlibrary.plugins.openlibrary.libraries import LoanStats from openlibrary.plugins.worksearch.subjects import SubjectEngine -from libraries import LoanStats class borrow(delegate.page): path = "/borrow" From 6af0ff80ca25cc962c416dd597f9df9832947515 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 Jan 2020 00:43:43 +0100 Subject: [PATCH 36/47] from six import StringIO --- openlibrary/plugins/search/solr_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openlibrary/plugins/search/solr_client.py b/openlibrary/plugins/search/solr_client.py index 89d64ed60ef..3d08722cbe5 100644 --- a/openlibrary/plugins/search/solr_client.py +++ b/openlibrary/plugins/search/solr_client.py @@ -1,19 +1,20 @@ #!/usr/bin/python from xml.etree.cElementTree import ElementTree -from cStringIO import StringIO import os import re from collections import defaultdict import cgi import web import simplejson -from facet_hash import facet_token import pdb import six +from six import StringIO from six.moves.urllib.parse import quote_plus from six.moves.urllib.request import urlopen +from openlibrary.plugins.search.facet_hash import facet_token + php_location = "/petabox/setup.inc" # server_addr = ('pharosdb.us.archive.org', 8983) From aec85216d69a42a5d5bc542af845212ce669e705 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 Jan 2020 00:48:51 +0100 Subject: [PATCH 37/47] from six import StringIO --- openlibrary/catalog/oca/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/catalog/oca/parse.py b/openlibrary/catalog/oca/parse.py index 05c04fd7104..9ff31d7f1af 100644 --- a/openlibrary/catalog/oca/parse.py +++ b/openlibrary/catalog/oca/parse.py @@ -1,6 +1,6 @@ from __future__ import print_function import sys -from cStringIO import StringIO +from six import StringIO from xml.parsers.expat import error as xml_error from elementtree import ElementTree from types import * From 1a6e8579afb04be3ab5e2cb62c5cbbb5947eaf15 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 Jan 2020 00:50:05 +0100 Subject: [PATCH 38/47] from six import StringIO --- openlibrary/coverstore/tests/test_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/coverstore/tests/test_code.py b/openlibrary/coverstore/tests/test_code.py index bb8f8e7b7e2..09b46afb2cf 100644 --- a/openlibrary/coverstore/tests/test_code.py +++ b/openlibrary/coverstore/tests/test_code.py @@ -1,5 +1,5 @@ from .. import code -from cStringIO import StringIO +from six import StringIO import web import datetime From 84110106f651f45878a407c40034d8c251f00adc Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 Jan 2020 00:51:38 +0100 Subject: [PATCH 39/47] from six import StringIO --- openlibrary/plugins/openlibrary/libraries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/plugins/openlibrary/libraries.py b/openlibrary/plugins/openlibrary/libraries.py index d8e38a48af3..eb2b95917b4 100644 --- a/openlibrary/plugins/openlibrary/libraries.py +++ b/openlibrary/plugins/openlibrary/libraries.py @@ -5,7 +5,7 @@ import logging import datetime import itertools -from cStringIO import StringIO +from six import StringIO import csv import simplejson From ab5083765b8b865f10956eab8205f7cf9b516153 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 Jan 2020 01:56:00 +0100 Subject: [PATCH 40/47] from openlibrary.plugins.importapi import import_edition_builder --- openlibrary/plugins/importapi/import_opds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/plugins/importapi/import_opds.py b/openlibrary/plugins/importapi/import_opds.py index 32145ea64bd..0f317c499ea 100644 --- a/openlibrary/plugins/importapi/import_opds.py +++ b/openlibrary/plugins/importapi/import_opds.py @@ -2,7 +2,7 @@ OL Import API OPDS parser """ -import import_edition_builder +from openlibrary.plugins.importapi import import_edition_builder import six From 48b5ebf9ebdbcc4b896bb5586ce553e560ee8ee7 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 Jan 2020 01:56:44 +0100 Subject: [PATCH 41/47] Update import_rdf.py --- openlibrary/plugins/importapi/import_rdf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/plugins/importapi/import_rdf.py b/openlibrary/plugins/importapi/import_rdf.py index 82e0fef6c46..63080c6da39 100644 --- a/openlibrary/plugins/importapi/import_rdf.py +++ b/openlibrary/plugins/importapi/import_rdf.py @@ -2,7 +2,7 @@ OL Import API RDF parser """ -import import_edition_builder +from openlibrary.plugins.importapi import import_edition_builder import six From 27886df3ba4f3d27a17b621b50fdf34604184d0c Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 Jan 2020 01:58:19 +0100 Subject: [PATCH 42/47] from openlibrary.plugins.importapi.import_edition_builder import import_edition_builder --- openlibrary/plugins/importapi/metaxml_to_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/plugins/importapi/metaxml_to_json.py b/openlibrary/plugins/importapi/metaxml_to_json.py index 6b500bfd461..b9da98a3846 100755 --- a/openlibrary/plugins/importapi/metaxml_to_json.py +++ b/openlibrary/plugins/importapi/metaxml_to_json.py @@ -10,7 +10,7 @@ """ from __future__ import print_function -from import_edition_builder import import_edition_builder +from openlibrary.plugins.importapi.import_edition_builder import import_edition_builder def parse_collection(collection): collection_dict = { From eb90a71b1e4d6396512a479f94ed8205f82ac911 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 Jan 2020 10:21:21 +0100 Subject: [PATCH 43/47] urllib.splittype(), urllib.splithost(), urllib.splitport() were removed in Python 3 --- openlibrary/coverstore/warc.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/openlibrary/coverstore/warc.py b/openlibrary/coverstore/warc.py index caa8f7f41b7..c3e41111625 100644 --- a/openlibrary/coverstore/warc.py +++ b/openlibrary/coverstore/warc.py @@ -4,18 +4,19 @@ http://archive-access.sourceforge.net/warc/warc_file_format-0.10.html """ -import urllib -import httplib import datetime +from six.moves.http_client import HTTPConnection +from six.moves.urllib.parse import urlparse + WARC_VERSION = "0.10" CRLF = "\r\n" class WARCReader: """Reader to read records from a warc file. - >>> import StringIO - >>> f = StringIO.StringIO() + >>> from six import StringIO + >>> f = StringIO() >>> r1 = WARCRecord("resource", "subject_uri", "image/jpeg", {"hello": "world"}, "foo") >>> r2 = WARCRecord("resource", "subject_uri", "image/jpeg", {"hello": "world"}, "bar") >>> w = WARCWriter(f) @@ -111,7 +112,7 @@ def _readuntil(self, condition): def read(self, size): protocol, host, port, path = self.urlsplit(self.url) - conn = httplib.HTTPConnection(host, port) + conn = HTTPConnection(host, port) headers = {'Range': 'bytes=%d-%d' % (self.offset, self.offset + size - 1)} conn.request('GET', path, None, headers) response = conn.getresponse() @@ -123,13 +124,16 @@ def urlsplit(self, url): """Splits url into protocol, host, port and path. >>> f = HTTPFile('') + >>> f.urlsplit("http://www.google.com") + ('http', 'www.google.com', None, '') >>> f.urlsplit("http://www.google.com/search?q=hello") ('http', 'www.google.com', None, '/search?q=hello') + >>> f.urlsplit("http://www.google.com:80/search?q=hello") + ('http', 'www.google.com', 80, '/search?q=hello') """ - protocol, rest = urllib.splittype(url) - hostport, path = urllib.splithost(rest) - host, port = urllib.splitport(hostport) - return protocol, host, port, path + p = urlparse(url) + fullpath = "?".join((p.path, p.query)) if p.path or p.query else "" + return p.scheme, p.hostname, p.port, p.path + fullpath class WARCHeader: r"""WARCHeader class represents the header in the WARC file format. @@ -219,9 +223,9 @@ def __repr__(self): class LazyWARCRecord(WARCRecord): """Class to create WARCRecord lazily. - >>> import StringIO + >>> from six import StringIO >>> r1 = WARCRecord("resource", "subject_uri", "image/jpeg", {"hello": "world"}, "foo bar", creation_date="20080808080808", record_id="record_42") - >>> f = StringIO.StringIO(str(r1)) + >>> f = StringIO(str(r1)) >>> offset = len(str(r1.get_header())) >>> r2 = LazyWARCRecord(f, offset, r1.get_header()) >>> r1 == r2 @@ -247,8 +251,9 @@ def get_data(self): class WARCWriter: r"""Writes to write warc records to file. - >>> import re, StringIO - >>> f = StringIO.StringIO() + >>> import re + >>> from six import StringIO + >>> f = StringIO() >>> r1 = WARCRecord("resource", "subject_uri", "image/jpeg", {"hello": "world"}, "foo", creation_date="20080808080808", record_id="record_42") >>> r2 = WARCRecord("resource", "subject_uri", "image/jpeg", {"hello": "world"}, "bar", creation_date="20080808090909", record_id="record_43") >>> w = WARCWriter(f) From 14513c0867878cd577c93112db469c2ec57ef277 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 Jan 2020 10:31:05 +0100 Subject: [PATCH 44/47] Update warc.py --- openlibrary/coverstore/warc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openlibrary/coverstore/warc.py b/openlibrary/coverstore/warc.py index c3e41111625..1d77a7cb798 100644 --- a/openlibrary/coverstore/warc.py +++ b/openlibrary/coverstore/warc.py @@ -132,8 +132,8 @@ def urlsplit(self, url): ('http', 'www.google.com', 80, '/search?q=hello') """ p = urlparse(url) - fullpath = "?".join((p.path, p.query)) if p.path or p.query else "" - return p.scheme, p.hostname, p.port, p.path + fullpath + fullpath = "?".join((p.path, p.query)) if (p.path or p.query) else "" + return p.scheme, p.hostname, p.port, fullpath class WARCHeader: r"""WARCHeader class represents the header in the WARC file format. From aa5f81447b5fb149ad6e5ddef3e853e6383b3edc Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 Jan 2020 11:34:21 +0100 Subject: [PATCH 45/47] Update test_find_works.py --- openlibrary/catalog/works/test_find_works.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openlibrary/catalog/works/test_find_works.py b/openlibrary/catalog/works/test_find_works.py index f74e135d362..1f82cae7c0e 100644 --- a/openlibrary/catalog/works/test_find_works.py +++ b/openlibrary/catalog/works/test_find_works.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from openlibrary.catalog.merge.normalize import normalize -from openlibrary.catalog.works.find_works import top_rev_wt, has_dot, freq_dict_top, find_works, get_books, find_works2, build_work_title_map, find_works3, find_work_sort> +from openlibrary.catalog.works.find_works import top_rev_wt, has_dot, freq_dict_top, find_works, get_books, find_works2, build_work_title_map, find_works3, find_work_sort def test_has_dot(): From 777326fd1de77c0d3098181d76ddfbac3ff8737d Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 Jan 2020 11:35:33 +0100 Subject: [PATCH 46/47] Update marc_binary.py --- openlibrary/catalog/marc/marc_binary.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openlibrary/catalog/marc/marc_binary.py b/openlibrary/catalog/marc/marc_binary.py index cd72d5d6b4c..c1d7c7184d0 100644 --- a/openlibrary/catalog/marc/marc_binary.py +++ b/openlibrary/catalog/marc/marc_binary.py @@ -4,6 +4,7 @@ from openlibrary.catalog.marc import mnemonics from openlibrary.catalog.marc.marc_base import MarcBase, MarcException, BadMARC + import six From 2e8d1bcec6fc81d13686761a3dc3b0db09c0814f Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 28 Jan 2020 17:22:16 +0100 Subject: [PATCH 47/47] Revert changes to mock_ol.py --- openlibrary/mocks/mock_ol.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/openlibrary/mocks/mock_ol.py b/openlibrary/mocks/mock_ol.py index 10375ac834f..19a587eafb6 100644 --- a/openlibrary/mocks/mock_ol.py +++ b/openlibrary/mocks/mock_ol.py @@ -2,11 +2,6 @@ import pytest import re import web -try: # newer versions of web.py - from web.browser import AppBrowser -except ImportError: # older versions of web.py - from web import AppBrowser - from infogami import config from infogami.infobase import client from infogami.utils import delegate @@ -39,11 +34,11 @@ def extract_links(self): """Extracts link from the email message.""" return re.findall(r"http://[^\s]*", self.message) -class OLBrowser(AppBrowser): +class OLBrowser(web.AppBrowser): def get_text(self, e=None, name=None, **kw): if name or kw: e = self.get_soup().find(name=name, **kw) - return AppBrowser.get_text(self, e) + return web.AppBrowser.get_text(self, e) class OL: """Mock OL object for all tests.