porting to python3

application/__init__.py

@@ -10,15 +10,15 @@
 builtins.__dict__['_'] = gettext
 
 #创建并初始化Flask wsgi对象
-def init_app(name, debug=False):
+def init_app(name, cfgMap, debug=False):
     thisDir = os.path.dirname(os.path.abspath(__file__))
     rootDir = os.path.abspath(os.path.join(thisDir, '..'))
     template_folder = os.path.join(thisDir, 'templates')
     static_folder = os.path.join(thisDir, 'static')
     i18n_folder = os.path.join(thisDir, 'translations')
 
     app = Flask(name, template_folder=template_folder, static_folder=static_folder)
-    app.config.from_pyfile(os.path.join(rootDir, 'config.py'))
+    app.config.from_mapping(cfgMap)
     app.config['MAX_CONTENT_LENGTH'] = 32 * 1024 * 1024 #32MB
 
     from .view import setting
@@ -31,13 +31,14 @@ def init_app(name, debug=False):
     from .back_end.task_queue_adpt import init_task_queue_service
     init_task_queue_service(app)
 
-    from .back_end.db_models import connect_database, close_database
+    from .back_end.db_models import create_database_tables, connect_database, close_database
+    create_database_tables()
 
     @app.before_request
     def BeforeRequest():
         g.version = appVer
         g.now = datetime.datetime.utcnow
-        g.allowSignup = app.config['ALLOW_SIGNUP']
+        g.allowSignup = app.config['ALLOW_SIGNUP'] == 'yes'
         connect_database()
 
     @app.teardown_request

application/back_end/db_models.py

@@ -259,12 +259,9 @@ def set_value(cls, name, value):
 
 #创建数据库表格，一个数据库只需要创建一次
 def create_database_tables():
-    #with dbInstance.connection_context():
-    #connect_database()
     dbInstance.create_tables([KeUser, UserBlob, Recipe, BookedRecipe, DeliverLog, WhiteList,
         SharedRss, SharedRssCategory, LastDelivered, InBox, AppInfo], safe=True)
     if not AppInfo.get_value(AppInfo.dbSchemaVersion):
         AppInfo.set_value(AppInfo.dbSchemaVersion, appVer)
-    #close_database()
 
     return 'Created database tables successfully'

application/back_end/send_mail_adpt.py

@@ -6,12 +6,12 @@
 #https://cloud.google.com/appengine/docs/standard/python3/reference/services/bundled/google/appengine/api/mail
 #https://cloud.google.com/appengine/docs/standard/python3/services/mail
 import os, datetime, zipfile, base64
-from ..utils import local_time, ke_decrypt
+from ..utils import local_time, ke_decrypt, str_to_bool
 from ..base_handler import save_delivery_log
 from .db_models import KeUser
 
 #google.appengine will apply patch for os.env module
-hideMailLocal = os.getenv('HIDE_MAIL_TO_LOCAL')
+hideMailLocal = str_to_bool(os.getenv('HIDE_MAIL_TO_LOCAL'))
 
 #判断是否是部署在gae平台
 if os.getenv('DATABASE_URL') == 'datastore':
@@ -105,7 +105,7 @@ def send_mail(user, to, subject, body, attachments=None, html=None):
     elif srv_type == 'smtp':
         data['host'] = sm_service.get('host', '')
         data['port'] = sm_service.get('port', 587)
-        data['username'] = user.sender
+        data['username'] = sm_service.get('username', '')
         data['password'] = ke_decrypt(sm_service.get('password', ''), user.secret_key)
         smtp_send_mail(**data)
     elif srv_type == 'local':

application/base_handler.py

@@ -17,7 +17,7 @@ def login_required(forAjax=False):
     def decorator(func):
         @wraps(func)
         def wrapper(*args, **kwargs):
-            if (session.get('login', '') == 1) and session.get('userName', ''):
+            if (session.get('login', '') == 1) and get_login_user():
                 return func(*args, **kwargs)
             else:
                 return redirect(url_for("bpLogin.NeedLoginAjax") if forAjax else url_for("bpLogin.Login"))

application/lib/build_ebook.py

@@ -79,7 +79,7 @@ def ke_opts(user, options=None):
     opt.setdefault('dont_split_on_page_breaks', True)
     opt['user'] = user
 
-    #opt.setdefault('debug_pipeline', os.getenv('TEMP_DIR'))
+    #opt.setdefault('debug_pipeline', os.getenv('KE_TEMP_DIR'))
     #opt.setdefault('verbose', 1)
     #opt.setdefault('test', 1)
     return opt

application/lib/calibre/ebooks/conversion/plumber.py

@@ -308,8 +308,13 @@ def set_profile(profiles, which):
 
         self.opts.no_inline_navbars = self.opts.output_profile.supports_mobi_indexing \
                 and self.output_fmt == 'mobi'
-        if self.opts.verbose:
-            self.log.filter_level = self.log.DEBUG
+
+        levelNames = {'CRITICAL': self.log.ERROR, 'FATAL': self.log.ERROR, 'ERROR': self.log.ERROR, 
+                'WARN': self.log.WARN, 'WARNING': self.log.WARN, 'INFO': self.log.INFO,
+                'DEBUG': self.log.DEBUG}
+        level = levelNames.get(os.getenv('LOG_LEVEL', '').upper(), self.log.WARN)
+        self.log.filter_level = self.log.DEBUG if self.opts.verbose else level
+
         if self.opts.verbose > 1:
             self.log.debug('Resolved conversion options')
             try:
@@ -358,8 +363,6 @@ def run(self):
         '''
         # Setup baseline option values
         self.setup_options()
-        if self.opts.verbose:
-            self.log.filter_level = self.log.DEBUG
 
         css_parser.log.setLevel(logging.WARN)
         #get_types_map()  # Ensure the mimetypes module is initialized
@@ -378,7 +381,7 @@ def run(self):
 
         self.output_plugin.specialize_options(self.log, self.opts, self.input_fmt)
         #根据需要，创建临时目录或创建内存缓存
-        system_temp_dir = os.environ.get('TEMP_DIR')
+        system_temp_dir = os.environ.get('KE_TEMP_DIR')
         if system_temp_dir and self.input_fmt != 'html':
             tdir = PersistentTemporaryDirectory(prefix='plumber_', dir=system_temp_dir)
             fs = FsDictStub(tdir)

application/lib/calibre/ebooks/oeb/base.py

@@ -1084,7 +1084,7 @@ def unload_data_from_memory(self, memory=None):
             if isinstance(self._data, bytes):
                 if memory is None:
                     from calibre.ptempfile import PersistentTemporaryFile
-                    temp_dir = os.environ.get('TEMP_DIR')
+                    temp_dir = os.environ.get('KE_TEMP_DIR')
                     pt = PersistentTemporaryFile(suffix='_oeb_base_mem_unloader.img', dir=temp_dir)
                     with pt:
                         pt.write(self._data)

application/lib/calibre/utils/img.py

@@ -141,6 +141,8 @@ def image_to_data(img, compression_quality=95, fmt='JPEG', png_compression_level
     if fmt == 'GIF':
         return png_data_to_gif_data(img)
     else:
+        if img.mode != 'RGB':
+            img = img.convert('RGB')
         data = BytesIO()
         img.save(data, fmt)
         return data.getvalue()

application/lib/calibre/web/feeds/news.py

@@ -397,7 +397,7 @@ class BasicNewsRecipe(Recipe):
     #: parameters are left at their default values, JPEG images will be scaled to fit
     #: in the screen dimensions set by the output profile and compressed to size at
     #: most (w * h)/16 where w x h are the scaled image dimensions.
-    compress_news_images = False
+    compress_news_images = True
 
     #: The factor used when auto compressing JPEG images. If set to None,
     #: auto compression is disabled. Otherwise, the images will be reduced in size to
@@ -410,7 +410,7 @@ class BasicNewsRecipe(Recipe):
     #: first be scaled and then its quality lowered until its size is less than
     #: (w * h)/factor where w and h are now the *scaled* image dimensions. In
     #: other words, this compression happens after scaling.
-    compress_news_images_auto_size = 16
+    compress_news_images_auto_size = None #16
 
     #: Set JPEG quality so images do not exceed the size given (in KBytes).
     #: If set, this parameter overrides auto compression via compress_news_images_auto_size.

application/lib/calibre/web/fetch/simple.py

@@ -458,11 +458,11 @@ def process_images(self, soup, baseurl):
                     if itype not in {'png', 'jpg', 'jpeg'}:
                         itype = 'png' if itype == 'gif' else 'jpeg'
                         data = image_to_data(img, fmt=itype)
-                    if self.compress_news_images and itype in {'jpg','jpeg'}:
+                    if self.compress_news_images: # and itype in {'jpg','jpeg'}:
                         try:
                             data = self.rescale_image(data)
                         except Exception:
-                            self.log.exception('failed to compress image '+iurl)
+                            self.log.warning('failed to compress image ' + iurl)
                     # Moon+ apparently cannot handle .jpeg files
                     if itype == 'jpeg':
                         itype = 'jpg'

application/lib/ebook_translator/engines/base.py

@@ -184,20 +184,29 @@ def _is_auto_lang(self):
 
     def _get_api_key(self):
         if self.need_api_key and self.api_keys:
-            return self.api_keys.pop(0)
+            return self.api_keys.pop(0).strip()
         return None
 
-    def get_result(self, url, data=None, headers={}, method='GET',
+    def get_result(self, url, data=None, headers=None, method='GET',
                    stream=False, silence=False, callback=None):
         result = ''
         br = UrlOpener(headers=headers, timeout=self.request_timeout)
         resp = br.open(url, data=data, method=method, stream=stream)
         if resp.status_code == 200:
-            text = resp.text
+            text = []
+            if stream:
+                for line in resp.iter_content(chunk_size=None, decode_unicode=True):
+                    text.append(line)
+                text = ''.join(text)
+            else:
+                text = resp.text
+
             try:
                 return callback(text) if callback else text
             except:
                 raise Exception('Can not parse translation. Raw data: {text}')
+            finally:
+                resp.close()
         elif silence:
             return None
         else:

application/lib/smtp_mail.py

@@ -14,7 +14,7 @@ def smtp_send_mail(sender, to, subject, body, host, username, password, port=Non
         host, port = host.split(':', 2)
         port = int(port)
     else:
-        port = 587 #587-TLS, 465-SSL
+        port = 587 #587-TLS, 465-SSL, 25-Nocrpt
 
     to = to if isinstance(to, list) else [to]
     message = MIMEMultipart('alternative') if html else MIMEMultipart()
@@ -33,11 +33,13 @@ def smtp_send_mail(sender, to, subject, body, host, username, password, port=Non
         message.attach(part)
 
     klass = smtplib.SMTP if port != 465 else smtplib.SMTP_SSL
-    with klass(host=host, port=port) as smtp_server:
-        smtp_server.connect(host, port)
-        smtp_server.ehlo()
-        smtp_server.starttls()
-        smtp_server.ehlo()
-        smtp_server.login(user=username, password=password)
-        smtp_server.sendmail(sender, to, message.as_string())
+    with klass(host=host, port=port) as server:
+        server.set_debuglevel(0) #0-no debug info, 1-base, 2- verbose
+        server.connect(host, port)
+        server.ehlo()
+        if port == 587:
+            server.starttls()
+            server.ehlo()
+        server.login(user=username, password=password)
+        server.sendmail(sender, to, message.as_string())
 

application/templates/base.html

@@ -92,7 +92,7 @@
     abbrSep: '{{_("Sep")|safe}}',
     abbrLog: '{{_("Log")|safe}}',
     abbrEmb: '{{_("Emb")|safe}}',
-    testEmailOk: '{{_("The test email has been successfully sent to [{0}]. Please check your inbox or spam folder to confirm its delivery. Depending on your email server, there may be a slight delay.")|safe}}'
+    testEmailOk: '{{_("The test email has been successfully sent to the following addresses. Please check your inbox or spam folder to confirm its delivery. Depending on your email server, there may be a slight delay.")|safe}}'
   };
   </script>
   {% block javascript_inhead %}

application/templates/setting.html

@@ -216,7 +216,7 @@
       </div>
       <div class="pure-control-group" id="sm_username">
         <label>{{_("Username")}}</label>
-        <input type="text" name="sm_username" value="{{user.sender}}" class="pure-u-1 pure-u-sm-1-2" readonly />
+        <input type="text" name="sm_username" value="{{sm_srv.get('username', '')}}" class="pure-u-1 pure-u-sm-1-2" />
       </div>
       <div class="pure-control-group" id="sm_password">
         <label>{{_("Password")}}</label>
@@ -317,7 +317,7 @@
 function SendTestEmail() {
   $.post("/send_test_email", {url: window.location.href}, function (data) {
     if (data.status == "ok") {
-      ShowSimpleModalDialog('<p>{0}</p>'.format(i18n.testEmailOk.format(data.email)));
+      ShowSimpleModalDialog('<p>{0}<br/><hr/>{1}</p>'.format(i18n.testEmailOk, data.emails.join('<br/>')));
     } else {
       alert(data.status);
     }

application/utils.py

@@ -129,6 +129,24 @@ def filesizeformat(value, binary=False, suffix='B'):
         value /= base
     return f"{value:.1f} {unit}{suffix}"
 
+
+#将字符串安全转义到xml格式，有标准库函数xml.sax.saxutils.escape()，但是简单的功能就简单的函数就好
+def xml_escape(txt):
+    txt = txt.replace("&", "&")
+    txt = txt.replace("<", "&lt;")
+    txt = txt.replace(">", "&gt;")
+    txt = txt.replace('"', "&quot;")
+    txt = txt.replace("'", "&apos;")
+    return txt
+
+def xml_unescape(txt):
+    txt = txt.replace("&amp;", "&")
+    txt = txt.replace("&lt;", "<")
+    txt = txt.replace("&gt;", ">")
+    txt = txt.replace("&quot;", '"')
+    txt = txt.replace("&apos;", "'")
+    return txt
+
 #-----------以下几个函数为安全相关的
 def new_secret_key(length=12):
     allchars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXZYabcdefghijklmnopqrstuvwxyz'

application/view/adv.py

@@ -2,14 +2,14 @@
 # -*- coding:utf-8 -*-
 #一些高级设置功能页面
 
-import datetime, hashlib, io
+import datetime, hashlib, io, textwrap
 from urllib.parse import quote, unquote, urljoin, urlparse
 from flask import Blueprint, url_for, render_template, redirect, session, send_file, abort, current_app as app
 from flask_babel import gettext as _
 from PIL import Image
 from ..base_handler import *
 from ..back_end.db_models import *
-from ..utils import local_time, ke_encrypt, ke_decrypt, str_to_bool, safe_eval
+from ..utils import local_time, ke_encrypt, ke_decrypt, str_to_bool, safe_eval, xml_escape, xml_unescape
 from ..lib.pocket import Pocket
 from ..lib.urlopener import UrlOpener
 
@@ -148,7 +148,7 @@ def AdvImportPost():
             return adv_render_template('adv_import.html', 'import', user=user, tips=str(e))
 
         for o in walkOpmlOutline(rssList):
-            title, url, isfulltext = o.text, unquote(o.xmlUrl), o.isFulltext #isFulltext为非标准属性
+            title, url, isfulltext = xml_unescape(o.text), xml_unescape(o.xmlUrl), o.isFulltext #isFulltext为非标准属性
             if isfulltext:
                 isfulltext = str_to_bool(isfulltext)
             else:
@@ -190,7 +190,8 @@ def AdvExport():
     user = get_login_user()
 
     #为了简单起见，就不用其他库生成xml，而直接使用字符串格式化生成
-    opmlTpl = """<?xml version="1.0" encoding="utf-8" ?>
+    opmlTpl = textwrap.dedent("""\
+    <?xml version="1.0" encoding="utf-8" ?>
     <opml version="2.0">
     <head>
       <title>KindleEar.opml</title>
@@ -201,7 +202,7 @@ def AdvExport():
     <body>
     {outLines}
     </body>
-    </opml>"""
+    </opml>""")
 
     date = local_time('%a, %d %b %Y %H:%M:%S GMT', user.timezone)
     #添加时区信息
@@ -210,14 +211,10 @@ def AdvExport():
     outLines = []
     for feed in user.all_custom_rss():
         outLines.append('<outline type="rss" text="{}" xmlUrl="{}" isFulltext="{}" />'.format(
-            feed.title, quote(feed.url), feed.isfulltext))
+            xml_escape(feed.title), xml_escape(feed.url), feed.isfulltext))
     outLines = '\n'.join(outLines)
 
-    opmlFile = opmlTpl.format(date=date, outLines=outLines)
-    outLines = []
-    for line in opmlFile.split('\n'):
-        outLines.append(line[4:] if line.startswith('  ') else line)
-    opmlFile = '\n'.join(outLines).encode('utf-8')
+    opmlFile = opmlTpl.format(date=date, outLines=outLines).encode('utf-8')
     return send_file(io.BytesIO(opmlFile), mimetype="text/xml", as_attachment=True, download_name="KindleEar_subscription.xml")
 
 #在本地选择一个图片上传做为自定义RSS书籍的封面