porting to python3

application/back_end/db_models.py

@@ -215,7 +215,7 @@ def categories(self):
 #Buffer for category of shared rss [for kindleear.appspot.com only]
 class SharedRssCategory(MyBaseModel):
     name = CharField()
-    language = CharField()
+    language = CharField(default='')
     last_updated = DateTimeField(default=datetime.datetime.utcnow)
 
 class LastDelivered(MyBaseModel):
@@ -225,6 +225,16 @@ class LastDelivered(MyBaseModel):
     record = CharField(default='')
     datetime = DateTimeField(default=datetime.datetime.utcnow)
 
+class InBox(MyBaseModel):
+    user = CharField()
+    sender = CharField()
+    to = CharField()
+    subject = CharField()
+    size = IntegerField(default=0)
+    datetime = DateTimeField(default=datetime.datetime.utcnow)
+    body = TextField(default='', index=False)
+    attachments = CharField(default='') #存放UserBlob的数据库id，逗号分割
+
 class AppInfo(MyBaseModel):
     name = CharField(unique=True)
     value = CharField(default='')
@@ -251,9 +261,9 @@ def create_database_tables():
     #with dbInstance.connection_context():
     #connect_database()
     dbInstance.create_tables([KeUser, UserBlob, Recipe, BookedRecipe, DeliverLog, WhiteList,
-        SharedRss, SharedRssCategory, LastDelivered, AppInfo], safe=True)
+        SharedRss, SharedRssCategory, LastDelivered, InBox, AppInfo], safe=True)
     if not AppInfo.get_value(AppInfo.dbSchemaVersion):
         AppInfo.set_value(AppInfo.dbSchemaVersion, appVer)
     #close_database()
-        
+
     return 'Created database tables successfully'

application/back_end/db_models_sql.py

@@ -16,7 +16,10 @@
     if not fileName.startswith('/'):
         dbName = 'sqlite:///{}'.format(os.path.join(appDir, fileName))
 
-dbInstance = connect(dbName)
+if dbName == 'sqlite://:memory:':
+    dbInstance = SqliteDatabase(':memory:')
+else:
+    dbInstance = connect(dbName)
 
 #调用此函数正式连接到数据库（打开数据库）
 def connect_database():

application/lib/calibre/web/feeds/__init__.py

@@ -10,6 +10,8 @@
 import re
 import time
 import traceback
+import json
+import datetime
 
 from calibre import entity_to_unicode, force_unicode, strftime
 from calibre.utils.cleantext import clean_ascii_chars, clean_xml_chars
@@ -56,7 +58,7 @@ def __init__(self, id, title, url, author, summary, published, content):
         self.text_summary = clean_ascii_chars(summary)
         self.author = author
         self.content = content
-        self.date = published
+        self.date = published #time.struct_time
         self.utctime = dt_factory(self.date, assume_utc=True, as_utc=True)
         self.localtime = self.utctime.astimezone(local_tz)
         self._formatted_date = None
@@ -143,6 +145,27 @@ def populate_from_feed(self, feed, title=None, oldest_article=7,
                 break
             self.parse_article(item)
 
+    #added by cdhigh
+    def populate_from_json(self, feed, title=None, oldest_article=7, max_articles_per_feed=100):
+        self.title        = feed.get('title', _('Unknown section')) if not title else title
+        self.description  = feed.get('description', '')
+        self.image_url    = feed.get('icon', None) or feed.get('favicon', None)
+        self.image_width  = 88
+        self.image_height = 31
+        self.image_alt    = ''
+
+        self.articles = []
+        self.id_counter = 0
+        self.added_articles = []
+
+        self.oldest_article = oldest_article
+
+        entries = feed.get('items')
+        for item in entries:
+            if len(self.articles) >= max_articles_per_feed:
+                break
+            self.parse_article_json(item)
+
     def populate_from_preparsed_feed(self, title, articles, oldest_article=7,
                            max_articles_per_feed=100):
         self.title      = str(title if title else _('Unknown feed'))
@@ -240,6 +263,50 @@ def parse_article(self, item):
                     title = title.decode('utf-8', 'replace')
                 self.logger.debug('Skipping article %s as it is too old'%title)
 
+    #added by cdhigh
+    def parse_article_json(self, item):
+        self.id_counter += 1
+        id_ = item.get('id', None) or f'internal id#{self.id_counter}'
+        if id_ in self.added_articles:
+            return
+
+        self.added_articles.append(id_)
+        published = item.get('date_modified', None) or item.get('date_published', None)
+        if published:
+            import dateutil
+            try:
+                published = dateutil.parser.parse(published).timetuple()
+            except:
+                published = time.gmtime()
+        else:
+            published = time.gmtime()
+
+        title = item.get('title', _('Untitled article'))
+        if title.startswith('<'):
+            title = re.sub(r'<.+?>', '', title)
+        link  = item.get('url', None) or item.get('external_url', None)
+        description = item.get('summary', None)
+        authors = item.get('authors', [])
+        author = ' '.join([aut.get('name', '') for aut in authors])
+        if not author:
+            author = item.get('author', {}).get('name', '')
+        content = item.get('content_html', None) or item.get('content_text', None)
+        if not link and not content:
+            return
+
+        article = Article(id_, title, link, author, description, published, content)
+        delta = utcnow() - article.utctime
+        if (self.oldest_article == 0) or (delta.days*24*3600 + delta.seconds <= 24*3600*self.oldest_article):
+            self.articles.append(article)
+        else:
+            try:
+                self.logger.debug('Skipping article %s (%s) from feed %s as it is too old.'%
+                                  (title, article.localtime.strftime('%a, %d %b, %Y %H:%M'), self.title))
+            except UnicodeDecodeError:
+                if not isinstance(title, str):
+                    title = title.decode('utf-8', 'replace')
+                self.logger.debug('Skipping article %s as it is too old'%title)
+
     def reverse(self):
         self.articles.reverse()
 
@@ -343,18 +410,37 @@ def feed_from_xml(raw_xml, title=None, oldest_article=7,
                   max_articles_per_feed=100,
                   get_article_url=lambda item: item.get('link', None),
                   log=default_log):
+
     from feedparser import parse
 
     # Handle unclosed escaped entities. They trip up feedparser and HBR for one
     # generates them
-    raw_xml = re.sub(br'(&amp;#\d+)([^0-9;])', br'\1;\2', raw_xml)
+    raw_xml = re.sub(r'(&amp;#\d+)([^0-9;])', r'\1;\2', raw_xml)
     feed = parse(raw_xml)
     pfeed = Feed(get_article_url=get_article_url, log=log)
     pfeed.populate_from_feed(feed, title=title,
                             oldest_article=oldest_article,
                             max_articles_per_feed=max_articles_per_feed)
     return pfeed
 
+#added by cdhigh
+def feed_from_json(raw_json, title=None, oldest_article=7,
+                  max_articles_per_feed=100,
+                  get_article_url=lambda item: item.get('link', None),
+                  log=default_log):
+
+    pfeed = Feed(get_article_url=get_article_url, log=log)
+
+    try:
+        feed = json.loads(raw_json)
+    except Exception as e:
+        log.warning('Parse json feed failed {}: {}'.format(title, str(e)))
+        return pfeed
+
+    pfeed.populate_from_json(feed, title=title,
+                            oldest_article=oldest_article,
+                            max_articles_per_feed=max_articles_per_feed)
+    return pfeed
 
 def feeds_from_index(index, oldest_article=7, max_articles_per_feed=100,
         log=default_log):

application/lib/calibre/web/feeds/news.py

@@ -29,7 +29,7 @@
 from calibre.utils.logging import ThreadSafeWrapper
 from calibre.utils.threadpool import NoResultsPending, ThreadPool, WorkRequest
 from calibre.web import Recipe
-from calibre.web.feeds import Feed, Article, feed_from_xml, feeds_from_index, templates
+from calibre.web.feeds import Feed, Article, feed_from_xml, feeds_from_index, templates, feed_from_json
 from calibre.web.fetch.simple import AbortArticle, RecursiveFetcher
 from calibre.web.fetch.utils import prepare_masthead_image
 from polyglot.builtins import string_or_bytes
@@ -1853,9 +1853,10 @@ def parse_feeds(self):
                 #        br.add_password(url, purl.username, purl.password)
                 resp = br.open(url, timeout=self.timeout)
                 if resp.status_code == 200:
-                    raw = resp.content
-                    feed = feed_from_xml(raw, title=title, log=self.log, oldest_article=self.oldest_article,
-                        max_articles_per_feed=self.max_articles_per_feed, get_article_url=self.get_article_url)
+                    raw = resp.text.lstrip()
+                    pFunc = feed_from_json if raw and raw[0] == '{' else feed_from_xml
+                    feed = pFunc(raw, title=title, log=self.log, oldest_article=self.oldest_article,
+                            max_articles_per_feed=self.max_articles_per_feed, get_article_url=self.get_article_url)
                     parsed_feeds.append(feed)
                 else:
                     raise Exception(f'Cannot fetch {url}:{resp.status_code}')

application/routes.py

@@ -3,18 +3,9 @@
 #主页和其他路由
 import os
 from flask import Blueprint, render_template, send_from_directory, current_app
-from .view import login
-from .view import admin
-from .view import adv
-from .view import deliver
-from .view import library
-from .view import library_offical
-from .view import logs
-from .view import setting
-from .view import share
-from .view import subscribe
-from .work import worker
-from .work import url2book
+from .view import (login, admin, adv, deliver, library, library_offical, logs, setting, share, 
+    subscribe, inbound_email)
+from .work import worker, url2book
 
 bpHome = Blueprint('bpHome', __name__)
 
@@ -62,10 +53,10 @@ def register_routes(app):
         app.register_blueprint(worker.bpWorker)
         app.register_blueprint(url2book.bpUrl2Book)
         app.register_blueprint(library_offical.bpLibraryOffical)
+        app.register_blueprint(inbound_email.bpInBoundEmail)
 
-        #使用GAE来接收邮件
-        if app.config['INBOUND_EMAIL_SERVICE'] == 'gae':
+        #启用GAE邮件服务如果部署在GAE平台
+        if app.config['DATABASE_URL'] == 'datastore':
             from google.appengine.api import wrap_wsgi_app
-            from application.view.inbound_email import bpInBoundEmail
-            app.wsgi_app = wrap_wsgi_app(app.wsgi_app)  #启用GAE邮件服务
-            app.register_blueprint(bpInBoundEmail)
+            app.wsgi_app = wrap_wsgi_app(app.wsgi_app)
+

application/templates/adv_base.html

@@ -36,14 +36,13 @@
         <ul class="pure-menu-list submenu-box">
           {% if advCurr=='deliverNow' -%}
           <li class="pure-menu-item pure-menu-selected">
-            <span>{{_("Deliver now")}}</span>
+            <span>{{_("Deliver Now")}}</span>
           </li>
           {% else -%}
           <li class="pure-menu-item">
-            <a href="/adv/delivernow" class="pure-menu-link">{{_("Deliver now")}} </a>
+            <a href="/adv/delivernow" class="pure-menu-link">{{_("Deliver Now")}} </a>
           </li>
           {% endif -%}
-          {% if in_email_service == 'gae' -%}
           {% if advCurr=='whitelist' -%}
           <li class="pure-menu-item pure-menu-selected">
             <span>{{_("White List")}}</span>
@@ -53,7 +52,6 @@
             <a href="/adv/whitelist" class="pure-menu-link">{{_("White List")}} </a>
           </li>
           {% endif -%}
-          {% endif -%}
           {% if advCurr=='archive' -%}
           <li class="pure-menu-item pure-menu-selected">
             <span>{{_("Archive")}}</span>
@@ -92,11 +90,11 @@
           {% endif -%}
           {% if advCurr=='calibreOptions' -%}
           <li class="pure-menu-item pure-menu-selected">
-            <span>{{_("Calibre options")}}</span>
+            <span>{{_("Calibre Options")}}</span>
           </li>
           {% else -%}
           <li class="pure-menu-item">
-            <a href="/adv/calibre" class="pure-menu-link">{{_("Calibre options")}}</a>
+            <a href="/adv/calibre" class="pure-menu-link">{{_("Calibre Options")}}</a>
           </li>
           {% endif -%}
         </ul>

application/templates/adv_calibre_options.html

@@ -9,7 +9,7 @@
     {% if tips -%}
     <div class="notice-box error">{{tips|safe}}</div>
     {% endif -%}
-    <legend><h3>{{_("Calibre options")}}</h3></legend>
+    <legend><h3>{{_("Calibre Options")}}</h3></legend>
     <p><small>{{_("Set the parameters for Calibre, in JSON dictionary format.")}}</small></p>
     <div class="box-list">
       <textarea name="options" rows="5" cols="50">{{options}}</textarea>

application/templates/adv_delivernow.html

@@ -5,8 +5,8 @@
 {% block advcontent -%}
 <div class="pure-form pure-form-aligned">
   <fieldset>
-    <legend><h3>{{ _("Deliver now") }}</h3></legend>
-    <p><small>{{ _("Deliver selected recipes now") }}</small></p>
+    <legend><h3>{{ _("Deliver Now") }}</h3></legend>
+    <p><small>{{ _("Deliver selected recipes now.") }}</small></p>
     <div class="box-list">
       {% if recipes|length == 0 -%}
         <div class="box">{{ _("There are no recipes subscribed") }}</div>

application/templates/adv_whitelist.html

@@ -12,7 +12,7 @@
   <fieldset>
     <legend><h3>{{_("White List")}}</h3></legend>
     <p> <small>
-    {{_("Emails sent to %(name)sxxx@appid.appspotmail.com will be transferred to your email.", name='' if (user.name == adminName) else user.name + '__')}}
+    {{_("Emails sent to %(name)sxxx@%(mailHost)s will be transferred to your kindle email.", name='' if (user.name == adminName) else user.name + '__', mailHost=mailHost)}}
     </small></p>
     <div class="box-list">
       {%- for wl in user.white_lists() -%}

application/view/adv.py

@@ -3,7 +3,7 @@
 #一些高级设置功能页面
 
 import datetime, hashlib, io
-from urllib.parse import quote, unquote, urljoin
+from urllib.parse import quote, unquote, urljoin, urlparse
 from flask import Blueprint, url_for, render_template, redirect, session, send_file, abort, current_app as app
 from flask_babel import gettext as _
 from PIL import Image
@@ -19,7 +19,6 @@ def adv_render_template(tpl, advCurr, **kwargs):
     kwargs.setdefault('tab', 'advset')
     kwargs.setdefault('tips', '')
     kwargs.setdefault('adminName', app.config['ADMIN_NAME'])
-    kwargs.setdefault('in_email_service', app.config['INBOUND_EMAIL_SERVICE'])
     return render_template(tpl, advCurr=advCurr, **kwargs)
 
 #现在推送
@@ -35,28 +34,27 @@ def AdvDeliverNow():
 @bpAdv.route("/adv/whitelist", endpoint='AdvWhiteList')
 @login_required()
 def AdvWhiteList():
-    if app.config['INBOUND_EMAIL_SERVICE'] == 'gae':
-        user = get_login_user()
-        return adv_render_template('adv_whitelist.html', 'whitelist', user=user)
+    user = get_login_user()
+    if app.config['DATABASE_URL'] == 'datastore':
+        mailHost = 'appid.appspotmail.com'
     else:
-        abort(404)
-
+        mailHost = urlparse(app.config['APP_DOMAIN']).netloc.split(':')[0]
+
+    return adv_render_template('adv_whitelist.html', 'whitelist', user=user, mailHost=mailHost)
+
 @bpAdv.post("/adv/whitelist", endpoint='AdvWhiteListPost')
 @login_required()
 def AdvWhiteListPost():
-    if app.config['INBOUND_EMAIL_SERVICE'] == 'gae':
-        user = get_login_user()
-        wlist = request.form.get('wlist')
+    user = get_login_user()
+    wlist = request.form.get('wlist')
+    if wlist:
+        wlist = wlist.replace('"', "").replace("'", "").strip()
+        if wlist.startswith('*@'): #输入*@xx.xx则修改为@xx.xx
+            wlist = wlist[1:]
         if wlist:
-            wlist = wlist.replace('"', "").replace("'", "").strip()
-            if wlist.startswith('*@'): #输入*@xx.xx则修改为@xx.xx
-                wlist = wlist[1:]
-            if wlist:
-                WhiteList.get_or_create(mail=wlist, user=user.name)
-        return redirect(url_for('bpAdv.AdvWhiteList'))
-    else:
-        abort(404)
-
+            WhiteList.get_or_create(mail=wlist, user=user.name)
+    return redirect(url_for('bpAdv.AdvWhiteList'))
+
 #删除白名单项目
 @bpAdv.route("/advdel", endpoint='AdvDel')
 @login_required()