Init

.github/workflows/pythonapp.yml

@@ -0,0 +1,36 @@
+name: Python application
+
+#on:
+#  schedule:
+#    - cron:  '0 */3 * * *' # every two hour
+#  push:
+#    branches:
+#      - master
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Run
+        env:
+          muser: ${{ secrets.muser }}
+          mpass: ${{ secrets.mpass }}
+          murl: ${{ secrets.murl }}
+          bot_token: ${{ secrets.bot_token }}
+          channel_id: ${{ secrets.channel_id }}
+          chat_id: ${{ secrets.chat_id }}
+          footer: ${{ secrets.footer }}
+        run: |
+          python rss.py
+          python bot.py

.gitignore

@@ -0,0 +1,3 @@
+.idea
+venv
+__pycache__

bot.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+
+import db
+import telegram
+
+if __name__ == '__main__':
+
+    unpublished_query = {'is_pub': False}
+    set_published_query = {'$set': {'is_pub': True}}
+    for x in db.feeds.find(unpublished_query):
+        title = x['title']
+        link = x['link']
+        date = x['date']
+        telegram.send_article(title, link, date)
+        db.feeds.update_one({'link': link}, set_published_query)

db.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+import os
+
+import pymongo
+
+db_url = 'mongodb+srv://{user}:{pwd}@{murl}/test?retryWrites=true&w=majority'
+db_user = os.environ['muser']
+db_pass = os.environ['mpass']
+db_domain = os.environ['murl']
+db_name = 'andev'
+db_feeds_collection = 'feeds'
+db_url = db_url.format(user=db_user, pwd=db_pass, murl=db_domain)
+client = pymongo.MongoClient(db_url)
+
+database = client[db_name]
+feeds = database[db_feeds_collection]

feeds.txt

@@ -0,0 +1,109 @@
+#android weekly
+http://us2.campaign-archive1.com/feed?u=887caf4f48db76fd91e20a06d&id=4eb677ad19
+http://adavis.info/feed
+http://alexzh.com/feed/
+http://andraskindler.com/feed.xml
+http://andreamaglie.com/feed.xml
+http://android.cyrilmottier.com/?feed=rss2
+http://android.programmerguru.com/feed/
+http://androidahead.com/feed/
+http://androidresearch.wordpress.com/feed/
+http://androiduiux.com/feed/
+http://andydyer.org/index.xml
+http://angrybyte.me/rss
+http://antonioleiva.com/feed/
+http://arnaud-camus.fr/feed.xml
+http://artemzin.com/blog/rss/
+http://baraabytes.com/feed/
+http://blog.danlew.net/rss/
+http://blog.egorand.me/rss/
+http://blog.feedpresso.com/feed.xml
+http://blog.karumi.com/rss/
+http://blog.nimbledroid.com/feed.xml
+http://blog.propaneapps.com/feed/
+http://feeds.feedburner.com/sqisland
+http://blog.stylingandroid.com/feed
+http://brianattwell.com/rss/
+http://catinean.com/rss/
+http://chintanrathod.com/feed/
+http://chris.banes.me/atom.xml
+http://chrisarriola.me/atom.xml
+http://cketti.de/feed.xml
+http://code.tutsplus.com/categories/android-sdk.atom
+http://commonsware.com/blog/feed.atom
+http://doridori.github.io/feed.xml
+http://emuneee.com/blog/feed/
+http://endlesswhileloop.com/feed.xml
+http://engineering.letsnurture.com/android/feed/
+http://feeds.feedburner.com/AndroidSnippetsLatest
+http://feeds.feedburner.com/AndroidTales
+http://feeds.feedburner.com/DeveloperPhil
+http://feeds.feedburner.com/GrokkingAndroid
+http://feeds.feedburner.com/blogspot/hsDu
+http://feeds.feedburner.com/survwithandroid
+http://frogermcs.github.io/feed.xml
+https://helw.net/index.xml
+http://ivankocijan.xyz/feed/
+http://jeremie-martinez.com/feed.xml
+http://jeroenmols.com/atom.xml
+http://josiassena.com/feed/
+http://konmik.com/index.xml
+http://kyubid.com/blog/feed/
+http://lomza.totem-soft.com/feed/
+http://michaelevans.org/atom.xml
+http://myhexaville.com/feed/
+http://mylifewithandroid.blogspot.com/feeds/posts/default
+http://mzgreen.github.io/atom.xml
+http://novoda.com/blog/blog/feed
+http://pguardiola.com/feed.xml
+http://ph0b.com/feed/
+http://philosophicalhacker.com/index.xml
+http://riggaroo.co.za/feed/
+http://szagurskii.com/feed.xml
+http://wiresareobsolete.com/feed/
+http://www.androidgig.com/feed/
+http://www.androidhive.info/feed/
+http://www.androidhub4you.com/feeds/posts/default
+http://www.androprogrammer.com/feeds/posts/default?alt=rss
+http://www.curious-creature.org/category/android/feed/
+http://www.hannesdorfmann.com//feed.xml
+http://www.hidroh.com/feed.xml
+http://www.myandroidsolutions.com/feed/
+http://www.rscottcarson.com/category/android/feed/
+http://www.sitepoint.com/mobile/android/feed/
+http://www.ssaurel.com/blog/feed/
+http://www.stkent.com/feed.xml
+http://www.truiton.com/feed/
+http://www.uwanttolearn.com/feed/
+http://www.willowtreeapps.com/feed/
+http://www.zoftino.com/feed
+http://yayandroid.com/feed/
+https://academy.realm.io/blog/feed.xml
+https://alexfu.github.io/feed.xml
+https://ar-g.github.io/feed.xml
+https://articles.caster.io/feed/
+https://barta.me/feed/
+https://blog.devdroids.com/feed
+https://blog.devknox.io/feed/
+https://blog.jayway.com/category/android/feed/
+https://jasonatwood.io/feed
+https://jebware.com/blog/?feed=rss2
+https://kanawish.com/index.xml
+https://mayojava.github.io/feed.xml
+https://medium.com/feed/android-news
+https://medium.com/feed/androiddevelopers
+https://medium.com/feed/google-developer-experts
+https://medium.com/feed/google-developers
+https://medium.com/feed/mindorks/tagged/android
+https://nullpointer.wtf/feed/
+https://overflow.buffer.com/category/android/feed/
+https://proandroiddev.com/feed
+https://rubensousa.github.io/feed.xml
+https://ryanharter.com/index.xml
+https://segunfamisa.github.io/feed.xml
+https://upday.github.io/feed.xml
+https://wajahatkarim.com/index.xml
+https://www.hrupin.com/feed
+https://www.nan-labs.com/blog/category/mobile/feed/
+https://yarikx.github.io/feed.xml
+http://saulmm.github.io/atom.xml

rss.py

@@ -0,0 +1,79 @@
+import feedparser
+import requests
+
+import db
+import telegram
+
+
+def get_feeds_url():
+    return [line.rstrip('\n') for line in open('feeds.txt')]
+
+
+def is_url_ok(url):
+    try:
+        r = requests.head(url)
+        if r.status_code == 200:
+            print('Url is OK: ' + url)
+            return True
+        else:
+            print('StatusCode is {}: {} '.format(r.status_code, url))
+            telegram.msg_to_admin('❌ StatusCode is ' + str(r.status_code) + ':\n' + url)
+            return True
+    except requests.ConnectionError:
+        print('Failed to connect: ' + url)
+        telegram.msg_to_admin('❌ Failed to connect:\n' + url)
+        return False
+
+
+def read_article_feed(feed_url):
+    if not is_url_ok(feed_url):
+        return
+    try:
+        feed = feedparser.parse(feed_url)
+        print('Count is ' + str(len(feed['entries'])))
+        for article in feed['entries']:
+            title = article['title']
+            link = article['link']
+
+            if 'published' in article:
+                date = article['published']
+            else:
+                date = article['updated']
+
+            if 'feedproxy.google.com' in link:
+                link = get_redirect_url(link)
+
+            if not is_article_in_db(link):
+                add_article_to_db(title, link, date, True)
+    except:
+        telegram.msg_to_admin('❌ Failed to parse:\n' + feed_url)
+
+
+def get_redirect_url(url):
+    return requests.get(url).url
+
+
+def is_article_in_db(url):
+    query = {'link': url}
+    if db.feeds.count_documents(query) == 0:
+        return False
+    else:
+        return True
+
+
+def add_article_to_db(title, link, date, is_pub):
+    article = {'title': title, 'link': link, 'date': date, 'is_pub': is_pub}
+    x = db.feeds.insert_one(article)
+    print(x.inserted_id)
+
+
+if __name__ == '__main__':
+    lines = get_feeds_url()
+    print('Feeds count: ' + str(len(lines)))
+    index = 0
+    for line in lines:
+        if line.startswith('#'):
+            continue
+        index += 1
+        print('Processing feed #' + str(index) + ' : ' + line)
+        read_article_feed(line)

telegram.py

@@ -0,0 +1,26 @@
+import os
+
+import requests
+
+bot_token = os.environ['bot_token']
+channel_id = os.environ['channel_id']
+admin_id = os.environ['chat_id']
+footer = os.environ['footer']
+
+
+def msg_to_admin(args):
+    text_caps = ''.join(args)
+    requests.post(
+        url='https://api.telegram.org/bot{0}/{1}'.format(bot_token, 'sendMessage'),
+        data={'chat_id': admin_id, 'text': text_caps}
+    )
+
+
+def send_article(title, link, date):
+    message = '<a href="{link}">{title}</a>\n\n<pre>{date}</pre>\n\n{footer}' \
+        .format(title=title, link=link, date=date,
+                footer=footer)
+    return requests.post(
+        url='https://api.telegram.org/bot{0}/{1}'.format(bot_token, 'sendMessage'),
+        data={'chat_id': channel_id, 'text': message, 'parse_mode': 'HTML'}
+    )