From 1b70cc1d41773924e9504e1d2fd9018e8f375ce2 Mon Sep 17 00:00:00 2001 From: runo280 Date: Thu, 12 Dec 2019 20:36:05 +0330 Subject: [PATCH] Init --- .github/workflows/pythonapp.yml | 36 +++++++++++ .gitignore | 3 + bot.py | 15 +++++ db.py | 16 +++++ feeds.txt | 109 ++++++++++++++++++++++++++++++++ requirements.txt | Bin 0 -> 278 bytes rss.py | 79 +++++++++++++++++++++++ telegram.py | 26 ++++++++ 8 files changed, 284 insertions(+) create mode 100644 .github/workflows/pythonapp.yml create mode 100644 .gitignore create mode 100644 bot.py create mode 100644 db.py create mode 100644 feeds.txt create mode 100644 requirements.txt create mode 100644 rss.py create mode 100644 telegram.py diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml new file mode 100644 index 0000000..a9407db --- /dev/null +++ b/.github/workflows/pythonapp.yml @@ -0,0 +1,36 @@ +name: Python application + +#on: +# schedule: +# - cron: '0 */3 * * *' # every two hour +# push: +# branches: +# - master + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Run + env: + muser: ${{ secrets.muser }} + mpass: ${{ secrets.mpass }} + murl: ${{ secrets.murl }} + bot_token: ${{ secrets.bot_token }} + channel_id: ${{ secrets.channel_id }} + chat_id: ${{ secrets.chat_id }} + footer: ${{ secrets.footer }} + run: | + python rss.py + python bot.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..26d0710 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea +venv +__pycache__ diff --git a/bot.py b/bot.py new file mode 100644 index 0000000..180b6b9 --- /dev/null +++ b/bot.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +import db +import telegram + +if __name__ == '__main__': + + unpublished_query = {'is_pub': False} + set_published_query = {'$set': {'is_pub': True}} + for x in db.feeds.find(unpublished_query): + title = x['title'] + link = x['link'] + date = x['date'] + telegram.send_article(title, link, date) + db.feeds.update_one({'link': link}, set_published_query) diff --git a/db.py b/db.py new file mode 100644 index 0000000..c809541 --- /dev/null +++ b/db.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +import os + +import pymongo + +db_url = 'mongodb+srv://{user}:{pwd}@{murl}/test?retryWrites=true&w=majority' +db_user = os.environ['muser'] +db_pass = os.environ['mpass'] +db_domain = os.environ['murl'] +db_name = 'andev' +db_feeds_collection = 'feeds' +db_url = db_url.format(user=db_user, pwd=db_pass, murl=db_domain) +client = pymongo.MongoClient(db_url) + +database = client[db_name] +feeds = database[db_feeds_collection] diff --git a/feeds.txt b/feeds.txt new file mode 100644 index 0000000..609d110 --- /dev/null +++ b/feeds.txt @@ -0,0 +1,109 @@ +#android weekly +http://us2.campaign-archive1.com/feed?u=887caf4f48db76fd91e20a06d&id=4eb677ad19 +http://adavis.info/feed +http://alexzh.com/feed/ +http://andraskindler.com/feed.xml +http://andreamaglie.com/feed.xml +http://android.cyrilmottier.com/?feed=rss2 +http://android.programmerguru.com/feed/ +http://androidahead.com/feed/ +http://androidresearch.wordpress.com/feed/ +http://androiduiux.com/feed/ +http://andydyer.org/index.xml +http://angrybyte.me/rss +http://antonioleiva.com/feed/ +http://arnaud-camus.fr/feed.xml +http://artemzin.com/blog/rss/ +http://baraabytes.com/feed/ +http://blog.danlew.net/rss/ +http://blog.egorand.me/rss/ +http://blog.feedpresso.com/feed.xml +http://blog.karumi.com/rss/ +http://blog.nimbledroid.com/feed.xml +http://blog.propaneapps.com/feed/ +http://feeds.feedburner.com/sqisland +http://blog.stylingandroid.com/feed +http://brianattwell.com/rss/ +http://catinean.com/rss/ +http://chintanrathod.com/feed/ +http://chris.banes.me/atom.xml +http://chrisarriola.me/atom.xml +http://cketti.de/feed.xml +http://code.tutsplus.com/categories/android-sdk.atom +http://commonsware.com/blog/feed.atom +http://doridori.github.io/feed.xml +http://emuneee.com/blog/feed/ +http://endlesswhileloop.com/feed.xml +http://engineering.letsnurture.com/android/feed/ +http://feeds.feedburner.com/AndroidSnippetsLatest +http://feeds.feedburner.com/AndroidTales +http://feeds.feedburner.com/DeveloperPhil +http://feeds.feedburner.com/GrokkingAndroid +http://feeds.feedburner.com/blogspot/hsDu +http://feeds.feedburner.com/survwithandroid +http://frogermcs.github.io/feed.xml +https://helw.net/index.xml +http://ivankocijan.xyz/feed/ +http://jeremie-martinez.com/feed.xml +http://jeroenmols.com/atom.xml +http://josiassena.com/feed/ +http://konmik.com/index.xml +http://kyubid.com/blog/feed/ +http://lomza.totem-soft.com/feed/ +http://michaelevans.org/atom.xml +http://myhexaville.com/feed/ +http://mylifewithandroid.blogspot.com/feeds/posts/default +http://mzgreen.github.io/atom.xml +http://novoda.com/blog/blog/feed +http://pguardiola.com/feed.xml +http://ph0b.com/feed/ +http://philosophicalhacker.com/index.xml +http://riggaroo.co.za/feed/ +http://szagurskii.com/feed.xml +http://wiresareobsolete.com/feed/ +http://www.androidgig.com/feed/ +http://www.androidhive.info/feed/ +http://www.androidhub4you.com/feeds/posts/default +http://www.androprogrammer.com/feeds/posts/default?alt=rss +http://www.curious-creature.org/category/android/feed/ +http://www.hannesdorfmann.com//feed.xml +http://www.hidroh.com/feed.xml +http://www.myandroidsolutions.com/feed/ +http://www.rscottcarson.com/category/android/feed/ +http://www.sitepoint.com/mobile/android/feed/ +http://www.ssaurel.com/blog/feed/ +http://www.stkent.com/feed.xml +http://www.truiton.com/feed/ +http://www.uwanttolearn.com/feed/ +http://www.willowtreeapps.com/feed/ +http://www.zoftino.com/feed +http://yayandroid.com/feed/ +https://academy.realm.io/blog/feed.xml +https://alexfu.github.io/feed.xml +https://ar-g.github.io/feed.xml +https://articles.caster.io/feed/ +https://barta.me/feed/ +https://blog.devdroids.com/feed +https://blog.devknox.io/feed/ +https://blog.jayway.com/category/android/feed/ +https://jasonatwood.io/feed +https://jebware.com/blog/?feed=rss2 +https://kanawish.com/index.xml +https://mayojava.github.io/feed.xml +https://medium.com/feed/android-news +https://medium.com/feed/androiddevelopers +https://medium.com/feed/google-developer-experts +https://medium.com/feed/google-developers +https://medium.com/feed/mindorks/tagged/android +https://nullpointer.wtf/feed/ +https://overflow.buffer.com/category/android/feed/ +https://proandroiddev.com/feed +https://rubensousa.github.io/feed.xml +https://ryanharter.com/index.xml +https://segunfamisa.github.io/feed.xml +https://upday.github.io/feed.xml +https://wajahatkarim.com/index.xml +https://www.hrupin.com/feed +https://www.nan-labs.com/blog/category/mobile/feed/ +https://yarikx.github.io/feed.xml +http://saulmm.github.io/atom.xml \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f09d09c1512191c63ddbcd8d8a0afb7220bf060f GIT binary patch literal 278 zcmX|+OAf*?3`A#*#8s-q{1+Ss`k|4al%`#9d0;#RWW`eC=aM=w{_>&|UR zm{_M0iH^jar}!pt_f6&tyU#p9RED4o1APxn?3pR@f8XSunTdb6vJ%eR|7A2K*vaw% Dfww3z literal 0 HcmV?d00001 diff --git a/rss.py b/rss.py new file mode 100644 index 0000000..123bf4e --- /dev/null +++ b/rss.py @@ -0,0 +1,79 @@ +import feedparser +import requests + +import db +import telegram + + +def get_feeds_url(): + return [line.rstrip('\n') for line in open('feeds.txt')] + + +def is_url_ok(url): + try: + r = requests.head(url) + if r.status_code == 200: + print('Url is OK: ' + url) + return True + else: + print('StatusCode is {}: {} '.format(r.status_code, url)) + telegram.msg_to_admin('❌ StatusCode is ' + str(r.status_code) + ':\n' + url) + return True + except requests.ConnectionError: + print('Failed to connect: ' + url) + telegram.msg_to_admin('❌ Failed to connect:\n' + url) + return False + + +def read_article_feed(feed_url): + if not is_url_ok(feed_url): + return + try: + feed = feedparser.parse(feed_url) + print('Count is ' + str(len(feed['entries']))) + for article in feed['entries']: + title = article['title'] + link = article['link'] + + if 'published' in article: + date = article['published'] + else: + date = article['updated'] + + if 'feedproxy.google.com' in link: + link = get_redirect_url(link) + + if not is_article_in_db(link): + add_article_to_db(title, link, date, True) + except: + telegram.msg_to_admin('❌ Failed to parse:\n' + feed_url) + + +def get_redirect_url(url): + return requests.get(url).url + + +def is_article_in_db(url): + query = {'link': url} + if db.feeds.count_documents(query) == 0: + return False + else: + return True + + +def add_article_to_db(title, link, date, is_pub): + article = {'title': title, 'link': link, 'date': date, 'is_pub': is_pub} + x = db.feeds.insert_one(article) + print(x.inserted_id) + + +if __name__ == '__main__': + lines = get_feeds_url() + print('Feeds count: ' + str(len(lines))) + index = 0 + for line in lines: + if line.startswith('#'): + continue + index += 1 + print('Processing feed #' + str(index) + ' : ' + line) + read_article_feed(line) diff --git a/telegram.py b/telegram.py new file mode 100644 index 0000000..7aa9ef9 --- /dev/null +++ b/telegram.py @@ -0,0 +1,26 @@ +import os + +import requests + +bot_token = os.environ['bot_token'] +channel_id = os.environ['channel_id'] +admin_id = os.environ['chat_id'] +footer = os.environ['footer'] + + +def msg_to_admin(args): + text_caps = ''.join(args) + requests.post( + url='https://api.telegram.org/bot{0}/{1}'.format(bot_token, 'sendMessage'), + data={'chat_id': admin_id, 'text': text_caps} + ) + + +def send_article(title, link, date): + message = '{title}\n\n
{date}
\n\n{footer}' \ + .format(title=title, link=link, date=date, + footer=footer) + return requests.post( + url='https://api.telegram.org/bot{0}/{1}'.format(bot_token, 'sendMessage'), + data={'chat_id': channel_id, 'text': message, 'parse_mode': 'HTML'} + )