Update

.github/workflows/forced_walker.yml

@@ -0,0 +1,32 @@
+name: Forced Walk and Publish
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+      - name: Run
+        env:
+          muser: ${{ secrets.muser }}
+          mpass: ${{ secrets.mpass }}
+          murl: ${{ secrets.murl }}
+          bot_token: ${{ secrets.bot_token }}
+          channel_id: ${{ secrets.channel_id }}
+          chat_id: ${{ secrets.chat_id }}
+          footer: ${{ secrets.footer }}
+        run: |
+          python rss.py
+#          python bot.py

.github/workflows/walker.yml

@@ -10,14 +10,14 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v1
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: '3.10'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -33,4 +33,4 @@ jobs:
           footer: ${{ secrets.footer }}
         run: |
           python rss.py
-          python bot.py
+#          python bot.py

.gitignore

@@ -1,3 +1,6 @@
 .idea
 venv
 __pycache__
+.gitignore
+1001268302234
+poetry.lock

db.py

@@ -15,4 +15,12 @@
 
 database = client[db_name]
 feeds = database[db_feeds_collection]
-urls = database[db_url_collection]
+urls = database[db_url_collection]
+
+
+def mark_as_read_all():
+    unpublished_query = {'is_pub': False}
+    set_published_query = {'$set': {'is_pub': True}}
+    for x in feeds.find(unpublished_query):
+        link = x['link']
+        feeds.update_one({'link': link}, set_published_query)

pyproject.toml

@@ -0,0 +1,23 @@
+[tool.poetry]
+name = "tg-andev"
+version = "0.1.0"
+description = ""
+authors = ["runo280 <[email protected]>"]
+readme = "README.md"
+packages = [{include = "tg_andev"}]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+dnspython = "^2.2.1"
+chardet = "^5.1.0"
+certifi = "^2022.12.7"
+feedparser = "^6.0.10"
+idna = "^3.4"
+pymongo = "^4.3.3"
+requests = "^2.28.1"
+urllib3 = "^1.26.13"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -1,8 +1,8 @@
-certifi==2022.12.7
-chardet==3.0.4
-dnspython==1.16.0
-feedparser==5.2.1
-idna==2.8
-pymongo==3.10.0
-requests==2.22.0
-urllib3
+certifi
+chardet
+dnspython
+feedparser
+idna
+pymongo
+requests
+urllib3

rss.py

@@ -6,7 +6,7 @@
 
 
 def get_feeds_url():
-    return [line.rstrip('\n') for line in open('feeds.txt')]
+    return [row.rstrip('\n') for row in open('feeds.txt')]
 
 
 def is_url_ok(url):
@@ -17,11 +17,11 @@ def is_url_ok(url):
             return True
         else:
             print('StatusCode is {}: {} '.format(r.status_code, url))
-            #telegram.msg_to_admin('❌ StatusCode is ' + str(r.status_code) + ':\n' + url)
+            telegram.msg_to_admin('❌ StatusCode is ' + str(r.status_code) + ':\n' + url)
             return True
     except requests.ConnectionError:
         print('Failed to connect: ' + url)
-        #telegram.msg_to_admin('❌ Failed to connect:\n' + url)
+        telegram.msg_to_admin('❌ Failed to connect:\n' + url)
         return False
 
 
@@ -46,9 +46,9 @@ def read_article_feed(feed_url):
 
             if not is_article_in_db(link):
                 add_article_to_db(title, link, date, first_crawl)
-    except:
+    except():
         print()
-        #telegram.msg_to_admin('❌ Failed to parse:\n' + feed_url)
+        telegram.msg_to_admin('❌ Failed to parse:\n' + feed_url)
 
 
 def get_redirect_url(url):
@@ -62,6 +62,7 @@ def is_article_in_db(url):
     else:
         return True
 
+
 def should_published(url):
     query = {'link': url}
     if db.urls.count_documents(query) == 0:
@@ -71,6 +72,7 @@ def should_published(url):
     else:
         return False
 
+
 def add_article_to_db(title, link, date, is_pub):
     article = {'title': title, 'link': link, 'date': date, 'is_pub': is_pub}
     x = db.feeds.insert_one(article)
@@ -87,3 +89,5 @@ def add_article_to_db(title, link, date, is_pub):
         index += 1
         print('Processing feed #' + str(index) + ' : ' + line)
         read_article_feed(line)
+
+    db.mark_as_read_all()