-
Notifications
You must be signed in to change notification settings - Fork 25
/
main.py
executable file
·100 lines (81 loc) · 2.89 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
import json
import logging
import shortener
from google.appengine.api import urlfetch, memcache
from google.appengine.ext import deferred, ndb
from flask import Flask, redirect, abort, make_response
from apis.hn import topstories, item_async
from database import StoryPost
app = Flask(__name__)
SCORE_TARGET = 150 # based on score inflation (see https://instruments.digital/inflation-adjusted-hn/)
@app.route('/s/<short_id>')
def story_redirect(short_id):
"""Redirect to story url"""
try:
story_id = str(shortener.decode(short_id))
except:
return abort(400)
redirect_url = memcache.get(story_id)
if not redirect_url:
story = ndb.Key(StoryPost, story_id).get()
if not story:
return make_response('<h1>Service Unavailable</h1><p>Try again later</p>', 503, {'Retry-After': 5})
story.add_memcache()
redirect_url = story.url
return redirect(redirect_url)
@app.route('/c/<short_id>')
def comments_redirect(short_id):
"""Redirect to comments url"""
try:
story_id = str(shortener.decode(short_id))
except:
return abort(400)
hn_url = "https://news.ycombinator.com/item?id={}".format(story_id)
return redirect(hn_url)
def task(stories):
def check_story(rpc):
try:
result = rpc.get_result()
story = json.loads(result.content)
if story and story.get('score') >= SCORE_TARGET:
StoryPost.add(story)
elif story and story.get('score'):
# api returned a comment once (issue with id 21447853)
logging.info('STOP: {id} has low score ({score})'.format(**story))
elif story:
logging.info('STOP: {id} has no score'.format(**story))
else:
logging.info("STOP: story was probably deleted/flagged")
except urlfetch.DownloadError as ex:
logging.exception(ex)
except ValueError as ex:
logging.info(result.content)
logging.exception(ex)
# stringify ids for use in memcache and convert to set for later
ids = set(str(story_id) for story_id in stories)
logging.info('checking stories: {}'.format(ids))
# get stories that we already posted to reduce the number of requests
cached_stories = set(memcache.get_multi(ids).keys())
logging.info('cached stories: {}'.format(cached_stories))
# remove stories we know about from stories that we need to check
stories_to_check = ids.difference(cached_stories)
rpcs = map(lambda id: item_async(id, check_story), stories_to_check)
for rpc in rpcs:
rpc.wait()
def chunkify(lst, n):
return [lst[i::n] for i in xrange(n)]
@app.route('/cron')
def cron():
stories = topstories()
chunks = chunkify(stories, 20)
for chunk in chunks:
deferred.defer(task, chunk)
return 'OK'
@app.errorhandler(404)
def page_not_found(e):
"""Return a custom 404 error."""
return 'Sorry, Nothing at this URL.', 404
@app.errorhandler(500)
def application_error(e):
"""Return a custom 500 error."""
return 'Sorry, unexpected error: {}'.format(e), 500