Cha 14 -爬虫实战二：知乎Live.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 13.3 项目实施"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 13.3.1 获取所有Live"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"error\": {\"message\": \"\\u672a\\u8bbe\\u7f6e\\u9a8c\\u8bc1\\u65b9\\u5f0f\", \"code\": 602}}\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "def scrapy(link):\n",
    "    headers = {\n",
    "        'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'\n",
    "    } \n",
    "    r = requests.get(link, headers= headers)\n",
    "    return (r.text)\n",
    "\n",
    "link = \"https://api.zhihu.com/lives/homefeed?includes=live\"\n",
    "html = scrapy(link)\n",
    "print (html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"paging\": {\"is_end\": false, \"next\": \"https://api.zhihu.com/lives/homefeed?limit=10&offset=1494594000\", \"previous\": \"\"}, \"data\": [{\"object_type\": \"live\", \"source_type\": \"hot_live\", \"live\": {\"status\": \"ended\", \"ends_at\": 1494945036, \"liked\": false, \"ends_in\": 0, \"speaker_message_count\": 99, \"tags\": [{\"score\": 0, \"name\": \"体育\", \"short_name\": \"体育\", \"available_num\": 136, \"created_at\": 1469691129, \"id\": 104, \"live_num\": 151}], \"review\": {\"count\": 9, \"status\": \"normal\", \"score\": 5.0, \"has_reviewed\": false, \"previous\": {\"status\": \"nonexistent\", \"score\": 0}}, \"liked_num\": 103, \"live_type\": \"audio\", \"is_public\": 1, \"video\": {}, \"cospeakers\": [{\"member\": {\"name\": \"胡海清\", \"url\": \"https://api.zhihu.com/people/4c29d9b23e17e872ac364fb4d264eebf\", \"type\": \"people\", \"user_type\": \"people\", \"headline\": \"浅石创投\", \"avatar_url\": \"https://pic4.zhimg.com/v2-f59d02a3bcb16843677b49be87c03bf7_s.jpg\", \"gender\": 1, \"url_token\": \"hu-hai-qing-33\", \"id\": \"4c29d9b23e17e872ac364fb4d264eebf\"}, \"bio\": \"浅石创投\", \"description\": \"我是胡海清，浅石创投合伙人,曾任经纬中国投资董事，参与投资和管理的博纳影业、果麦文化、北半球等泛娱乐类知名企业。\"}], \"fee\": {\"original_price\": 999, \"amount\": 999, \"unit\": \"RMB\"}, \"speaker\": {\"member\": {\"name\": \"王涛\", \"url\": \"https://api.zhihu.com/people/c47dfb36c0b63dd2ed5880e15585321d\", \"type\": \"people\", \"user_type\": \"people\", \"headline\": \"北半球国际传媒CEO，前央视足球解说员\", \"avatar_url\": \"https://pic2.zhimg.com/v2-7fe8fa06491e77e896de7c40846f7f11_s.jpg\", \"gender\": 1, \"url_token\": \"wang-tao-48-71\", \"id\": \"c47dfb36c0b63dd2ed5880e15585321d\"}, \"bio\": \"北半球国际传媒CEO，前央视足球解说员\", \"description\": \"我是王涛，原 CCTV5 《天下足球》创始团队之一，《实况足球》和《 FIFAOL3 》的中文解说员，留下了《疯狂的足球》、《爆笑体育》、《蓝色狂想》等个人色彩浓烈的作品。 2014 年从央视离职创立北半球传媒，专注于体育视频内容制作，制作出了《绿茵继承者》、《足球各种嘿》、《球迷朋友圈》等多个知名体育 IP ，如今创造了自媒体矩阵「骚客」。并于 2015 年获浅石创投和经纬中国的联合投资。我最大的爱好是足球、游戏、解说，很幸运，我的职业从未离开爱好。我在足球圈、电竞圈浸淫了 14 年，最大的财富就是对自己喜欢事业的孜孜不倦，并坚持用梦想创造财富。\"}, \"seats\": {\"taken\": 147, \"max\": 500}, \"starts_at\": 1494936000, \"role\": \"visitor\", \"in_promotion\": false, \"id\": \"840520148518592512\", \"subject\": \"从央视名嘴到体育内容创业者\"}, \"action\": \"new_hot_live\", \"source_id\": 0, \"id\": \"live840520148518592512\"}, {\"object_type\": \"live\", \"source_type\": \"hot_live\", \"live\": {\"status\": \"ended\", \"ends_at\": 1494949124, \"liked\": false, \"ends_in\": 0, \"speaker_message_count\": 145, \"tags\": [{\"score\": 0, \"name\": \"科学技术\", \"short_name\": \"科学技术\", \"available_num\": 253, \"created_at\": 1469691129, \"id\": 201, \"live_num\": 262}], \"review\": {\"count\": 236, \"status\": \"normal\", \"score\": 4.9, \"has_reviewed\": false, \"previous\": {\"status\": \"nonexistent\", \"score\": 0}}, \"liked_num\": 2489, \"live_type\": \"audio\", \"is_public\": 1, \"video\": {}, \"cospeakers\": [], \"fee\": {\"original_price\": 1900, \"amount\": 1900, \"unit\": \"RMB\"}, \"speaker\": {\"member\": {\"name\": \"金旭亮\", \"url\": \"https://api.zhihu.com/people/d127d540d194cf37387ebd4ea250ff6a\", \"type\": \"people\", \"user_type\": \"people\", \"headline\": \"喜欢写代码的老师\", \"avatar_url\": \"https://pic2.zhimg.com/e34780301_s.jpg\", \"gender\": 1, \"url_token\": \"jin-xu-liang\", \"id\": \"d127d540d194cf37387ebd4ea250ff6a\"}, \"bio\": \"北京理工大学计算机学院教师，教书匠一个，老码农一名\", \"description\": \"我是金旭亮，二十四年前，我从广西大学电气工程系毕业，误打误撞地，不可救药地爱上了编程，从此开始了漫长的自学之路，后考上北京理工大学计算机研究生，毕业留校任教至今。另外我写了 6 本软件书籍和教材。\"}, \"seats\": {\"taken\": 2714, \"max\": 500}, \"starts_at\": 1494936000, \"role\": \"visitor\", \"in_promotion\": false, \"id\": \"837669764146003968\", \"subject\": \"如何自学计算机专业课程？\"}, \"action\": \"new_hot_live\", \"source_id\": 0, \"id\": \"live837669764146003968\"}, {\"object_type\": \"live\", \"source_type\": \"hot_live\", \"live\": {\"status\": \"ended\", \"ends_at\": 1494857577, \"liked\": false, \"ends_in\": 0, \"speaker_message_count\": 194, \"tags\": [{\"score\": 0, \"name\": \"体育\", \"short_name\": \"体育\", \"available_num\": 136, \"created_at\": 1469691129, \"id\": 104, \"live_num\": 151}], \"review\": {\"count\": 21, \"status\": \"normal\", \"score\": 4.95, \"has_reviewed\": false, \"previous\": {\"status\": \"nonexistent\", \"score\": 0}}, \"liked_num\": 415, \"live_type\": \"audio\", \"is_public\": 1, \"video\": {}, \"cospeakers\": [], \"fee\": {\"original_price\": 1900, \"amount\": 1900, \"unit\": \"RMB\"}, \"speaker\": {\"member\": {\"name\": \"李明威\", \"url\": \"https://api.zhihu.com/people/440876e577bc9391b22ee2e04db078c3\", \"type\": \"people\", \"user_type\": \"people\", \"headline\": \"让人人都能享受到国家队专业的运动康复资源。我在北京，创业运动康复。\", \"avatar_url\": \"https://pic2.zhimg.com/7ca0f8ae4ecf51abfca357ffd8d60b65_s.jpg\", \"gender\": 1, \"url_token\": \"ming-wei-shuo\", \"id\": \"440876e577bc9391b22ee2e04db078c3\"}, \"bio\": \"让人人都能享受到国家队专业的运动康复资源。我在北京，创业运动康复。\", \"description\": \"我是李明威，毕业于北京体育大学，主要研究运动康复与健康，也是北京健行者运动康复的创始人。\"}, \"seats\": {\"taken\": 362, \"max\": 500}, \"starts_at\": 1494849600, \"role\": \"visitor\", \"in_promotion\": false, \"id\": \"837693085466107904\", \"subject\": \"腰间盘突出的分析和运动康复\"}, \"action\": \"new_hot_live\", \"source_id\": 0, \"id\": \"live837693085466107904\"}, {\"object_type\": \"live\", \"source_type\": \"hot_live\", \"live\": {\"status\": \"ended\", \"ends_at\": 1494685704, \"liked\": false, \"ends_in\": 0, \"speaker_message_count\": 205, \"tags\": [{\"score\": 0, \"name\": \"教育\", \"short_name\": \"教育\", \"available_num\": 563, \"created_at\": 1469691129, \"id\": 303, \"live_num\": 586}], \"review\": {\"count\": 19, \"status\": \"normal\", \"score\": 4.89, \"has_reviewed\": false, \"previous\": {\"status\": \"nonexistent\", \"score\": 0}}, \"liked_num\": 268, \"live_type\": \"audio\", \"is_public\": 1, \"video\": {}, \"cospeakers\": [], \"fee\": {\"original_price\": 999, \"amount\": 999, \"unit\": \"RMB\"}, \"speaker\": {\"member\": {\"name\": \"田昕\", \"url\": \"https://api.zhihu.com/people/c242b52075b4bde011dc1a76e1cb02fc\", \"type\": \"people\", \"user_type\": \"people\", \"headline\": \"研究生 厨娘 学跳舞\", \"avatar_url\": \"https://pic1.zhimg.com/v2-659787e3fadee5ea180703a509dae8d4_s.jpg\", \"gender\": 0, \"url_token\": \"tian-xin-51-51\", \"id\": \"c242b52075b4bde011dc1a76e1cb02fc\"}, \"bio\": \"研究生 厨娘  学跳舞\", \"description\": \"我在 2017 年考研中，考入中国人民大学法律硕士（法学）专业，历时 130 天，英语第一，面试第一，总分第六。我非学霸，只是用对了计划和方法。我的专业课押题本基本覆盖主观题，我的英语客观题只扣掉 4 份，我对人大复试的诸多细节有深刻体会，我对考研也有一些教训要分享。\"}, \"seats\": {\"taken\": 182, \"max\": 500}, \"starts_at\": 1494680400, \"role\": \"visitor\", \"in_promotion\": false, \"id\": \"843124614065434624\", \"subject\": \"130 天备考人大法律硕士\"}, \"action\": \"new_hot_live\", \"source_id\": 0, \"id\": \"live843124614065434624\"}, {\"object_type\": \"live\", \"source_type\": \"hot_live\", \"live\": {\"status\": \"ended\", \"ends_at\": 1494699100, \"liked\": false, \"ends_in\": 0, \"speaker_message_count\": 205, \"tags\": [{\"score\": 0, \"name\": \"设计\", \"short_name\": \"设计\", \"available_num\": 187, \"created_at\": 1469691129, \"id\": 106, \"live_num\": 190}], \"review\": {\"count\": 21, \"status\": \"normal\", \"score\": 5.0, \"has_reviewed\": false, \"previous\": {\"status\": \"nonexistent\", \"score\": 0}}, \"liked_num\": 366, \"live_type\": \"audio\", \"is_public\": 1, \"video\": {}, \"cospeakers\": [], \"fee\": {\"original_price\": 4990, \"amount\": 4990, \"unit\": \"RMB\"}, \"speaker\": {\"member\": {\"name\": \"SunLau\", \"url\": \"https://api.zhihu.com/people/6cf8d594290b3f3b0c3737bb14904385\", \"type\": \"people\", \"user_type\": \"people\", \"headline\": \"室内设计师 ，自由码字人\", \"avatar_url\": \"https://pic1.zhimg.com/36625a7b4_s.jpg\", \"gender\": 1, \"url_token\": \"sunlau\", \"id\": \"6cf8d594290b3f3b0c3737bb14904385\"}, \"bio\": \"室内设计师 ，自由码字人\", \"description\": \"我是一名室内设计师，从业 12 年，2008 年成为自由职业至今。关注家居、生活方式、互联网和一切有趣的事。2011 年注册知乎以来，回答了 800 多个有关家居的问题。\"}, \"seats\": {\"taken\": 255, \"max\": 500}, \"starts_at\": 1494676800, \"role\": \"visitor\", \"in_promotion\": false, \"id\": \"842731562154987520\", \"subject\": \"给家点颜色，家居配色基础课\"}, \"action\": \"new_hot_live\", \"source_id\": 0, \"id\": \"live842731562154987520\"}, {\"object_type\": \"live\", \"source_type\": \"hot_live\", \"live\": {\"status\": \"ended\", \"ends_at\": 1494691484, \"liked\": false, \"ends_in\": 0, \"speaker_message_count\": 154, \"tags\": [{\"score\": 0, \"name\": \"艺术\", \"short_name\": \"艺术\", \"available_num\": 238, \"created_at\": 1469691129, \"id\": 103, \"live_num\": 240}], \"review\": {\"count\": 16, \"status\": \"normal\", \"score\": 4.94, \"has_reviewed\": false, \"previous\": {\"status\": \"nonexistent\", \"score\": 0}}, \"liked_num\": 429, \"live_type\": \"audio\", \"is_public\": 1, \"video\": {}, \"cospeakers\": [], \"fee\": {\"original_price\": 999, \"amount\": 999, \"unit\": \"RMB\"}, \"speaker\": {\"member\": {\"name\": \"王彦翔\", \"url\": \"https://api.zhihu.com/people/927740dfdc0ab4f77a42b2574caefb2a\", \"type\": \"people\", \"user_type\": \"people\", \"headline\": \"一个过气煮池人。\", \"avatar_url\": \"https://pic2.zhimg.com/e131ad325248d5b7b13e254d28afd2e1_s.jpg\", \"gender\": 1, \"url_token\": \"wangyanxiang\", \"id\": \"927740dfdc0ab4f77a42b2574caefb2a\"}, \"bio\": \"一个过气煮池人。\", \"description\": \"播音主持专业科班毕业，普通话等级一级甲等。原中央电视台财经频道外景主持人，后于四川新闻频率FM106.1任栏目主播，现已辞职，下海创业！公司麋鹿映画成立教育事业部专门针对于对普通话有所需求的人群进行一对一线上教学，先后已完成3000余名学员课程，协助他们让普通话等级从原来的70-80分段，跨越至90甚至更高分段。\"}, \"seats\": {\"taken\": 333, \"max\": 500}, \"starts_at\": 1494676800, \"role\": \"visitor\", \"in_promotion\": false, \"id\": \"839869490635366400\", \"subject\": \"科学发声，好好说话。\"}, \"action\": \"new_hot_live\", \"source_id\": 0, \"id\": \"live839869490635366400\"}, {\"object_type\": \"live\", \"source_type\": \"hot_live\", \"live\": {\"status\": \"ended\", \"ends_at\": 1494686293, \"liked\": false, \"ends_in\": 0, \"speaker_message_count\": 171, \"tags\": [{\"score\": 0, \"name\": \"教育\", \"short_name\": \"教育\", \"available_num\": 563, \"created_at\": 1469691129, \"id\": 303, \"live_num\": 586}], \"review\": {\"count\": 69, \"status\": \"normal\", \"score\": 4.71, \"has_reviewed\": false, \"previous\": {\"status\": \"nonexistent\", \"score\": 0}}, \"liked_num\": 579, \"live_type\": \"audio\", \"is_public\": 1, \"video\": {}, \"cospeakers\": [], \"fee\": {\"original_price\": 1998, \"amount\": 1998, \"unit\": \"RMB\"}, \"speaker\": {\"member\": {\"name\": \"姜小白\", \"url\": \"https://api.zhihu.com/people/fa14835a18c410298a67c7e3714f7f80\", \"type\": \"people\", \"user_type\": \"people\", \"headline\": \"零基础学习英语的学渣 瘦子长肌肉的健身爱好者\", \"avatar_url\": \"https://pic2.zhimg.com/4cf519f905ad71116f5011a48b3732dd_s.jpg\", \"gender\": 1, \"url_token\": \"learingenglish\", \"id\": \"fa14835a18c410298a67c7e3714f7f80\"}, \"bio\": \"零基础学习英语的学渣 瘦子长肌肉的健身爱好者\", \"description\": \"我是姜小白，我在知乎专门分享零基础自学英语的经验。\"}, \"seats\": {\"taken\": 364, \"max\": 500}, \"starts_at\": 1494676800, \"role\": \"visitor\", \"in_promotion\": false, \"id\": \"839804209770692608\", \"subject\": \"英语零基础自学入门经验分享\"}, \"action\": \"new_hot_live\", \"source_id\": 0, \"id\": \"live839804209770692608\"}, {\"object_type\": \"live\", \"source_type\": \"hot_live\", \"live\": {\"status\": \"ended\", \"ends_at\": 1494678574, \"liked\": false, \"ends_in\": 0, \"speaker_message_count\": 287, \"tags\": [{\"score\": 0, \"name\": \"阅读与写作\", \"short_name\": \"阅读写作\", \"available_num\": 182, \"created_at\": 1483690424, \"id\": 109, \"live_num\": 186}], \"review\": {\"count\": 36, \"status\": \"normal\", \"score\": 4.31, \"has_reviewed\": false, \"previous\": {\"status\": \"nonexistent\", \"score\": 0}}, \"liked_num\": 328, \"live_type\": \"audio\", \"is_public\": 1, \"video\": {}, \"cospeakers\": [], \"fee\": {\"original_price\": 1985, \"amount\": 1985, \"unit\": \"RMB\"}, \"speaker\": {\"member\": {\"name\": \"郑枫\", \"url\": \"https://api.zhihu.com/people/0a5e625d69eb04a9ef745b1719e78cf1\", \"type\": \"people\", \"user_type\": \"people\", \"headline\": \"战略忽悠局党委副书记\", \"avatar_url\": \"https://pic2.zhimg.com/28e70672fc251a48aecc9b690a02f30d_s.jpg\", \"gender\": 1, \"url_token\": \"jue-hen-37\", \"id\": \"0a5e625d69eb04a9ef745b1719e78cf1\"}, \"bio\": \"战略忽悠局党委副书记\", \"description\": \"我是郑枫，资深国学与军事爱好者，2015年毕业于南京航空航天大学，现于某校读研并准备考博中，专栏《高考语文阅读及写作满分技巧》与live《轻松应对2017年高考语文作文》广受各位同学们的好评。\"}, \"seats\": {\"taken\": 224, \"max\": 500}, \"starts_at\": 1494671400, \"role\": \"visitor\", \"in_promotion\": false, \"id\": \"843110642910117888\", \"subject\": \"17高考语文现代文阅读攻略\"}, \"action\": \"new_hot_live\", \"source_id\": 0, \"id\": \"live843110642910117888\"}, {\"object_type\": \"live\", \"source_type\": \"hot_live\", \"live\": {\"status\": \"ended\", \"ends_at\": 1494683379, \"liked\": false, \"ends_in\": 0, \"speaker_message_count\": 282, \"tags\": [{\"score\": 0, \"name\": \"教育\", \"short_name\": \"教育\", \"available_num\": 563, \"created_at\": 1469691129, \"id\": 303, \"live_num\": 586}], \"review\": {\"count\": 27, \"status\": \"normal\", \"score\": 4.7, \"has_reviewed\": false, \"previous\": {\"status\": \"nonexistent\", \"score\": 0}}, \"liked_num\": 466, \"live_type\": \"audio\", \"is_public\": 1, \"video\": {}, \"cospeakers\": [], \"fee\": {\"original_price\": 999, \"amount\": 999, \"unit\": \"RMB\"}, \"speaker\": {\"member\": {\"name\": \"松松\", \"url\": \"https://api.zhihu.com/people/a7c03c52bfe726076b83ef07e0a6d8ad\", \"type\": \"people\", \"user_type\": \"people\", \"headline\": \"公众号：和我一起写作文/欢迎加入作文群：606196156\", \"avatar_url\": \"https://pic3.zhimg.com/v2-f143c02a516c78364d937fa98fc1904e_s.jpg\", \"gender\": 1, \"url_token\": \"huo-chai-tian-tang-81\", \"id\": \"a7c03c52bfe726076b83ef07e0a6d8ad\"}, \"bio\": \"公众号：和我一起写作文/欢迎加入作文群：606196156\", \"description\": \"我是松松，北京大学本科生，文学爱好者，经常写高分作文。专栏《和我一起写高考作文》备受好评，知乎作文答案帮助很多高中生获得高分。\"}, \"seats\": {\"taken\": 315, \"max\": 500}, \"starts_at\": 1494669600, \"role\": \"visitor\", \"in_promotion\": false, \"id\": \"840570499422650368\", \"subject\": \"高分作文方法概论\"}, \"action\": \"new_hot_live\", \"source_id\": 0, \"id\": \"live840570499422650368\"}, {\"object_type\": \"live\", \"source_type\": \"hot_live\", \"live\": {\"status\": \"ended\", \"ends_at\": 1494602524, \"liked\": false, \"ends_in\": 0, \"speaker_message_count\": 145, \"tags\": [{\"score\": 0, \"name\": \"音乐、影视和游戏\", \"short_name\": \"音影游\", \"available_num\": 331, \"created_at\": 1469691129, \"id\": 102, \"live_num\": 340}], \"review\": {\"count\": 50, \"status\": \"normal\", \"score\": 4.68, \"has_reviewed\": false, \"previous\": {\"status\": \"nonexistent\", \"score\": 0}}, \"liked_num\": 347, \"live_type\": \"audio\", \"is_public\": 1, \"video\": {}, \"cospeakers\": [], \"fee\": {\"original_price\": 999, \"amount\": 999, \"unit\": \"RMB\"}, \"speaker\": {\"member\": {\"name\": \"Johnny Zhu\", \"url\": \"https://api.zhihu.com/people/da20be0340548946af69eadd706fec0d\", \"type\": \"people\", \"user_type\": \"people\", \"headline\": \"欢迎参加我主讲的摇滚乐Live\", \"avatar_url\": \"https://pic3.zhimg.com/efaf57544a81c84c27131363316fbf3a_s.jpg\", \"gender\": 1, \"url_token\": \"zhu-johnny-66\", \"id\": \"da20be0340548946af69eadd706fec0d\"}, \"bio\": \"知乎摇滚和民谣话题获得赞同数量最多的作者前三甲之一。\", \"description\": \"我是 Johnny Zhu ，知乎摇滚民谣话题获得赞同数量最多的作者之一。毕业于北京师范大学，现在美国攻读博士学位。业余乐评人，已在各媒体发表乐评数十万字。旅居美国，深入摇滚诞生和现代民谣复兴的起源地，探究音乐的文化内涵，搜寻音乐人的往事。在此将音乐结合作品背后反映的社会文化现象与深入的思考分享给大家，和同道中人一同探索并丰富精神世界。\"}, \"seats\": {\"taken\": 269, \"max\": 500}, \"starts_at\": 1494594000, \"role\": \"visitor\", \"in_promotion\": false, \"id\": \"838157855663230976\", \"subject\": \"摇滚乐最灿烂的一年追忆（上）\"}, \"action\": \"new_hot_live\", \"source_id\": 0, \"id\": \"live838157855663230976\"}]}\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "def scrapy(link):\n",
    "    headers = {\n",
    "        'Host' : 'api.zhihu.com',\n",
    "        'Origin' : 'https://www.zhihu.com',\n",
    "        'Referer' : 'https://www.zhihu.com/lives',\n",
    "        'authorization' : 'oauth 8274ffb553d511e6a7fdacbc328e205d',\n",
    "        'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'\n",
    "    } \n",
    "    r = requests.get(link, headers= headers)\n",
    "    return (r.text)\n",
    "\n",
    "link = \"https://api.zhihu.com/lives/homefeed?includes=live\"\n",
    "html = scrapy(link)\n",
    "print (html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://api.zhihu.com/lives/homefeed?limit=10&offset=1494594000\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "decodejson = json.loads(html)\n",
    "next_page = decodejson['paging']['next']\n",
    "is_end = decodejson['paging']['is_end']\n",
    "print (next_page)\n",
    "print (is_end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from pymongo import MongoClient\n",
    "import json\n",
    "import time\n",
    "import random\n",
    "\n",
    "client = MongoClient('localhost',27017)\n",
    "db = client.zhihu_database\n",
    "collection = db.live\n",
    "\n",
    "def scrapy(link):\n",
    "    headers = {\n",
    "        'Host' : 'api.zhihu.com',\n",
    "        'Origin' : 'https://www.zhihu.com',\n",
    "        'Referer' : 'https://www.zhihu.com/lives',\n",
    "        'authorization' : 'oauth 8274ffb553d511e6a7fdacbc328e205d',\n",
    "        'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'\n",
    "    } \n",
    "    r = requests.get(link, headers= headers)\n",
    "    return (r.text)\n",
    "\n",
    "link = \"https://api.zhihu.com/lives/homefeed?includes=live\"\n",
    "is_end = False\n",
    "while not is_end:\n",
    "    html = scrapy(link)\n",
    "    decodejson = json.loads(html)\n",
    "    collection.insert_one(decodejson)\n",
    "    \n",
    "    link = decodejson['paging']['next']\n",
    "    is_end = decodejson['paging']['is_end']\n",
    "    time.sleep(random.randint(2,3) + random.random())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 13.3.2 获取Live的听众"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "840520148518592512\n",
      "837669764146003968\n",
      "837693085466107904\n",
      "843124614065434624\n",
      "842731562154987520\n",
      "839869490635366400\n",
      "839804209770692608\n",
      "843110642910117888\n",
      "840570499422650368\n",
      "838157855663230976\n"
     ]
    }
   ],
   "source": [
    "from pymongo import MongoClient\n",
    "client = MongoClient('localhost',27017)\n",
    "db = client.zhihu_database\n",
    "collection = db.live\n",
    "\n",
    "first_page = collection.find_one()\n",
    "for each in first_page['data']:\n",
    "    print (each['live']['id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pymongo import MongoClient\n",
    "client = MongoClient('localhost',27017)\n",
    "db = client.zhihu_database\n",
    "    \n",
    "live_id = '840520148518592512'\n",
    "\n",
    "def get_audience(live_id):\n",
    "    headers = {\n",
    "        'Host' : 'api.zhihu.com',\n",
    "        'Origin' : 'https://www.zhihu.com',\n",
    "        'Referer' : 'https://www.zhihu.com/lives',\n",
    "        'authorization' : 'oauth 8274ffb553d511e6a7fdacbc328e205d',\n",
    "        'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36'\n",
    "    } \n",
    "    link = 'https://api.zhihu.com/lives/' + live_id + '/members?limit=10&offset=0'\n",
    "    \n",
    "    is_end = False\n",
    "    while not is_end:\n",
    "        r = requests.get(link, headers= headers)\n",
    "        html = r.text\n",
    "        decodejson = json.loads(html)\n",
    "        decodejson['live_id'] = live_id\n",
    "        db.live_audience.insert_one(decodejson)\n",
    "\n",
    "        link = decodejson['paging']['next']\n",
    "        is_end = decodejson['paging']['is_end']\n",
    "        time.sleep(random.randint(2,3) + random.random())\n",
    "\n",
    "get_audience(live_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "840520148518592512\n",
      "837669764146003968\n",
      "837693085466107904\n",
      "843124614065434624\n",
      "842731562154987520\n",
      "839869490635366400\n",
      "839804209770692608\n",
      "843110642910117888\n",
      "840570499422650368\n",
      "838157855663230976\n",
      "840886183490912256\n",
      "840879245080276992\n",
      "837680104363679744\n",
      "841344265740255232\n",
      "842735083944308736\n",
      "842735925179088896\n",
      "839875793512456192\n",
      "839859178712465408\n",
      "835436065190993920\n",
      "838163090607591424\n",
      "841289257443274752\n",
      "837679382431662080\n",
      "838159504024682496\n",
      "831457860721479680\n",
      "840984548597714944\n",
      "839864761947652096\n",
      "833702327444725760\n",
      "838388999964987392\n",
      "835139866269474816\n",
      "842740340824879104\n",
      "819509716299436032\n",
      "839869964864348160\n",
      "837682828706746368\n",
      "838405704879607808\n",
      "827151989283635200\n",
      "835125363007946752\n",
      "836344346444967936\n",
      "837373383242043392\n",
      "835136598269112320\n",
      "830380491126501376\n",
      "835121024906432512\n",
      "824618968198291456\n",
      "831104976066609152\n"
     ]
    },
    {
     "ename": "JSONDecodeError",
     "evalue": "Expecting value: line 1 column 1 (char 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mJSONDecodeError\u001b[0m                           Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-30-8324dc7c7798>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0mlive_id\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0meach\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'live'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mprint\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mlive_id\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m         \u001b[0mget_audience\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlive_id\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-29-8a2c34e0d07e>\u001b[0m in \u001b[0;36mget_audience\u001b[0;34m(live_id)\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlink\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0mheaders\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0mhtml\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m         \u001b[0mdecodejson\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhtml\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m         \u001b[0mdecodejson\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'live_id'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlive_id\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0mdb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlive_audience\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minsert_one\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdecodejson\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\ProgramData\\Anaconda3\\lib\\json\\__init__.py\u001b[0m in \u001b[0;36mloads\u001b[0;34m(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    352\u001b[0m             \u001b[0mparse_int\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mparse_float\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    353\u001b[0m             parse_constant is None and object_pairs_hook is None and not kw):\n\u001b[0;32m--> 354\u001b[0;31m         \u001b[1;32mreturn\u001b[0m \u001b[0m_default_decoder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    355\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0mcls\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mJSONDecoder\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\ProgramData\\Anaconda3\\lib\\json\\decoder.py\u001b[0m in \u001b[0;36mdecode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m    337\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    338\u001b[0m         \"\"\"\n\u001b[0;32m--> 339\u001b[0;31m         \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraw_decode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    340\u001b[0m         \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    341\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\ProgramData\\Anaconda3\\lib\\json\\decoder.py\u001b[0m in \u001b[0;36mraw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m    355\u001b[0m             \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscan_once\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[1;32mraise\u001b[0m \u001b[0mJSONDecodeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Expecting value\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)"
     ]
    }
   ],
   "source": [
    "from pymongo import MongoClient\n",
    "client = MongoClient('localhost',27017)\n",
    "db = client.zhihu_database\n",
    "\n",
    "for each_page in db.live.find():\n",
    "    for each in each_page['data']:\n",
    "        live_id = each['live']['id']\n",
    "        print (live_id)        \n",
    "        get_audience(live_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}