merge bleeding into master

CHANGELOG.txt

@@ -1,3 +1,7 @@
+2016-02-01  Version 3.1.0
+=========================
+- [#32] apply 163 new api
+
 2016-02-01  Version 3.0.4
 =========================
 - [#29] check the key (position) in json dict before using it. 163 changed json structure.

README.md

@@ -28,6 +28,7 @@ zhuaxia 是一个基于命令行的虾米音乐 ( www.xiami.com 以下简称[虾
 - requests module
 - mutagen module
 - beautifulsoup4 module
+- pycrypto module
 
 ##Features
 - 自动识别解析URL. 目前支持：

README_EN.md

@@ -28,6 +28,7 @@ zhuaxia(抓虾) (MIT Licensed) is a little tool to batch download music resource
 - requests module
 - mutagen module
 - beautifulsoup4 module
+- pycrypto module
 
 ## Features
 

setup.py

@@ -4,7 +4,7 @@
 setup(
     name = 'zhuaxia',
     version = zxver.version,
-    install_requires=[ 'requests','mutagen','beautifulsoup4' ],
+    install_requires=['pycrypto', 'requests','mutagen','beautifulsoup4' ],
     packages = find_packages(),
     package_data={'zhuaxia':['conf/default.*']},
     #data_files=[('conf',glob.glob('conf/*.*'))],

zhuaxia/downloader.py

@@ -193,19 +193,24 @@ def download_url_urllib(url,filepath,show_progress=False, proxy=None):
     """
 
     if ( not filepath ) or (not url):
-        LOG.err( 'Url or filepath is not valid, resouce cannot be downloaded.')
+        LOG.error( 'Url or filepath is not valid, resouce cannot be downloaded.')
         return 1
 
     fname = path.basename(filepath)
 
     try:
         proxyServer = urllib2.ProxyHandler(proxy) if proxy else None
         opener = urllib2.build_opener()
-        if proxyServer:
-            opener = urllib2.build_opener(proxyServer)
+        # for downloading, ignore the proxy, it seems that if 
+        # we got the real link, both 163 and xiami don't need a CN proxy to
+        # download the songs
+
+        # if proxyServer:
+            # opener = urllib2.build_opener(proxyServer)
 
         urllib2.install_opener(opener)
         r = urllib2.urlopen(url, timeout=30)
+
         if r.getcode() == 200:
             total_length = int(r.info().getheader('Content-Length').strip())
 
@@ -224,6 +229,7 @@ def download_url_urllib(url,filepath,show_progress=False, proxy=None):
         else:
             LOG.debug("[DL_URL] HTTP Status %d . Song: %s " % (r.status_code,fname))
             return 1
+
     except Exception, err:
         LOG.debug("[DL_URL] downloading song %s timeout!" % fname)
         LOG.debug(traceback.format_exc())
@@ -236,7 +242,7 @@ def download_url(url,filepath,show_progress=False, proxy=None):
     http.get timeout: 30s
     """
     if ( not filepath ) or (not url):
-        LOG.err( 'Url or filepath is not valid, resouce cannot be downloaded.')
+        LOG.error( 'Url or filepath is not valid, resouce cannot be downloaded.')
         return 1
 
     fname = path.basename(filepath)
@@ -269,14 +275,14 @@ def download_single_song(song):
     """
     global done, progress
 
-
     if ( not song.filename ) or (not song.dl_link):
-        LOG.err( 'Song [id:%s] cannot be downloaded' % song.song_id)
+        LOG.error( 'Song [id:%s] cannot be downloaded' % song.song_id)
         return
     mp3_file = song.abs_path
 
     retry = 5
     dl_result = -1 # download return code
+    LOG.debug("[DL_Song] downloading: %s " % song.dl_link)
     while retry > 0 :
         retry -= 1
         LOG.debug("[DL_Song] start downloading: %s retry: %d" % (mp3_file, 5-retry))

zhuaxia/netease.py

@@ -3,7 +3,9 @@
 import re
 import requests
 import log, config, util
+import json
 import md5
+import os
 from os import path
 import downloader
 from obj import Song, Handler
@@ -17,20 +19,23 @@
 
 #163 music api url
 url_163="http://music.163.com"
-url_mp3="http://m1.music.126.net/%s/%s.mp3"
+#url_mp3="http://m1.music.126.net/%s/%s.mp3" #not valid any longer
 url_album="http://music.163.com/api/album/%s/"
 url_song="http://music.163.com/api/song/detail/?id=%s&ids=[%s]"
 url_playlist="http://music.163.com/api/playlist/detail?id=%s"
 url_artist_top_song = "http://music.163.com/api/artist/%s"
 url_lyric = "http://music.163.com/api/song/lyric?id=%s&lv=1"
+url_mp3_post = 'http://music.163.com/weapi/song/enhance/player/url?csrf_token='
 
 #agent string for http request header
 AGENT= 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36'
 
-#headers
-HEADERS = {'User-Agent':AGENT}
-HEADERS['Referer'] = url_163
-HEADERS['Cookie'] = 'appver=1.7.3'
+#this block is kind of magical secret.....No idea why the keys, modulus have those values ( for building the post request parameters. The encryption logic was take from https://github.com/Catofes/musicbox/blob/new_api/NEMbox/api.py)
+modulus = '00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7'
+nonce = '0CoJUm6Qyw8W8jud'
+pubKey = '010001'
+
+
 
 class NeteaseSong(Song):
     """
@@ -67,7 +72,7 @@ def init_by_json(self,js):
         self.song_id = js['id']
         #name
         self.song_name = util.decode_html(js['name'])
-
+        LOG.debug("parsing song %s ...."%self.song_name)
 
         # artist_name
         self.artist_name = js['artists'][0]['name']
@@ -82,15 +87,23 @@ def init_by_json(self,js):
 
         # download link
         dfsId = ''
+        bitrate = 0
         if self.handler.is_hq and js['hMusic']:
             dfsId = js['hMusic']['dfsId']
+            quality = 'HD'
+            bitrate = js['hMusic']['bitrate']
         elif js['mMusic']:
             dfsId = js['mMusic']['dfsId']
+            quality = 'MD'
+            bitrate = js['mMusic']['bitrate']
         elif js['lMusic']:
             LOG.warning(msg.head_163 + msg.fmt_quality_fallback %self.song_name)
             dfsId = js['lMusic']['dfsId']
+            quality = 'LD'
+            bitrate = js['lMusic']['bitrate']
         if dfsId:
-            self.dl_link = url_mp3 % (self.handler.encrypt_dfsId(dfsId), dfsId)
+            # self.dl_link = url_mp3 % (self.handler.encrypt_dfsId(dfsId), dfsId)
+            self.dl_link = self.handler.get_mp3_dl_link(self.song_id, bitrate)
         else:
             LOG.warning(msg.head_163 + msg.fmt_err_song_parse %self.song_name)
 
@@ -199,18 +212,23 @@ def __init__(self, option):
         Handler.__init__(self,option.proxies)
         self.is_hq = option.is_hq
         self.dl_lyric = option.dl_lyric
+        #headers
+        self.HEADERS = {'User-Agent':AGENT}
+        self.HEADERS['Referer'] = url_163
+        self.HEADERS['Cookie'] = 'appver=1.7.3'
 
     def read_link(self, link):
 
         retVal = None
+        requests_proxy = {}
         if config.CHINA_PROXY_HTTP:
             requests_proxy = { 'http':config.CHINA_PROXY_HTTP}
         if self.need_proxy_pool:
             requests_proxy = {'http':self.proxies.get_proxy()}
 
             while True:
                 try:
-                    retVal =  requests.get(link, headers=HEADERS, proxies=requests_proxy)
+                    retVal =  requests.get(link, headers=self.HEADERS, proxies=requests_proxy)
                     break 
                 except requests.exceptions.ConnectionError:
                     LOG.debug('invalid proxy detected, removing from pool')
@@ -222,7 +240,7 @@ def read_link(self, link):
                         raise
                         break
         else:
-            retVal =  requests.get(link, headers=HEADERS)
+            retVal =  requests.get(link, headers=self.HEADERS, proxies=requests_proxy)
         return retVal
 
     def encrypt_dfsId(self,dfsId):
@@ -237,3 +255,36 @@ def encrypt_dfsId(self,dfsId):
         result = result.replace('/', '_')
         result = result.replace('+', '-')
         return result
+
+    def createSecretKey(self, size):
+        return (''.join(map(lambda xx: (hex(ord(xx))[2:]), os.urandom(size))))[0:16]
+
+    def encrypt_post_param(self,req_dict):
+        text = json.dumps(req_dict)
+        secKey = self.createSecretKey(16)
+        encText = util.aes_encrypt(util.aes_encrypt(text, nonce), secKey)
+        encSecKey = util.rsa_encrypt(secKey, pubKey, modulus)
+        result = {
+            'params': encText,
+            'encSecKey': encSecKey
+        }
+        return result
+
+    def get_mp3_dl_link(self, song_id,  bitrate):
+        req = {
+                "ids": [song_id],
+                "br": bitrate,
+                "csrf_token": ""
+            }
+        page = requests.post(url_mp3_post, data=self.encrypt_post_param(req), headers=self.HEADERS, timeout=30)
+        result = page.json()["data"][0]["url"]
+
+        #the redirect.....
+        if result:
+
+            r = self.read_link(result)
+            if r.history:
+                return  r.history[0].headers['Location']
+
+        return result
+

zhuaxia/obj.py

@@ -21,6 +21,7 @@ class Handler(object):
     def __init__(self, proxies = None):
         self.proxies = proxies
         self.need_proxy_pool = self.proxies != None
+        self.HEADERS = {}
 
 class History(object):
     """

zhuaxia/util.py

@@ -10,6 +10,9 @@
 
 #used by get_terminal_size
 import fcntl, termios, struct
+#used by Netease post request parameter encoding
+from Crypto.Cipher import AES
+import base64
 
 def get_terminal_size(fd=1):
     """
@@ -60,3 +63,17 @@ def rjust(s,n,fillchar=' '):
     no_ascii_list = re.findall(r'[^\x00-\x7F]+', s)
     ln = len(''.join(no_ascii_list))
     return s.rjust(n-ln, fillchar)
+
+def rsa_encrypt(text, pubKey, modulus):
+    text = text[::-1]
+    rs = int(text.encode('hex'), 16) ** int(pubKey, 16) % int(modulus, 16)
+    return format(rs, 'x').zfill(256)
+
+def aes_encrypt(text, secKey):
+    pad = 16 - len(text) % 16
+    text = text + pad * chr(pad)
+    encryptor = AES.new(secKey, 2, '0102030405060708')
+    ciphertext = encryptor.encrypt(text)
+    ciphertext = base64.b64encode(ciphertext)
+    return ciphertext
+

zhuaxia/zxver.py

@@ -1 +1 @@
-version='3.0.5'
+version='3.1.0'