diff --git a/CHANGELOG.txt b/CHANGELOG.txt index d206898..7e11638 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,7 @@ +2016-02-01 Version 3.1.0 +========================= +- [#32] apply 163 new api + 2016-02-01 Version 3.0.4 ========================= - [#29] check the key (position) in json dict before using it. 163 changed json structure. diff --git a/README.md b/README.md index 866d557..0d1c8ca 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ zhuaxia 是一个基于命令行的虾米音乐 ( www.xiami.com 以下简称[虾 - requests module - mutagen module - beautifulsoup4 module +- pycrypto module ##Features - 自动识别解析URL. 目前支持： diff --git a/README_EN.md b/README_EN.md index 7f35236..eb4c279 100644 --- a/README_EN.md +++ b/README_EN.md @@ -28,6 +28,7 @@ zhuaxia(抓虾) (MIT Licensed) is a little tool to batch download music resource - requests module - mutagen module - beautifulsoup4 module +- pycrypto module ## Features diff --git a/setup.py b/setup.py index c53317a..7c0abb8 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name = 'zhuaxia', version = zxver.version, - install_requires=[ 'requests','mutagen','beautifulsoup4' ], + install_requires=['pycrypto', 'requests','mutagen','beautifulsoup4' ], packages = find_packages(), package_data={'zhuaxia':['conf/default.*']}, #data_files=[('conf',glob.glob('conf/*.*'))], diff --git a/zhuaxia/downloader.py b/zhuaxia/downloader.py index 3d3ad17..ad16ea7 100644 --- a/zhuaxia/downloader.py +++ b/zhuaxia/downloader.py @@ -193,7 +193,7 @@ def download_url_urllib(url,filepath,show_progress=False, proxy=None): """ if ( not filepath ) or (not url): - LOG.err( 'Url or filepath is not valid, resouce cannot be downloaded.') + LOG.error( 'Url or filepath is not valid, resouce cannot be downloaded.') return 1 fname = path.basename(filepath) @@ -201,11 +201,16 @@ def download_url_urllib(url,filepath,show_progress=False, proxy=None): try: proxyServer = urllib2.ProxyHandler(proxy) if proxy else None opener = urllib2.build_opener() - if proxyServer: - opener = urllib2.build_opener(proxyServer) + # for downloading, ignore the proxy, it seems that if + # we got the real link, both 163 and xiami don't need a CN proxy to + # download the songs + + # if proxyServer: + # opener = urllib2.build_opener(proxyServer) urllib2.install_opener(opener) r = urllib2.urlopen(url, timeout=30) + if r.getcode() == 200: total_length = int(r.info().getheader('Content-Length').strip()) @@ -224,6 +229,7 @@ def download_url_urllib(url,filepath,show_progress=False, proxy=None): else: LOG.debug("[DL_URL] HTTP Status %d . Song: %s " % (r.status_code,fname)) return 1 + except Exception, err: LOG.debug("[DL_URL] downloading song %s timeout!" % fname) LOG.debug(traceback.format_exc()) @@ -236,7 +242,7 @@ def download_url(url,filepath,show_progress=False, proxy=None): http.get timeout: 30s """ if ( not filepath ) or (not url): - LOG.err( 'Url or filepath is not valid, resouce cannot be downloaded.') + LOG.error( 'Url or filepath is not valid, resouce cannot be downloaded.') return 1 fname = path.basename(filepath) @@ -269,14 +275,14 @@ def download_single_song(song): """ global done, progress - if ( not song.filename ) or (not song.dl_link): - LOG.err( 'Song [id:%s] cannot be downloaded' % song.song_id) + LOG.error( 'Song [id:%s] cannot be downloaded' % song.song_id) return mp3_file = song.abs_path retry = 5 dl_result = -1 # download return code + LOG.debug("[DL_Song] downloading: %s " % song.dl_link) while retry > 0 : retry -= 1 LOG.debug("[DL_Song] start downloading: %s retry: %d" % (mp3_file, 5-retry)) diff --git a/zhuaxia/netease.py b/zhuaxia/netease.py index 90636d8..5847318 100644 --- a/zhuaxia/netease.py +++ b/zhuaxia/netease.py @@ -3,7 +3,9 @@ import re import requests import log, config, util +import json import md5 +import os from os import path import downloader from obj import Song, Handler @@ -17,20 +19,23 @@ #163 music api url url_163="http://music.163.com" -url_mp3="http://m1.music.126.net/%s/%s.mp3" +#url_mp3="http://m1.music.126.net/%s/%s.mp3" #not valid any longer url_album="http://music.163.com/api/album/%s/" url_song="http://music.163.com/api/song/detail/?id=%s&ids=[%s]" url_playlist="http://music.163.com/api/playlist/detail?id=%s" url_artist_top_song = "http://music.163.com/api/artist/%s" url_lyric = "http://music.163.com/api/song/lyric?id=%s&lv=1" +url_mp3_post = 'http://music.163.com/weapi/song/enhance/player/url?csrf_token=' #agent string for http request header AGENT= 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36' -#headers -HEADERS = {'User-Agent':AGENT} -HEADERS['Referer'] = url_163 -HEADERS['Cookie'] = 'appver=1.7.3' +#this block is kind of magical secret.....No idea why the keys, modulus have those values ( for building the post request parameters. The encryption logic was take from https://github.com/Catofes/musicbox/blob/new_api/NEMbox/api.py) +modulus = '00e0b509f6259df8642dbc35662901477df22677ec152b5ff68ace615bb7b725152b3ab17a876aea8a5aa76d2e417629ec4ee341f56135fccf695280104e0312ecbda92557c93870114af6c9d05c4f7f0c3685b7a46bee255932575cce10b424d813cfe4875d3e82047b97ddef52741d546b8e289dc6935b3ece0462db0a22b8e7' +nonce = '0CoJUm6Qyw8W8jud' +pubKey = '010001' + + class NeteaseSong(Song): """ @@ -67,7 +72,7 @@ def init_by_json(self,js): self.song_id = js['id'] #name self.song_name = util.decode_html(js['name']) - + LOG.debug("parsing song %s ...."%self.song_name) # artist_name self.artist_name = js['artists'][0]['name'] @@ -82,15 +87,23 @@ def init_by_json(self,js): # download link dfsId = '' + bitrate = 0 if self.handler.is_hq and js['hMusic']: dfsId = js['hMusic']['dfsId'] + quality = 'HD' + bitrate = js['hMusic']['bitrate'] elif js['mMusic']: dfsId = js['mMusic']['dfsId'] + quality = 'MD' + bitrate = js['mMusic']['bitrate'] elif js['lMusic']: LOG.warning(msg.head_163 + msg.fmt_quality_fallback %self.song_name) dfsId = js['lMusic']['dfsId'] + quality = 'LD' + bitrate = js['lMusic']['bitrate'] if dfsId: - self.dl_link = url_mp3 % (self.handler.encrypt_dfsId(dfsId), dfsId) + # self.dl_link = url_mp3 % (self.handler.encrypt_dfsId(dfsId), dfsId) + self.dl_link = self.handler.get_mp3_dl_link(self.song_id, bitrate) else: LOG.warning(msg.head_163 + msg.fmt_err_song_parse %self.song_name) @@ -199,10 +212,15 @@ def __init__(self, option): Handler.__init__(self,option.proxies) self.is_hq = option.is_hq self.dl_lyric = option.dl_lyric + #headers + self.HEADERS = {'User-Agent':AGENT} + self.HEADERS['Referer'] = url_163 + self.HEADERS['Cookie'] = 'appver=1.7.3' def read_link(self, link): retVal = None + requests_proxy = {} if config.CHINA_PROXY_HTTP: requests_proxy = { 'http':config.CHINA_PROXY_HTTP} if self.need_proxy_pool: @@ -210,7 +228,7 @@ def read_link(self, link): while True: try: - retVal = requests.get(link, headers=HEADERS, proxies=requests_proxy) + retVal = requests.get(link, headers=self.HEADERS, proxies=requests_proxy) break except requests.exceptions.ConnectionError: LOG.debug('invalid proxy detected, removing from pool') @@ -222,7 +240,7 @@ def read_link(self, link): raise break else: - retVal = requests.get(link, headers=HEADERS) + retVal = requests.get(link, headers=self.HEADERS, proxies=requests_proxy) return retVal def encrypt_dfsId(self,dfsId): @@ -237,3 +255,36 @@ def encrypt_dfsId(self,dfsId): result = result.replace('/', '_') result = result.replace('+', '-') return result + + def createSecretKey(self, size): + return (''.join(map(lambda xx: (hex(ord(xx))[2:]), os.urandom(size))))[0:16] + + def encrypt_post_param(self,req_dict): + text = json.dumps(req_dict) + secKey = self.createSecretKey(16) + encText = util.aes_encrypt(util.aes_encrypt(text, nonce), secKey) + encSecKey = util.rsa_encrypt(secKey, pubKey, modulus) + result = { + 'params': encText, + 'encSecKey': encSecKey + } + return result + + def get_mp3_dl_link(self, song_id, bitrate): + req = { + "ids": [song_id], + "br": bitrate, + "csrf_token": "" + } + page = requests.post(url_mp3_post, data=self.encrypt_post_param(req), headers=self.HEADERS, timeout=30) + result = page.json()["data"][0]["url"] + + #the redirect..... + if result: + + r = self.read_link(result) + if r.history: + return r.history[0].headers['Location'] + + return result + diff --git a/zhuaxia/obj.py b/zhuaxia/obj.py index e85b657..21f44b3 100644 --- a/zhuaxia/obj.py +++ b/zhuaxia/obj.py @@ -21,6 +21,7 @@ class Handler(object): def __init__(self, proxies = None): self.proxies = proxies self.need_proxy_pool = self.proxies != None + self.HEADERS = {} class History(object): """ diff --git a/zhuaxia/util.py b/zhuaxia/util.py index b4fc328..18280e6 100644 --- a/zhuaxia/util.py +++ b/zhuaxia/util.py @@ -10,6 +10,9 @@ #used by get_terminal_size import fcntl, termios, struct +#used by Netease post request parameter encoding +from Crypto.Cipher import AES +import base64 def get_terminal_size(fd=1): """ @@ -60,3 +63,17 @@ def rjust(s,n,fillchar=' '): no_ascii_list = re.findall(r'[^\x00-\x7F]+', s) ln = len(''.join(no_ascii_list)) return s.rjust(n-ln, fillchar) + +def rsa_encrypt(text, pubKey, modulus): + text = text[::-1] + rs = int(text.encode('hex'), 16) ** int(pubKey, 16) % int(modulus, 16) + return format(rs, 'x').zfill(256) + +def aes_encrypt(text, secKey): + pad = 16 - len(text) % 16 + text = text + pad * chr(pad) + encryptor = AES.new(secKey, 2, '0102030405060708') + ciphertext = encryptor.encrypt(text) + ciphertext = base64.b64encode(ciphertext) + return ciphertext + diff --git a/zhuaxia/zxver.py b/zhuaxia/zxver.py index f10162a..6df34dd 100644 --- a/zhuaxia/zxver.py +++ b/zhuaxia/zxver.py @@ -1 +1 @@ -version='3.0.5' +version='3.1.0'