check_v2ray.py

#! /usr/bin/env python3

__author__ = 'JZ'
__webpage__ = 'https://github.com/Justsoos'

import re
import sys,os
# import socks
import socket
import multiprocessing
import threading
import logging
import json
import time
import requests
import argparse
import subprocess
import base64

from glob import glob

from pprint import pprint
from requests.adapters import HTTPAdapter
from multiprocessing import Pool

logging.getLogger().setLevel(logging.DEBUG)
logging.debug('')

def run_v(conf, t_conf):
	if int(conf.get('configType')) != 1:
		return None
	try:
		users = {}
		users['id'] = conf.get('id')
		users['alterId'] = int(conf.get('alterId', '0'))
		users['security'] = conf.get('security', 'aes-128-gcm')
		u = []
		u.append(users)

		vnext = {}
		vnext['address'] = conf.get('address')
		vnext['port'] = int(conf.get('port'))
		vnext['users'] = u

		v = []
		v.append(vnext)

		t_conf['outbound']['settings']['vnext'] = v

		t_conf['outbound']['streamSettings']['network'] = conf.get('network')

		t_conf['outbound']['streamSettings']['wsSettings'] = None
		t_conf['outbound']['streamSettings']['kcpSettings'] = None
		t_conf['outbound']['streamSettings']['tcpSettings'] = None
		
		network = conf.get('network')
		if network == 'ws':
			t_conf['outbound']['streamSettings']['wsSettings'] = \
				{
					'connectionReuse': True,
					'path': None,
					'headers': None
				}
			t_conf['outbound']['streamSettings']['wsSettings']['headers'] = (conf.get('headerType'), None)[conf.get('headerType') == 'none']

			re = conf.get('requestHost')
			if ';' in re:
				r = re.split(';')
				t_conf['outbound']['streamSettings']['wsSettings']['path'] = r[0]
				t_conf['outbound']['streamSettings']['wsSettings']['headers']['Host'] = r[1]
			else:
				t_conf['outbound']['streamSettings']['wsSettings']['path'] = re

		elif network == 'kcp':
			t_conf['outbound']['streamSettings']['kcpSettings'] = \
				{
					'mtu': 1350,
					'tti': 10,
					'uplinkCapacity': 20,
					'downlinkCapacity': 100,
					'congestion': True,
					'readBufferSize': 4,
					'writeBufferSize': 4,
					'header': {
						'type': None,
						'request': None,
						'response': None
					}
				}

			t_conf['outbound']['streamSettings']['kcpSettings']['header']['type'] = (conf.get('headerType'), 'none')[conf.get('headerType') == 'none']

		elif network == 'tcp':
			t_conf['outbound']['streamSettings']['tcpSettings'] = \
				{
					'connectionReuse': True,
					'header': {
						'type': None,
						'request': {
							'version': '1.1',
							'method': 'GET',
							'path': [
								'/'
							],
							'headers': {
								'Host': [
									''
								],
								'User-Agent': [
									'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36',
									'Mozilla/5.0 (iPhone; CPU iPhone OS 10_0_2 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/53.0.2785.109 Mobile/14A456 Safari/601.1.46'
								],
								'Accept-Encoding': [
									'gzip,deflate'
								],
								'Connection': [
									'keep-alive'
								],
								'Pragma': 'no-cache'
							}
						},
						'response': {
							'version': '1.1',
							'status': '200',
							'reason': 'OK',
							'headers': {
								'Content-Type': [
									'application/octet-stream',
									'video/mpeg'
								],
								'Transfer-Encoding': [
									'chunked'
								],
								'Connection': [
									'keep-alive'
								],
								'Pragma': 'no-cache'
							}
						}
					}
				}
			if conf.get('headerType') == 'none' or conf.get('headerType') == '':
				t_conf['outbound']['streamSettings']['tcpSettings'] = None
			else:
				t_conf['outbound']['streamSettings']['tcpSettings']['header']['type'] = (conf.get('headerType'), None)[conf.get('headerType') == 'none']
				t_conf['outbound']['streamSettings']['tcpSettings']['header']['request']['headers']['Host'] = conf.get('requestHost', '').split(',')

		elif network == 'h2' or network == 'http':

			t_conf['outbound']['streamSettings']['httpSettings'] = \
				{
					'path': None,
					'host': []
				}

			t_conf['outbound']['streamSettings']['httpSettings']['path'] = conf.get('path')
			t_conf['outbound']['streamSettings']['httpSettings']['Host'] = list(conf.get('requestHost'))

		else:
			raise NameError("unkonwn network", network)

		t_conf['outbound']['streamSettings']['security'] = (conf.get('streamSecurity'), '')[conf.get('streamSecurity') == None]

		t_conf['outbound']['protocol'] = 'vmess'

		port = get_free_tcp_port()
		if not port:
			raise Exception('Error getting local port.')
		t_conf['inbound']['port'] = int(port)
		
		t_conf['inbound']['listen'] = '127.0.0.1'
		t_conf['inbound']['protocol'] = 'socks'
		t_conf['inbound']['settings']['auth'] = 'noauth'
		t_conf['inbound']['settings']['ip'] = '127.0.0.1'
	except:
		raise
	#remarks = conf.get('remarks', None)
	temp_file = 'temp_file_{}_{}.json'.format(int(round(time.time() * 1000)), port)
	with open(temp_file, 'w') as f:
		json.dump(t_conf, f)

	if not os.path.isfile(temp_file):
		logging.debug('here missing missing: '.format(temp_file))

	cmd_line = 'v2ray.exe --config={}'.format(temp_file)
	logging.debug('checking id: {} with port {}'.format(users['id'], port))
	try:
		p = subprocess.Popen(cmd_line, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL)
	except:
		raise
	time.sleep(0.8)
	return p, port, temp_file

def test_connect(port):
	perfect = 9
	sum_r = 0
	time.sleep(2)
	get_latency(port)
	time.sleep(2)
	for i in range(1,10):
		time.sleep(0.1)
		r, p = get_latency(port)
		if p is True:
			print('.', end='')
			sum_r += r
			perfect -= 1
	if perfect != 9:
		times = 9 - int(perfect)
		s = sum_r / times
		latency = format(s, '0.2f')
	else:
		latency = 0
	return perfect, latency

def get_latency(port):
	test_urls = 'https://www.google.com/'
	headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36'}
	proxies = {}
	proxies['http'] = 'socks5h://127.0.0.1:{}'.format(port)
	proxies['https'] = 'socks5h://127.0.0.1:{}'.format(port)

	start_time = time.time()
	s = requests.Session()
	try:
		s.mount(test_urls, HTTPAdapter(max_retries=0))
		r = s.get(test_urls, proxies=proxies, headers=headers, verify=True, timeout=(7,15), allow_redirects=False, cookies={'':''})
		r.raise_for_status()
		connectivity = True
	except Exception as err:
		print(err)
		connectivity = False
	end_time = time.time()
	
	latency_time = end_time - start_time
	return latency_time, connectivity

def sub_proc(proc, single_json, t_conf):
	proc, port, temp_file = run_v(single_json, t_conf)

	try:
		perfect, latency = test_connect(port)
	except:
		raise
	
	proc.kill()

	logging.debug('process what ? {}'.format(proc))
	stdout, stderr = proc.communicate()
	logging.debug('stdout of process{}'.format(stdout))
	logging.debug('stdERR of process{}'.format(stderr))

	if not os.path.isfile(temp_file):
		logging.debug('missing temp config file: {}'.format(temp_file))
		raise ValueError('config file missing...')

	os.remove(temp_file)

	return single_json, perfect, latency

def multi_proc(configs):
	global t_conf
	multiprocessing.freeze_support()
	proc = multiprocessing.Pool(16)

	proc_result = []
	if isinstance(configs, dict):
		t = []
		t.append(configs.copy())
		configs = t

	for i, ei in enumerate(configs):
		r = proc.apply_async(sub_proc, args=(i, ei, t_conf))
		proc_result.append(r)

	proc.close()
	proc.join()

	configs_all = []
	for k in proc_result:
		configs_all.append(k.get())
	
	info = []
	configs_good_temp = []
	configs_bad_temp = []
	configs_bad = []
	configs_good = []
	for j in configs_all:
		info.append((j[1],j[2]))
		if j[1] == 9:
			configs_bad_temp.append(j[0])
		else:
			configs_good_temp.append(j)

	if configs_good_temp:
		configs_good_temp.sort(key = lambda x:x[2])
		configs_good_temp.sort(key = lambda x:x[1])
		for i in configs_good_temp:
			r = re.match('^\d_\d\.\d{2}_(.*)', i[0].get('remarks'))
			if r:
				remarks = r.group(1)
			else:
				remarks = i[0].get('remarks')
			remarks = '{}_{}_{}'.format(i[1], i[2], remarks)
			i[0]['remarks'] = remarks[:60]
			configs_good.append(i[0])

	if configs_bad_temp:
		for k in configs_bad_temp:
			r = re.match('^\d_\d\.\d{2}_(.*)', k.get('remarks'))
			if r:
				remarks = r.group(1)
			else:
				remarks = k.get('remarks')
			remarks = '{}_{}_HCR_{}'.format('9', '9.99', remarks)
			k['remarks'] = remarks[:60]
			configs_bad.append(k)
	
	return configs_good, configs_bad, info

def deDup(conf):
	dest_list = []
	dup_list = []
	global other_list
	other_list = []
	try:
		for i, ei in enumerate(conf):
			if int(ei.get('configType')) != 1:
				other_list.append(ei)
				continue
			for j, ej in enumerate(conf[i+1:]):
				if (
					(ei['address'] == ej['address']) and
					(int(ei['port']) == int(ej['port'])) and
					(ei['id'] == ej['id']) and
					(int(ei['alterId']) == int(ej['alterId'])) and
					(ei['network'] == ej['network']) and
					(ei['headerType'] == ej['headerType']) and
					(ei['requestHost'] == ej['requestHost']) and
					(ei['streamSecurity'] == ej['streamSecurity']) and
					(int(ei['configType']) == int(ej['configType']))
					):
					dup_list.append(ei)
					dest_found = False
					break
				else:
					dest_found = True
			if dest_found:
				dest_list.append(ei)
	except KeyError as err:
		print('The No.{} record seems wrong with {}...'.format((i+j), err))
		raise
	deDup_info = '**** All records: {}, found dups: {}, unique VMESS records: {}, non VMESS records: {}'.format(len(conf), len(dup_list), len(dest_list), len(other_list))
	return dest_list, dup_list, deDup_info

def get_free_tcp_port():
	tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	tcp.bind(('', 0))
	addr, port = tcp.getsockname()
	tcp.close()
	return port

def rewrite_socks_dns(address, timeout=None, source_address=None):
	sock = socks.socksocket()
	sock.connect(address)
	return sock

def kill():
	cmd_task = 'taskkill /f /t /im'
	kill_list = ['v2rayN.exe', 'v2ray.exe', 'v2ctl.exe', 'wv2ray.exe']
	try:
		for i in kill_list:
			cmd = '{} {}'.format(cmd_task, i)
			s = subprocess.call(cmd)
			print(s)
	except:
		pass

def files(wildcard_file_args):
	if wildcard_file_args:
		file_list = (glob(name) for name in wildcard_file_args)
		file_list = [i for j in file_list for i in j]
		file_list = list(set(file_list))
	else:
		file_list = None
	return file_list

def main_dev():

	global json_files
	global uri_files
	global only_test

	parser = argparse.ArgumentParser( description='de-duplicate, merge, test, benchmark and backup tools for v2ray with v2rayN, M$ Windows')
	parser.add_argument('-j', metavar='input JSON filenames', dest='json_files', default=False, type=str, nargs='+', help='Input json filenames')
	parser.add_argument('-l', metavar='input Link filenames', dest='uri_files', default=False, type=str, nargs='+', help='Input uri link filenames')
	parser.add_argument('-t', dest='only_test', action='store_true', default=False, help='Just test, no output file')
	parser.add_argument('-v',action='version', version='0.2')
	args = parser.parse_args()

	json_files = files(args.json_files)
	uri_files = files(args.uri_files)
	only_test = args.only_test

	if not args.json_files and not args.uri_files:
		json_files = ['guiNConfig.json']

def main():
	# socket.socket = socks.socksocket
	# socket.create_connection = rewrite_socks_dns

	global json_files
	global uri_files
	global only_test
	global other_list

	main_dev()

	kill()

	global t_conf
	t_conf = \
	{
		'log': {
			'access': None,
			'error': None,
			'loglevel': None
		},
		'inbound': {
			'port': 48080,
			'listen': '127.0.0.1',
			'protocol': 'socks',
			'settings': {
				'auth': 'noauth',
				'udp': False,
				'ip': '127.0.0.1',
				'clients': None
			},
			'streamSettings': None
		},
		'outbound': {
			'tag': 'agentout',
			'protocol': 'vmess',
			'settings': {
				'vnext': [
					{
						'address': '213.213.213.213',
						'port': 23333,
						'users': [
							{
								'id': 'dddda000-bbbb-4444-2222-fffff6666666',
								'alterId': 100,
								'security': 'aes-128-gcm'
							}
						]
					}
				],
				'servers': None
			},
			'streamSettings': {
				'network': None,
				'security': None,
				'tcpSettings': None,
				'kcpSettings': None,
				'wsSettings': None
			},
			'mux': {
				'enabled': False
			}
		},
		'inboundDetour': None,
		'outboundDetour': [
			{
				'protocol': 'freedom',
				'settings': {
					'response': None
				},
				'tag': 'direct'
			},
			{
				'protocol': 'blackhole',
				'settings': {
					'response': {
						'type': 'http'
					}
				},
				'tag': 'blockout'
			}
		],
		'dns': {
			'servers': [
				'8.8.8.8',
				'8.8.4.4',
				'114.114.114.114'
			]
		},
		'routing': {
			'strategy': 'rules',
			'settings': {
				'domainStrategy': 'IPIfNonMatch',
				'rules': [
					{
						'type': 'field',
						'port': None,
						'outboundTag': 'direct',
						'ip': [
							'0.0.0.0/8',
							'10.0.0.0/8',
							'100.64.0.0/10',
							'127.0.0.0/8',
							'169.254.0.0/16',
							'172.16.0.0/12',
							'192.0.0.0/24',
							'192.0.2.0/24',
							'192.168.0.0/16',
							'198.18.0.0/15',
							'198.51.100.0/24',
							'203.0.113.0/24',
							'::1/128',
							'fc00::/7',
							'fe80::/10'
						],
						'domain': None
					}
				]
			}
		}
	}

	t_guiNConfig = \
		{
			"inbound": [{
				"localPort": 28080,
				"protocol": "socks",
				"udpEnabled": False
			}],
			"logEnabled": False,
			"loglevel": "error",
			"index": 78,
			"vmess": [],
			"muxEnabled": False,
			"chinasites": False,
			"chinaip": False,
			"useragent": [],
			"userdirect": [],
			"userblock": [],
			"kcpItem": {
				"mtu": 1350,
				"tti": 10,
				"uplinkCapacity": 20,
				"downlinkCapacity": 100,
				"congestion": True,
				"readBufferSize": 4,
				"writeBufferSize": 4
			},
			"autoSyncTime": False,
			"sysAgentEnabled": False,
			"listenerType": 1,
			"urlGFWList": ""
		}
	
	configs = []
	links = []
	guiNConfig = None
	data = None
	vmess = None

	if json_files:
		try:
			for i in json_files:
				if not os.path.isfile(i):
					print('****Bad file path or name: {} ...'.format(i))
					break
				else:
					with open(i, 'r', encoding='utf-8') as f:
						data = json.load(f)
					if not data:
						print('can not found specified format on {}'.format(i))
						break

				if isinstance(data, dict) and data.get('inbound') and data.get('vmess'):
					vmess = data.get('vmess')
					configs.extend(vmess)
					guiNConfig = data
				elif isinstance(data, list) and data[0].get('address'):
					configs.extend(data)
				else:
					print('can not found specified format on {}'.format(i))
		except:
			raise

	guiNConfig = t_guiNConfig if not guiNConfig else guiNConfig

	if configs:
		conf, _, deDup_info = deDup(configs)
	else:
		print('No legal data input...')
		sys.exit()

	print(deDup_info)
	try:
		input('**** Waiting for comfirm, Ctrl+C to interrupt or continue with Enter...')
	except KeyboardInterrupt:
		sys.exit()

	configs_good, configs_bad, _ = multi_proc(conf)

	kill()
	
	all_list = []
	if configs_good:
		all_list.extend(configs_good)
	if configs_bad:
		all_list.extend(configs_bad)
	if other_list:
		all_list.extend(other_list)

	if deDup_info:
		print(deDup_info)

	if only_test:
		print('**** Only test, result: deDuped records {}, good ones {}, bad ones {}.'.format(len(all_list), len(configs_good), len(configs_bad)))
	else:
		guiNConfig['vmess'] = all_list
		guiNConfig_new = 'guiNConfig_{}_.json'.format(time.strftime('%Y-%m-%d_%H-%M-%S'))
		with open(guiNConfig_new, 'w') as f:
			json.dump(guiNConfig, f)
			print('***** Output to {}, {} records. good ones {}, bad ones {}. *****'.format(guiNConfig_new, len(all_list), len(configs_good), len(configs_bad)))


if __name__ == '__main__':
	run_start = time.time()
	main()
	run_end = time.time()
	duration = run_end - run_start
	m, s = divmod(duration, 60)
	print('Cost time: {:.0f} mins {:.0f} seconds. '.format(m, s))