From 91a121fee69bcf48590a9fdf4222897283568454 Mon Sep 17 00:00:00 2001 From: micheal Date: Wed, 15 May 2024 20:54:31 +1200 Subject: [PATCH] 5.10.5 Socks5 accept UDP proxy for DNS. --- code/default/launcher/config.py | 5 +- code/default/lib/noarch/utils.py | 4 +- code/default/lib/noarch/xconfig.py | 6 +- code/default/lib/noarch/xlog.py | 46 ++++++----- code/default/smart_router/local/__init__.py | 1 + code/default/smart_router/local/dns_server.py | 81 ++++++++++++++++++- .../smart_router/local/proxy_handler.py | 23 ++++-- code/default/version.txt | 2 +- code/default/x_tunnel/local/config.py | 2 + code/default/x_tunnel/local/proxy_session.py | 13 ++- code/default/x_tunnel/local/upload_logs.py | 21 ++--- code/default/x_tunnel/web_ui/config.html | 2 +- 12 files changed, 158 insertions(+), 48 deletions(-) diff --git a/code/default/launcher/config.py b/code/default/launcher/config.py index 5c23b3b0be..39314f0eb3 100644 --- a/code/default/launcher/config.py +++ b/code/default/launcher/config.py @@ -82,7 +82,10 @@ config.set_var("global_proxy_username", "") config.set_var("global_proxy_password", "") -config.load() +try: + config.load() +except Exception as e: + xlog.warn("loading config e:%r", e) app_name = "XX-Net" valid_language = ['en_US', 'fa_IR', 'zh_CN', 'ru_RU'] diff --git a/code/default/lib/noarch/utils.py b/code/default/lib/noarch/utils.py index e04e91a19f..cf13e5902f 100644 --- a/code/default/lib/noarch/utils.py +++ b/code/default/lib/noarch/utils.py @@ -117,8 +117,8 @@ def check_domain_valid(hostname): def str2hex(data): - data = to_str(data) - return ":".join("{:02x}".format(ord(c)) for c in data) + data = to_bytes(data) + return data.hex(':') def get_ip_maskc(ip_str): diff --git a/code/default/lib/noarch/xconfig.py b/code/default/lib/noarch/xconfig.py index b555c524e1..4c5e8ff99c 100644 --- a/code/default/lib/noarch/xconfig.py +++ b/code/default/lib/noarch/xconfig.py @@ -24,7 +24,7 @@ def check_change(self): def load(self): self.last_load_time = time.time() if os.path.isfile(self.config_path): - with open(self.config_path, 'r') as f: + with open(self.config_path, 'r', encoding='utf-8') as f: content = f.read() content = content.strip() content = content.replace("\r", "") @@ -51,8 +51,8 @@ def save(self): else: self.file_config[var_name] = getattr(self, var_name) - with open(self.config_path, "w") as f: - f.write(json.dumps(self.file_config, indent=2)) + with open(self.config_path, "w", encoding='utf-8') as f: + f.write(json.dumps(self.file_config, indent=2, ensure_ascii=False)) def set_var(self, var_name, default_value): self.default_config[var_name] = default_value diff --git a/code/default/lib/noarch/xlog.py b/code/default/lib/noarch/xlog.py index 9a63034074..202d51cc35 100644 --- a/code/default/lib/noarch/xlog.py +++ b/code/default/lib/noarch/xlog.py @@ -21,6 +21,11 @@ DEBUG = 10 NOTSET = 0 +# full_log set by server, upload full log for debug (maybe next time start session), remove old log file on reset log +full_log = False + +# keep log set by UI, keep all logs, never delete old log, also upload log to server. + class Logger(): def __init__(self, name, buffer_size=0, file_name=None, roll_num=1, @@ -45,8 +50,8 @@ def __init__(self, name, buffer_size=0, file_name=None, roll_num=1, if log_path and save_start_log: now = datetime.now() time_str = now.strftime("%Y-%m-%d_%H-%M-%S") - log_fn = os.path.join(log_path, "start_log_%s_%s.log" % (name, time_str)) - self.start_log = open(log_fn, "w") + self.log_fn = os.path.join(log_path, "start_log_%s_%s.log" % (name, time_str)) + self.start_log = open(self.log_fn, "w") else: self.start_log = None @@ -75,33 +80,32 @@ def set_buffer(self, buffer_size): pass def reset_log_files(self): - if self.keep_log: - return - - if self.start_log: - self.start_log.close() - self.start_log = None + if not (self.keep_log or full_log): + if self.start_log: + self.start_log.close() + self.start_log = None - if self.warning_log: - self.warning_log.close() - self.warning_log = None + if self.warning_log: + self.warning_log.close() + self.warning_log = None - if self.log_path: + if self.log_path and not self.keep_log: for filename in os.listdir(self.log_path): - if not filename.endswith(".log"): + fp = os.path.join(self.log_path, filename) + if not filename.endswith(".log") or fp == self.log_fn or not filename.startswith("start_log_%s" % self.name): continue - fp = os.path.join(self.log_path, filename) try: os.remove(fp) except: pass - if self.warning_log_fn: + if self.warning_log_fn and not self.keep_log: self.warning_log = open(self.warning_log_fn, "a") def keep_logs(self): self.keep_log = True + # self.debug("keep log for %s", self.name) if not self.log_path: return @@ -217,7 +221,7 @@ def log(self, level, console_color, html_color, fmt, *args, **kwargs): pass self.start_log_num += 1 - if self.start_log_num > self.save_start_log and not self.keep_log: + if self.start_log_num > self.save_start_log and not self.keep_log and not full_log: self.start_log.close() self.start_log = None @@ -357,9 +361,13 @@ def reset_log_files(): log.reset_log_files() -def keep_log(): - for name, log in loggerDict.items(): - log.keep_logs() +def keep_log(temp=False): + global full_log + if temp: + full_log = True + else: + for name, log in loggerDict.items(): + log.keep_logs(temp) default_log = getLogger() diff --git a/code/default/smart_router/local/__init__.py b/code/default/smart_router/local/__init__.py index 71ce003beb..e98efdf435 100644 --- a/code/default/smart_router/local/__init__.py +++ b/code/default/smart_router/local/__init__.py @@ -68,6 +68,7 @@ def load_config(): config.set_var("dns_bind_ip", "127.0.0.1") config.set_var("dns_port", 53) config.set_var("dns_backup_port", 8053) + config.set_var("udp_relay_port", 8086) config.set_var("proxy_bind_ip", "127.0.0.1") config.set_var("proxy_port", 8086) diff --git a/code/default/smart_router/local/dns_server.py b/code/default/smart_router/local/dns_server.py index d9aea39bf6..89d97eddad 100644 --- a/code/default/smart_router/local/dns_server.py +++ b/code/default/smart_router/local/dns_server.py @@ -7,6 +7,7 @@ import socket import time import select +import struct current_path = os.path.dirname(os.path.abspath(__file__)) root_path = os.path.abspath(os.path.join(current_path, os.pardir, os.pardir)) @@ -30,6 +31,8 @@ class DnsServer(object): def __init__(self, bind_ip="127.0.0.1", port=53, backup_port=8053, ttl=24*3600): self.sockets = [] + self.udp_relay_sock = None + self.udp_relay_port = 0 self.listen_port = port self.running = False if isinstance(bind_ip, str): @@ -51,6 +54,7 @@ def init_socket(self): listen_all_v4 and '.' in ip or listen_all_v6 and ':' in ip): continue + self.bing_udp_relay(ip) self.bing_listen(ip) def bing_listen(self, bind_ip): @@ -84,7 +88,25 @@ def bing_listen(self, bind_ip): xlog.warn("Then: sudo setcap 'cap_net_bind_service=+ep' /usr/bin/python2.7") xlog.warn("Or run as root") - def on_udp_query(self, rsock, req_data, addr): + def bing_udp_relay(self, bind_ip): + if ":" in bind_ip: + sock = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + else: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + port = g.config.udp_relay_port + for port in range(port, port + 20): + try: + sock.bind((bind_ip, port)) + xlog.info("start UDP relay server at %s:%d", bind_ip, port) + self.sockets.append(sock) + self.udp_relay_sock = sock + self.udp_relay_port = port + return + except: + xlog.warn("bind UDP %s:%d fail", bind_ip, self.port) + + def dns_query(self, req_data, addr): start_time = time.time() try: request = DNSRecord.parse(req_data) @@ -118,12 +140,62 @@ def on_udp_query(self, rsock, req_data, addr): reply.add_answer(RR(domain, rtype=dns_type, ttl=60, rdata=NS(ip))) res_data = reply.pack() - rsock.sendto(res_data, addr) xlog.debug("query:%s type:%d from:%s, return ip num:%d cost:%d", domain, dns_type, addr, len(reply.rr), (time.time()-start_time)*1000) + return res_data except Exception as e: xlog.exception("on_query except:%r", e) + def on_udp_query(self, rsock, req_data, addr): + res_data = self.dns_query(req_data, addr) + rsock.sendto(res_data, addr) + + def on_udp_relay(self, rsock, req_data, from_addr): + # We currently only support DNS query for UDP relay + + # SOCKS5 UDP forward request + # reserved, frag, addr_type, domain_len, domain, port, data + try: + reserved = struct.unpack(">H", req_data[0:2])[0] + frag = ord(req_data[2:3]) + if reserved != 0 or frag != 0: + xlog.warn("reserved:%d frag:%d", reserved, frag) + return + + addr_type = ord(req_data[3:4]) + if addr_type == 1: # IPv4 + addr_pack = req_data[4:8] + addr = socket.inet_ntoa(addr_pack) + port = struct.unpack(">H", req_data[8:10])[0] + data = req_data[10:] + elif addr_type == 3: # Domain name + domain_len_pack = req_data[4:5] + domain_len = ord(domain_len_pack) + domain = req_data[5:5 + domain_len] + addr = domain + port = struct.unpack(">H", req_data[5 + domain_len:5 + domain_len + 2])[0] + data = req_data[5 + domain_len + 2:] + elif addr_type == 4: # IPv6 + addr_pack = req_data[4:20] + addr = socket.inet_ntop(socket.AF_INET6, addr_pack) + port = struct.unpack(">H", req_data[20:22])[0] + data = req_data[22:] + else: + xlog.warn("request address type unknown:%d", addr_type) + return + + xlog.debug("UDP relay from %s size:%d to:%s:%d", from_addr, len(data), addr, port) + head_length = len(req_data) - len(data) + head = req_data[:head_length] + res_data = self.dns_query(data, from_addr) + if not res_data: + return + + rsock.sendto(head + res_data, from_addr) + xlog.debug("UDP relay from %s size:%d to:%s:%d res len:%d", from_addr, len(data), addr, port, len(res_data)) + except Exception as e: + xlog.exception("on_udp_relay data:[%s] except:%r", utils.str2hex(req_data), e) + def server_forever(self): while self.running: r, w, e = select.select(self.sockets, [], [], 1) @@ -137,7 +209,10 @@ def server_forever(self): xlog.warn("recv except: %r", e) break - threading.Thread(target=self.on_udp_query, args=(rsock, data, addr), name="DNSServer_udp_handler").start() + if rsock == self.udp_relay_sock: + threading.Thread(target=self.on_udp_relay, args=(rsock, data, addr), name="UDP_relay").start() + else: + threading.Thread(target=self.on_udp_query, args=(rsock, data, addr), name="DNSServer_udp_handler").start() self.th = None diff --git a/code/default/smart_router/local/proxy_handler.py b/code/default/smart_router/local/proxy_handler.py index 933db03478..6580c80642 100644 --- a/code/default/smart_router/local/proxy_handler.py +++ b/code/default/smart_router/local/proxy_handler.py @@ -233,6 +233,15 @@ def socks4_handler(self): else: handle_ip_proxy(sock, addr, port, self.client_address) + def handle_udp_associate(self, sock, addr, port, addrtype_pack, addr_pack): + udp_relay_port = g.dns_srv.udp_relay_port + xlog.debug("socks5 from:%r udp associate to %s:%d use udp_relay_port:%d", self.client_address, addr, port, udp_relay_port) + reply = b"\x05\x00\x00" + addrtype_pack + addr_pack + struct.pack(">H", udp_relay_port) + sock.send(reply) + + self.rfile.read(1) + xlog.debug("socks5 from:%r udp associate to %s:%d closed", self.client_address, addr, port) + def socks5_handler(self): sock = self.conn socks_version = ord(self.read_bytes(1)) @@ -253,11 +262,6 @@ def socks5_handler(self): return command = ord(data[1:2]) - if command != 1: # 1. Tcp connect - xlog.warn("request not supported command mode:%d", command) - sock.send(b"\x05\x07\x00\x01") # Command not supported - return - addrtype_pack = data[3:4] addrtype = ord(addrtype_pack) if addrtype == 1: # IPv4 @@ -276,9 +280,16 @@ def socks5_handler(self): xlog.warn("request address type unknown:%d", addrtype) sock.send(b"\x05\x07\x00\x01") # Command not supported return - port = struct.unpack('>H', self.rfile.read(2))[0] + if command == 3: # 3. UDP associate + return self.handle_udp_associate(sock, addr, port, addrtype_pack, addr_pack) + + if command != 1: # 1. Tcp connect + xlog.warn("request not supported command mode:%d", command) + sock.send(b"\x05\x07\x00\x01") # Command not supported + return + # xlog.debug("socks5 %r connect to %s:%d", self.client_address, addr, port) reply = b"\x05\x00\x00" + addrtype_pack + addr_pack + struct.pack(">H", port) sock.send(reply) diff --git a/code/default/version.txt b/code/default/version.txt index c55621169c..a16dcb354a 100644 --- a/code/default/version.txt +++ b/code/default/version.txt @@ -1 +1 @@ -5.9.10 \ No newline at end of file +5.10.5 \ No newline at end of file diff --git a/code/default/x_tunnel/local/config.py b/code/default/x_tunnel/local/config.py index a8b053e43f..f854028c05 100644 --- a/code/default/x_tunnel/local/config.py +++ b/code/default/x_tunnel/local/config.py @@ -25,6 +25,8 @@ def load_config(): config.set_var("write_log_file", 0) config.set_var("save_start_log", 1500) config.set_var("show_debug", 0) + config.set_var("delay_collect_log", 3 * 60) + config.set_var("delay_collect_log2", 30) config.set_var("encrypt_data", 0) config.set_var("encrypt_password", "encrypt_pass") diff --git a/code/default/x_tunnel/local/proxy_session.py b/code/default/x_tunnel/local/proxy_session.py index 036d5be3c2..c718e650bc 100644 --- a/code/default/x_tunnel/local/proxy_session.py +++ b/code/default/x_tunnel/local/proxy_session.py @@ -5,7 +5,7 @@ import xstruct as struct import hashlib -from xlog import getLogger +from xlog import getLogger, keep_log xlog = getLogger("x_tunnel") import utils @@ -422,6 +422,15 @@ def login_session(self): xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message) return False + try: + msg_info = json.loads(message) + if msg_info.get("full_log"): + xlog.debug("keep full log") + keep_log(temp=True) + except Exception as e: + xlog.warn("login_session %s json error:%r", message, e) + msg_info = {} + g.last_api_error = "" xlog.info("login_session %s time:%d msg:%s", self.session_id, 1000 * time_cost, message) return True @@ -581,7 +590,7 @@ def get_send_data(self, work_id): self.wait_queue.wait(work_id) xlog.debug("get_send_data on stop") - return "", "" + return b"", b"" def ack_process(self, ack): self.lock.acquire() diff --git a/code/default/x_tunnel/local/upload_logs.py b/code/default/x_tunnel/local/upload_logs.py index c2b0e86ec8..2cb88bd9e3 100644 --- a/code/default/x_tunnel/local/upload_logs.py +++ b/code/default/x_tunnel/local/upload_logs.py @@ -110,7 +110,7 @@ def list_files(): return other_files + log_files_list -def pack_logs(max_size=800 * 1024): +def pack_logs(max_size=4 * 1024 * 1024): content_size = 0 collect_debug_and_log() @@ -118,11 +118,10 @@ def pack_logs(max_size=800 * 1024): try: files = list_files() zip_buffer = io.BytesIO() - with zipfile.ZipFile(zip_buffer, mode="w") as zfd: + with zipfile.ZipFile(zip_buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as zfd: for src_file in files: file_size = os.path.getsize(src_file) - content_size += file_size - if content_size > max_size: + if content_size + file_size > max_size: break relate_path = src_file[len(data_path) + 1:] @@ -133,30 +132,32 @@ def pack_logs(max_size=800 * 1024): zfd.writestr(relate_path, content) else: zfd.write(src_file, arcname=relate_path) + content_size += file_size - if content_size > max_size: - break - return zip_buffer.getvalue() + compressed_data = zip_buffer.getvalue() + xlog.debug("compress log size:%d to %d", content_size, len(compressed_data)) + return compressed_data except Exception as e: xlog.exception("packing logs except:%r", e) return None def upload_logs_thread(): - sleep(3 * 60) + sleep(g.config.delay_collect_log) while g.running: if not g.running or not g.server_host or not g.session or g.session.last_receive_time == 0: time.sleep(10) else: break - sleep(30) + sleep(g.config.delay_collect_log2) if not g.running: return session_id = utils.to_str(g.session.session_id) data = pack_logs() - upload(session_id, data) + if data: + upload(session_id, data) def upload(session_id, data): diff --git a/code/default/x_tunnel/web_ui/config.html b/code/default/x_tunnel/web_ui/config.html index fe45c674c2..9b476892c3 100644 --- a/code/default/x_tunnel/web_ui/config.html +++ b/code/default/x_tunnel/web_ui/config.html @@ -35,7 +35,7 @@

{{ _("Login") }}

-

+