You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
shadowsocks-456-mod-py/web_transfer.py

642 lines
26 KiB
Python

#!/usr/bin/python
# -*- coding: UTF-8 -*-
import logging
import time
import sys
import os
import socket
from server_pool import ServerPool
import traceback
from shadowsocks import common, shell, lru_cache
from configloader import load_config, get_config
import importloader
switchrule = None
db_instance = None
class WebTransfer(object):
def __init__(self):
import threading
self.last_update_transfer = {}
self.event = threading.Event()
self.port_uid_table = {}
self.uid_port_table = {}
self.node_speedlimit = 0.00
self.traffic_rate = 0.0
self.detect_text_list = {}
self.detect_hex_list = {}
self.mu_only = False
self.node_offset = 0
self.ss_method = ''
self.ss_protocol = ''
self.ss_obfs = ''
self.is_relay = False
self.relay_rule_list = {}
self.mu_port_list = []
self.has_stopped = False
def update_all_user(self, dt_transfer):
global webapi
alive_user_count = 0
bandwidth_thistime = 0
data = []
for id in dt_transfer.keys():
if dt_transfer[id][0] == 0 and dt_transfer[id][1] == 0:
continue
data.append({'u': dt_transfer[id][0], 'd': dt_transfer[
id][1], 'user_id': self.port_uid_table[id]})
webapi.postApi('users/traffic',
{'node_id': get_config().NODE_ID},
{'data': data})
webapi.postApi(
'nodes/%d/info' %
(get_config().NODE_ID), {
'node_id': get_config().NODE_ID}, {
'uptime': str(
self.uptime()), 'load': str(
self.load())})
online_iplist = ServerPool.get_instance().get_servers_iplist()
data = []
for port in online_iplist.keys():
for ip in online_iplist[port]:
data.append({'ip': ip, 'user_id': self.port_uid_table[port]})
webapi.postApi('users/aliveip',
{'node_id': get_config().NODE_ID},
{'data': data})
def uptime(self):
with open('/proc/uptime', 'r') as f:
return float(f.readline().split()[0])
def load(self):
import os
return os.popen(
"cat /proc/loadavg | awk '{ print $1\" \"$2\" \"$3 }'").readlines()[0]
def trafficShow(self, Traffic):
if Traffic < 1024:
return str(round((Traffic), 2)) + "B"
if Traffic < 1024 * 1024:
return str(round((Traffic / 1024), 2)) + "KB"
if Traffic < 1024 * 1024 * 1024:
return str(round((Traffic / 1024 / 1024), 2)) + "MB"
return str(round((Traffic / 1024 / 1024 / 1024), 2)) + "GB"
def push_db_all_user(self):
# 更新用户流量到数据库
last_transfer = self.last_update_transfer
curr_transfer = ServerPool.get_instance().get_servers_transfer()
# 上次和本次的增量
dt_transfer = {}
for id in curr_transfer.keys():
if id in last_transfer:
if curr_transfer[id][0] + curr_transfer[id][1] - \
last_transfer[id][0] - last_transfer[id][1] <= 0:
continue
if last_transfer[id][0] <= curr_transfer[id][0] and \
last_transfer[id][1] <= curr_transfer[id][1]:
dt_transfer[id] = [
curr_transfer[id][0] - last_transfer[id][0],
curr_transfer[id][1] - last_transfer[id][1]]
else:
dt_transfer[id] = [
curr_transfer[id][0], curr_transfer[id][1]]
else:
if curr_transfer[id][0] + curr_transfer[id][1] <= 0:
continue
dt_transfer[id] = [curr_transfer[id][0], curr_transfer[id][1]]
for id in dt_transfer.keys():
last = last_transfer.get(id, [0, 0])
last_transfer[id] = [
last[0] + dt_transfer[id][0],
last[1] + dt_transfer[id][1]]
self.last_update_transfer = last_transfer.copy()
self.update_all_user(dt_transfer)
def pull_db_all_user(self):
global webapi
nodeinfo = webapi.getApi(
'nodes/%d/info' %
(get_config().NODE_ID))
if not nodeinfo:
rows = []
return rows
self.node_speedlimit = nodeinfo['node_speedlimit']
self.traffic_rate = nodeinfo['traffic_rate']
self.mu_only = nodeinfo['mu_only']
self.node_offset = int(nodeinfo['node_offset'])
if get_config().NODE_CUSTOM_OBFS == 1:
self.ss_method = nodeinfo['ss_method']
self.ss_protocol = nodeinfo['ss_protocol']
self.ss_obfs = nodeinfo['ss_obfs']
if nodeinfo['sort'] == 10:
self.is_relay = True
else:
self.is_relay = False
data = webapi.getApi('users', {'node_id': get_config().NODE_ID})
if not data:
rows = []
return rows
rows = data
# 读取审计规则,数据包匹配部分
self.detect_text_list = {}
self.detect_hex_list = {}
data = webapi.getApi('func/detect_rules')
for rule in data:
d = {}
d['id'] = int(rule['id'])
d['regex'] = str(rule['regex'])
if int(rule['type']) == 1:
self.detect_text_list[d['id']] = d.copy()
else:
self.detect_hex_list[d['id']] = d.copy()
# 读取中转规则,如果是中转节点的话
if self.is_relay:
self.relay_rule_list = {}
data = webapi.getApi(
'func/relay_rules', {'node_id': get_config().NODE_ID})
for rule in data:
d = {}
d['id'] = int(rule['id'])
d['user_id'] = int(rule['user_id'])
d['dist_ip'] = str(rule['dist_ip'])
d['port'] = int(rule['port'])
d['priority'] = int(rule['priority'])
self.relay_rule_list[d['id']] = d.copy()
for row in rows:
row['port'] = row['port'] + self.node_offset
if get_config().NODE_CUSTOM_OBFS == 1 :
if self.ss_method:
row['method'] = self.ss_method
if self.ss_protocol:
row['protocol'] = self.ss_protocol
if self.ss_obfs:
row['obfs'] = self.ss_obfs
if row['obfs'] == "plain" and row['is_multi_user'] == 2:
row['obfs_param'] = ""
#logging.error('obfs: %s obfs_param: %s' % (row['obfs'], row['obfs_param'] ))
return rows
def cmp(self, val1, val2):
if isinstance(val1, bytes):
val1 = common.to_str(val1)
if isinstance(val2, bytes):
val2 = common.to_str(val2)
return val1 == val2
def del_server_out_of_bound_safe(self, last_rows, rows):
#stop service out of traffic
#start service which is normal
#load switchrule dynamic
try:
switchrule = importloader.load('switchrule')
except Exception as e:
logging.error('load switchrule.py fail')
cur_servers = {}
new_servers = {}
md5_users = {}
self.mu_port_list = []
for row in rows:
self.port_uid_table[row['port']] = row['id']
self.uid_port_table[row['id']] = row['port']
if row['is_multi_user'] != 0:
self.mu_port_list.append(int(row['port']))
continue
md5_users[row['id']] = row.copy()
del md5_users[row['id']]['u']
del md5_users[row['id']]['d']
if md5_users[row['id']]['disconnect_ip'] is None:
md5_users[row['id']]['disconnect_ip'] = ''
if md5_users[row['id']]['forbidden_ip'] is None:
md5_users[row['id']]['forbidden_ip'] = ''
if md5_users[row['id']]['forbidden_port'] is None:
md5_users[row['id']]['forbidden_port'] = ''
md5_users[row['id']]['md5'] = common.get_md5(
str(row['id']) + row['passwd'] + row['method'] + row['obfs'] + row['protocol'])
if self.mu_only == 1:
i = 0
while i < len(rows):
if rows[i]['is_multi_user'] == 0:
rows.pop(i)
i -= 1
else:
pass
i += 1
if self.mu_only == -1:
i = 0
while i < len(rows):
if rows[i]['is_multi_user'] != 0:
rows.pop(i)
i -= 1
else:
pass
i += 1
for row in rows:
port = row['port']
user_id = row['id']
passwd = common.to_bytes(row['passwd'])
cfg = {'password': passwd}
read_config_keys = [
'method',
'obfs',
'obfs_param',
'protocol',
'protocol_param',
'forbidden_ip',
'forbidden_port',
'node_speedlimit',
'disconnect_ip',
'is_multi_user']
for name in read_config_keys:
if name in row and row[name]:
cfg[name] = row[name]
merge_config_keys = ['password'] + read_config_keys
for name in cfg.keys():
if hasattr(cfg[name], 'encode'):
try:
cfg[name] = cfg[name].encode('utf-8')
except Exception as e:
logging.warning(
'encode cfg key "%s" fail, val "%s"' % (name, cfg[name]))
if 'node_speedlimit' in cfg:
# if both setting speedlimit, just choose the smaller one
if float(
self.node_speedlimit) > 0.0 and float(
cfg['node_speedlimit']) > 0.0:
cfg['node_speedlimit'] = min(
float(
self.node_speedlimit), float(
cfg['node_speedlimit']))
else:
cfg['node_speedlimit'] = max(
float(self.node_speedlimit), float(
cfg['node_speedlimit']))
else:
cfg['node_speedlimit'] = max(
float(self.node_speedlimit), float(0.00))
if 'disconnect_ip' not in cfg:
cfg['disconnect_ip'] = ''
if 'forbidden_ip' not in cfg:
cfg['forbidden_ip'] = ''
if 'forbidden_port' not in cfg:
cfg['forbidden_port'] = ''
if 'protocol_param' not in cfg:
cfg['protocol_param'] = ''
if 'obfs_param' not in cfg:
cfg['obfs_param'] = ''
if 'is_multi_user' not in cfg:
cfg['is_multi_user'] = 0
if port not in cur_servers:
cur_servers[port] = passwd
else:
logging.error(
'more than one user use the same port [%s]' %
(port,))
continue
if cfg['is_multi_user'] != 0:
cfg['users_table'] = md5_users.copy()
cfg['detect_hex_list'] = self.detect_hex_list.copy()
cfg['detect_text_list'] = self.detect_text_list.copy()
if self.is_relay and row['is_multi_user'] != 2:
temp_relay_rules = {}
for id in self.relay_rule_list:
if ((self.relay_rule_list[id]['user_id'] == user_id or self.relay_rule_list[id]['user_id'] == 0) or row[
'is_multi_user'] != 0) and (self.relay_rule_list[id]['port'] == 0 or self.relay_rule_list[id]['port'] == port):
has_higher_priority = False
for priority_id in self.relay_rule_list:
if (
(
self.relay_rule_list[priority_id]['priority'] > self.relay_rule_list[id]['priority'] and self.relay_rule_list[id]['id'] != self.relay_rule_list[priority_id]['id']) or (
self.relay_rule_list[priority_id]['priority'] == self.relay_rule_list[id]['priority'] and self.relay_rule_list[id]['id'] > self.relay_rule_list[priority_id]['id'])) and (
self.relay_rule_list[priority_id]['user_id'] == user_id or self.relay_rule_list[priority_id]['user_id'] == 0) and (
self.relay_rule_list[priority_id]['port'] == port or self.relay_rule_list[priority_id]['port'] == 0):
has_higher_priority = True
continue
if has_higher_priority:
continue
if self.relay_rule_list[id]['dist_ip'] == '0.0.0.0' and row['is_multi_user'] == 0:
continue
temp_relay_rules[id] = self.relay_rule_list[id]
cfg['relay_rules'] = temp_relay_rules.copy()
else:
temp_relay_rules = {}
cfg['relay_rules'] = temp_relay_rules.copy()
if ServerPool.get_instance().server_is_run(port) > 0:
cfgchange = False
if port in ServerPool.get_instance().tcp_servers_pool:
ServerPool.get_instance().tcp_servers_pool[
port].modify_detect_text_list(self.detect_text_list)
ServerPool.get_instance().tcp_servers_pool[
port].modify_detect_hex_list(self.detect_hex_list)
if port in ServerPool.get_instance().tcp_ipv6_servers_pool:
ServerPool.get_instance().tcp_ipv6_servers_pool[
port].modify_detect_text_list(self.detect_text_list)
ServerPool.get_instance().tcp_ipv6_servers_pool[
port].modify_detect_hex_list(self.detect_hex_list)
if port in ServerPool.get_instance().udp_servers_pool:
ServerPool.get_instance().udp_servers_pool[
port].modify_detect_text_list(self.detect_text_list)
ServerPool.get_instance().udp_servers_pool[
port].modify_detect_hex_list(self.detect_hex_list)
if port in ServerPool.get_instance().udp_ipv6_servers_pool:
ServerPool.get_instance().udp_ipv6_servers_pool[
port].modify_detect_text_list(self.detect_text_list)
ServerPool.get_instance().udp_ipv6_servers_pool[
port].modify_detect_hex_list(self.detect_hex_list)
if row['is_multi_user'] != 0:
if port in ServerPool.get_instance().tcp_servers_pool:
ServerPool.get_instance().tcp_servers_pool[
port].modify_multi_user_table(md5_users)
if port in ServerPool.get_instance().tcp_ipv6_servers_pool:
ServerPool.get_instance().tcp_ipv6_servers_pool[
port].modify_multi_user_table(md5_users)
if port in ServerPool.get_instance().udp_servers_pool:
ServerPool.get_instance().udp_servers_pool[
port].modify_multi_user_table(md5_users)
if port in ServerPool.get_instance().udp_ipv6_servers_pool:
ServerPool.get_instance().udp_ipv6_servers_pool[
port].modify_multi_user_table(md5_users)
if self.is_relay and row['is_multi_user'] != 2:
temp_relay_rules = {}
for id in self.relay_rule_list:
if ((self.relay_rule_list[id]['user_id'] == user_id or self.relay_rule_list[id]['user_id'] == 0) or row[
'is_multi_user'] != 0) and (self.relay_rule_list[id]['port'] == 0 or self.relay_rule_list[id]['port'] == port):
has_higher_priority = False
for priority_id in self.relay_rule_list:
if (
(
self.relay_rule_list[priority_id]['priority'] > self.relay_rule_list[id]['priority'] and self.relay_rule_list[id]['id'] != self.relay_rule_list[priority_id]['id']) or (
self.relay_rule_list[priority_id]['priority'] == self.relay_rule_list[id]['priority'] and self.relay_rule_list[id]['id'] > self.relay_rule_list[priority_id]['id'])) and (
self.relay_rule_list[priority_id]['user_id'] == user_id or self.relay_rule_list[priority_id]['user_id'] == 0) and (
self.relay_rule_list[priority_id]['port'] == port or self.relay_rule_list[priority_id]['port'] == 0):
has_higher_priority = True
continue
if has_higher_priority:
continue
if self.relay_rule_list[id]['dist_ip'] == '0.0.0.0' and row['is_multi_user'] == 0:
continue
temp_relay_rules[id] = self.relay_rule_list[id]
if port in ServerPool.get_instance().tcp_servers_pool:
ServerPool.get_instance().tcp_servers_pool[
port].push_relay_rules(temp_relay_rules)
if port in ServerPool.get_instance().tcp_ipv6_servers_pool:
ServerPool.get_instance().tcp_ipv6_servers_pool[
port].push_relay_rules(temp_relay_rules)
if port in ServerPool.get_instance().udp_servers_pool:
ServerPool.get_instance().udp_servers_pool[
port].push_relay_rules(temp_relay_rules)
if port in ServerPool.get_instance().udp_ipv6_servers_pool:
ServerPool.get_instance().udp_ipv6_servers_pool[
port].push_relay_rules(temp_relay_rules)
else:
temp_relay_rules = {}
if port in ServerPool.get_instance().tcp_servers_pool:
ServerPool.get_instance().tcp_servers_pool[
port].push_relay_rules(temp_relay_rules)
if port in ServerPool.get_instance().tcp_ipv6_servers_pool:
ServerPool.get_instance().tcp_ipv6_servers_pool[
port].push_relay_rules(temp_relay_rules)
if port in ServerPool.get_instance().udp_servers_pool:
ServerPool.get_instance().udp_servers_pool[
port].push_relay_rules(temp_relay_rules)
if port in ServerPool.get_instance().udp_ipv6_servers_pool:
ServerPool.get_instance().udp_ipv6_servers_pool[
port].push_relay_rules(temp_relay_rules)
if port in ServerPool.get_instance().tcp_servers_pool:
relay = ServerPool.get_instance().tcp_servers_pool[port]
for name in merge_config_keys:
if name in cfg and not self.cmp(
cfg[name], relay._config[name]):
cfgchange = True
break
if not cfgchange and port in ServerPool.get_instance().tcp_ipv6_servers_pool:
relay = ServerPool.get_instance().tcp_ipv6_servers_pool[
port]
for name in merge_config_keys:
if name in cfg and not self.cmp(
cfg[name], relay._config[name]):
cfgchange = True
break
# config changed
if cfgchange:
self.del_server(port, "config changed")
new_servers[port] = (passwd, cfg)
elif ServerPool.get_instance().server_run_status(port) is False:
# new_servers[port] = passwd
self.new_server(port, passwd, cfg)
for row in last_rows:
if row['port'] in cur_servers:
pass
else:
self.del_server(row['port'], "port not exist")
if len(new_servers) > 0:
from shadowsocks import eventloop
self.event.wait(eventloop.TIMEOUT_PRECISION +
eventloop.TIMEOUT_PRECISION / 2)
for port in new_servers.keys():
passwd, cfg = new_servers[port]
self.new_server(port, passwd, cfg)
ServerPool.get_instance().push_uid_port_table(self.uid_port_table)
def del_server(self, port, reason):
logging.info(
'db stop server at port [%s] reason: %s!' %
(port, reason))
ServerPool.get_instance().cb_del_server(port)
if port in self.last_update_transfer:
del self.last_update_transfer[port]
for mu_user_port in self.mu_port_list:
if mu_user_port in ServerPool.get_instance().tcp_servers_pool:
ServerPool.get_instance().tcp_servers_pool[
mu_user_port].reset_single_multi_user_traffic(self.port_uid_table[port])
if mu_user_port in ServerPool.get_instance().tcp_ipv6_servers_pool:
ServerPool.get_instance().tcp_ipv6_servers_pool[
mu_user_port].reset_single_multi_user_traffic(self.port_uid_table[port])
if mu_user_port in ServerPool.get_instance().udp_servers_pool:
ServerPool.get_instance().udp_servers_pool[
mu_user_port].reset_single_multi_user_traffic(self.port_uid_table[port])
if mu_user_port in ServerPool.get_instance().udp_ipv6_servers_pool:
ServerPool.get_instance().udp_ipv6_servers_pool[
mu_user_port].reset_single_multi_user_traffic(self.port_uid_table[port])
def new_server(self, port, passwd, cfg):
protocol = cfg.get(
'protocol',
ServerPool.get_instance().config.get(
'protocol',
'origin'))
method = cfg.get(
'method',
ServerPool.get_instance().config.get(
'method',
'None'))
obfs = cfg.get(
'obfs',
ServerPool.get_instance().config.get(
'obfs',
'plain'))
logging.info(
'db start server at port [%s] pass [%s] protocol [%s] method [%s] obfs [%s]' %
(port, passwd, protocol, method, obfs))
ServerPool.get_instance().new_server(port, cfg)
@staticmethod
def del_servers():
global db_instance
for port in [
v for v in ServerPool.get_instance().tcp_servers_pool.keys()]:
if ServerPool.get_instance().server_is_run(port) > 0:
ServerPool.get_instance().cb_del_server(port)
if port in db_instance.last_update_transfer:
del db_instance.last_update_transfer[port]
for port in [
v for v in ServerPool.get_instance().tcp_ipv6_servers_pool.keys()]:
if ServerPool.get_instance().server_is_run(port) > 0:
ServerPool.get_instance().cb_del_server(port)
if port in db_instance.last_update_transfer:
del db_instance.last_update_transfer[port]
@staticmethod
def thread_db(obj):
import socket
import time
import webapi_utils
global db_instance
global webapi
timeout = 60
socket.setdefaulttimeout(timeout)
last_rows = []
db_instance = obj()
webapi = webapi_utils.WebApi()
shell.log_shadowsocks_version()
try:
import resource
logging.info(
'current process RLIMIT_NOFILE resource: soft %d hard %d' %
resource.getrlimit(
resource.RLIMIT_NOFILE))
except:
pass
try:
while True:
load_config()
try:
ping = webapi.getApi('func/ping')
if ping is None:
logging.error(
'something wrong with your http api, please check your config and website status and try again later.')
else:
db_instance.push_db_all_user()
rows = db_instance.pull_db_all_user()
db_instance.del_server_out_of_bound_safe(
last_rows, rows)
last_rows = rows
except Exception as e:
trace = traceback.format_exc()
logging.error(trace)
# logging.warn('db thread except:%s' % e)
if db_instance.event.wait(60) or not db_instance.is_all_thread_alive():
break
if db_instance.has_stopped:
break
except KeyboardInterrupt as e:
pass
db_instance.del_servers()
ServerPool.get_instance().stop()
db_instance = None
@staticmethod
def thread_db_stop():
global db_instance
db_instance.has_stopped = True
db_instance.event.set()
def is_all_thread_alive(self):
if not ServerPool.get_instance().thread.is_alive():
return False
return True