Memory leak with high user load #4054
Description
Integrity requirements
- I confirm that I have read the documentation, understand the meaning of all the configuration items I wrote, and did not pile up seemingly useful options or default values.
- I provided the complete config and logs, rather than just providing the truncated parts based on my own judgment.
- I searched issues and did not find any similar issues.
- The problem can be successfully reproduced in the latest Release
Description
Hello everyone! I'm experiencing a memory leak issue with a large number of active users. When the problem occurs, there's a sharp increase in memory consumption which then drops back to normal levels after about 2 minutes. However, after each spike, the baseline memory usage incrementally increases, indicating a potential memory leak.
I'm using the latest version of 3x-ui, and I've also tried rolling back to several previous versions, but this didn't resolve the issue. I monitored the traffic on xray's API interface during the problem occurrence, and there were no suspicious requests from 3x-ui to xray during these spikes.
Using pprof for memory analysis during these spikes revealed potential issues, but I'm not sure how to address them in the code. Also, there are no log records (function calls or similar) while the problem occurs.
System details:
- Average number of active users: 2-3k
- Protocol: exclusively vless
- mux and warp are disabled in the panel
- Server resources: 128 cores, 64GB RAM
- Network traffic bandwidth is 5Gb/s
Has anyone encountered similar issues or can suggest possible solutions? I can share pprof analysis results if needed.
Reproduction Method
Client config
Details
Applications that supports vless protocol
Server config
Details
{
"log": {
"access": "./access.log",
"dnsLog": true,
"error": "./error.log",
"loglevel": "debug"
},
"api": {
"tag": "api",
"services": [
"HandlerService",
"LoggerService",
"StatsService"
]
},
"inbounds": [
{
"tag": "api",
"listen": "0.0.0.0",
"port": 62789,
"protocol": "dokodemo-door",
"settings": {
"address": "0.0.0.0"
}
}
],
"outbounds": [
{
"tag": "direct",
"protocol": "freedom",
"settings": {
"domainStrategy": "UseIP"
}
},
{
"tag": "blocked",
"protocol": "blackhole",
"settings": {}
},
{
"tag": "",
"protocol": "vless",
"settings": {
"vnext": [
{
"address": "",
"port": 443,
"users": [
{
"id": "",
"flow": "xtls-rprx-vision",
"encryption": "none"
}
]
}
]
},
"streamSettings": {
"network": "tcp",
"security": "reality",
"realitySettings": {
"publicKey": "",
"fingerprint": "chrome",
"serverName": "google.com",
"shortId": "",
"spiderX": "/"
},
"tcpSettings": {
"header": {
"type": "none"
}
}
}
},
{
"tag": "",
"protocol": "vless",
"settings": {
"vnext": [
{
"address": "",
"port": 443,
"users": [
{
"id": "",
"flow": "xtls-rprx-vision",
"encryption": "none"
}
]
}
]
},
"streamSettings": {
"network": "tcp",
"security": "reality",
"realitySettings": {
"publicKey": "",
"fingerprint": "chrome",
"serverName": "google.com",
"shortId": "",
"spiderX": "/"
},
"tcpSettings": {
"header": {
"type": "none"
}
}
}
},
{
"tag": "",
"protocol": "vless",
"settings": {
"vnext": [
{
"address": "",
"port": 443,
"users": [
{
"id": "",
"flow": "xtls-rprx-vision",
"encryption": "none"
}
]
}
]
},
"streamSettings": {
"network": "tcp",
"security": "reality",
"realitySettings": {
"publicKey": "",
"fingerprint": "chrome",
"serverName": "google.com",
"shortId": "",
"spiderX": "/"
},
"tcpSettings": {
"header": {
"type": "none"
}
}
}
},
{
"tag": "",
"protocol": "vless",
"settings": {
"vnext": [
{
"address": "",
"port": 443,
"users": [
{
"id": "",
"flow": "xtls-rprx-vision",
"encryption": "none"
}
]
}
]
},
"streamSettings": {
"network": "tcp",
"security": "reality",
"realitySettings": {
"publicKey": "",
"fingerprint": "chrome",
"serverName": "google.com",
"shortId": "",
"spiderX": "/"
},
"tcpSettings": {
"header": {
"type": "none"
}
}
}
},
{
"tag": "",
"protocol": "vless",
"settings": {
"vnext": [
{
"address": "",
"port": 443,
"users": [
{
"id": "",
"flow": "xtls-rprx-vision",
"encryption": "none"
}
]
}
]
},
"streamSettings": {
"network": "tcp",
"security": "reality",
"realitySettings": {
"publicKey": "",
"fingerprint": "chrome",
"serverName": "google.com",
"shortId": "",
"spiderX": "/"
},
"tcpSettings": {
"header": {
"type": "none"
}
}
}
}
],
"policy": {
"levels": {
"0": {
"statsUserDownlink": true,
"statsUserUplink": true
}
},
"system": {
"statsInboundDownlink": true,
"statsInboundUplink": true,
"statsOutboundDownlink": true,
"statsOutboundUplink": true
}
},
"routing": {
"domainStrategy": "AsIs",
"rules": [
{
"type": "field",
"inboundTag": [
"api"
],
"outboundTag": "api"
},
{
"type": "field",
"outboundTag": "blocked",
"ip": [
"geoip:private"
]
},
{
"type": "field",
"network": "TCP,UDP",
"inboundTag": [
"inbound-443"
],
"protocol": [
"bittorrent"
],
"outboundTag": "blocked"
},
{
"type": "field",
"port": "6881,6882,6883,6884,6885,6886,6887,6888,6889,6969,1337",
"network": "TCP,UDP",
"inboundTag": [
"inbound-443"
],
"outboundTag": "blocked"
},
{
"type": "field",
"domainMatcher": "hybrid",
"domain": [
... some-domains ...
],
"network": "TCP,UDP",
"inboundTag": [
"inbound-443"
],
"protocol": [
"http",
"tls",
"bittorrent"
],
"outboundTag": "blocked"
},
{
"type": "field",
"domain": [
"geosite:category-ads-all",
"ext:geosite_IR.dat:category-ads-all"
],
"outboundTag": "blocked"
},
{
"type": "field",
"domainMatcher": "hybrid",
"domain": [
... some-domains ...
],
"network": "TCP,UDP",
"inboundTag": [
"inbound-443"
],
"protocol": [
"tls",
"http"
],
"balancerTag": ""
},
{
"type": "field",
"domainMatcher": "hybrid",
"inboundTag": [
"inbound-443"
],
"balancerTag": ""
},
{
"type": "field",
"domainMatcher": "hybrid",
"network": "tcp",
"inboundTag": [
"inbound-444"
],
"protocol": [
"http",
"tls"
],
"balancerTag": ""
},
{
"type": "field",
"domainMatcher": "hybrid",
"domain": [
... some-domains ...
],
"network": "TCP,UDP",
"inboundTag": [
"inbound-8443"
],
"protocol": [
"http",
"tls"
],
"balancerTag": ""
},
{
"type": "field",
"domainMatcher": "hybrid",
"network": "TCP,UDP",
"inboundTag": [
"inbound-8443"
],
"protocol": [
"http",
"tls"
],
"outboundTag": "direct"
}
],
"balancers": [
{
"tag": "",
"selector": [
"",
"",
""
],
"fallbackTag": "direct",
"strategy": {
"type": "roundRobin"
}
},
{
"tag": "",
"selector": [
""
],
"fallbackTag": "direct"
},
{
"tag": "",
"selector": [
"",
""
],
"fallbackTag": "direct",
"strategy": {
"type": "roundRobin"
}
}
]
},
"stats": {}
}
Sample client inbound configuration (in 3x-ui located outside of main server config):
Details
{
"email": "",
"flow": "xtls-rprx-vision",
"id": ""
},
{
"email": "",
"flow": "xtls-rprx-vision",
"id": ""
},
{
"email": "",
"flow": "xtls-rprx-vision",
"id": ""
},
{
"email": "",
"flow": "xtls-rprx-vision",
"id": ""
},
Client log
Details
-
Server log
Details
-