using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Caching.Memory; using Newtonsoft.Json.Linq; using Shared; using Shared.Engine; using Shared.Models; using System; using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; namespace Lampac.Engine.Middlewares { public class ProxyCub { #region ProxyCub static FileSystemWatcher fileWatcher; static readonly ConcurrentDictionary cacheFiles = new (); public static int Stat_ContCacheFiles => cacheFiles.IsEmpty ? 0 : cacheFiles.Count; static Timer cleanupTimer; static ProxyCub() { Directory.CreateDirectory("cache/cub"); foreach (string path in Directory.EnumerateFiles("cache/cub", "*")) { using (var handle = File.OpenHandle(path)) cacheFiles.TryAdd(Path.GetFileName(path), (int)RandomAccess.GetLength(handle)); } fileWatcher = new FileSystemWatcher { Path = "cache/cub", NotifyFilter = NotifyFilters.FileName, EnableRaisingEvents = true }; //fileWatcher.Created += (s, e) => { cacheFiles.TryAdd(e.Name, 0); }; fileWatcher.Deleted += (s, e) => { cacheFiles.TryRemove(e.Name, out _); }; cleanupTimer = new Timer(cleanup, null, TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(20)); } static void cleanup(object state) { try { foreach (string md5fileName in cacheFiles.Keys) { if (!File.Exists(Path.Combine("cache", "cub", md5fileName))) cacheFiles.TryRemove(md5fileName, out _); } } catch { } } public ProxyCub(RequestDelegate next) { } #endregion async public Task InvokeAsync(HttpContext httpContext, IMemoryCache memoryCache) { using (var ctsHttp = CancellationTokenSource.CreateLinkedTokenSource(httpContext.RequestAborted)) { ctsHttp.CancelAfter(TimeSpan.FromSeconds(30)); var requestInfo = httpContext.Features.Get(); var hybridCache = IHybridCache.Get(requestInfo); var init = AppInit.conf.cub; string domain = init.domain; string path = httpContext.Request.Path.Value.Replace("/cub/", "", StringComparison.OrdinalIgnoreCase); string query = httpContext.Request.QueryString.Value; string uri = Regex.Match(path, "^[^/]+/(.*)", RegexOptions.IgnoreCase).Groups[1].Value + query; if (!init.enable || domain == "ws") { httpContext.Response.Redirect($"https://{path}/{query}"); return; } if (path.Split(".")[0] is "geo" or "tmdb" or "tmapi" or "apitmdb" or "imagetmdb" or "cdn" or "ad" or "ws") domain = $"{path.Split(".")[0]}.{domain}"; if (domain.StartsWith("geo", StringComparison.OrdinalIgnoreCase)) { string country = requestInfo.Country; if (string.IsNullOrEmpty(country)) { var ipify = await Http.Get("https://api.ipify.org/?format=json"); if (ipify != null || !string.IsNullOrEmpty(ipify.Value("ip"))) country = GeoIP2.Country(ipify.Value("ip")); } await httpContext.Response.WriteAsync(country ?? "", ctsHttp.Token); return; } #region checker if (path.StartsWith("api/checker", StringComparison.OrdinalIgnoreCase) || uri.StartsWith("api/checker", StringComparison.OrdinalIgnoreCase)) { if (HttpMethods.IsPost(httpContext.Request.Method)) { if (httpContext.Request.ContentType != null && httpContext.Request.ContentType.StartsWith("application/x-www-form-urlencoded", StringComparison.OrdinalIgnoreCase)) { using (var reader = new StreamReader(httpContext.Request.Body, leaveOpen: true, bufferSize: PoolInvk.bufferSize)) { string form = await reader.ReadToEndAsync(); var match = Regex.Match(form, @"(?:^|&)data=([^&]+)"); if (match.Success) { string dataValue = Uri.UnescapeDataString(match.Groups[1].Value); await httpContext.Response.WriteAsync(dataValue, ctsHttp.Token); return; } } } } await httpContext.Response.WriteAsync("ok", ctsHttp.Token); return; } #endregion #region blacklist if (uri.StartsWith("api/plugins/blacklist", StringComparison.OrdinalIgnoreCase)) { httpContext.Response.ContentType = "application/json; charset=utf-8"; await httpContext.Response.WriteAsync("[]", ctsHttp.Token); return; } #endregion #region ads/log/metric if (uri.StartsWith("api/metric/", StringComparison.OrdinalIgnoreCase) || uri.StartsWith("api/ad/stat", StringComparison.OrdinalIgnoreCase)) { await httpContext.Response.WriteAsJsonAsync(new { secuses = true }); return; } if (uri.StartsWith("api/ad/vast", StringComparison.OrdinalIgnoreCase)) { await httpContext.Response.WriteAsJsonAsync(new { secuses = true, ad = new string[] { }, day_of_month = DateTime.Now.Day, days_in_month = 31, month = DateTime.Now.Month }); return; } #endregion var proxyManager = new ProxyManager("cub_api", init); var proxy = proxyManager.Get(); bool isMedia = Regex.IsMatch(uri, "\\.(jpe?g|png|gif|webp|ico|svg|mp4|js|css)", RegexOptions.IgnoreCase); if (0 >= init.cache_api || !HttpMethods.IsGet(httpContext.Request.Method) || isMedia || (path.Split(".")[0] is "imagetmdb" or "cdn" or "ad") || httpContext.Request.Headers.ContainsKey("token") || httpContext.Request.Headers.ContainsKey("profile")) { #region bypass or media cache string md5key = CrypTo.md5($"{domain}:{uri}"); string outFile = Path.Combine("cache", "cub", md5key); if (cacheFiles.ContainsKey(md5key)) { httpContext.Response.Headers["X-Cache-Status"] = "HIT"; httpContext.Response.ContentType = getContentType(uri); if (init.responseContentLength && cacheFiles.ContainsKey(md5key)) httpContext.Response.ContentLength = cacheFiles[md5key]; await httpContext.Response.SendFileAsync(outFile, ctsHttp.Token).ConfigureAwait(false); return; } var handler = new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.None, AllowAutoRedirect = true }; handler.ServerCertificateCustomValidationCallback += (sender, cert, chain, sslPolicyErrors) => true; if (proxy != null) { handler.UseProxy = true; handler.Proxy = proxy; } else { handler.UseProxy = false; } var client = FrendlyHttp.MessageClient("proxyRedirect", handler); var request = CreateProxyHttpRequest(httpContext, new Uri($"{init.scheme}://{domain}/{uri}"), requestInfo, init.viewru && path.Split(".")[0] == "tmdb"); using (var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctsHttp.Token).ConfigureAwait(false)) { if (init.cache_img > 0 && isMedia && HttpMethods.IsGet(httpContext.Request.Method) && AppInit.conf.mikrotik == false && response.StatusCode == HttpStatusCode.OK) { #region cache httpContext.Response.ContentType = getContentType(uri); httpContext.Response.Headers["X-Cache-Status"] = "MISS"; if (init.responseContentLength && response.Content?.Headers?.ContentLength > 0) { if (!AppInit.CompressionMimeTypes.Contains(httpContext.Response.ContentType)) httpContext.Response.ContentLength = response.Content.Headers.ContentLength.Value; } var semaphore = new SemaphorManager(outFile, ctsHttp.Token); try { await semaphore.WaitAsync().ConfigureAwait(false); byte[] buffer = ArrayPool.Shared.Rent(PoolInvk.rentChunk); try { int cacheLength = 0; using (var cacheStream = new FileStream(outFile, FileMode.Create, FileAccess.Write, FileShare.None, PoolInvk.bufferSize)) { using (var responseStream = await response.Content.ReadAsStreamAsync(ctsHttp.Token).ConfigureAwait(false)) { int bytesRead; while ((bytesRead = await responseStream.ReadAsync(buffer, ctsHttp.Token).ConfigureAwait(false)) > 0) { cacheLength += bytesRead; await cacheStream.WriteAsync(buffer, 0, bytesRead).ConfigureAwait(false); await httpContext.Response.Body.WriteAsync(buffer, 0, bytesRead, ctsHttp.Token).ConfigureAwait(false); } } } if (!response.Content.Headers.ContentLength.HasValue || response.Content.Headers.ContentLength.Value == cacheLength) { cacheFiles[md5key] = cacheLength; } else { File.Delete(outFile); } } catch { File.Delete(outFile); throw; } finally { ArrayPool.Shared.Return(buffer); } } finally { semaphore.Release(); } #endregion } else { httpContext.Response.Headers["X-Cache-Status"] = "bypass"; await CopyProxyHttpResponse(httpContext, response, ctsHttp.Token).ConfigureAwait(false); } } #endregion } else { #region cache string string memkey = $"cubproxy:key2:{domain}:{uri}"; (byte[] content, int statusCode, string contentType) cache = default; var semaphore = new SemaphorManager(memkey, ctsHttp.Token); try { await semaphore.WaitAsync().ConfigureAwait(false); if (!hybridCache.TryGetValue(memkey, out cache, inmemory: false)) { var headers = HeadersModel.Init(); if (!string.IsNullOrEmpty(requestInfo.Country)) headers.Add(new HeadersModel("cf-connecting-ip", requestInfo.IP)); if (path.Split(".")[0] == "tmdb") { if (init.viewru) headers.Add(new HeadersModel("cookie", "viewru=1")); headers.Add(new HeadersModel("user-agent", httpContext.Request.Headers.UserAgent.ToString())); } else { foreach (var header in httpContext.Request.Headers) { if (header.Key.ToLower() is "cookie" or "user-agent") headers.Add(new HeadersModel(header.Key, header.Value.ToString())); } } var result = await Http.BaseGet($"{init.scheme}://{domain}/{uri}", timeoutSeconds: 10, proxy: proxy, headers: headers, statusCodeOK: false, useDefaultHeaders: false).ConfigureAwait(false); if (string.IsNullOrEmpty(result.content)) { proxyManager.Refresh(); httpContext.Response.StatusCode = (int)result.response.StatusCode; return; } cache.content = Encoding.UTF8.GetBytes(result.content); cache.statusCode = (int)result.response.StatusCode; cache.contentType = result.response.Content?.Headers?.ContentType?.ToString() ?? getContentType(uri); if (domain.StartsWith("tmdb") || domain.StartsWith("tmapi") || domain.StartsWith("apitmdb")) { if (result.content == "{\"blocked\":true}") { var header = HeadersModel.Init(("localrequest", AppInit.rootPasswd)); string json = await Http.Get($"http://{AppInit.conf.listen.localhost}:{AppInit.conf.listen.port}/tmdb/api/{uri}", timeoutSeconds: 5, headers: header).ConfigureAwait(false); if (!string.IsNullOrEmpty(json)) { cache.statusCode = 200; cache.contentType = "application/json; charset=utf-8"; cache.content = Encoding.UTF8.GetBytes(json); } } } httpContext.Response.Headers["X-Cache-Status"] = "MISS"; if (cache.statusCode == 200) { proxyManager.Success(); hybridCache.Set(memkey, cache, DateTime.Now.AddMinutes(init.cache_api), inmemory: false); } } else { httpContext.Response.Headers["X-Cache-Status"] = "HIT"; } } finally { semaphore.Release(); } if (init.responseContentLength && !AppInit.CompressionMimeTypes.Contains(cache.contentType)) httpContext.Response.ContentLength = cache.content.Length; httpContext.Response.StatusCode = cache.statusCode; httpContext.Response.ContentType = cache.contentType; await httpContext.Response.Body.WriteAsync(cache.content, ctsHttp.Token).ConfigureAwait(false); #endregion } } } #region getContentType static string getContentType(string uri) { return Path.GetExtension(uri).ToLowerInvariant() switch { ".jpg" or ".jpeg" => "image/jpeg", ".png" => "image/png", ".gif" => "image/gif", ".webp" => "image/webp", ".ico" => "image/x-icon", ".svg" => "image/svg+xml", ".mp4" => "video/mp4", ".js" => "application/javascript", ".css" => "text/css", _ => "application/octet-stream" }; } #endregion #region CreateProxyHttpRequest HttpRequestMessage CreateProxyHttpRequest(HttpContext context, Uri uri, RequestModel requestInfo, bool viewru) { var request = context.Request; var requestMessage = new HttpRequestMessage(); var requestMethod = request.Method; if (HttpMethods.IsPost(requestMethod)) { var streamContent = new StreamContent(request.Body); requestMessage.Content = streamContent; } if (viewru) request.Headers["Cookie"] = "viewru=1"; if (!string.IsNullOrEmpty(requestInfo.Country)) request.Headers["Cf-Connecting-Ip"] = requestInfo.IP; #region Headers foreach (var header in request.Headers) { string key = header.Key; if (key.Equals("host", StringComparison.OrdinalIgnoreCase) || key.Equals("origin", StringComparison.OrdinalIgnoreCase) || key.Equals("referer", StringComparison.OrdinalIgnoreCase) || key.Equals("content-disposition", StringComparison.OrdinalIgnoreCase) || key.Equals("accept-encoding", StringComparison.OrdinalIgnoreCase)) continue; if (viewru && key.Equals("cookie", StringComparison.OrdinalIgnoreCase)) continue; if (key.StartsWith("x-", StringComparison.OrdinalIgnoreCase)) continue; if (!requestMessage.Headers.TryAddWithoutValidation(header.Key, header.Value.ToArray())) { if (requestMessage.Content?.Headers != null) requestMessage.Content.Headers.TryAddWithoutValidation(header.Key, header.Value.ToArray()); } } #endregion requestMessage.Headers.Host = uri.Authority; requestMessage.RequestUri = uri; requestMessage.Method = new HttpMethod(request.Method); //requestMessage.Version = new Version(2, 0); return requestMessage; } #endregion #region CopyProxyHttpResponse async Task CopyProxyHttpResponse(HttpContext context, HttpResponseMessage responseMessage, CancellationToken cancellationToken) { var response = context.Response; response.StatusCode = (int)responseMessage.StatusCode; #region responseContentLength if (AppInit.conf.cub.responseContentLength && responseMessage.Content?.Headers?.ContentLength > 0) { IEnumerable contentType = null; if (responseMessage.Content?.Headers != null) responseMessage.Content.Headers.TryGetValues("Content-Type", out contentType); string type = contentType?.FirstOrDefault()?.ToLower(); if (string.IsNullOrEmpty(type) || !AppInit.CompressionMimeTypes.Contains(type)) response.ContentLength = responseMessage.Content.Headers.ContentLength; } #endregion #region UpdateHeaders void UpdateHeaders(HttpHeaders headers) { foreach (var header in headers) { string key = header.Key; if (key.Equals("server", StringComparison.OrdinalIgnoreCase) || key.Equals("transfer-encoding", StringComparison.OrdinalIgnoreCase) || key.Equals("etag", StringComparison.OrdinalIgnoreCase) || key.Equals("connection", StringComparison.OrdinalIgnoreCase) || key.Equals("content-security-policy", StringComparison.OrdinalIgnoreCase) || key.Equals("content-disposition", StringComparison.OrdinalIgnoreCase) || key.Equals("content-length", StringComparison.OrdinalIgnoreCase)) continue; if (key.StartsWith("x-", StringComparison.OrdinalIgnoreCase) || key.StartsWith("alt-", StringComparison.OrdinalIgnoreCase)) continue; if (key.StartsWith("access-control", StringComparison.OrdinalIgnoreCase)) continue; var values = header.Value; using (var e = values.GetEnumerator()) { if (!e.MoveNext()) continue; var first = e.Current; response.Headers[key] = e.MoveNext() ? string.Join("; ", values) : first; } } } #endregion UpdateHeaders(responseMessage.Headers); UpdateHeaders(responseMessage.Content.Headers); using (var responseStream = await responseMessage.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false)) { if (response.Body == null) throw new ArgumentNullException("destination"); if (!responseStream.CanRead && !responseStream.CanWrite) throw new ObjectDisposedException("ObjectDisposed_StreamClosed"); if (!response.Body.CanRead && !response.Body.CanWrite) throw new ObjectDisposedException("ObjectDisposed_StreamClosed"); if (!responseStream.CanRead) throw new NotSupportedException("NotSupported_UnreadableStream"); if (!response.Body.CanWrite) throw new NotSupportedException("NotSupported_UnwritableStream"); byte[] buffer = ArrayPool.Shared.Rent(PoolInvk.rentChunk); try { int bytesRead; while ((bytesRead = await responseStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) != 0) await response.Body.WriteAsync(buffer, 0, bytesRead, cancellationToken).ConfigureAwait(false); } finally { ArrayPool.Shared.Return(buffer); } } } #endregion } }