commit a15d15d6b0bc1d0e681ca41b0e71ae2772b4ba38 Author: rudolf Date: Wed Sep 24 22:40:21 2025 +0200 reverseproxy diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dd989cb --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +venv +.venv +cache +__pycache__ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..efa79f9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.11-slim +WORKDIR /app + +EXPOSE 8080 + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD ["python", "proxy.py", "http://backend:8000"] diff --git a/TEST.md b/TEST.md new file mode 100644 index 0000000..5b087cb --- /dev/null +++ b/TEST.md @@ -0,0 +1,31 @@ +dd if=orig.mkv bs=1 skip=0 count=13484 status=none | md5sum +dd if=orig.mkv bs=1 skip=13484 count=1424 status=none | md5sum +dd if=orig.mkv bs=1 skip=14908 count=4272 status=none | md5sum +dd if=orig.mkv bs=1 skip=19180 count=2848 status=none | md5sum +dd if=orig.mkv bs=1 skip=22028 count=11392 status=none | md5sum +dd if=orig.mkv bs=1 skip=33420 count=2848 status=none | md5sum +dd if=orig.mkv bs=1 skip=36268 count=4272 status=none | md5sum +md5sum 653134336/0000000000 +md5sum 653134336/0000013484 +md5sum 653134336/0000014908 +md5sum 653134336/0000019180 +md5sum 653134336/0000022028 +md5sum 653134336/0000033420 +md5sum 653134336/0000036268 + +bffeb107d791300457e1c9f0d4280d86 - +b70c3b6a906eaaf8701969869591c79b - +06e1e3fd434112d3c810cba2d6798d4b - +c2806cb0cbb566a4936841e46b6f29b2 - +88cae8a40480453c26f14afb29153589 - +584684b9bf0a4d39a78b342db6d8bc64 - +f7e7dac5dcea561b801750e69e29a296 - +bffeb107d791300457e1c9f0d4280d86 653134336/0000000000 +b70c3b6a906eaaf8701969869591c79b 653134336/0000013484 +06e1e3fd434112d3c810cba2d6798d4b 653134336/0000014908 +c2806cb0cbb566a4936841e46b6f29b2 653134336/0000019180 +88cae8a40480453c26f14afb29153589 653134336/0000022028 +584684b9bf0a4d39a78b342db6d8bc64 653134336/0000033420 +f7e7dac5dcea561b801750e69e29a296 653134336/0000036268 + +OK \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..47b7362 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,27 @@ +version: '3.8' +services: + reverseproxy: + build: . + image: registry.rudolf.keenetic.link/torr_cache:${version} + volumes: + - data:/app/cache:rw + ports: + - "8080:8080" + environment: + - PYTHONUNBUFFERED=1 + command: ["python", "proxy.py", "http://95.142.46.84:5665"] + fileserver: + image: python:3.11-slim + working_dir: /data + volumes: + - data:/data:ro + command: ["python3", "-m", "http.server", "8888"] + ports: + - "8888:8888" +volumes: + data: + driver: local + driver_opts: + type: nfs + o: addr=192.168.1.96 + device: :/media/DATA/runtime/torrCacher diff --git a/m3u_download.py b/m3u_download.py new file mode 100755 index 0000000..d32493e --- /dev/null +++ b/m3u_download.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 + +import os +import sys +import asyncio +import traceback +import aiohttp +import aiofiles +import urllib.parse +from urllib.parse import urljoin, urlparse +from tqdm.asyncio import tqdm + + +def is_url(string): + """Check if the provided string is a URL.""" + try: + result = urlparse(string) + return all([result.scheme, result.netloc]) + except: + return False + +async def download_head(session, url): + """Download a file from URL to the specified destination asynchronously. + If head is a number, download only the first head bytes. + """ + try: + print(f"Downloading: {url} (HEAD)") + headers = {} + headers['Range'] = f'bytes=0-100' + headers['NoPriority'] = f'NoPriority' + async with session.get(url, headers=headers) as response: + response.raise_for_status() + total_size = int(response.headers.get('content-length', 0)) + downloaded = 0 + async for chunk in response.content.iter_chunked(8192): + if chunk: + downloaded += len(chunk) + return True + except Exception as e: + print(f"Error downloading {url}: {str(e)}") + return False + + + +async def download_file(session, url, destination): + """Download a file from URL to the specified destination asynchronously.""" + try: + if destination is not None: + os.makedirs(os.path.dirname(destination), exist_ok=True) + if os.path.exists(destination): + print(f"File already exists: {destination}") + return True + print(f"Downloading: {url} -> {destination}") + async with session.get(url) as response: + response.raise_for_status() + total_size = int(response.headers.get('content-length', 0)) + print("Len ", total_size) + + block_size = 8192 + if destination is not None: + async with aiofiles.open(destination, 'wb') as f: + pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc=os.path.basename(destination)) + async for chunk in response.content.iter_chunked(block_size): + if chunk: + await f.write(chunk) + pbar.update(len(chunk)) + pbar.close() + else: + pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc="To None") + pbar.update(0) + async for chunk in response.content.iter_chunked(block_size): + if chunk: + pbar.update(len(chunk)) + pbar.close() + return True + except Exception as e: + print(f"Error downloading {url}: {str(e)}") + return False + + +async def get_m3u_urls(m3u_url, session, visited=None): + """Recursively collect all file URLs from an M3U playlist.""" + if visited is None: + visited = set() + urls = [] + print(f"Adding m3u {m3u_url}") + try: + async with session.get(m3u_url) as response: + response.raise_for_status() + content = await response.text() + if "#EXTM3U" not in content: + raise ValueError("The provided file is not a valid M3U playlist.") + base_url = m3u_url.rsplit('/', 1)[0] + '/' + for line in content.splitlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + file_url = line if is_url(line) else urljoin(base_url, line) + path_part = urlparse(file_url).path + if '.m3u' in path_part and file_url not in visited: + visited.add(file_url) + nested_urls = await get_m3u_urls(file_url, session=session, visited=visited) + urls.extend(nested_urls) + continue + urls.append(file_url) + return urls + except Exception as e: + print(f"Error processing M3U file: {str(e)}") + traceback.print_exc() + return [] + + +async def process_m3u(m3u_url, output_dir, session=None): + """Process an M3U file: get all URLs and download them.""" + with aiohttp.ClientSession() as session: + urls = await get_m3u_urls(m3u_url, session=session) + result = [] + if session is None: + session = aiohttp.ClientSession() + for file_url in urls: + try: + local_path = None + if output_dir is not None: + path_part = urlparse(file_url).path + if path_part.startswith('/'): + path_part = path_part[1:] + path_part = urllib.parse.unquote(path_part) + local_path = None if output_dir is None else os.path.join(output_dir, path_part) + await download_file(session, file_url, local_path) + except Exception as e: + print(f"Error downloading URLs: {str(e)}") + traceback.print_exc() + return False + + return result + + +if __name__ == "__main__": + m3u = "http://192.168.1.96:8080/playlistall/all.m3u" + if len(sys.argv) < 2: + m3u_url = m3u + else: + m3u_url = sys.argv[1] + output_dir = sys.argv[2] if len(sys.argv) > 2 else None + print(f"Downloading content from {m3u_url} to {output_dir}") + asyncio.run(process_m3u(m3u_url, output_dir)) diff --git a/proxy.py b/proxy.py new file mode 100644 index 0000000..963a32a --- /dev/null +++ b/proxy.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +import asyncio +from tracemalloc import start +import aiohttp +from aiohttp import request, web +import sys +import json +import os +import traceback +from sortedcontainers import SortedDict + +from throttler import DownloadSpeedLimiter + +CHUNK_SIZE = 1024 * 1024 # Default chunk size: 1MB + + +cache_path = "cache" +if not os.path.exists(cache_path): + os.makedirs(cache_path) +class Chunk: + def __init__(self, path, offset, data = None): + self.path = path + self.offset = offset + self.file = os.path.join(cache_path, path, str(offset).zfill(15)) + if data is not None: + with open(self.file, "wb") as f: + f.write(data) + self._len = os.path.getsize(self.file) + def len(self): + return self._len + + def data(self, start = 0, size = -1): + with open(self.file, "rb") as f: + f.seek(start) + dt = f.read(min(self.len() - start, size) if size >= 0 else None) + return dt + def append(self, data): + with open(self.file, "r+b") as f: + f.seek(0, os.SEEK_END) + f.write(data) + f.flush() + self._len = os.path.getsize(self.file) + + def __repr__(self): + return f"Chunk(offset={self.offset}, file={self.file}, length={self.len()})" + + +priority = {} +async def stats_handler(request): + items = await response_cache.allItems() + cache_dump = {} + for k, v in items.items(): + cache_dump[str(k)] = { + 'key': str(v.key), + 'len': f"{v.len:_}", + 'chd': f"{sum(i.len() for i in v.chunks.values()):_}", + 'url': v.url, + 'headers': dict(v.headers), + 'params': dict(v.params), + 'method': v.method, + 'cachePath': v.cachePath, + 'chunks': f"{len(v.chunks.keys()):_}", + } + stats = { + 'size': { + 'total_size': f"{sum([v.len for k,v in items.items()]):_}", + 'chunks_size': f"{sum([v1.len() for k,v in items.items() for v1 in v.chunks.values()]):_}", + 'chunks_cnt': f"{sum([len(v.chunks) for k,v in items.items()]):_}", + 'total_cnt': f"{sum([1 for k,v in items.items()]):_}", + 'download_speed': f"{limiter._estimated_speed:_}" + }, + 'priority': {str(k): v for k, v in priority.items()}, + 'items': cache_dump, + } + return web.Response( + text=json.dumps(stats, indent=2, ensure_ascii=False), + content_type='application/json' + ) + +class CacheEntry: + def __init__(self, key, headReponseHeaders, url, headers, params, method): + self.lock = asyncio.Lock() + self.key = key + self.initialResponseHeaders = headReponseHeaders + + self.len = int(headReponseHeaders['Content-Length']) + self.url = url + self.headers = headers + self.params = params + self.method = method + self.cachePath = (self.url + self.initialResponseHeaders['Etag'][:20]).replace("/", "_").replace(":", "_").replace("?", "_").replace("&", "_").replace("=", "_") + os.makedirs(os.path.join(cache_path, self.cachePath), exist_ok=True) + self.chunks: SortedDict[int, Chunk] = SortedDict({ + int(l): Chunk(self.cachePath, int(l), None) + for l in os.listdir(os.path.join(cache_path, self.cachePath)) + }) + # print(f"CacheEntry created for {url} with {self.chunks} chunks") + + async def set(self, offset, chunk: bytes): + async with self.lock: + self.chunks[offset] = Chunk(self.cachePath, offset, chunk) + return self.chunks[offset] + + def get(self, offset): + async def get_chunk_length(offset: int): + while True: + await asyncio.sleep(0) + if offset >= self.len: + break + async with self.lock: + found_index = self.chunks.bisect_right(offset) - 1 + # print(f"Found index {found_index} for offset {offset}") + found_chunk: Chunk = None + if found_index >= 0: + found_chunk = self.chunks[self.chunks.keys()[found_index]] + # print(f"Found chunk {found_chunk}, {found_chunk.offset}, {found_chunk.len()}") + if found_chunk is not None and found_chunk.offset + found_chunk.len() <= offset: + found_chunk = None + + # found_key = max((k for k, c in self.chunks.items() if k <= offset and c.offset + c.len() > offset), default=None) + # print(f"Raising priority {self.key}, {offset}") + if found_chunk is None: + # print(f"Waiting {offset}") + # log = {k: (c.offset, c.offset + c.len()) for k, c in self.chunks.items()} + # print(f"Chunk keys {log}") + await asyncio.sleep(1) + yield b"" + continue + + dt = found_chunk.data(offset - found_chunk.offset, CHUNK_SIZE) + if len(dt) == 0: + raise ValueError("Empty chunk data") + yield dt + offset += len(dt) + + return get_chunk_length(offset) + +class Cache: + def __init__(self): + self.store: dict[any, CacheEntry] = {} + self.lock = asyncio.Lock() + + async def allItems(self) -> dict[any, CacheEntry]: + async with self.lock: + return dict(self.store) + + async def getOrCreate(self, key, headResponseHeaders, method, url, headers, params) -> CacheEntry: + async with self.lock: + if key not in self.store: + self.store[key] = CacheEntry(key, headReponseHeaders=headResponseHeaders, method=method, url=url, headers=headers, params=params) + return self.store.get(key) + async def mergeAnyTwo(self): + async with self.lock: + for entry in self.store.values(): + async with entry.lock: + keysI = list(entry.chunks.keys()) + for keyI in range(len(keysI) - 1): + chunk1:Chunk = entry.chunks[keysI[keyI]] + chunk2:Chunk = entry.chunks[keysI[keyI+1]] + if chunk2 is None: + print(f"No chunk2 for {keysI[keyI]} in {entry.key}") + continue + if chunk1.offset >= chunk2.offset: + print(f"Chunk1 {chunk1} is not before Chunk2 {chunk2} in {entry.key}") + continue + if chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len(): + print(f"Chunk1 {chunk1} already covers Chunk2 {chunk2} in {entry.key}: {chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len()}") + continue + if chunk1.offset + chunk1.len() < chunk2.offset: + # print(f"Chunk1 {chunk1} has gap after Chunk2 {chunk2} in {entry.key}: {chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len()}") + continue + offset1 = chunk1.offset + chunk1.len() - chunk2.offset + while offset1 < chunk2.len(): + print(f"Merging {chunk1}+{chunk2}[{offset1}:] from {entry.key}") + chunk1.append(chunk2.data(offset1, CHUNK_SIZE)) + offset1 += CHUNK_SIZE + del entry.chunks[chunk2.offset] + os.remove(chunk2.file) + return True + return False + +response_cache = Cache() + +async def simple_proxy_handler(request): + backend_url = request.app['backend_url'] + headers = dict(request.headers) + params = request.rel_url.query + path = request.rel_url.path + method = request.method + body = await request.read() + + async with aiohttp.ClientSession() as session: + async with session.request(method, backend_url + path, headers=headers, params=params, data=body) as resp: + # print("Response status:", resp.headers) + response = web.StreamResponse(status=resp.status, headers=resp.headers) + await response.prepare(request) + async for chunk in resp.content.iter_chunked(CHUNK_SIZE): + await response.write(chunk) + await response.write_eof() + return response + +def find_hole(entry: CacheEntry, start_offset=0): + chunks = entry.chunks + if len(chunks) == 0: + return (0, entry.len) + # TODO Lock with merger + keys = chunks.bisect_right(start_offset) - 1 + keys = list(chunks.islice(keys, len(chunks))) + for i in range(len(keys) - 1): + if keys[i + 1] < start_offset: + continue + curr: Chunk = chunks[keys[i]] + if curr.offset + curr.len() < keys[i + 1]: + return (curr.offset + curr.len(), keys[i + 1]) + if keys[-1] + chunks[keys[-1]].len() < entry.len: + return (keys[-1] + chunks[keys[-1]].len(), entry.len) + return (None, None) + +async def merger(): + while True: + try: + if await stream_lock.get() > 0: + await asyncio.sleep(30) + continue + cnt = 0 + if await response_cache.mergeAnyTwo() and cnt < 1000: + cnt += 1 + if cnt > 0: + await asyncio.sleep(0.1) + continue + await asyncio.sleep(30) + except Exception as e: + print(f"merger error: {e}", file=sys.stderr) + traceback.print_exc() + +async def verifirer(): + # TODO + # TODO Locks + pass + +limiter = DownloadSpeedLimiter(200_000) + +async def downloader(): + global priority + while True: + try: + async with aiohttp.ClientSession() as session: + while True: + await asyncio.sleep(0) + print(f"Checking for downloads...") + keys = await response_cache.allItems() + queuekeys = [(k, v, priority[k], find_hole(v, priority[k])[0]) for k, v in keys.items() if k in priority.keys()] + queuekeys = [x for x in queuekeys if x[3] is not None] + queuekeys.sort(key=lambda x: x[3]) + if limiter.max_speed == 0 and len(queuekeys) == 0: + print(f"Download disabled, sleeping") + limiter._estimated_speed = 0 + await asyncio.sleep(10) + continue + + queuekeys += [(k, v, None, find_hole(v, 0)[0]) for k, v in keys.items()] + queuekeys = [(k,v,p,h) for k,v,p,h in queuekeys if h is not None] + + if len(queuekeys) == 0: + print(f"Nothing to download, sleeping") + await asyncio.sleep(10) + continue + priority = {} + for key, cached, startFrom, h in queuekeys: + startOffset = None + endOffset = None + if startFrom is not None: + (startOffset, endOffset) = find_hole(cached, startFrom) + if startOffset is not None: + startOffset = max(startOffset, startFrom) + endOffset = startOffset + CHUNK_SIZE * 10 + + if startOffset is None: + (startOffset, endOffset) = find_hole(cached) + + if startOffset is None: + continue + endOffset = min(endOffset, startOffset + CHUNK_SIZE) + + print(f"Downloading startOffset {startOffset}..{endOffset} for key {key} ({startOffset * 100.0/cached.len}%)") + rangeHeader = {"Range": f"bytes={startOffset}-{endOffset-1}"} + headers_with_range = dict(cached.headers) + headers_with_range.update(rangeHeader) + async with session.request(cached.method, cached.url, headers=headers_with_range, params=cached.params) as resp: + + # Stream response in chunks + # print(f"Response headers: {resp.headers}") + if "Content-Range" not in resp.headers: + raise ValueError("No Content-Range in response for Range request") + if "bytes" not in resp.headers["Content-Range"]: + raise ValueError("Only bytes Content-Range supported") + (start_end, size) = resp.headers["Content-Range"].replace("bytes ", "").split("/") + if start_end == "*": + (start, end) = (0, size - 1) + else: + (start, end) = [int(i) for i in start_end.split("-")] + # print(f"Content-Range: {start}-{end}/{size}") + # 1newChunk: Chunk = None + receivedBytes: bytes = b"" + receivedBytesTotal = 0 + + async for receivedBytesLoc in resp.content.iter_chunked(CHUNK_SIZE): + receivedBytes += receivedBytesLoc + receivedBytesTotal += len(receivedBytesLoc) + + await cached.set(start, receivedBytes) # 2 + limiter.consumed(receivedBytesTotal) + delay = limiter.delay() + print(f"Sleep {delay}, {receivedBytesTotal}, est {limiter._estimated_speed} bytes/second") + if startFrom is None: + await asyncio.sleep(delay) + else: + print(f"Skip sleep for priority download") + await asyncio.sleep(0) + + print(f"Downloading done") + break + except Exception as e: + print(f"downloader error: {e}", file=sys.stderr) + await asyncio.sleep(10) + traceback.print_exc() + +# Python does not have a built-in atomic integer type. +# For atomic operations, you can use threading.Lock or asyncio.Lock to protect access to an integer. +# Alternatively, you can use multiprocessing.Value for atomicity across processes. +# For asyncio, you can implement a simple atomic integer like this: + +class AtomicInteger: + def __init__(self, initial=0): + self._value = initial + self._lock = asyncio.Lock() + + async def get(self): + async with self._lock: + return self._value + + async def set(self, value): + async with self._lock: + self._value = value + + async def increment(self, delta=1): + async with self._lock: + self._value += delta + return self._value + + async def decrement(self, delta=1): + async with self._lock: + self._value -= delta + return self._value +stream_lock = AtomicInteger() +async def proxy_handler(request): + try: + backend_url = request.app['backend_url'] + headers = dict(request.headers) + params = request.rel_url.query + path = request.rel_url.path + method = request.method + body = await request.read() + # if "Range" not in headers.keys(): + if "stream" not in path or "m3u" in path: + # print(f"Simple handler {method}, {path}, {tuple(sorted(params.items()))}, {body}, {headers}") + try: + return await simple_proxy_handler(request) + except Exception as e: + print(f"simple_proxy_handler error: {e}", file=sys.stderr) + # traceback.print_exc() + return web.Response(status=500, text=f"Internal Server Error: {e}") + + if "Range" not in headers.keys(): + headers["Range"] = "bytes=0-" + + range = headers.get("Range") + if range is not None: + if "bytes" not in range: + raise ValueError("Only bytes range supported") + range = range.replace("bytes=", "") + def to_int_or_none(s): + try: + return int(s) + except ValueError: + return None + range = [to_int_or_none(i) for r in range.split(",") for i in r.split("-") ] + + # Make a HEAD request to get resource info + async with aiohttp.ClientSession() as session: + head_headers = dict(headers) + head_headers.pop("Range", None) + head_resp = await session.head(backend_url + path, headers=head_headers, params=params) + # print(f"HEAD response status: {head_resp.status}") + # print(f"HEAD response headers: {head_resp.headers}") + headHeaders = head_resp.headers + if head_resp.status != 200: + return web.Response(status=head_resp.status, text=f"Upstream server returned status {head_resp.status}") + + cache_key = (method, path, (headHeaders['Etag'][:20].__hash__())) + # cache_key = (method, path, (tuple(sorted(params.items())), body, tuple(sorted(headHeaders.items()))).__hash__()) + print(f"Cache key: {cache_key}") + # print(f"Range requested: {range}") + cached = await response_cache.getOrCreate(cache_key, headHeaders, method=method, url = backend_url + path, headers=head_headers, params=params) + + start = (range and range[0]) or 0 + headHeaders = dict(headHeaders) + if range is not None: + end = range[1] if len(range) > 1 and range[1] is not None else cached.len - 1 + content_range_header = f"bytes {start}-{end}/{cached.len}" + headHeaders["Content-Range"] = content_range_header + headHeaders["Content-Length"] = f"{end - start + 1}" + headHeaders["Accept-Ranges"] = 'bytes' + # print(f"Serving with headers: {headHeaders}") + stream_lock.increment() + try: + response = web.StreamResponse(status=206, headers=headHeaders) + await response.prepare(request) + # print("Sending chunks") + stream = cached.get(start) + leftBytes = (end - start + 1) if range is not None else cached.len - start + async for chunk in stream: + if 'NoPriority' not in headers.keys(): + priority[cache_key] = start + await response.write(chunk[:min(len(chunk), leftBytes)]) + leftBytes -= len(chunk) + # print(f"Sending chunks {cache_key} {len(chunk)}") + if leftBytes <= 0: + break + await asyncio.sleep(0) + await response.write_eof() + finally: + stream_lock.decrement() + return response + + except aiohttp.ClientConnectionResetError as e: + print(f"ClientConnectionResetError: {e}", file=sys.stderr) + return web.Response(status=500, text=f"Internal Server Error: {e}") + # except aiohttp.ConnectionResetError as e: + # print(f"ConnectionResetError: {e}", file=sys.stderr) + # return web.Response(status=500, text=f"Internal Server Error: {e}") + except Exception as e: + print(f"proxy_handler error: {e}", file=sys.stderr) + traceback.print_exc() + return web.Response(status=500, text=f"Internal Server Error: {e}") + + +async def server(app): + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, '0.0.0.0', 8080) + await site.start() + +async def init_m3u(request): + from m3u_download import process_m3u,get_m3u_urls, download_head + while await stream_lock.get() > 0: + await asyncio.sleep(0.1) + tail = request.match_info['tail'] + params = request.rel_url.query_string + url = f"http://0.0.0.0:8080/{tail}" + if params: + url += f"?{params}" + + success = [] + async with aiohttp.ClientSession() as session: + urls = await get_m3u_urls(url, session=session) + for url in urls: + print(f" URL: {url}") + if await download_head(session, url): + print(f" URL ok: {url}") + success.append(url) + + print(f"Processed {len(success)} URLs") + return web.Response(text=f"OK\nProcessed {len(success)} URLs\n", content_type='text/plain') + + +def main(): + if len(sys.argv) < 2: + backend_url = "http://95.142.46.84:5665" + else: + backend_url = sys.argv[1] + app = web.Application() + app['backend_url'] = backend_url + app.router.add_get('/stats', stats_handler) + + # http://192.168.1.96:8080/add_m3u/playlistall/all.m3u + # http://192.168.1.96:8080/add_m3u/stream/Squid%20Game%20S03%20%5B1080p%5D.m3u?link=732f0756f5d6304adfa8532a8c1f6ddd3a753ac6&m3u + # http://192.168.1.96:8080/add_m3u/stream/Tears%20of%20Steel.m3u?link=209c8226b299b308beaf2b9cd3fb49212dbd13ec&m3u + app.router.add_get('/add_m3u/{tail:.*}', init_m3u) + app.router.add_route('*', '/{tail:.*}', proxy_handler) + loop = asyncio.new_event_loop() + loop.create_task(downloader()) + loop.create_task(merger()) + loop.create_task(server(app)) + + loop.run_forever() + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..142ae17 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +aiohttp +sortedcontainers +aiofiles +aiohttp +tqdm diff --git a/throttler.py b/throttler.py new file mode 100644 index 0000000..db65843 --- /dev/null +++ b/throttler.py @@ -0,0 +1,68 @@ +from random import randint +import time + +class DownloadSpeedLimiter: + def __init__(self, max_speed_bytes_per_sec): + self.max_speed = max_speed_bytes_per_sec + # self._start_time = time.time() + self._prev_time = time.time() - 10 + self._estimated_speed = 0 + self._ratio = 0.5 + self._delay = 0.0 + + def consumed(self, bytes_count: int): + now = time.time() + elapsed = now - self._prev_time + self._prev_time = now + self._estimated_speed = self._estimated_speed * self._ratio + (1.0 - self._ratio) * (bytes_count + 0.0)/elapsed + + def delay(self) -> float: + if self.max_speed == 0: + return 0.0 + newDelay = (self._estimated_speed - self.max_speed) / self.max_speed + self._delay = self._delay + 0.2 * newDelay + return max(0.0, self._delay) + + +if __name__ == "__main__": + start = time.time() + limiter = DownloadSpeedLimiter(2000) + counterStart = time.time() + consumed = 0 + def consume(number): + global consumed, counterStart + if time.time() - counterStart > 10: + consumed = 0 + counterStart = time.time() + else: + consumed += number + limiter.consumed(number) + delay = limiter.delay() + + print(f"Consumed {number} bytes, sleeping for {delay} seconds") + print(f"estimated speed {limiter._estimated_speed}") + print(f"internal delay {limiter._delay}") + print(f"Rate {consumed / (time.time() - counterStart)}") + time.sleep(delay) + + while time.time() - start < 10: + number = randint(1000, 2000) / 2 + time.sleep(0.01) + consume(number) + + print(f"Consuming zeros") + start = time.time() + while time.time() - start < 10: + number = 0 + time.sleep(0.01) + consume(number) + + print(f"Consuming 100") + start = time.time() + while time.time() - start < 10: + number = 10000 + time.sleep(0.01) + consume(number) + print(f"Consumed {consumed} bytes") + print(f"estimated speed {limiter._estimated_speed}") + print(f"Rate {consumed / (time.time() - counterStart)}") \ No newline at end of file