#!/usr/bin/env python3 import asyncio from tracemalloc import start import aiohttp from aiohttp import request, web import sys import json import os import traceback from sortedcontainers import SortedDict from throttler import DownloadSpeedLimiter CHUNK_SIZE = 1024 * 1024 # Default chunk size: 1MB cache_path = "cache" if not os.path.exists(cache_path): os.makedirs(cache_path) class Chunk: def __init__(self, path, offset, data = None): self.path = path self.offset = offset self.file = os.path.join(cache_path, path, str(offset).zfill(15)) if data is not None: with open(self.file, "wb") as f: f.write(data) self._len = os.path.getsize(self.file) def len(self): return self._len def data(self, start = 0, size = -1): with open(self.file, "rb") as f: f.seek(start) dt = f.read(min(self.len() - start, size) if size >= 0 else None) return dt def append(self, data): with open(self.file, "r+b") as f: f.seek(0, os.SEEK_END) f.write(data) f.flush() self._len = os.path.getsize(self.file) def __repr__(self): return f"Chunk(offset={self.offset}, file={self.file}, length={self.len()})" priority = {} async def stats_handler(request): items = await response_cache.allItems() cache_dump = {} for k, v in items.items(): cache_dump[str(k)] = { 'key': str(v.key), 'len': f"{v.len:_}", 'chd': f"{sum(i.len() for i in v.chunks.values()):_}", 'url': v.url, 'headers': dict(v.headers), 'params': dict(v.params), 'method': v.method, 'cachePath': v.cachePath, 'chunks': f"{len(v.chunks.keys()):_}", } stats = { 'size': { 'total_size': f"{sum([v.len for k,v in items.items()]):_}", 'chunks_size': f"{sum([v1.len() for k,v in items.items() for v1 in v.chunks.values()]):_}", 'chunks_cnt': f"{sum([len(v.chunks) for k,v in items.items()]):_}", 'total_cnt': f"{sum([1 for k,v in items.items()]):_}", 'download_speed': f"{limiter._estimated_speed:_}" }, 'priority': {str(k): v for k, v in priority.items()}, 'items': cache_dump, } return web.Response( text=json.dumps(stats, indent=2, ensure_ascii=False), content_type='application/json' ) class CacheEntry: def __init__(self, key, headReponseHeaders, url, headers, params, method): self.lock = asyncio.Lock() self.key = key self.initialResponseHeaders = headReponseHeaders self.len = int(headReponseHeaders['Content-Length']) self.url = url self.headers = headers self.params = params self.method = method self.cachePath = (self.url + self.initialResponseHeaders['Etag'][:20]).replace("/", "_").replace(":", "_").replace("?", "_").replace("&", "_").replace("=", "_") os.makedirs(os.path.join(cache_path, self.cachePath), exist_ok=True) self.chunks: SortedDict[int, Chunk] = SortedDict({ int(l): Chunk(self.cachePath, int(l), None) for l in os.listdir(os.path.join(cache_path, self.cachePath)) }) # print(f"CacheEntry created for {url} with {self.chunks} chunks") async def set(self, offset, chunk: bytes): async with self.lock: self.chunks[offset] = Chunk(self.cachePath, offset, chunk) return self.chunks[offset] def get(self, offset): async def get_chunk_length(offset: int): while True: await asyncio.sleep(0) if offset >= self.len: break async with self.lock: found_index = self.chunks.bisect_right(offset) - 1 # print(f"Found index {found_index} for offset {offset}") found_chunk: Chunk = None if found_index >= 0: found_chunk = self.chunks[self.chunks.keys()[found_index]] # print(f"Found chunk {found_chunk}, {found_chunk.offset}, {found_chunk.len()}") if found_chunk is not None and found_chunk.offset + found_chunk.len() <= offset: found_chunk = None # found_key = max((k for k, c in self.chunks.items() if k <= offset and c.offset + c.len() > offset), default=None) # print(f"Raising priority {self.key}, {offset}") if found_chunk is None: # print(f"Waiting {offset}") # log = {k: (c.offset, c.offset + c.len()) for k, c in self.chunks.items()} # print(f"Chunk keys {log}") await asyncio.sleep(1) yield b"" continue dt = found_chunk.data(offset - found_chunk.offset, CHUNK_SIZE) if len(dt) == 0: raise ValueError("Empty chunk data") yield dt offset += len(dt) return get_chunk_length(offset) class Cache: def __init__(self): self.store: dict[any, CacheEntry] = {} self.lock = asyncio.Lock() async def allItems(self) -> dict[any, CacheEntry]: async with self.lock: return dict(self.store) async def getOrCreate(self, key, headResponseHeaders, method, url, headers, params) -> CacheEntry: async with self.lock: if key not in self.store: self.store[key] = CacheEntry(key, headReponseHeaders=headResponseHeaders, method=method, url=url, headers=headers, params=params) return self.store.get(key) async def mergeAnyTwo(self): async with self.lock: for entry in self.store.values(): async with entry.lock: keysI = list(entry.chunks.keys()) for keyI in range(len(keysI) - 1): chunk1:Chunk = entry.chunks[keysI[keyI]] chunk2:Chunk = entry.chunks[keysI[keyI+1]] if chunk2 is None: print(f"No chunk2 for {keysI[keyI]} in {entry.key}") continue if chunk1.offset >= chunk2.offset: print(f"Chunk1 {chunk1} is not before Chunk2 {chunk2} in {entry.key}") continue if chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len(): print(f"Chunk1 {chunk1} already covers Chunk2 {chunk2} in {entry.key}: {chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len()}") continue if chunk1.offset + chunk1.len() < chunk2.offset: # print(f"Chunk1 {chunk1} has gap after Chunk2 {chunk2} in {entry.key}: {chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len()}") continue offset1 = chunk1.offset + chunk1.len() - chunk2.offset while offset1 < chunk2.len(): print(f"Merging {chunk1}+{chunk2}[{offset1}:] from {entry.key}") chunk1.append(chunk2.data(offset1, CHUNK_SIZE)) offset1 += CHUNK_SIZE del entry.chunks[chunk2.offset] os.remove(chunk2.file) return True return False response_cache = Cache() async def simple_proxy_handler(request): backend_url = request.app['backend_url'] headers = dict(request.headers) params = request.rel_url.query path = request.rel_url.path method = request.method body = await request.read() async with aiohttp.ClientSession() as session: async with session.request(method, backend_url + path, headers=headers, params=params, data=body) as resp: # print("Response status:", resp.headers) response = web.StreamResponse(status=resp.status, headers=resp.headers) await response.prepare(request) async for chunk in resp.content.iter_chunked(CHUNK_SIZE): await response.write(chunk) await response.write_eof() return response def find_hole(entry: CacheEntry, start_offset=0): chunks = entry.chunks if len(chunks) == 0: return (0, entry.len) # TODO Lock with merger keys = chunks.bisect_right(start_offset) - 1 keys = list(chunks.islice(keys, len(chunks))) for i in range(len(keys) - 1): if keys[i + 1] < start_offset: continue curr: Chunk = chunks[keys[i]] if curr.offset + curr.len() < keys[i + 1]: return (curr.offset + curr.len(), keys[i + 1]) if keys[-1] + chunks[keys[-1]].len() < entry.len: return (keys[-1] + chunks[keys[-1]].len(), entry.len) return (None, None) async def merger(): while True: try: if await stream_lock.get() > 0: await asyncio.sleep(30) continue cnt = 0 if await response_cache.mergeAnyTwo() and cnt < 1000: cnt += 1 if cnt > 0: await asyncio.sleep(0.1) continue await asyncio.sleep(30) except Exception as e: print(f"merger error: {e}", file=sys.stderr) traceback.print_exc() async def verifirer(): # TODO # TODO Locks pass limiter = DownloadSpeedLimiter(200_000) async def downloader(): global priority while True: try: async with aiohttp.ClientSession() as session: while True: await asyncio.sleep(0) print(f"Checking for downloads...") keys = await response_cache.allItems() queuekeys = [(k, v, priority[k], find_hole(v, priority[k])[0]) for k, v in keys.items() if k in priority.keys()] queuekeys = [x for x in queuekeys if x[3] is not None] queuekeys.sort(key=lambda x: x[3]) if limiter.max_speed == 0 and len(queuekeys) == 0: print(f"Download disabled, sleeping") limiter._estimated_speed = 0 await asyncio.sleep(10) continue queuekeys += [(k, v, None, find_hole(v, 0)[0]) for k, v in keys.items()] queuekeys = [(k,v,p,h) for k,v,p,h in queuekeys if h is not None] if len(queuekeys) == 0: print(f"Nothing to download, sleeping") await asyncio.sleep(10) continue priority = {} for key, cached, startFrom, h in queuekeys: startOffset = None endOffset = None if startFrom is not None: (startOffset, endOffset) = find_hole(cached, startFrom) if startOffset is not None: startOffset = max(startOffset, startFrom) endOffset = startOffset + CHUNK_SIZE * 10 if startOffset is None: (startOffset, endOffset) = find_hole(cached) if startOffset is None: continue endOffset = min(endOffset, startOffset + CHUNK_SIZE) print(f"Downloading startOffset {startOffset}..{endOffset} for key {key} ({startOffset * 100.0/cached.len}%)") rangeHeader = {"Range": f"bytes={startOffset}-{endOffset-1}"} headers_with_range = dict(cached.headers) headers_with_range.update(rangeHeader) async with session.request(cached.method, cached.url, headers=headers_with_range, params=cached.params) as resp: # Stream response in chunks # print(f"Response headers: {resp.headers}") if "Content-Range" not in resp.headers: raise ValueError("No Content-Range in response for Range request") if "bytes" not in resp.headers["Content-Range"]: raise ValueError("Only bytes Content-Range supported") (start_end, size) = resp.headers["Content-Range"].replace("bytes ", "").split("/") if start_end == "*": (start, end) = (0, size - 1) else: (start, end) = [int(i) for i in start_end.split("-")] # print(f"Content-Range: {start}-{end}/{size}") # 1newChunk: Chunk = None receivedBytes: bytes = b"" receivedBytesTotal = 0 async for receivedBytesLoc in resp.content.iter_chunked(CHUNK_SIZE): receivedBytes += receivedBytesLoc receivedBytesTotal += len(receivedBytesLoc) await cached.set(start, receivedBytes) # 2 limiter.consumed(receivedBytesTotal) delay = limiter.delay() print(f"Sleep {delay}, {receivedBytesTotal}, est {limiter._estimated_speed} bytes/second") if startFrom is None: await asyncio.sleep(delay) else: print(f"Skip sleep for priority download") await asyncio.sleep(0) print(f"Downloading done") break except Exception as e: print(f"downloader error: {e}", file=sys.stderr) await asyncio.sleep(10) traceback.print_exc() # Python does not have a built-in atomic integer type. # For atomic operations, you can use threading.Lock or asyncio.Lock to protect access to an integer. # Alternatively, you can use multiprocessing.Value for atomicity across processes. # For asyncio, you can implement a simple atomic integer like this: class AtomicInteger: def __init__(self, initial=0): self._value = initial self._lock = asyncio.Lock() async def get(self): async with self._lock: return self._value async def set(self, value): async with self._lock: self._value = value async def increment(self, delta=1): async with self._lock: self._value += delta return self._value async def decrement(self, delta=1): async with self._lock: self._value -= delta return self._value stream_lock = AtomicInteger() async def proxy_handler(request): try: backend_url = request.app['backend_url'] headers = dict(request.headers) params = request.rel_url.query path = request.rel_url.path method = request.method body = await request.read() # if "Range" not in headers.keys(): if "stream" not in path or "m3u" in path: # print(f"Simple handler {method}, {path}, {tuple(sorted(params.items()))}, {body}, {headers}") try: return await simple_proxy_handler(request) except Exception as e: print(f"simple_proxy_handler error: {e}", file=sys.stderr) # traceback.print_exc() return web.Response(status=500, text=f"Internal Server Error: {e}") if "Range" not in headers.keys(): headers["Range"] = "bytes=0-" range = headers.get("Range") if range is not None: if "bytes" not in range: raise ValueError("Only bytes range supported") range = range.replace("bytes=", "") def to_int_or_none(s): try: return int(s) except ValueError: return None range = [to_int_or_none(i) for r in range.split(",") for i in r.split("-") ] # Make a HEAD request to get resource info async with aiohttp.ClientSession() as session: head_headers = dict(headers) head_headers.pop("Range", None) head_resp = await session.head(backend_url + path, headers=head_headers, params=params) # print(f"HEAD response status: {head_resp.status}") # print(f"HEAD response headers: {head_resp.headers}") headHeaders = head_resp.headers if head_resp.status != 200: return web.Response(status=head_resp.status, text=f"Upstream server returned status {head_resp.status}") cache_key = (method, path, (headHeaders['Etag'][:20].__hash__())) # cache_key = (method, path, (tuple(sorted(params.items())), body, tuple(sorted(headHeaders.items()))).__hash__()) print(f"Cache key: {cache_key}") # print(f"Range requested: {range}") cached = await response_cache.getOrCreate(cache_key, headHeaders, method=method, url = backend_url + path, headers=head_headers, params=params) start = (range and range[0]) or 0 headHeaders = dict(headHeaders) if range is not None: end = range[1] if len(range) > 1 and range[1] is not None else cached.len - 1 content_range_header = f"bytes {start}-{end}/{cached.len}" headHeaders["Content-Range"] = content_range_header headHeaders["Content-Length"] = f"{end - start + 1}" headHeaders["Accept-Ranges"] = 'bytes' # print(f"Serving with headers: {headHeaders}") stream_lock.increment() try: response = web.StreamResponse(status=206, headers=headHeaders) await response.prepare(request) # print("Sending chunks") stream = cached.get(start) leftBytes = (end - start + 1) if range is not None else cached.len - start async for chunk in stream: if 'NoPriority' not in headers.keys(): priority[cache_key] = start await response.write(chunk[:min(len(chunk), leftBytes)]) leftBytes -= len(chunk) # print(f"Sending chunks {cache_key} {len(chunk)}") if leftBytes <= 0: break await asyncio.sleep(0) await response.write_eof() finally: stream_lock.decrement() return response except aiohttp.ClientConnectionResetError as e: print(f"ClientConnectionResetError: {e}", file=sys.stderr) return web.Response(status=500, text=f"Internal Server Error: {e}") # except aiohttp.ConnectionResetError as e: # print(f"ConnectionResetError: {e}", file=sys.stderr) # return web.Response(status=500, text=f"Internal Server Error: {e}") except Exception as e: print(f"proxy_handler error: {e}", file=sys.stderr) traceback.print_exc() return web.Response(status=500, text=f"Internal Server Error: {e}") async def server(app): runner = web.AppRunner(app) await runner.setup() site = web.TCPSite(runner, '0.0.0.0', 8080) await site.start() async def init_m3u(request): from m3u_download import process_m3u,get_m3u_urls, download_head while await stream_lock.get() > 0: await asyncio.sleep(0.1) tail = request.match_info['tail'] params = request.rel_url.query_string url = f"http://0.0.0.0:8080/{tail}" if params: url += f"?{params}" success = [] async with aiohttp.ClientSession() as session: urls = await get_m3u_urls(url, session=session) for url in urls: print(f" URL: {url}") if await download_head(session, url): print(f" URL ok: {url}") success.append(url) print(f"Processed {len(success)} URLs") return web.Response(text=f"OK\nProcessed {len(success)} URLs\n", content_type='text/plain') def main(): if len(sys.argv) < 2: backend_url = "http://95.142.46.84:5665" else: backend_url = sys.argv[1] app = web.Application() app['backend_url'] = backend_url app.router.add_get('/stats', stats_handler) # http://192.168.1.96:8080/add_m3u/playlistall/all.m3u # http://192.168.1.96:8080/add_m3u/stream/Squid%20Game%20S03%20%5B1080p%5D.m3u?link=732f0756f5d6304adfa8532a8c1f6ddd3a753ac6&m3u # http://192.168.1.96:8080/add_m3u/stream/Tears%20of%20Steel.m3u?link=209c8226b299b308beaf2b9cd3fb49212dbd13ec&m3u app.router.add_get('/add_m3u/{tail:.*}', init_m3u) app.router.add_route('*', '/{tail:.*}', proxy_handler) loop = asyncio.new_event_loop() loop.create_task(downloader()) loop.create_task(merger()) loop.create_task(server(app)) loop.run_forever() if __name__ == "__main__": main()