Files
2025-09-24 22:40:21 +02:00

501 lines
21 KiB
Python

#!/usr/bin/env python3
import asyncio
from tracemalloc import start
import aiohttp
from aiohttp import request, web
import sys
import json
import os
import traceback
from sortedcontainers import SortedDict
from throttler import DownloadSpeedLimiter
CHUNK_SIZE = 1024 * 1024 # Default chunk size: 1MB
cache_path = "cache"
if not os.path.exists(cache_path):
os.makedirs(cache_path)
class Chunk:
def __init__(self, path, offset, data = None):
self.path = path
self.offset = offset
self.file = os.path.join(cache_path, path, str(offset).zfill(15))
if data is not None:
with open(self.file, "wb") as f:
f.write(data)
self._len = os.path.getsize(self.file)
def len(self):
return self._len
def data(self, start = 0, size = -1):
with open(self.file, "rb") as f:
f.seek(start)
dt = f.read(min(self.len() - start, size) if size >= 0 else None)
return dt
def append(self, data):
with open(self.file, "r+b") as f:
f.seek(0, os.SEEK_END)
f.write(data)
f.flush()
self._len = os.path.getsize(self.file)
def __repr__(self):
return f"Chunk(offset={self.offset}, file={self.file}, length={self.len()})"
priority = {}
async def stats_handler(request):
items = await response_cache.allItems()
cache_dump = {}
for k, v in items.items():
cache_dump[str(k)] = {
'key': str(v.key),
'len': f"{v.len:_}",
'chd': f"{sum(i.len() for i in v.chunks.values()):_}",
'url': v.url,
'headers': dict(v.headers),
'params': dict(v.params),
'method': v.method,
'cachePath': v.cachePath,
'chunks': f"{len(v.chunks.keys()):_}",
}
stats = {
'size': {
'total_size': f"{sum([v.len for k,v in items.items()]):_}",
'chunks_size': f"{sum([v1.len() for k,v in items.items() for v1 in v.chunks.values()]):_}",
'chunks_cnt': f"{sum([len(v.chunks) for k,v in items.items()]):_}",
'total_cnt': f"{sum([1 for k,v in items.items()]):_}",
'download_speed': f"{limiter._estimated_speed:_}"
},
'priority': {str(k): v for k, v in priority.items()},
'items': cache_dump,
}
return web.Response(
text=json.dumps(stats, indent=2, ensure_ascii=False),
content_type='application/json'
)
class CacheEntry:
def __init__(self, key, headReponseHeaders, url, headers, params, method):
self.lock = asyncio.Lock()
self.key = key
self.initialResponseHeaders = headReponseHeaders
self.len = int(headReponseHeaders['Content-Length'])
self.url = url
self.headers = headers
self.params = params
self.method = method
self.cachePath = (self.url + self.initialResponseHeaders['Etag'][:20]).replace("/", "_").replace(":", "_").replace("?", "_").replace("&", "_").replace("=", "_")
os.makedirs(os.path.join(cache_path, self.cachePath), exist_ok=True)
self.chunks: SortedDict[int, Chunk] = SortedDict({
int(l): Chunk(self.cachePath, int(l), None)
for l in os.listdir(os.path.join(cache_path, self.cachePath))
})
# print(f"CacheEntry created for {url} with {self.chunks} chunks")
async def set(self, offset, chunk: bytes):
async with self.lock:
self.chunks[offset] = Chunk(self.cachePath, offset, chunk)
return self.chunks[offset]
def get(self, offset):
async def get_chunk_length(offset: int):
while True:
await asyncio.sleep(0)
if offset >= self.len:
break
async with self.lock:
found_index = self.chunks.bisect_right(offset) - 1
# print(f"Found index {found_index} for offset {offset}")
found_chunk: Chunk = None
if found_index >= 0:
found_chunk = self.chunks[self.chunks.keys()[found_index]]
# print(f"Found chunk {found_chunk}, {found_chunk.offset}, {found_chunk.len()}")
if found_chunk is not None and found_chunk.offset + found_chunk.len() <= offset:
found_chunk = None
# found_key = max((k for k, c in self.chunks.items() if k <= offset and c.offset + c.len() > offset), default=None)
# print(f"Raising priority {self.key}, {offset}")
if found_chunk is None:
# print(f"Waiting {offset}")
# log = {k: (c.offset, c.offset + c.len()) for k, c in self.chunks.items()}
# print(f"Chunk keys {log}")
await asyncio.sleep(1)
yield b""
continue
dt = found_chunk.data(offset - found_chunk.offset, CHUNK_SIZE)
if len(dt) == 0:
raise ValueError("Empty chunk data")
yield dt
offset += len(dt)
return get_chunk_length(offset)
class Cache:
def __init__(self):
self.store: dict[any, CacheEntry] = {}
self.lock = asyncio.Lock()
async def allItems(self) -> dict[any, CacheEntry]:
async with self.lock:
return dict(self.store)
async def getOrCreate(self, key, headResponseHeaders, method, url, headers, params) -> CacheEntry:
async with self.lock:
if key not in self.store:
self.store[key] = CacheEntry(key, headReponseHeaders=headResponseHeaders, method=method, url=url, headers=headers, params=params)
return self.store.get(key)
async def mergeAnyTwo(self):
async with self.lock:
for entry in self.store.values():
async with entry.lock:
keysI = list(entry.chunks.keys())
for keyI in range(len(keysI) - 1):
chunk1:Chunk = entry.chunks[keysI[keyI]]
chunk2:Chunk = entry.chunks[keysI[keyI+1]]
if chunk2 is None:
print(f"No chunk2 for {keysI[keyI]} in {entry.key}")
continue
if chunk1.offset >= chunk2.offset:
print(f"Chunk1 {chunk1} is not before Chunk2 {chunk2} in {entry.key}")
continue
if chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len():
print(f"Chunk1 {chunk1} already covers Chunk2 {chunk2} in {entry.key}: {chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len()}")
continue
if chunk1.offset + chunk1.len() < chunk2.offset:
# print(f"Chunk1 {chunk1} has gap after Chunk2 {chunk2} in {entry.key}: {chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len()}")
continue
offset1 = chunk1.offset + chunk1.len() - chunk2.offset
while offset1 < chunk2.len():
print(f"Merging {chunk1}+{chunk2}[{offset1}:] from {entry.key}")
chunk1.append(chunk2.data(offset1, CHUNK_SIZE))
offset1 += CHUNK_SIZE
del entry.chunks[chunk2.offset]
os.remove(chunk2.file)
return True
return False
response_cache = Cache()
async def simple_proxy_handler(request):
backend_url = request.app['backend_url']
headers = dict(request.headers)
params = request.rel_url.query
path = request.rel_url.path
method = request.method
body = await request.read()
async with aiohttp.ClientSession() as session:
async with session.request(method, backend_url + path, headers=headers, params=params, data=body) as resp:
# print("Response status:", resp.headers)
response = web.StreamResponse(status=resp.status, headers=resp.headers)
await response.prepare(request)
async for chunk in resp.content.iter_chunked(CHUNK_SIZE):
await response.write(chunk)
await response.write_eof()
return response
def find_hole(entry: CacheEntry, start_offset=0):
chunks = entry.chunks
if len(chunks) == 0:
return (0, entry.len)
# TODO Lock with merger
keys = chunks.bisect_right(start_offset) - 1
keys = list(chunks.islice(keys, len(chunks)))
for i in range(len(keys) - 1):
if keys[i + 1] < start_offset:
continue
curr: Chunk = chunks[keys[i]]
if curr.offset + curr.len() < keys[i + 1]:
return (curr.offset + curr.len(), keys[i + 1])
if keys[-1] + chunks[keys[-1]].len() < entry.len:
return (keys[-1] + chunks[keys[-1]].len(), entry.len)
return (None, None)
async def merger():
while True:
try:
if await stream_lock.get() > 0:
await asyncio.sleep(30)
continue
cnt = 0
if await response_cache.mergeAnyTwo() and cnt < 1000:
cnt += 1
if cnt > 0:
await asyncio.sleep(0.1)
continue
await asyncio.sleep(30)
except Exception as e:
print(f"merger error: {e}", file=sys.stderr)
traceback.print_exc()
async def verifirer():
# TODO
# TODO Locks
pass
limiter = DownloadSpeedLimiter(200_000)
async def downloader():
global priority
while True:
try:
async with aiohttp.ClientSession() as session:
while True:
await asyncio.sleep(0)
print(f"Checking for downloads...")
keys = await response_cache.allItems()
queuekeys = [(k, v, priority[k], find_hole(v, priority[k])[0]) for k, v in keys.items() if k in priority.keys()]
queuekeys = [x for x in queuekeys if x[3] is not None]
queuekeys.sort(key=lambda x: x[3])
if limiter.max_speed == 0 and len(queuekeys) == 0:
print(f"Download disabled, sleeping")
limiter._estimated_speed = 0
await asyncio.sleep(10)
continue
queuekeys += [(k, v, None, find_hole(v, 0)[0]) for k, v in keys.items()]
queuekeys = [(k,v,p,h) for k,v,p,h in queuekeys if h is not None]
if len(queuekeys) == 0:
print(f"Nothing to download, sleeping")
await asyncio.sleep(10)
continue
priority = {}
for key, cached, startFrom, h in queuekeys:
startOffset = None
endOffset = None
if startFrom is not None:
(startOffset, endOffset) = find_hole(cached, startFrom)
if startOffset is not None:
startOffset = max(startOffset, startFrom)
endOffset = startOffset + CHUNK_SIZE * 10
if startOffset is None:
(startOffset, endOffset) = find_hole(cached)
if startOffset is None:
continue
endOffset = min(endOffset, startOffset + CHUNK_SIZE)
print(f"Downloading startOffset {startOffset}..{endOffset} for key {key} ({startOffset * 100.0/cached.len}%)")
rangeHeader = {"Range": f"bytes={startOffset}-{endOffset-1}"}
headers_with_range = dict(cached.headers)
headers_with_range.update(rangeHeader)
async with session.request(cached.method, cached.url, headers=headers_with_range, params=cached.params) as resp:
# Stream response in chunks
# print(f"Response headers: {resp.headers}")
if "Content-Range" not in resp.headers:
raise ValueError("No Content-Range in response for Range request")
if "bytes" not in resp.headers["Content-Range"]:
raise ValueError("Only bytes Content-Range supported")
(start_end, size) = resp.headers["Content-Range"].replace("bytes ", "").split("/")
if start_end == "*":
(start, end) = (0, size - 1)
else:
(start, end) = [int(i) for i in start_end.split("-")]
# print(f"Content-Range: {start}-{end}/{size}")
# 1newChunk: Chunk = None
receivedBytes: bytes = b""
receivedBytesTotal = 0
async for receivedBytesLoc in resp.content.iter_chunked(CHUNK_SIZE):
receivedBytes += receivedBytesLoc
receivedBytesTotal += len(receivedBytesLoc)
await cached.set(start, receivedBytes) # 2
limiter.consumed(receivedBytesTotal)
delay = limiter.delay()
print(f"Sleep {delay}, {receivedBytesTotal}, est {limiter._estimated_speed} bytes/second")
if startFrom is None:
await asyncio.sleep(delay)
else:
print(f"Skip sleep for priority download")
await asyncio.sleep(0)
print(f"Downloading done")
break
except Exception as e:
print(f"downloader error: {e}", file=sys.stderr)
await asyncio.sleep(10)
traceback.print_exc()
# Python does not have a built-in atomic integer type.
# For atomic operations, you can use threading.Lock or asyncio.Lock to protect access to an integer.
# Alternatively, you can use multiprocessing.Value for atomicity across processes.
# For asyncio, you can implement a simple atomic integer like this:
class AtomicInteger:
def __init__(self, initial=0):
self._value = initial
self._lock = asyncio.Lock()
async def get(self):
async with self._lock:
return self._value
async def set(self, value):
async with self._lock:
self._value = value
async def increment(self, delta=1):
async with self._lock:
self._value += delta
return self._value
async def decrement(self, delta=1):
async with self._lock:
self._value -= delta
return self._value
stream_lock = AtomicInteger()
async def proxy_handler(request):
try:
backend_url = request.app['backend_url']
headers = dict(request.headers)
params = request.rel_url.query
path = request.rel_url.path
method = request.method
body = await request.read()
# if "Range" not in headers.keys():
if "stream" not in path or "m3u" in path:
# print(f"Simple handler {method}, {path}, {tuple(sorted(params.items()))}, {body}, {headers}")
try:
return await simple_proxy_handler(request)
except Exception as e:
print(f"simple_proxy_handler error: {e}", file=sys.stderr)
# traceback.print_exc()
return web.Response(status=500, text=f"Internal Server Error: {e}")
if "Range" not in headers.keys():
headers["Range"] = "bytes=0-"
range = headers.get("Range")
if range is not None:
if "bytes" not in range:
raise ValueError("Only bytes range supported")
range = range.replace("bytes=", "")
def to_int_or_none(s):
try:
return int(s)
except ValueError:
return None
range = [to_int_or_none(i) for r in range.split(",") for i in r.split("-") ]
# Make a HEAD request to get resource info
async with aiohttp.ClientSession() as session:
head_headers = dict(headers)
head_headers.pop("Range", None)
head_resp = await session.head(backend_url + path, headers=head_headers, params=params)
# print(f"HEAD response status: {head_resp.status}")
# print(f"HEAD response headers: {head_resp.headers}")
headHeaders = head_resp.headers
if head_resp.status != 200:
return web.Response(status=head_resp.status, text=f"Upstream server returned status {head_resp.status}")
cache_key = (method, path, (headHeaders['Etag'][:20].__hash__()))
# cache_key = (method, path, (tuple(sorted(params.items())), body, tuple(sorted(headHeaders.items()))).__hash__())
print(f"Cache key: {cache_key}")
# print(f"Range requested: {range}")
cached = await response_cache.getOrCreate(cache_key, headHeaders, method=method, url = backend_url + path, headers=head_headers, params=params)
start = (range and range[0]) or 0
headHeaders = dict(headHeaders)
if range is not None:
end = range[1] if len(range) > 1 and range[1] is not None else cached.len - 1
content_range_header = f"bytes {start}-{end}/{cached.len}"
headHeaders["Content-Range"] = content_range_header
headHeaders["Content-Length"] = f"{end - start + 1}"
headHeaders["Accept-Ranges"] = 'bytes'
# print(f"Serving with headers: {headHeaders}")
stream_lock.increment()
try:
response = web.StreamResponse(status=206, headers=headHeaders)
await response.prepare(request)
# print("Sending chunks")
stream = cached.get(start)
leftBytes = (end - start + 1) if range is not None else cached.len - start
async for chunk in stream:
if 'NoPriority' not in headers.keys():
priority[cache_key] = start
await response.write(chunk[:min(len(chunk), leftBytes)])
leftBytes -= len(chunk)
# print(f"Sending chunks {cache_key} {len(chunk)}")
if leftBytes <= 0:
break
await asyncio.sleep(0)
await response.write_eof()
finally:
stream_lock.decrement()
return response
except aiohttp.ClientConnectionResetError as e:
print(f"ClientConnectionResetError: {e}", file=sys.stderr)
return web.Response(status=500, text=f"Internal Server Error: {e}")
# except aiohttp.ConnectionResetError as e:
# print(f"ConnectionResetError: {e}", file=sys.stderr)
# return web.Response(status=500, text=f"Internal Server Error: {e}")
except Exception as e:
print(f"proxy_handler error: {e}", file=sys.stderr)
traceback.print_exc()
return web.Response(status=500, text=f"Internal Server Error: {e}")
async def server(app):
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, '0.0.0.0', 8080)
await site.start()
async def init_m3u(request):
from m3u_download import process_m3u,get_m3u_urls, download_head
while await stream_lock.get() > 0:
await asyncio.sleep(0.1)
tail = request.match_info['tail']
params = request.rel_url.query_string
url = f"http://0.0.0.0:8080/{tail}"
if params:
url += f"?{params}"
success = []
async with aiohttp.ClientSession() as session:
urls = await get_m3u_urls(url, session=session)
for url in urls:
print(f" URL: {url}")
if await download_head(session, url):
print(f" URL ok: {url}")
success.append(url)
print(f"Processed {len(success)} URLs")
return web.Response(text=f"OK\nProcessed {len(success)} URLs\n", content_type='text/plain')
def main():
if len(sys.argv) < 2:
backend_url = "http://95.142.46.84:5665"
else:
backend_url = sys.argv[1]
app = web.Application()
app['backend_url'] = backend_url
app.router.add_get('/stats', stats_handler)
# http://192.168.1.96:8080/add_m3u/playlistall/all.m3u
# http://192.168.1.96:8080/add_m3u/stream/Squid%20Game%20S03%20%5B1080p%5D.m3u?link=732f0756f5d6304adfa8532a8c1f6ddd3a753ac6&m3u
# http://192.168.1.96:8080/add_m3u/stream/Tears%20of%20Steel.m3u?link=209c8226b299b308beaf2b9cd3fb49212dbd13ec&m3u
app.router.add_get('/add_m3u/{tail:.*}', init_m3u)
app.router.add_route('*', '/{tail:.*}', proxy_handler)
loop = asyncio.new_event_loop()
loop.create_task(downloader())
loop.create_task(merger())
loop.create_task(server(app))
loop.run_forever()
if __name__ == "__main__":
main()