501 lines
21 KiB
Python
501 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
import asyncio
|
|
from tracemalloc import start
|
|
import aiohttp
|
|
from aiohttp import request, web
|
|
import sys
|
|
import json
|
|
import os
|
|
import traceback
|
|
from sortedcontainers import SortedDict
|
|
|
|
from throttler import DownloadSpeedLimiter
|
|
|
|
CHUNK_SIZE = 1024 * 1024 # Default chunk size: 1MB
|
|
|
|
|
|
cache_path = "cache"
|
|
if not os.path.exists(cache_path):
|
|
os.makedirs(cache_path)
|
|
class Chunk:
|
|
def __init__(self, path, offset, data = None):
|
|
self.path = path
|
|
self.offset = offset
|
|
self.file = os.path.join(cache_path, path, str(offset).zfill(15))
|
|
if data is not None:
|
|
with open(self.file, "wb") as f:
|
|
f.write(data)
|
|
self._len = os.path.getsize(self.file)
|
|
def len(self):
|
|
return self._len
|
|
|
|
def data(self, start = 0, size = -1):
|
|
with open(self.file, "rb") as f:
|
|
f.seek(start)
|
|
dt = f.read(min(self.len() - start, size) if size >= 0 else None)
|
|
return dt
|
|
def append(self, data):
|
|
with open(self.file, "r+b") as f:
|
|
f.seek(0, os.SEEK_END)
|
|
f.write(data)
|
|
f.flush()
|
|
self._len = os.path.getsize(self.file)
|
|
|
|
def __repr__(self):
|
|
return f"Chunk(offset={self.offset}, file={self.file}, length={self.len()})"
|
|
|
|
|
|
priority = {}
|
|
async def stats_handler(request):
|
|
items = await response_cache.allItems()
|
|
cache_dump = {}
|
|
for k, v in items.items():
|
|
cache_dump[str(k)] = {
|
|
'key': str(v.key),
|
|
'len': f"{v.len:_}",
|
|
'chd': f"{sum(i.len() for i in v.chunks.values()):_}",
|
|
'url': v.url,
|
|
'headers': dict(v.headers),
|
|
'params': dict(v.params),
|
|
'method': v.method,
|
|
'cachePath': v.cachePath,
|
|
'chunks': f"{len(v.chunks.keys()):_}",
|
|
}
|
|
stats = {
|
|
'size': {
|
|
'total_size': f"{sum([v.len for k,v in items.items()]):_}",
|
|
'chunks_size': f"{sum([v1.len() for k,v in items.items() for v1 in v.chunks.values()]):_}",
|
|
'chunks_cnt': f"{sum([len(v.chunks) for k,v in items.items()]):_}",
|
|
'total_cnt': f"{sum([1 for k,v in items.items()]):_}",
|
|
'download_speed': f"{limiter._estimated_speed:_}"
|
|
},
|
|
'priority': {str(k): v for k, v in priority.items()},
|
|
'items': cache_dump,
|
|
}
|
|
return web.Response(
|
|
text=json.dumps(stats, indent=2, ensure_ascii=False),
|
|
content_type='application/json'
|
|
)
|
|
|
|
class CacheEntry:
|
|
def __init__(self, key, headReponseHeaders, url, headers, params, method):
|
|
self.lock = asyncio.Lock()
|
|
self.key = key
|
|
self.initialResponseHeaders = headReponseHeaders
|
|
|
|
self.len = int(headReponseHeaders['Content-Length'])
|
|
self.url = url
|
|
self.headers = headers
|
|
self.params = params
|
|
self.method = method
|
|
self.cachePath = (self.url + self.initialResponseHeaders['Etag'][:20]).replace("/", "_").replace(":", "_").replace("?", "_").replace("&", "_").replace("=", "_")
|
|
os.makedirs(os.path.join(cache_path, self.cachePath), exist_ok=True)
|
|
self.chunks: SortedDict[int, Chunk] = SortedDict({
|
|
int(l): Chunk(self.cachePath, int(l), None)
|
|
for l in os.listdir(os.path.join(cache_path, self.cachePath))
|
|
})
|
|
# print(f"CacheEntry created for {url} with {self.chunks} chunks")
|
|
|
|
async def set(self, offset, chunk: bytes):
|
|
async with self.lock:
|
|
self.chunks[offset] = Chunk(self.cachePath, offset, chunk)
|
|
return self.chunks[offset]
|
|
|
|
def get(self, offset):
|
|
async def get_chunk_length(offset: int):
|
|
while True:
|
|
await asyncio.sleep(0)
|
|
if offset >= self.len:
|
|
break
|
|
async with self.lock:
|
|
found_index = self.chunks.bisect_right(offset) - 1
|
|
# print(f"Found index {found_index} for offset {offset}")
|
|
found_chunk: Chunk = None
|
|
if found_index >= 0:
|
|
found_chunk = self.chunks[self.chunks.keys()[found_index]]
|
|
# print(f"Found chunk {found_chunk}, {found_chunk.offset}, {found_chunk.len()}")
|
|
if found_chunk is not None and found_chunk.offset + found_chunk.len() <= offset:
|
|
found_chunk = None
|
|
|
|
# found_key = max((k for k, c in self.chunks.items() if k <= offset and c.offset + c.len() > offset), default=None)
|
|
# print(f"Raising priority {self.key}, {offset}")
|
|
if found_chunk is None:
|
|
# print(f"Waiting {offset}")
|
|
# log = {k: (c.offset, c.offset + c.len()) for k, c in self.chunks.items()}
|
|
# print(f"Chunk keys {log}")
|
|
await asyncio.sleep(1)
|
|
yield b""
|
|
continue
|
|
|
|
dt = found_chunk.data(offset - found_chunk.offset, CHUNK_SIZE)
|
|
if len(dt) == 0:
|
|
raise ValueError("Empty chunk data")
|
|
yield dt
|
|
offset += len(dt)
|
|
|
|
return get_chunk_length(offset)
|
|
|
|
class Cache:
|
|
def __init__(self):
|
|
self.store: dict[any, CacheEntry] = {}
|
|
self.lock = asyncio.Lock()
|
|
|
|
async def allItems(self) -> dict[any, CacheEntry]:
|
|
async with self.lock:
|
|
return dict(self.store)
|
|
|
|
async def getOrCreate(self, key, headResponseHeaders, method, url, headers, params) -> CacheEntry:
|
|
async with self.lock:
|
|
if key not in self.store:
|
|
self.store[key] = CacheEntry(key, headReponseHeaders=headResponseHeaders, method=method, url=url, headers=headers, params=params)
|
|
return self.store.get(key)
|
|
async def mergeAnyTwo(self):
|
|
async with self.lock:
|
|
for entry in self.store.values():
|
|
async with entry.lock:
|
|
keysI = list(entry.chunks.keys())
|
|
for keyI in range(len(keysI) - 1):
|
|
chunk1:Chunk = entry.chunks[keysI[keyI]]
|
|
chunk2:Chunk = entry.chunks[keysI[keyI+1]]
|
|
if chunk2 is None:
|
|
print(f"No chunk2 for {keysI[keyI]} in {entry.key}")
|
|
continue
|
|
if chunk1.offset >= chunk2.offset:
|
|
print(f"Chunk1 {chunk1} is not before Chunk2 {chunk2} in {entry.key}")
|
|
continue
|
|
if chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len():
|
|
print(f"Chunk1 {chunk1} already covers Chunk2 {chunk2} in {entry.key}: {chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len()}")
|
|
continue
|
|
if chunk1.offset + chunk1.len() < chunk2.offset:
|
|
# print(f"Chunk1 {chunk1} has gap after Chunk2 {chunk2} in {entry.key}: {chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len()}")
|
|
continue
|
|
offset1 = chunk1.offset + chunk1.len() - chunk2.offset
|
|
while offset1 < chunk2.len():
|
|
print(f"Merging {chunk1}+{chunk2}[{offset1}:] from {entry.key}")
|
|
chunk1.append(chunk2.data(offset1, CHUNK_SIZE))
|
|
offset1 += CHUNK_SIZE
|
|
del entry.chunks[chunk2.offset]
|
|
os.remove(chunk2.file)
|
|
return True
|
|
return False
|
|
|
|
response_cache = Cache()
|
|
|
|
async def simple_proxy_handler(request):
|
|
backend_url = request.app['backend_url']
|
|
headers = dict(request.headers)
|
|
params = request.rel_url.query
|
|
path = request.rel_url.path
|
|
method = request.method
|
|
body = await request.read()
|
|
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.request(method, backend_url + path, headers=headers, params=params, data=body) as resp:
|
|
# print("Response status:", resp.headers)
|
|
response = web.StreamResponse(status=resp.status, headers=resp.headers)
|
|
await response.prepare(request)
|
|
async for chunk in resp.content.iter_chunked(CHUNK_SIZE):
|
|
await response.write(chunk)
|
|
await response.write_eof()
|
|
return response
|
|
|
|
def find_hole(entry: CacheEntry, start_offset=0):
|
|
chunks = entry.chunks
|
|
if len(chunks) == 0:
|
|
return (0, entry.len)
|
|
# TODO Lock with merger
|
|
keys = chunks.bisect_right(start_offset) - 1
|
|
keys = list(chunks.islice(keys, len(chunks)))
|
|
for i in range(len(keys) - 1):
|
|
if keys[i + 1] < start_offset:
|
|
continue
|
|
curr: Chunk = chunks[keys[i]]
|
|
if curr.offset + curr.len() < keys[i + 1]:
|
|
return (curr.offset + curr.len(), keys[i + 1])
|
|
if keys[-1] + chunks[keys[-1]].len() < entry.len:
|
|
return (keys[-1] + chunks[keys[-1]].len(), entry.len)
|
|
return (None, None)
|
|
|
|
async def merger():
|
|
while True:
|
|
try:
|
|
if await stream_lock.get() > 0:
|
|
await asyncio.sleep(30)
|
|
continue
|
|
cnt = 0
|
|
if await response_cache.mergeAnyTwo() and cnt < 1000:
|
|
cnt += 1
|
|
if cnt > 0:
|
|
await asyncio.sleep(0.1)
|
|
continue
|
|
await asyncio.sleep(30)
|
|
except Exception as e:
|
|
print(f"merger error: {e}", file=sys.stderr)
|
|
traceback.print_exc()
|
|
|
|
async def verifirer():
|
|
# TODO
|
|
# TODO Locks
|
|
pass
|
|
|
|
limiter = DownloadSpeedLimiter(200_000)
|
|
|
|
async def downloader():
|
|
global priority
|
|
while True:
|
|
try:
|
|
async with aiohttp.ClientSession() as session:
|
|
while True:
|
|
await asyncio.sleep(0)
|
|
print(f"Checking for downloads...")
|
|
keys = await response_cache.allItems()
|
|
queuekeys = [(k, v, priority[k], find_hole(v, priority[k])[0]) for k, v in keys.items() if k in priority.keys()]
|
|
queuekeys = [x for x in queuekeys if x[3] is not None]
|
|
queuekeys.sort(key=lambda x: x[3])
|
|
if limiter.max_speed == 0 and len(queuekeys) == 0:
|
|
print(f"Download disabled, sleeping")
|
|
limiter._estimated_speed = 0
|
|
await asyncio.sleep(10)
|
|
continue
|
|
|
|
queuekeys += [(k, v, None, find_hole(v, 0)[0]) for k, v in keys.items()]
|
|
queuekeys = [(k,v,p,h) for k,v,p,h in queuekeys if h is not None]
|
|
|
|
if len(queuekeys) == 0:
|
|
print(f"Nothing to download, sleeping")
|
|
await asyncio.sleep(10)
|
|
continue
|
|
priority = {}
|
|
for key, cached, startFrom, h in queuekeys:
|
|
startOffset = None
|
|
endOffset = None
|
|
if startFrom is not None:
|
|
(startOffset, endOffset) = find_hole(cached, startFrom)
|
|
if startOffset is not None:
|
|
startOffset = max(startOffset, startFrom)
|
|
endOffset = startOffset + CHUNK_SIZE * 10
|
|
|
|
if startOffset is None:
|
|
(startOffset, endOffset) = find_hole(cached)
|
|
|
|
if startOffset is None:
|
|
continue
|
|
endOffset = min(endOffset, startOffset + CHUNK_SIZE)
|
|
|
|
print(f"Downloading startOffset {startOffset}..{endOffset} for key {key} ({startOffset * 100.0/cached.len}%)")
|
|
rangeHeader = {"Range": f"bytes={startOffset}-{endOffset-1}"}
|
|
headers_with_range = dict(cached.headers)
|
|
headers_with_range.update(rangeHeader)
|
|
async with session.request(cached.method, cached.url, headers=headers_with_range, params=cached.params) as resp:
|
|
|
|
# Stream response in chunks
|
|
# print(f"Response headers: {resp.headers}")
|
|
if "Content-Range" not in resp.headers:
|
|
raise ValueError("No Content-Range in response for Range request")
|
|
if "bytes" not in resp.headers["Content-Range"]:
|
|
raise ValueError("Only bytes Content-Range supported")
|
|
(start_end, size) = resp.headers["Content-Range"].replace("bytes ", "").split("/")
|
|
if start_end == "*":
|
|
(start, end) = (0, size - 1)
|
|
else:
|
|
(start, end) = [int(i) for i in start_end.split("-")]
|
|
# print(f"Content-Range: {start}-{end}/{size}")
|
|
# 1newChunk: Chunk = None
|
|
receivedBytes: bytes = b""
|
|
receivedBytesTotal = 0
|
|
|
|
async for receivedBytesLoc in resp.content.iter_chunked(CHUNK_SIZE):
|
|
receivedBytes += receivedBytesLoc
|
|
receivedBytesTotal += len(receivedBytesLoc)
|
|
|
|
await cached.set(start, receivedBytes) # 2
|
|
limiter.consumed(receivedBytesTotal)
|
|
delay = limiter.delay()
|
|
print(f"Sleep {delay}, {receivedBytesTotal}, est {limiter._estimated_speed} bytes/second")
|
|
if startFrom is None:
|
|
await asyncio.sleep(delay)
|
|
else:
|
|
print(f"Skip sleep for priority download")
|
|
await asyncio.sleep(0)
|
|
|
|
print(f"Downloading done")
|
|
break
|
|
except Exception as e:
|
|
print(f"downloader error: {e}", file=sys.stderr)
|
|
await asyncio.sleep(10)
|
|
traceback.print_exc()
|
|
|
|
# Python does not have a built-in atomic integer type.
|
|
# For atomic operations, you can use threading.Lock or asyncio.Lock to protect access to an integer.
|
|
# Alternatively, you can use multiprocessing.Value for atomicity across processes.
|
|
# For asyncio, you can implement a simple atomic integer like this:
|
|
|
|
class AtomicInteger:
|
|
def __init__(self, initial=0):
|
|
self._value = initial
|
|
self._lock = asyncio.Lock()
|
|
|
|
async def get(self):
|
|
async with self._lock:
|
|
return self._value
|
|
|
|
async def set(self, value):
|
|
async with self._lock:
|
|
self._value = value
|
|
|
|
async def increment(self, delta=1):
|
|
async with self._lock:
|
|
self._value += delta
|
|
return self._value
|
|
|
|
async def decrement(self, delta=1):
|
|
async with self._lock:
|
|
self._value -= delta
|
|
return self._value
|
|
stream_lock = AtomicInteger()
|
|
async def proxy_handler(request):
|
|
try:
|
|
backend_url = request.app['backend_url']
|
|
headers = dict(request.headers)
|
|
params = request.rel_url.query
|
|
path = request.rel_url.path
|
|
method = request.method
|
|
body = await request.read()
|
|
# if "Range" not in headers.keys():
|
|
if "stream" not in path or "m3u" in path:
|
|
# print(f"Simple handler {method}, {path}, {tuple(sorted(params.items()))}, {body}, {headers}")
|
|
try:
|
|
return await simple_proxy_handler(request)
|
|
except Exception as e:
|
|
print(f"simple_proxy_handler error: {e}", file=sys.stderr)
|
|
# traceback.print_exc()
|
|
return web.Response(status=500, text=f"Internal Server Error: {e}")
|
|
|
|
if "Range" not in headers.keys():
|
|
headers["Range"] = "bytes=0-"
|
|
|
|
range = headers.get("Range")
|
|
if range is not None:
|
|
if "bytes" not in range:
|
|
raise ValueError("Only bytes range supported")
|
|
range = range.replace("bytes=", "")
|
|
def to_int_or_none(s):
|
|
try:
|
|
return int(s)
|
|
except ValueError:
|
|
return None
|
|
range = [to_int_or_none(i) for r in range.split(",") for i in r.split("-") ]
|
|
|
|
# Make a HEAD request to get resource info
|
|
async with aiohttp.ClientSession() as session:
|
|
head_headers = dict(headers)
|
|
head_headers.pop("Range", None)
|
|
head_resp = await session.head(backend_url + path, headers=head_headers, params=params)
|
|
# print(f"HEAD response status: {head_resp.status}")
|
|
# print(f"HEAD response headers: {head_resp.headers}")
|
|
headHeaders = head_resp.headers
|
|
if head_resp.status != 200:
|
|
return web.Response(status=head_resp.status, text=f"Upstream server returned status {head_resp.status}")
|
|
|
|
cache_key = (method, path, (headHeaders['Etag'][:20].__hash__()))
|
|
# cache_key = (method, path, (tuple(sorted(params.items())), body, tuple(sorted(headHeaders.items()))).__hash__())
|
|
print(f"Cache key: {cache_key}")
|
|
# print(f"Range requested: {range}")
|
|
cached = await response_cache.getOrCreate(cache_key, headHeaders, method=method, url = backend_url + path, headers=head_headers, params=params)
|
|
|
|
start = (range and range[0]) or 0
|
|
headHeaders = dict(headHeaders)
|
|
if range is not None:
|
|
end = range[1] if len(range) > 1 and range[1] is not None else cached.len - 1
|
|
content_range_header = f"bytes {start}-{end}/{cached.len}"
|
|
headHeaders["Content-Range"] = content_range_header
|
|
headHeaders["Content-Length"] = f"{end - start + 1}"
|
|
headHeaders["Accept-Ranges"] = 'bytes'
|
|
# print(f"Serving with headers: {headHeaders}")
|
|
stream_lock.increment()
|
|
try:
|
|
response = web.StreamResponse(status=206, headers=headHeaders)
|
|
await response.prepare(request)
|
|
# print("Sending chunks")
|
|
stream = cached.get(start)
|
|
leftBytes = (end - start + 1) if range is not None else cached.len - start
|
|
async for chunk in stream:
|
|
if 'NoPriority' not in headers.keys():
|
|
priority[cache_key] = start
|
|
await response.write(chunk[:min(len(chunk), leftBytes)])
|
|
leftBytes -= len(chunk)
|
|
# print(f"Sending chunks {cache_key} {len(chunk)}")
|
|
if leftBytes <= 0:
|
|
break
|
|
await asyncio.sleep(0)
|
|
await response.write_eof()
|
|
finally:
|
|
stream_lock.decrement()
|
|
return response
|
|
|
|
except aiohttp.ClientConnectionResetError as e:
|
|
print(f"ClientConnectionResetError: {e}", file=sys.stderr)
|
|
return web.Response(status=500, text=f"Internal Server Error: {e}")
|
|
# except aiohttp.ConnectionResetError as e:
|
|
# print(f"ConnectionResetError: {e}", file=sys.stderr)
|
|
# return web.Response(status=500, text=f"Internal Server Error: {e}")
|
|
except Exception as e:
|
|
print(f"proxy_handler error: {e}", file=sys.stderr)
|
|
traceback.print_exc()
|
|
return web.Response(status=500, text=f"Internal Server Error: {e}")
|
|
|
|
|
|
async def server(app):
|
|
runner = web.AppRunner(app)
|
|
await runner.setup()
|
|
site = web.TCPSite(runner, '0.0.0.0', 8080)
|
|
await site.start()
|
|
|
|
async def init_m3u(request):
|
|
from m3u_download import process_m3u,get_m3u_urls, download_head
|
|
while await stream_lock.get() > 0:
|
|
await asyncio.sleep(0.1)
|
|
tail = request.match_info['tail']
|
|
params = request.rel_url.query_string
|
|
url = f"http://0.0.0.0:8080/{tail}"
|
|
if params:
|
|
url += f"?{params}"
|
|
|
|
success = []
|
|
async with aiohttp.ClientSession() as session:
|
|
urls = await get_m3u_urls(url, session=session)
|
|
for url in urls:
|
|
print(f" URL: {url}")
|
|
if await download_head(session, url):
|
|
print(f" URL ok: {url}")
|
|
success.append(url)
|
|
|
|
print(f"Processed {len(success)} URLs")
|
|
return web.Response(text=f"OK\nProcessed {len(success)} URLs\n", content_type='text/plain')
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 2:
|
|
backend_url = "http://95.142.46.84:5665"
|
|
else:
|
|
backend_url = sys.argv[1]
|
|
app = web.Application()
|
|
app['backend_url'] = backend_url
|
|
app.router.add_get('/stats', stats_handler)
|
|
|
|
# http://192.168.1.96:8080/add_m3u/playlistall/all.m3u
|
|
# http://192.168.1.96:8080/add_m3u/stream/Squid%20Game%20S03%20%5B1080p%5D.m3u?link=732f0756f5d6304adfa8532a8c1f6ddd3a753ac6&m3u
|
|
# http://192.168.1.96:8080/add_m3u/stream/Tears%20of%20Steel.m3u?link=209c8226b299b308beaf2b9cd3fb49212dbd13ec&m3u
|
|
app.router.add_get('/add_m3u/{tail:.*}', init_m3u)
|
|
app.router.add_route('*', '/{tail:.*}', proxy_handler)
|
|
loop = asyncio.new_event_loop()
|
|
loop.create_task(downloader())
|
|
loop.create_task(merger())
|
|
loop.create_task(server(app))
|
|
|
|
loop.run_forever()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|