reverseproxy
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
venv
|
||||
.venv
|
||||
cache
|
||||
__pycache__
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
FROM python:3.11-slim
|
||||
WORKDIR /app
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
CMD ["python", "proxy.py", "http://backend:8000"]
|
||||
@@ -0,0 +1,31 @@
|
||||
dd if=orig.mkv bs=1 skip=0 count=13484 status=none | md5sum
|
||||
dd if=orig.mkv bs=1 skip=13484 count=1424 status=none | md5sum
|
||||
dd if=orig.mkv bs=1 skip=14908 count=4272 status=none | md5sum
|
||||
dd if=orig.mkv bs=1 skip=19180 count=2848 status=none | md5sum
|
||||
dd if=orig.mkv bs=1 skip=22028 count=11392 status=none | md5sum
|
||||
dd if=orig.mkv bs=1 skip=33420 count=2848 status=none | md5sum
|
||||
dd if=orig.mkv bs=1 skip=36268 count=4272 status=none | md5sum
|
||||
md5sum 653134336/0000000000
|
||||
md5sum 653134336/0000013484
|
||||
md5sum 653134336/0000014908
|
||||
md5sum 653134336/0000019180
|
||||
md5sum 653134336/0000022028
|
||||
md5sum 653134336/0000033420
|
||||
md5sum 653134336/0000036268
|
||||
|
||||
bffeb107d791300457e1c9f0d4280d86 -
|
||||
b70c3b6a906eaaf8701969869591c79b -
|
||||
06e1e3fd434112d3c810cba2d6798d4b -
|
||||
c2806cb0cbb566a4936841e46b6f29b2 -
|
||||
88cae8a40480453c26f14afb29153589 -
|
||||
584684b9bf0a4d39a78b342db6d8bc64 -
|
||||
f7e7dac5dcea561b801750e69e29a296 -
|
||||
bffeb107d791300457e1c9f0d4280d86 653134336/0000000000
|
||||
b70c3b6a906eaaf8701969869591c79b 653134336/0000013484
|
||||
06e1e3fd434112d3c810cba2d6798d4b 653134336/0000014908
|
||||
c2806cb0cbb566a4936841e46b6f29b2 653134336/0000019180
|
||||
88cae8a40480453c26f14afb29153589 653134336/0000022028
|
||||
584684b9bf0a4d39a78b342db6d8bc64 653134336/0000033420
|
||||
f7e7dac5dcea561b801750e69e29a296 653134336/0000036268
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,27 @@
|
||||
version: '3.8'
|
||||
services:
|
||||
reverseproxy:
|
||||
build: .
|
||||
image: registry.rudolf.keenetic.link/torr_cache:${version}
|
||||
volumes:
|
||||
- data:/app/cache:rw
|
||||
ports:
|
||||
- "8080:8080"
|
||||
environment:
|
||||
- PYTHONUNBUFFERED=1
|
||||
command: ["python", "proxy.py", "http://95.142.46.84:5665"]
|
||||
fileserver:
|
||||
image: python:3.11-slim
|
||||
working_dir: /data
|
||||
volumes:
|
||||
- data:/data:ro
|
||||
command: ["python3", "-m", "http.server", "8888"]
|
||||
ports:
|
||||
- "8888:8888"
|
||||
volumes:
|
||||
data:
|
||||
driver: local
|
||||
driver_opts:
|
||||
type: nfs
|
||||
o: addr=192.168.1.96
|
||||
device: :/media/DATA/runtime/torrCacher
|
||||
Executable
+146
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
import traceback
|
||||
import aiohttp
|
||||
import aiofiles
|
||||
import urllib.parse
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from tqdm.asyncio import tqdm
|
||||
|
||||
|
||||
def is_url(string):
|
||||
"""Check if the provided string is a URL."""
|
||||
try:
|
||||
result = urlparse(string)
|
||||
return all([result.scheme, result.netloc])
|
||||
except:
|
||||
return False
|
||||
|
||||
async def download_head(session, url):
|
||||
"""Download a file from URL to the specified destination asynchronously.
|
||||
If head is a number, download only the first head bytes.
|
||||
"""
|
||||
try:
|
||||
print(f"Downloading: {url} (HEAD)")
|
||||
headers = {}
|
||||
headers['Range'] = f'bytes=0-100'
|
||||
headers['NoPriority'] = f'NoPriority'
|
||||
async with session.get(url, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
total_size = int(response.headers.get('content-length', 0))
|
||||
downloaded = 0
|
||||
async for chunk in response.content.iter_chunked(8192):
|
||||
if chunk:
|
||||
downloaded += len(chunk)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Error downloading {url}: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
|
||||
async def download_file(session, url, destination):
|
||||
"""Download a file from URL to the specified destination asynchronously."""
|
||||
try:
|
||||
if destination is not None:
|
||||
os.makedirs(os.path.dirname(destination), exist_ok=True)
|
||||
if os.path.exists(destination):
|
||||
print(f"File already exists: {destination}")
|
||||
return True
|
||||
print(f"Downloading: {url} -> {destination}")
|
||||
async with session.get(url) as response:
|
||||
response.raise_for_status()
|
||||
total_size = int(response.headers.get('content-length', 0))
|
||||
print("Len ", total_size)
|
||||
|
||||
block_size = 8192
|
||||
if destination is not None:
|
||||
async with aiofiles.open(destination, 'wb') as f:
|
||||
pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc=os.path.basename(destination))
|
||||
async for chunk in response.content.iter_chunked(block_size):
|
||||
if chunk:
|
||||
await f.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
pbar.close()
|
||||
else:
|
||||
pbar = tqdm(total=total_size, unit='B', unit_scale=True, desc="To None")
|
||||
pbar.update(0)
|
||||
async for chunk in response.content.iter_chunked(block_size):
|
||||
if chunk:
|
||||
pbar.update(len(chunk))
|
||||
pbar.close()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Error downloading {url}: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
async def get_m3u_urls(m3u_url, session, visited=None):
|
||||
"""Recursively collect all file URLs from an M3U playlist."""
|
||||
if visited is None:
|
||||
visited = set()
|
||||
urls = []
|
||||
print(f"Adding m3u {m3u_url}")
|
||||
try:
|
||||
async with session.get(m3u_url) as response:
|
||||
response.raise_for_status()
|
||||
content = await response.text()
|
||||
if "#EXTM3U" not in content:
|
||||
raise ValueError("The provided file is not a valid M3U playlist.")
|
||||
base_url = m3u_url.rsplit('/', 1)[0] + '/'
|
||||
for line in content.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
file_url = line if is_url(line) else urljoin(base_url, line)
|
||||
path_part = urlparse(file_url).path
|
||||
if '.m3u' in path_part and file_url not in visited:
|
||||
visited.add(file_url)
|
||||
nested_urls = await get_m3u_urls(file_url, session=session, visited=visited)
|
||||
urls.extend(nested_urls)
|
||||
continue
|
||||
urls.append(file_url)
|
||||
return urls
|
||||
except Exception as e:
|
||||
print(f"Error processing M3U file: {str(e)}")
|
||||
traceback.print_exc()
|
||||
return []
|
||||
|
||||
|
||||
async def process_m3u(m3u_url, output_dir, session=None):
|
||||
"""Process an M3U file: get all URLs and download them."""
|
||||
with aiohttp.ClientSession() as session:
|
||||
urls = await get_m3u_urls(m3u_url, session=session)
|
||||
result = []
|
||||
if session is None:
|
||||
session = aiohttp.ClientSession()
|
||||
for file_url in urls:
|
||||
try:
|
||||
local_path = None
|
||||
if output_dir is not None:
|
||||
path_part = urlparse(file_url).path
|
||||
if path_part.startswith('/'):
|
||||
path_part = path_part[1:]
|
||||
path_part = urllib.parse.unquote(path_part)
|
||||
local_path = None if output_dir is None else os.path.join(output_dir, path_part)
|
||||
await download_file(session, file_url, local_path)
|
||||
except Exception as e:
|
||||
print(f"Error downloading URLs: {str(e)}")
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
m3u = "http://192.168.1.96:8080/playlistall/all.m3u"
|
||||
if len(sys.argv) < 2:
|
||||
m3u_url = m3u
|
||||
else:
|
||||
m3u_url = sys.argv[1]
|
||||
output_dir = sys.argv[2] if len(sys.argv) > 2 else None
|
||||
print(f"Downloading content from {m3u_url} to {output_dir}")
|
||||
asyncio.run(process_m3u(m3u_url, output_dir))
|
||||
@@ -0,0 +1,500 @@
|
||||
#!/usr/bin/env python3
|
||||
import asyncio
|
||||
from tracemalloc import start
|
||||
import aiohttp
|
||||
from aiohttp import request, web
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import traceback
|
||||
from sortedcontainers import SortedDict
|
||||
|
||||
from throttler import DownloadSpeedLimiter
|
||||
|
||||
CHUNK_SIZE = 1024 * 1024 # Default chunk size: 1MB
|
||||
|
||||
|
||||
cache_path = "cache"
|
||||
if not os.path.exists(cache_path):
|
||||
os.makedirs(cache_path)
|
||||
class Chunk:
|
||||
def __init__(self, path, offset, data = None):
|
||||
self.path = path
|
||||
self.offset = offset
|
||||
self.file = os.path.join(cache_path, path, str(offset).zfill(15))
|
||||
if data is not None:
|
||||
with open(self.file, "wb") as f:
|
||||
f.write(data)
|
||||
self._len = os.path.getsize(self.file)
|
||||
def len(self):
|
||||
return self._len
|
||||
|
||||
def data(self, start = 0, size = -1):
|
||||
with open(self.file, "rb") as f:
|
||||
f.seek(start)
|
||||
dt = f.read(min(self.len() - start, size) if size >= 0 else None)
|
||||
return dt
|
||||
def append(self, data):
|
||||
with open(self.file, "r+b") as f:
|
||||
f.seek(0, os.SEEK_END)
|
||||
f.write(data)
|
||||
f.flush()
|
||||
self._len = os.path.getsize(self.file)
|
||||
|
||||
def __repr__(self):
|
||||
return f"Chunk(offset={self.offset}, file={self.file}, length={self.len()})"
|
||||
|
||||
|
||||
priority = {}
|
||||
async def stats_handler(request):
|
||||
items = await response_cache.allItems()
|
||||
cache_dump = {}
|
||||
for k, v in items.items():
|
||||
cache_dump[str(k)] = {
|
||||
'key': str(v.key),
|
||||
'len': f"{v.len:_}",
|
||||
'chd': f"{sum(i.len() for i in v.chunks.values()):_}",
|
||||
'url': v.url,
|
||||
'headers': dict(v.headers),
|
||||
'params': dict(v.params),
|
||||
'method': v.method,
|
||||
'cachePath': v.cachePath,
|
||||
'chunks': f"{len(v.chunks.keys()):_}",
|
||||
}
|
||||
stats = {
|
||||
'size': {
|
||||
'total_size': f"{sum([v.len for k,v in items.items()]):_}",
|
||||
'chunks_size': f"{sum([v1.len() for k,v in items.items() for v1 in v.chunks.values()]):_}",
|
||||
'chunks_cnt': f"{sum([len(v.chunks) for k,v in items.items()]):_}",
|
||||
'total_cnt': f"{sum([1 for k,v in items.items()]):_}",
|
||||
'download_speed': f"{limiter._estimated_speed:_}"
|
||||
},
|
||||
'priority': {str(k): v for k, v in priority.items()},
|
||||
'items': cache_dump,
|
||||
}
|
||||
return web.Response(
|
||||
text=json.dumps(stats, indent=2, ensure_ascii=False),
|
||||
content_type='application/json'
|
||||
)
|
||||
|
||||
class CacheEntry:
|
||||
def __init__(self, key, headReponseHeaders, url, headers, params, method):
|
||||
self.lock = asyncio.Lock()
|
||||
self.key = key
|
||||
self.initialResponseHeaders = headReponseHeaders
|
||||
|
||||
self.len = int(headReponseHeaders['Content-Length'])
|
||||
self.url = url
|
||||
self.headers = headers
|
||||
self.params = params
|
||||
self.method = method
|
||||
self.cachePath = (self.url + self.initialResponseHeaders['Etag'][:20]).replace("/", "_").replace(":", "_").replace("?", "_").replace("&", "_").replace("=", "_")
|
||||
os.makedirs(os.path.join(cache_path, self.cachePath), exist_ok=True)
|
||||
self.chunks: SortedDict[int, Chunk] = SortedDict({
|
||||
int(l): Chunk(self.cachePath, int(l), None)
|
||||
for l in os.listdir(os.path.join(cache_path, self.cachePath))
|
||||
})
|
||||
# print(f"CacheEntry created for {url} with {self.chunks} chunks")
|
||||
|
||||
async def set(self, offset, chunk: bytes):
|
||||
async with self.lock:
|
||||
self.chunks[offset] = Chunk(self.cachePath, offset, chunk)
|
||||
return self.chunks[offset]
|
||||
|
||||
def get(self, offset):
|
||||
async def get_chunk_length(offset: int):
|
||||
while True:
|
||||
await asyncio.sleep(0)
|
||||
if offset >= self.len:
|
||||
break
|
||||
async with self.lock:
|
||||
found_index = self.chunks.bisect_right(offset) - 1
|
||||
# print(f"Found index {found_index} for offset {offset}")
|
||||
found_chunk: Chunk = None
|
||||
if found_index >= 0:
|
||||
found_chunk = self.chunks[self.chunks.keys()[found_index]]
|
||||
# print(f"Found chunk {found_chunk}, {found_chunk.offset}, {found_chunk.len()}")
|
||||
if found_chunk is not None and found_chunk.offset + found_chunk.len() <= offset:
|
||||
found_chunk = None
|
||||
|
||||
# found_key = max((k for k, c in self.chunks.items() if k <= offset and c.offset + c.len() > offset), default=None)
|
||||
# print(f"Raising priority {self.key}, {offset}")
|
||||
if found_chunk is None:
|
||||
# print(f"Waiting {offset}")
|
||||
# log = {k: (c.offset, c.offset + c.len()) for k, c in self.chunks.items()}
|
||||
# print(f"Chunk keys {log}")
|
||||
await asyncio.sleep(1)
|
||||
yield b""
|
||||
continue
|
||||
|
||||
dt = found_chunk.data(offset - found_chunk.offset, CHUNK_SIZE)
|
||||
if len(dt) == 0:
|
||||
raise ValueError("Empty chunk data")
|
||||
yield dt
|
||||
offset += len(dt)
|
||||
|
||||
return get_chunk_length(offset)
|
||||
|
||||
class Cache:
|
||||
def __init__(self):
|
||||
self.store: dict[any, CacheEntry] = {}
|
||||
self.lock = asyncio.Lock()
|
||||
|
||||
async def allItems(self) -> dict[any, CacheEntry]:
|
||||
async with self.lock:
|
||||
return dict(self.store)
|
||||
|
||||
async def getOrCreate(self, key, headResponseHeaders, method, url, headers, params) -> CacheEntry:
|
||||
async with self.lock:
|
||||
if key not in self.store:
|
||||
self.store[key] = CacheEntry(key, headReponseHeaders=headResponseHeaders, method=method, url=url, headers=headers, params=params)
|
||||
return self.store.get(key)
|
||||
async def mergeAnyTwo(self):
|
||||
async with self.lock:
|
||||
for entry in self.store.values():
|
||||
async with entry.lock:
|
||||
keysI = list(entry.chunks.keys())
|
||||
for keyI in range(len(keysI) - 1):
|
||||
chunk1:Chunk = entry.chunks[keysI[keyI]]
|
||||
chunk2:Chunk = entry.chunks[keysI[keyI+1]]
|
||||
if chunk2 is None:
|
||||
print(f"No chunk2 for {keysI[keyI]} in {entry.key}")
|
||||
continue
|
||||
if chunk1.offset >= chunk2.offset:
|
||||
print(f"Chunk1 {chunk1} is not before Chunk2 {chunk2} in {entry.key}")
|
||||
continue
|
||||
if chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len():
|
||||
print(f"Chunk1 {chunk1} already covers Chunk2 {chunk2} in {entry.key}: {chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len()}")
|
||||
continue
|
||||
if chunk1.offset + chunk1.len() < chunk2.offset:
|
||||
# print(f"Chunk1 {chunk1} has gap after Chunk2 {chunk2} in {entry.key}: {chunk1.offset + chunk1.len() - chunk2.offset >= chunk2.len()}")
|
||||
continue
|
||||
offset1 = chunk1.offset + chunk1.len() - chunk2.offset
|
||||
while offset1 < chunk2.len():
|
||||
print(f"Merging {chunk1}+{chunk2}[{offset1}:] from {entry.key}")
|
||||
chunk1.append(chunk2.data(offset1, CHUNK_SIZE))
|
||||
offset1 += CHUNK_SIZE
|
||||
del entry.chunks[chunk2.offset]
|
||||
os.remove(chunk2.file)
|
||||
return True
|
||||
return False
|
||||
|
||||
response_cache = Cache()
|
||||
|
||||
async def simple_proxy_handler(request):
|
||||
backend_url = request.app['backend_url']
|
||||
headers = dict(request.headers)
|
||||
params = request.rel_url.query
|
||||
path = request.rel_url.path
|
||||
method = request.method
|
||||
body = await request.read()
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.request(method, backend_url + path, headers=headers, params=params, data=body) as resp:
|
||||
# print("Response status:", resp.headers)
|
||||
response = web.StreamResponse(status=resp.status, headers=resp.headers)
|
||||
await response.prepare(request)
|
||||
async for chunk in resp.content.iter_chunked(CHUNK_SIZE):
|
||||
await response.write(chunk)
|
||||
await response.write_eof()
|
||||
return response
|
||||
|
||||
def find_hole(entry: CacheEntry, start_offset=0):
|
||||
chunks = entry.chunks
|
||||
if len(chunks) == 0:
|
||||
return (0, entry.len)
|
||||
# TODO Lock with merger
|
||||
keys = chunks.bisect_right(start_offset) - 1
|
||||
keys = list(chunks.islice(keys, len(chunks)))
|
||||
for i in range(len(keys) - 1):
|
||||
if keys[i + 1] < start_offset:
|
||||
continue
|
||||
curr: Chunk = chunks[keys[i]]
|
||||
if curr.offset + curr.len() < keys[i + 1]:
|
||||
return (curr.offset + curr.len(), keys[i + 1])
|
||||
if keys[-1] + chunks[keys[-1]].len() < entry.len:
|
||||
return (keys[-1] + chunks[keys[-1]].len(), entry.len)
|
||||
return (None, None)
|
||||
|
||||
async def merger():
|
||||
while True:
|
||||
try:
|
||||
if await stream_lock.get() > 0:
|
||||
await asyncio.sleep(30)
|
||||
continue
|
||||
cnt = 0
|
||||
if await response_cache.mergeAnyTwo() and cnt < 1000:
|
||||
cnt += 1
|
||||
if cnt > 0:
|
||||
await asyncio.sleep(0.1)
|
||||
continue
|
||||
await asyncio.sleep(30)
|
||||
except Exception as e:
|
||||
print(f"merger error: {e}", file=sys.stderr)
|
||||
traceback.print_exc()
|
||||
|
||||
async def verifirer():
|
||||
# TODO
|
||||
# TODO Locks
|
||||
pass
|
||||
|
||||
limiter = DownloadSpeedLimiter(200_000)
|
||||
|
||||
async def downloader():
|
||||
global priority
|
||||
while True:
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
while True:
|
||||
await asyncio.sleep(0)
|
||||
print(f"Checking for downloads...")
|
||||
keys = await response_cache.allItems()
|
||||
queuekeys = [(k, v, priority[k], find_hole(v, priority[k])[0]) for k, v in keys.items() if k in priority.keys()]
|
||||
queuekeys = [x for x in queuekeys if x[3] is not None]
|
||||
queuekeys.sort(key=lambda x: x[3])
|
||||
if limiter.max_speed == 0 and len(queuekeys) == 0:
|
||||
print(f"Download disabled, sleeping")
|
||||
limiter._estimated_speed = 0
|
||||
await asyncio.sleep(10)
|
||||
continue
|
||||
|
||||
queuekeys += [(k, v, None, find_hole(v, 0)[0]) for k, v in keys.items()]
|
||||
queuekeys = [(k,v,p,h) for k,v,p,h in queuekeys if h is not None]
|
||||
|
||||
if len(queuekeys) == 0:
|
||||
print(f"Nothing to download, sleeping")
|
||||
await asyncio.sleep(10)
|
||||
continue
|
||||
priority = {}
|
||||
for key, cached, startFrom, h in queuekeys:
|
||||
startOffset = None
|
||||
endOffset = None
|
||||
if startFrom is not None:
|
||||
(startOffset, endOffset) = find_hole(cached, startFrom)
|
||||
if startOffset is not None:
|
||||
startOffset = max(startOffset, startFrom)
|
||||
endOffset = startOffset + CHUNK_SIZE * 10
|
||||
|
||||
if startOffset is None:
|
||||
(startOffset, endOffset) = find_hole(cached)
|
||||
|
||||
if startOffset is None:
|
||||
continue
|
||||
endOffset = min(endOffset, startOffset + CHUNK_SIZE)
|
||||
|
||||
print(f"Downloading startOffset {startOffset}..{endOffset} for key {key} ({startOffset * 100.0/cached.len}%)")
|
||||
rangeHeader = {"Range": f"bytes={startOffset}-{endOffset-1}"}
|
||||
headers_with_range = dict(cached.headers)
|
||||
headers_with_range.update(rangeHeader)
|
||||
async with session.request(cached.method, cached.url, headers=headers_with_range, params=cached.params) as resp:
|
||||
|
||||
# Stream response in chunks
|
||||
# print(f"Response headers: {resp.headers}")
|
||||
if "Content-Range" not in resp.headers:
|
||||
raise ValueError("No Content-Range in response for Range request")
|
||||
if "bytes" not in resp.headers["Content-Range"]:
|
||||
raise ValueError("Only bytes Content-Range supported")
|
||||
(start_end, size) = resp.headers["Content-Range"].replace("bytes ", "").split("/")
|
||||
if start_end == "*":
|
||||
(start, end) = (0, size - 1)
|
||||
else:
|
||||
(start, end) = [int(i) for i in start_end.split("-")]
|
||||
# print(f"Content-Range: {start}-{end}/{size}")
|
||||
# 1newChunk: Chunk = None
|
||||
receivedBytes: bytes = b""
|
||||
receivedBytesTotal = 0
|
||||
|
||||
async for receivedBytesLoc in resp.content.iter_chunked(CHUNK_SIZE):
|
||||
receivedBytes += receivedBytesLoc
|
||||
receivedBytesTotal += len(receivedBytesLoc)
|
||||
|
||||
await cached.set(start, receivedBytes) # 2
|
||||
limiter.consumed(receivedBytesTotal)
|
||||
delay = limiter.delay()
|
||||
print(f"Sleep {delay}, {receivedBytesTotal}, est {limiter._estimated_speed} bytes/second")
|
||||
if startFrom is None:
|
||||
await asyncio.sleep(delay)
|
||||
else:
|
||||
print(f"Skip sleep for priority download")
|
||||
await asyncio.sleep(0)
|
||||
|
||||
print(f"Downloading done")
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"downloader error: {e}", file=sys.stderr)
|
||||
await asyncio.sleep(10)
|
||||
traceback.print_exc()
|
||||
|
||||
# Python does not have a built-in atomic integer type.
|
||||
# For atomic operations, you can use threading.Lock or asyncio.Lock to protect access to an integer.
|
||||
# Alternatively, you can use multiprocessing.Value for atomicity across processes.
|
||||
# For asyncio, you can implement a simple atomic integer like this:
|
||||
|
||||
class AtomicInteger:
|
||||
def __init__(self, initial=0):
|
||||
self._value = initial
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def get(self):
|
||||
async with self._lock:
|
||||
return self._value
|
||||
|
||||
async def set(self, value):
|
||||
async with self._lock:
|
||||
self._value = value
|
||||
|
||||
async def increment(self, delta=1):
|
||||
async with self._lock:
|
||||
self._value += delta
|
||||
return self._value
|
||||
|
||||
async def decrement(self, delta=1):
|
||||
async with self._lock:
|
||||
self._value -= delta
|
||||
return self._value
|
||||
stream_lock = AtomicInteger()
|
||||
async def proxy_handler(request):
|
||||
try:
|
||||
backend_url = request.app['backend_url']
|
||||
headers = dict(request.headers)
|
||||
params = request.rel_url.query
|
||||
path = request.rel_url.path
|
||||
method = request.method
|
||||
body = await request.read()
|
||||
# if "Range" not in headers.keys():
|
||||
if "stream" not in path or "m3u" in path:
|
||||
# print(f"Simple handler {method}, {path}, {tuple(sorted(params.items()))}, {body}, {headers}")
|
||||
try:
|
||||
return await simple_proxy_handler(request)
|
||||
except Exception as e:
|
||||
print(f"simple_proxy_handler error: {e}", file=sys.stderr)
|
||||
# traceback.print_exc()
|
||||
return web.Response(status=500, text=f"Internal Server Error: {e}")
|
||||
|
||||
if "Range" not in headers.keys():
|
||||
headers["Range"] = "bytes=0-"
|
||||
|
||||
range = headers.get("Range")
|
||||
if range is not None:
|
||||
if "bytes" not in range:
|
||||
raise ValueError("Only bytes range supported")
|
||||
range = range.replace("bytes=", "")
|
||||
def to_int_or_none(s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
return None
|
||||
range = [to_int_or_none(i) for r in range.split(",") for i in r.split("-") ]
|
||||
|
||||
# Make a HEAD request to get resource info
|
||||
async with aiohttp.ClientSession() as session:
|
||||
head_headers = dict(headers)
|
||||
head_headers.pop("Range", None)
|
||||
head_resp = await session.head(backend_url + path, headers=head_headers, params=params)
|
||||
# print(f"HEAD response status: {head_resp.status}")
|
||||
# print(f"HEAD response headers: {head_resp.headers}")
|
||||
headHeaders = head_resp.headers
|
||||
if head_resp.status != 200:
|
||||
return web.Response(status=head_resp.status, text=f"Upstream server returned status {head_resp.status}")
|
||||
|
||||
cache_key = (method, path, (headHeaders['Etag'][:20].__hash__()))
|
||||
# cache_key = (method, path, (tuple(sorted(params.items())), body, tuple(sorted(headHeaders.items()))).__hash__())
|
||||
print(f"Cache key: {cache_key}")
|
||||
# print(f"Range requested: {range}")
|
||||
cached = await response_cache.getOrCreate(cache_key, headHeaders, method=method, url = backend_url + path, headers=head_headers, params=params)
|
||||
|
||||
start = (range and range[0]) or 0
|
||||
headHeaders = dict(headHeaders)
|
||||
if range is not None:
|
||||
end = range[1] if len(range) > 1 and range[1] is not None else cached.len - 1
|
||||
content_range_header = f"bytes {start}-{end}/{cached.len}"
|
||||
headHeaders["Content-Range"] = content_range_header
|
||||
headHeaders["Content-Length"] = f"{end - start + 1}"
|
||||
headHeaders["Accept-Ranges"] = 'bytes'
|
||||
# print(f"Serving with headers: {headHeaders}")
|
||||
stream_lock.increment()
|
||||
try:
|
||||
response = web.StreamResponse(status=206, headers=headHeaders)
|
||||
await response.prepare(request)
|
||||
# print("Sending chunks")
|
||||
stream = cached.get(start)
|
||||
leftBytes = (end - start + 1) if range is not None else cached.len - start
|
||||
async for chunk in stream:
|
||||
if 'NoPriority' not in headers.keys():
|
||||
priority[cache_key] = start
|
||||
await response.write(chunk[:min(len(chunk), leftBytes)])
|
||||
leftBytes -= len(chunk)
|
||||
# print(f"Sending chunks {cache_key} {len(chunk)}")
|
||||
if leftBytes <= 0:
|
||||
break
|
||||
await asyncio.sleep(0)
|
||||
await response.write_eof()
|
||||
finally:
|
||||
stream_lock.decrement()
|
||||
return response
|
||||
|
||||
except aiohttp.ClientConnectionResetError as e:
|
||||
print(f"ClientConnectionResetError: {e}", file=sys.stderr)
|
||||
return web.Response(status=500, text=f"Internal Server Error: {e}")
|
||||
# except aiohttp.ConnectionResetError as e:
|
||||
# print(f"ConnectionResetError: {e}", file=sys.stderr)
|
||||
# return web.Response(status=500, text=f"Internal Server Error: {e}")
|
||||
except Exception as e:
|
||||
print(f"proxy_handler error: {e}", file=sys.stderr)
|
||||
traceback.print_exc()
|
||||
return web.Response(status=500, text=f"Internal Server Error: {e}")
|
||||
|
||||
|
||||
async def server(app):
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, '0.0.0.0', 8080)
|
||||
await site.start()
|
||||
|
||||
async def init_m3u(request):
|
||||
from m3u_download import process_m3u,get_m3u_urls, download_head
|
||||
while await stream_lock.get() > 0:
|
||||
await asyncio.sleep(0.1)
|
||||
tail = request.match_info['tail']
|
||||
params = request.rel_url.query_string
|
||||
url = f"http://0.0.0.0:8080/{tail}"
|
||||
if params:
|
||||
url += f"?{params}"
|
||||
|
||||
success = []
|
||||
async with aiohttp.ClientSession() as session:
|
||||
urls = await get_m3u_urls(url, session=session)
|
||||
for url in urls:
|
||||
print(f" URL: {url}")
|
||||
if await download_head(session, url):
|
||||
print(f" URL ok: {url}")
|
||||
success.append(url)
|
||||
|
||||
print(f"Processed {len(success)} URLs")
|
||||
return web.Response(text=f"OK\nProcessed {len(success)} URLs\n", content_type='text/plain')
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
backend_url = "http://95.142.46.84:5665"
|
||||
else:
|
||||
backend_url = sys.argv[1]
|
||||
app = web.Application()
|
||||
app['backend_url'] = backend_url
|
||||
app.router.add_get('/stats', stats_handler)
|
||||
|
||||
# http://192.168.1.96:8080/add_m3u/playlistall/all.m3u
|
||||
# http://192.168.1.96:8080/add_m3u/stream/Squid%20Game%20S03%20%5B1080p%5D.m3u?link=732f0756f5d6304adfa8532a8c1f6ddd3a753ac6&m3u
|
||||
# http://192.168.1.96:8080/add_m3u/stream/Tears%20of%20Steel.m3u?link=209c8226b299b308beaf2b9cd3fb49212dbd13ec&m3u
|
||||
app.router.add_get('/add_m3u/{tail:.*}', init_m3u)
|
||||
app.router.add_route('*', '/{tail:.*}', proxy_handler)
|
||||
loop = asyncio.new_event_loop()
|
||||
loop.create_task(downloader())
|
||||
loop.create_task(merger())
|
||||
loop.create_task(server(app))
|
||||
|
||||
loop.run_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,5 @@
|
||||
aiohttp
|
||||
sortedcontainers
|
||||
aiofiles
|
||||
aiohttp
|
||||
tqdm
|
||||
@@ -0,0 +1,68 @@
|
||||
from random import randint
|
||||
import time
|
||||
|
||||
class DownloadSpeedLimiter:
|
||||
def __init__(self, max_speed_bytes_per_sec):
|
||||
self.max_speed = max_speed_bytes_per_sec
|
||||
# self._start_time = time.time()
|
||||
self._prev_time = time.time() - 10
|
||||
self._estimated_speed = 0
|
||||
self._ratio = 0.5
|
||||
self._delay = 0.0
|
||||
|
||||
def consumed(self, bytes_count: int):
|
||||
now = time.time()
|
||||
elapsed = now - self._prev_time
|
||||
self._prev_time = now
|
||||
self._estimated_speed = self._estimated_speed * self._ratio + (1.0 - self._ratio) * (bytes_count + 0.0)/elapsed
|
||||
|
||||
def delay(self) -> float:
|
||||
if self.max_speed == 0:
|
||||
return 0.0
|
||||
newDelay = (self._estimated_speed - self.max_speed) / self.max_speed
|
||||
self._delay = self._delay + 0.2 * newDelay
|
||||
return max(0.0, self._delay)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
start = time.time()
|
||||
limiter = DownloadSpeedLimiter(2000)
|
||||
counterStart = time.time()
|
||||
consumed = 0
|
||||
def consume(number):
|
||||
global consumed, counterStart
|
||||
if time.time() - counterStart > 10:
|
||||
consumed = 0
|
||||
counterStart = time.time()
|
||||
else:
|
||||
consumed += number
|
||||
limiter.consumed(number)
|
||||
delay = limiter.delay()
|
||||
|
||||
print(f"Consumed {number} bytes, sleeping for {delay} seconds")
|
||||
print(f"estimated speed {limiter._estimated_speed}")
|
||||
print(f"internal delay {limiter._delay}")
|
||||
print(f"Rate {consumed / (time.time() - counterStart)}")
|
||||
time.sleep(delay)
|
||||
|
||||
while time.time() - start < 10:
|
||||
number = randint(1000, 2000) / 2
|
||||
time.sleep(0.01)
|
||||
consume(number)
|
||||
|
||||
print(f"Consuming zeros")
|
||||
start = time.time()
|
||||
while time.time() - start < 10:
|
||||
number = 0
|
||||
time.sleep(0.01)
|
||||
consume(number)
|
||||
|
||||
print(f"Consuming 100")
|
||||
start = time.time()
|
||||
while time.time() - start < 10:
|
||||
number = 10000
|
||||
time.sleep(0.01)
|
||||
consume(number)
|
||||
print(f"Consumed {consumed} bytes")
|
||||
print(f"estimated speed {limiter._estimated_speed}")
|
||||
print(f"Rate {consumed / (time.time() - counterStart)}")
|
||||
Reference in New Issue
Block a user