273 lines
12 KiB
Python
273 lines
12 KiB
Python
# Самостоятельный доступ к базе прошивок: HTTPS, проверяемая загрузка, кэш и публикация в
|
||
# Gitea. Проверка SHA-256 связывает скачанные байты с записью каталога; сетевые полномочия
|
||
# передаются явно через клиент.
|
||
|
||
"""Standalone firmware database: HTTPS catalog, verified downloads and Gitea publishing.
|
||
|
||
No Qt, desktop application, credential store or MCU dependencies.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import base64
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import tempfile
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Callable
|
||
from urllib.error import HTTPError
|
||
from urllib.parse import quote, unquote, urljoin, urlsplit
|
||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||
|
||
from .firmware_catalog import MAX_MANIFEST_BYTES, FirmwareRelease, parse_firmware_catalog
|
||
from .firmware_publish import (
|
||
MAX_FIRMWARE_BYTES, FirmwarePublication, firmware_release_entry,
|
||
firmware_release_tag, sha256_file, update_firmware_manifest,
|
||
)
|
||
|
||
CHUNK = 128 * 1024
|
||
|
||
|
||
def _origin(url: str) -> tuple[str, str, int]:
|
||
parsed = urlsplit(url)
|
||
if (parsed.scheme != "https" or not parsed.hostname
|
||
or parsed.username is not None or parsed.password is not None):
|
||
raise ValueError("Expected an HTTPS URL without embedded credentials")
|
||
return parsed.scheme, parsed.hostname, parsed.port or 443
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Credentials:
|
||
login: str
|
||
password: str = field(repr=False)
|
||
|
||
|
||
class _NoRedirect(HTTPRedirectHandler):
|
||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||
return None
|
||
|
||
|
||
class HttpsClient:
|
||
"""Credentials are restricted to one origin, including across redirects."""
|
||
|
||
def __init__(self, credential_origin: str, credentials: Credentials | None = None):
|
||
self.origin = _origin(credential_origin)
|
||
self.credentials = credentials
|
||
self.opener = build_opener(_NoRedirect())
|
||
|
||
def open(self, url: str, *, method: str = "GET", data: bytes | None = None,
|
||
content_type: str = "application/json"):
|
||
for attempt in range(6):
|
||
origin = _origin(url)
|
||
headers = {"Accept": "application/json, application/octet-stream",
|
||
"Content-Type": content_type, "User-Agent": "templates-firmware-db/1"}
|
||
if self.credentials is not None and origin == self.origin:
|
||
raw = f"{self.credentials.login}:{self.credentials.password}".encode("utf-8")
|
||
headers["Authorization"] = "Basic " + base64.b64encode(raw).decode("ascii")
|
||
request = Request(url, data=data, headers=headers, method=method)
|
||
try:
|
||
return self.opener.open(request, timeout=180)
|
||
except HTTPError as error:
|
||
# Never replay a write or its body at a redirect destination.
|
||
if (method != "GET" or error.code not in (301, 302, 303, 307, 308)
|
||
or not error.headers.get("Location") or attempt == 5):
|
||
raise
|
||
destination = urljoin(url, error.headers["Location"])
|
||
error.close()
|
||
url = destination
|
||
raise RuntimeError("Too many redirects")
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class GiteaRepository:
|
||
server: str
|
||
owner: str
|
||
repository: str
|
||
branch: str = "main"
|
||
manifest_path: str = "update.json"
|
||
|
||
def __post_init__(self):
|
||
_origin(self.server)
|
||
parsed = urlsplit(self.server)
|
||
if parsed.query or parsed.fragment:
|
||
raise ValueError("Server URL must not contain a query or fragment")
|
||
if not all((self.owner, self.repository, self.branch, self.manifest_path)):
|
||
raise ValueError("Repository settings must not be empty")
|
||
|
||
@property
|
||
def web(self) -> str:
|
||
return f"{self.server.rstrip('/')}/{quote(self.owner, safe='')}/{quote(self.repository, safe='')}"
|
||
|
||
@property
|
||
def api(self) -> str:
|
||
return (f"{self.server.rstrip('/')}/api/v1/repos/"
|
||
f"{quote(self.owner, safe='')}/{quote(self.repository, safe='')}")
|
||
|
||
@property
|
||
def manifest_url(self) -> str:
|
||
return (f"{self.web}/raw/branch/{quote(self.branch, safe='')}/"
|
||
f"{quote(self.manifest_path, safe='/')}")
|
||
|
||
|
||
class FirmwareDatabase:
|
||
"""Blocking service; call from a worker when integrating with a GUI."""
|
||
|
||
def __init__(self, manifest_url: str, cache_dir: Path, *,
|
||
credentials: Credentials | None = None, client=None):
|
||
_origin(manifest_url)
|
||
self.manifest_url = manifest_url
|
||
self.cache_dir = Path(cache_dir)
|
||
self.client = client if client is not None else HttpsClient(manifest_url, credentials)
|
||
|
||
def read_catalog(self, *, product: str | None = None,
|
||
transport: str | None = None) -> list[FirmwareRelease]:
|
||
with self.client.open(self.manifest_url) as response:
|
||
data = response.read(MAX_MANIFEST_BYTES + 1)
|
||
releases = parse_firmware_catalog(data, self.manifest_url)
|
||
return [r for r in releases
|
||
if (product is None or r.product.casefold() == product.casefold())
|
||
and (transport is None or r.transport == transport.lower())]
|
||
|
||
def download(self, release: FirmwareRelease,
|
||
progress: Callable[[int], None] | None = None) -> Path:
|
||
# Validate even objects constructed directly by callers (including file names).
|
||
entry = {"product": release.product, "versionName": release.version,
|
||
"versionCode": release.version_code, "imageUrl": release.image_url,
|
||
"fileName": release.file_name, "sha256": release.sha256,
|
||
"transport": release.transport}
|
||
release = parse_firmware_catalog(
|
||
json.dumps({"firmware": [entry]}).encode(), self.manifest_url)[0]
|
||
directory = self.cache_dir / release.sha256
|
||
directory.mkdir(parents=True, exist_ok=True)
|
||
target = directory / release.file_name
|
||
if target.is_file() and sha256_file(target) == release.sha256:
|
||
if progress:
|
||
progress(100)
|
||
return target
|
||
# Unique staging files keep concurrent downloads independent.
|
||
with tempfile.NamedTemporaryFile(dir=directory, suffix=".part", delete=False) as output:
|
||
temporary = Path(output.name)
|
||
try:
|
||
digest = hashlib.sha256()
|
||
received = 0
|
||
with self.client.open(release.image_url) as response, temporary.open("wb") as output:
|
||
total = int(response.headers.get("Content-Length", "-1") or -1)
|
||
if total > MAX_FIRMWARE_BYTES:
|
||
raise ValueError("Firmware exceeds maximum size")
|
||
while chunk := response.read(CHUNK):
|
||
received += len(chunk)
|
||
if received > MAX_FIRMWARE_BYTES:
|
||
raise ValueError("Firmware exceeds maximum size")
|
||
digest.update(chunk)
|
||
output.write(chunk)
|
||
if progress and total > 0:
|
||
progress(min(99, received * 100 // total))
|
||
output.flush()
|
||
os.fsync(output.fileno())
|
||
if not received or digest.hexdigest() != release.sha256:
|
||
raise ValueError("Downloaded firmware is empty or SHA-256 does not match")
|
||
temporary.replace(target)
|
||
if progress:
|
||
progress(100)
|
||
return target
|
||
finally:
|
||
temporary.unlink(missing_ok=True)
|
||
|
||
|
||
class GiteaFirmwarePublisher:
|
||
"""Publish an immutable image, verify it, then commit the shared catalog."""
|
||
|
||
def __init__(self, repository: GiteaRepository, credentials: Credentials | None = None,
|
||
*, client=None):
|
||
self.repository = repository
|
||
self.client = client if client is not None else HttpsClient(repository.server, credentials)
|
||
|
||
def _call(self, method: str, path: str, payload=None):
|
||
binary = isinstance(payload, bytes)
|
||
data = payload if binary or payload is None else json.dumps(payload).encode("utf-8")
|
||
with self.client.open(
|
||
self.repository.api + path, method=method, data=data,
|
||
content_type="application/octet-stream" if binary else "application/json",
|
||
) as response:
|
||
raw = response.read(4 * MAX_MANIFEST_BYTES + 1)
|
||
if len(raw) > 4 * MAX_MANIFEST_BYTES:
|
||
raise ValueError("Gitea response exceeds maximum size")
|
||
return json.loads(raw) if raw else None
|
||
|
||
def preflight(self, publication: FirmwarePublication) -> dict:
|
||
publication.validate()
|
||
return self._entry(publication, sha256_file(publication.path))
|
||
|
||
def _entry(self, publication: FirmwarePublication, digest: str) -> dict:
|
||
tag = firmware_release_tag(publication)
|
||
# Different bytes never replace an asset used by an already published row.
|
||
asset_name = f"{publication.path.stem}-{digest}{publication.path.suffix.lower()}"
|
||
url = f"{self.repository.web}/releases/download/{quote(tag, safe='')}/{quote(asset_name, safe='')}"
|
||
entry = firmware_release_entry(publication, url, digest)
|
||
update_firmware_manifest({}, entry)
|
||
return entry
|
||
|
||
def publish(self, publication: FirmwarePublication) -> dict:
|
||
publication.validate()
|
||
# Snapshot the bytes once so a concurrent rebuild cannot change the upload.
|
||
with publication.path.open("rb") as source:
|
||
image = source.read(MAX_FIRMWARE_BYTES + 1)
|
||
if not image or len(image) > MAX_FIRMWARE_BYTES:
|
||
raise ValueError("Firmware is empty or exceeds maximum size")
|
||
entry = self._entry(publication, hashlib.sha256(image).hexdigest())
|
||
tag = firmware_release_tag(publication)
|
||
tag_path = f"/releases/tags/{quote(tag, safe='')}"
|
||
try:
|
||
release = self._call("GET", tag_path)
|
||
except HTTPError as error:
|
||
if error.code != 404:
|
||
raise
|
||
error.close()
|
||
release = self._call("POST", "/releases", {
|
||
"tag_name": tag, "target_commitish": self.repository.branch,
|
||
"name": f"{publication.product} {publication.version_name}",
|
||
"body": publication.notes, "draft": False, "prerelease": False,
|
||
})
|
||
asset_name = urlsplit(entry["imageUrl"]).path.rsplit("/", 1)[1]
|
||
assets_path = f"/releases/{int(release['id'])}/assets"
|
||
# Gitea assets are paginated; do not silently miss an existing image.
|
||
found = False
|
||
page = 1
|
||
while True:
|
||
assets = self._call("GET", f"{assets_path}?limit=50&page={page}")
|
||
if any(asset["name"] == unquote(asset_name) for asset in assets):
|
||
found = True
|
||
break
|
||
if len(assets) < 50:
|
||
break
|
||
page += 1
|
||
if not found:
|
||
self._call("POST", f"{assets_path}?name={asset_name}", image)
|
||
# Always verify against the server, never against a previous local cache.
|
||
with tempfile.TemporaryDirectory(prefix="firmware-verify-") as cache:
|
||
reader = FirmwareDatabase(self.repository.manifest_url, Path(cache), client=self.client)
|
||
parsed = parse_firmware_catalog(
|
||
json.dumps({"firmware": [entry]}).encode(), self.repository.manifest_url)[0]
|
||
reader.download(parsed)
|
||
|
||
path = f"/contents/{quote(self.repository.manifest_path, safe='/')}"
|
||
source = self._call("GET", f"{path}?ref={quote(self.repository.branch, safe='')}")
|
||
manifest = json.loads(base64.b64decode(source["content"]))
|
||
if not isinstance(manifest, dict):
|
||
raise ValueError("Manifest must be a JSON object")
|
||
updated = update_firmware_manifest(manifest, entry)
|
||
if updated != manifest:
|
||
self._call("PUT", path, {
|
||
"branch": self.repository.branch, "sha": source["sha"],
|
||
"message": f"Publish firmware {publication.product} {publication.version_name}",
|
||
"content": base64.b64encode(json.dumps(updated, ensure_ascii=False).encode()).decode(),
|
||
})
|
||
# Use the contents API to avoid a stale raw-file CDN cache on readback.
|
||
check = self._call("GET", f"{path}?ref={quote(self.repository.branch, safe='')}")
|
||
rows = parse_firmware_catalog(base64.b64decode(check["content"]), self.repository.manifest_url)
|
||
if parsed not in rows:
|
||
raise RuntimeError("Published catalog readback does not match the release")
|
||
return entry
|