269 lines
12 KiB
Python
269 lines
12 KiB
Python
"""Standalone firmware database: HTTPS catalog, verified downloads and Gitea publishing.
|
|
|
|
No Qt, desktop application, credential store or MCU dependencies.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import tempfile
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Callable
|
|
from urllib.error import HTTPError
|
|
from urllib.parse import quote, unquote, urljoin, urlsplit
|
|
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
|
|
|
from .firmware_catalog import MAX_MANIFEST_BYTES, FirmwareRelease, parse_firmware_catalog
|
|
from .firmware_publish import (
|
|
MAX_FIRMWARE_BYTES, FirmwarePublication, firmware_release_entry,
|
|
firmware_release_tag, sha256_file, update_firmware_manifest,
|
|
)
|
|
|
|
CHUNK = 128 * 1024
|
|
|
|
|
|
def _origin(url: str) -> tuple[str, str, int]:
|
|
parsed = urlsplit(url)
|
|
if (parsed.scheme != "https" or not parsed.hostname
|
|
or parsed.username is not None or parsed.password is not None):
|
|
raise ValueError("Expected an HTTPS URL without embedded credentials")
|
|
return parsed.scheme, parsed.hostname, parsed.port or 443
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Credentials:
|
|
login: str
|
|
password: str = field(repr=False)
|
|
|
|
|
|
class _NoRedirect(HTTPRedirectHandler):
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
return None
|
|
|
|
|
|
class HttpsClient:
|
|
"""Credentials are restricted to one origin, including across redirects."""
|
|
|
|
def __init__(self, credential_origin: str, credentials: Credentials | None = None):
|
|
self.origin = _origin(credential_origin)
|
|
self.credentials = credentials
|
|
self.opener = build_opener(_NoRedirect())
|
|
|
|
def open(self, url: str, *, method: str = "GET", data: bytes | None = None,
|
|
content_type: str = "application/json"):
|
|
for attempt in range(6):
|
|
origin = _origin(url)
|
|
headers = {"Accept": "application/json, application/octet-stream",
|
|
"Content-Type": content_type, "User-Agent": "templates-firmware-db/1"}
|
|
if self.credentials is not None and origin == self.origin:
|
|
raw = f"{self.credentials.login}:{self.credentials.password}".encode("utf-8")
|
|
headers["Authorization"] = "Basic " + base64.b64encode(raw).decode("ascii")
|
|
request = Request(url, data=data, headers=headers, method=method)
|
|
try:
|
|
return self.opener.open(request, timeout=180)
|
|
except HTTPError as error:
|
|
# Never replay a write or its body at a redirect destination.
|
|
if (method != "GET" or error.code not in (301, 302, 303, 307, 308)
|
|
or not error.headers.get("Location") or attempt == 5):
|
|
raise
|
|
destination = urljoin(url, error.headers["Location"])
|
|
error.close()
|
|
url = destination
|
|
raise RuntimeError("Too many redirects")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GiteaRepository:
|
|
server: str
|
|
owner: str
|
|
repository: str
|
|
branch: str = "main"
|
|
manifest_path: str = "update.json"
|
|
|
|
def __post_init__(self):
|
|
_origin(self.server)
|
|
parsed = urlsplit(self.server)
|
|
if parsed.query or parsed.fragment:
|
|
raise ValueError("Server URL must not contain a query or fragment")
|
|
if not all((self.owner, self.repository, self.branch, self.manifest_path)):
|
|
raise ValueError("Repository settings must not be empty")
|
|
|
|
@property
|
|
def web(self) -> str:
|
|
return f"{self.server.rstrip('/')}/{quote(self.owner, safe='')}/{quote(self.repository, safe='')}"
|
|
|
|
@property
|
|
def api(self) -> str:
|
|
return (f"{self.server.rstrip('/')}/api/v1/repos/"
|
|
f"{quote(self.owner, safe='')}/{quote(self.repository, safe='')}")
|
|
|
|
@property
|
|
def manifest_url(self) -> str:
|
|
return (f"{self.web}/raw/branch/{quote(self.branch, safe='')}/"
|
|
f"{quote(self.manifest_path, safe='/')}")
|
|
|
|
|
|
class FirmwareDatabase:
|
|
"""Blocking service; call from a worker when integrating with a GUI."""
|
|
|
|
def __init__(self, manifest_url: str, cache_dir: Path, *,
|
|
credentials: Credentials | None = None, client=None):
|
|
_origin(manifest_url)
|
|
self.manifest_url = manifest_url
|
|
self.cache_dir = Path(cache_dir)
|
|
self.client = client if client is not None else HttpsClient(manifest_url, credentials)
|
|
|
|
def read_catalog(self, *, product: str | None = None,
|
|
transport: str | None = None) -> list[FirmwareRelease]:
|
|
with self.client.open(self.manifest_url) as response:
|
|
data = response.read(MAX_MANIFEST_BYTES + 1)
|
|
releases = parse_firmware_catalog(data, self.manifest_url)
|
|
return [r for r in releases
|
|
if (product is None or r.product.casefold() == product.casefold())
|
|
and (transport is None or r.transport == transport.lower())]
|
|
|
|
def download(self, release: FirmwareRelease,
|
|
progress: Callable[[int], None] | None = None) -> Path:
|
|
# Validate even objects constructed directly by callers (including file names).
|
|
entry = {"product": release.product, "versionName": release.version,
|
|
"versionCode": release.version_code, "imageUrl": release.image_url,
|
|
"fileName": release.file_name, "sha256": release.sha256,
|
|
"transport": release.transport}
|
|
release = parse_firmware_catalog(
|
|
json.dumps({"firmware": [entry]}).encode(), self.manifest_url)[0]
|
|
directory = self.cache_dir / release.sha256
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
target = directory / release.file_name
|
|
if target.is_file() and sha256_file(target) == release.sha256:
|
|
if progress:
|
|
progress(100)
|
|
return target
|
|
# Unique staging files keep concurrent downloads independent.
|
|
with tempfile.NamedTemporaryFile(dir=directory, suffix=".part", delete=False) as output:
|
|
temporary = Path(output.name)
|
|
try:
|
|
digest = hashlib.sha256()
|
|
received = 0
|
|
with self.client.open(release.image_url) as response, temporary.open("wb") as output:
|
|
total = int(response.headers.get("Content-Length", "-1") or -1)
|
|
if total > MAX_FIRMWARE_BYTES:
|
|
raise ValueError("Firmware exceeds maximum size")
|
|
while chunk := response.read(CHUNK):
|
|
received += len(chunk)
|
|
if received > MAX_FIRMWARE_BYTES:
|
|
raise ValueError("Firmware exceeds maximum size")
|
|
digest.update(chunk)
|
|
output.write(chunk)
|
|
if progress and total > 0:
|
|
progress(min(99, received * 100 // total))
|
|
output.flush()
|
|
os.fsync(output.fileno())
|
|
if not received or digest.hexdigest() != release.sha256:
|
|
raise ValueError("Downloaded firmware is empty or SHA-256 does not match")
|
|
temporary.replace(target)
|
|
if progress:
|
|
progress(100)
|
|
return target
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
class GiteaFirmwarePublisher:
|
|
"""Publish an immutable image, verify it, then commit the shared catalog."""
|
|
|
|
def __init__(self, repository: GiteaRepository, credentials: Credentials | None = None,
|
|
*, client=None):
|
|
self.repository = repository
|
|
self.client = client if client is not None else HttpsClient(repository.server, credentials)
|
|
|
|
def _call(self, method: str, path: str, payload=None):
|
|
binary = isinstance(payload, bytes)
|
|
data = payload if binary or payload is None else json.dumps(payload).encode("utf-8")
|
|
with self.client.open(
|
|
self.repository.api + path, method=method, data=data,
|
|
content_type="application/octet-stream" if binary else "application/json",
|
|
) as response:
|
|
raw = response.read(4 * MAX_MANIFEST_BYTES + 1)
|
|
if len(raw) > 4 * MAX_MANIFEST_BYTES:
|
|
raise ValueError("Gitea response exceeds maximum size")
|
|
return json.loads(raw) if raw else None
|
|
|
|
def preflight(self, publication: FirmwarePublication) -> dict:
|
|
publication.validate()
|
|
return self._entry(publication, sha256_file(publication.path))
|
|
|
|
def _entry(self, publication: FirmwarePublication, digest: str) -> dict:
|
|
tag = firmware_release_tag(publication)
|
|
# Different bytes never replace an asset used by an already published row.
|
|
asset_name = f"{publication.path.stem}-{digest}{publication.path.suffix.lower()}"
|
|
url = f"{self.repository.web}/releases/download/{quote(tag, safe='')}/{quote(asset_name, safe='')}"
|
|
entry = firmware_release_entry(publication, url, digest)
|
|
update_firmware_manifest({}, entry)
|
|
return entry
|
|
|
|
def publish(self, publication: FirmwarePublication) -> dict:
|
|
publication.validate()
|
|
# Snapshot the bytes once so a concurrent rebuild cannot change the upload.
|
|
with publication.path.open("rb") as source:
|
|
image = source.read(MAX_FIRMWARE_BYTES + 1)
|
|
if not image or len(image) > MAX_FIRMWARE_BYTES:
|
|
raise ValueError("Firmware is empty or exceeds maximum size")
|
|
entry = self._entry(publication, hashlib.sha256(image).hexdigest())
|
|
tag = firmware_release_tag(publication)
|
|
tag_path = f"/releases/tags/{quote(tag, safe='')}"
|
|
try:
|
|
release = self._call("GET", tag_path)
|
|
except HTTPError as error:
|
|
if error.code != 404:
|
|
raise
|
|
error.close()
|
|
release = self._call("POST", "/releases", {
|
|
"tag_name": tag, "target_commitish": self.repository.branch,
|
|
"name": f"{publication.product} {publication.version_name}",
|
|
"body": publication.notes, "draft": False, "prerelease": False,
|
|
})
|
|
asset_name = urlsplit(entry["imageUrl"]).path.rsplit("/", 1)[1]
|
|
assets_path = f"/releases/{int(release['id'])}/assets"
|
|
# Gitea assets are paginated; do not silently miss an existing image.
|
|
found = False
|
|
page = 1
|
|
while True:
|
|
assets = self._call("GET", f"{assets_path}?limit=50&page={page}")
|
|
if any(asset["name"] == unquote(asset_name) for asset in assets):
|
|
found = True
|
|
break
|
|
if len(assets) < 50:
|
|
break
|
|
page += 1
|
|
if not found:
|
|
self._call("POST", f"{assets_path}?name={asset_name}", image)
|
|
# Always verify against the server, never against a previous local cache.
|
|
with tempfile.TemporaryDirectory(prefix="firmware-verify-") as cache:
|
|
reader = FirmwareDatabase(self.repository.manifest_url, Path(cache), client=self.client)
|
|
parsed = parse_firmware_catalog(
|
|
json.dumps({"firmware": [entry]}).encode(), self.repository.manifest_url)[0]
|
|
reader.download(parsed)
|
|
|
|
path = f"/contents/{quote(self.repository.manifest_path, safe='/')}"
|
|
source = self._call("GET", f"{path}?ref={quote(self.repository.branch, safe='')}")
|
|
manifest = json.loads(base64.b64decode(source["content"]))
|
|
if not isinstance(manifest, dict):
|
|
raise ValueError("Manifest must be a JSON object")
|
|
updated = update_firmware_manifest(manifest, entry)
|
|
if updated != manifest:
|
|
self._call("PUT", path, {
|
|
"branch": self.repository.branch, "sha": source["sha"],
|
|
"message": f"Publish firmware {publication.product} {publication.version_name}",
|
|
"content": base64.b64encode(json.dumps(updated, ensure_ascii=False).encode()).decode(),
|
|
})
|
|
# Use the contents API to avoid a stale raw-file CDN cache on readback.
|
|
check = self._call("GET", f"{path}?ref={quote(self.repository.branch, safe='')}")
|
|
rows = parse_firmware_catalog(base64.b64decode(check["content"]), self.repository.manifest_url)
|
|
if parsed not in rows:
|
|
raise RuntimeError("Published catalog readback does not match the release")
|
|
return entry
|