Добавить протокол Altera Logic и общие клиенты прошивки
This commit is contained in:
268
python/setprotocol/firmware_database.py
Normal file
268
python/setprotocol/firmware_database.py
Normal file
@@ -0,0 +1,268 @@
|
||||
"""Standalone firmware database: HTTPS catalog, verified downloads and Gitea publishing.
|
||||
|
||||
No Qt, desktop application, credential store or MCU dependencies.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import quote, unquote, urljoin, urlsplit
|
||||
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
||||
|
||||
from .firmware_catalog import MAX_MANIFEST_BYTES, FirmwareRelease, parse_firmware_catalog
|
||||
from .firmware_publish import (
|
||||
MAX_FIRMWARE_BYTES, FirmwarePublication, firmware_release_entry,
|
||||
firmware_release_tag, sha256_file, update_firmware_manifest,
|
||||
)
|
||||
|
||||
CHUNK = 128 * 1024
|
||||
|
||||
|
||||
def _origin(url: str) -> tuple[str, str, int]:
|
||||
parsed = urlsplit(url)
|
||||
if (parsed.scheme != "https" or not parsed.hostname
|
||||
or parsed.username is not None or parsed.password is not None):
|
||||
raise ValueError("Expected an HTTPS URL without embedded credentials")
|
||||
return parsed.scheme, parsed.hostname, parsed.port or 443
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Credentials:
|
||||
login: str
|
||||
password: str = field(repr=False)
|
||||
|
||||
|
||||
class _NoRedirect(HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
return None
|
||||
|
||||
|
||||
class HttpsClient:
|
||||
"""Credentials are restricted to one origin, including across redirects."""
|
||||
|
||||
def __init__(self, credential_origin: str, credentials: Credentials | None = None):
|
||||
self.origin = _origin(credential_origin)
|
||||
self.credentials = credentials
|
||||
self.opener = build_opener(_NoRedirect())
|
||||
|
||||
def open(self, url: str, *, method: str = "GET", data: bytes | None = None,
|
||||
content_type: str = "application/json"):
|
||||
for attempt in range(6):
|
||||
origin = _origin(url)
|
||||
headers = {"Accept": "application/json, application/octet-stream",
|
||||
"Content-Type": content_type, "User-Agent": "templates-firmware-db/1"}
|
||||
if self.credentials is not None and origin == self.origin:
|
||||
raw = f"{self.credentials.login}:{self.credentials.password}".encode("utf-8")
|
||||
headers["Authorization"] = "Basic " + base64.b64encode(raw).decode("ascii")
|
||||
request = Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
return self.opener.open(request, timeout=180)
|
||||
except HTTPError as error:
|
||||
# Never replay a write or its body at a redirect destination.
|
||||
if (method != "GET" or error.code not in (301, 302, 303, 307, 308)
|
||||
or not error.headers.get("Location") or attempt == 5):
|
||||
raise
|
||||
destination = urljoin(url, error.headers["Location"])
|
||||
error.close()
|
||||
url = destination
|
||||
raise RuntimeError("Too many redirects")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GiteaRepository:
|
||||
server: str
|
||||
owner: str
|
||||
repository: str
|
||||
branch: str = "main"
|
||||
manifest_path: str = "update.json"
|
||||
|
||||
def __post_init__(self):
|
||||
_origin(self.server)
|
||||
parsed = urlsplit(self.server)
|
||||
if parsed.query or parsed.fragment:
|
||||
raise ValueError("Server URL must not contain a query or fragment")
|
||||
if not all((self.owner, self.repository, self.branch, self.manifest_path)):
|
||||
raise ValueError("Repository settings must not be empty")
|
||||
|
||||
@property
|
||||
def web(self) -> str:
|
||||
return f"{self.server.rstrip('/')}/{quote(self.owner, safe='')}/{quote(self.repository, safe='')}"
|
||||
|
||||
@property
|
||||
def api(self) -> str:
|
||||
return (f"{self.server.rstrip('/')}/api/v1/repos/"
|
||||
f"{quote(self.owner, safe='')}/{quote(self.repository, safe='')}")
|
||||
|
||||
@property
|
||||
def manifest_url(self) -> str:
|
||||
return (f"{self.web}/raw/branch/{quote(self.branch, safe='')}/"
|
||||
f"{quote(self.manifest_path, safe='/')}")
|
||||
|
||||
|
||||
class FirmwareDatabase:
|
||||
"""Blocking service; call from a worker when integrating with a GUI."""
|
||||
|
||||
def __init__(self, manifest_url: str, cache_dir: Path, *,
|
||||
credentials: Credentials | None = None, client=None):
|
||||
_origin(manifest_url)
|
||||
self.manifest_url = manifest_url
|
||||
self.cache_dir = Path(cache_dir)
|
||||
self.client = client if client is not None else HttpsClient(manifest_url, credentials)
|
||||
|
||||
def read_catalog(self, *, product: str | None = None,
|
||||
transport: str | None = None) -> list[FirmwareRelease]:
|
||||
with self.client.open(self.manifest_url) as response:
|
||||
data = response.read(MAX_MANIFEST_BYTES + 1)
|
||||
releases = parse_firmware_catalog(data, self.manifest_url)
|
||||
return [r for r in releases
|
||||
if (product is None or r.product.casefold() == product.casefold())
|
||||
and (transport is None or r.transport == transport.lower())]
|
||||
|
||||
def download(self, release: FirmwareRelease,
|
||||
progress: Callable[[int], None] | None = None) -> Path:
|
||||
# Validate even objects constructed directly by callers (including file names).
|
||||
entry = {"product": release.product, "versionName": release.version,
|
||||
"versionCode": release.version_code, "imageUrl": release.image_url,
|
||||
"fileName": release.file_name, "sha256": release.sha256,
|
||||
"transport": release.transport}
|
||||
release = parse_firmware_catalog(
|
||||
json.dumps({"firmware": [entry]}).encode(), self.manifest_url)[0]
|
||||
directory = self.cache_dir / release.sha256
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
target = directory / release.file_name
|
||||
if target.is_file() and sha256_file(target) == release.sha256:
|
||||
if progress:
|
||||
progress(100)
|
||||
return target
|
||||
# Unique staging files keep concurrent downloads independent.
|
||||
with tempfile.NamedTemporaryFile(dir=directory, suffix=".part", delete=False) as output:
|
||||
temporary = Path(output.name)
|
||||
try:
|
||||
digest = hashlib.sha256()
|
||||
received = 0
|
||||
with self.client.open(release.image_url) as response, temporary.open("wb") as output:
|
||||
total = int(response.headers.get("Content-Length", "-1") or -1)
|
||||
if total > MAX_FIRMWARE_BYTES:
|
||||
raise ValueError("Firmware exceeds maximum size")
|
||||
while chunk := response.read(CHUNK):
|
||||
received += len(chunk)
|
||||
if received > MAX_FIRMWARE_BYTES:
|
||||
raise ValueError("Firmware exceeds maximum size")
|
||||
digest.update(chunk)
|
||||
output.write(chunk)
|
||||
if progress and total > 0:
|
||||
progress(min(99, received * 100 // total))
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
if not received or digest.hexdigest() != release.sha256:
|
||||
raise ValueError("Downloaded firmware is empty or SHA-256 does not match")
|
||||
temporary.replace(target)
|
||||
if progress:
|
||||
progress(100)
|
||||
return target
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
class GiteaFirmwarePublisher:
|
||||
"""Publish an immutable image, verify it, then commit the shared catalog."""
|
||||
|
||||
def __init__(self, repository: GiteaRepository, credentials: Credentials | None = None,
|
||||
*, client=None):
|
||||
self.repository = repository
|
||||
self.client = client if client is not None else HttpsClient(repository.server, credentials)
|
||||
|
||||
def _call(self, method: str, path: str, payload=None):
|
||||
binary = isinstance(payload, bytes)
|
||||
data = payload if binary or payload is None else json.dumps(payload).encode("utf-8")
|
||||
with self.client.open(
|
||||
self.repository.api + path, method=method, data=data,
|
||||
content_type="application/octet-stream" if binary else "application/json",
|
||||
) as response:
|
||||
raw = response.read(4 * MAX_MANIFEST_BYTES + 1)
|
||||
if len(raw) > 4 * MAX_MANIFEST_BYTES:
|
||||
raise ValueError("Gitea response exceeds maximum size")
|
||||
return json.loads(raw) if raw else None
|
||||
|
||||
def preflight(self, publication: FirmwarePublication) -> dict:
|
||||
publication.validate()
|
||||
return self._entry(publication, sha256_file(publication.path))
|
||||
|
||||
def _entry(self, publication: FirmwarePublication, digest: str) -> dict:
|
||||
tag = firmware_release_tag(publication)
|
||||
# Different bytes never replace an asset used by an already published row.
|
||||
asset_name = f"{publication.path.stem}-{digest}{publication.path.suffix.lower()}"
|
||||
url = f"{self.repository.web}/releases/download/{quote(tag, safe='')}/{quote(asset_name, safe='')}"
|
||||
entry = firmware_release_entry(publication, url, digest)
|
||||
update_firmware_manifest({}, entry)
|
||||
return entry
|
||||
|
||||
def publish(self, publication: FirmwarePublication) -> dict:
|
||||
publication.validate()
|
||||
# Snapshot the bytes once so a concurrent rebuild cannot change the upload.
|
||||
with publication.path.open("rb") as source:
|
||||
image = source.read(MAX_FIRMWARE_BYTES + 1)
|
||||
if not image or len(image) > MAX_FIRMWARE_BYTES:
|
||||
raise ValueError("Firmware is empty or exceeds maximum size")
|
||||
entry = self._entry(publication, hashlib.sha256(image).hexdigest())
|
||||
tag = firmware_release_tag(publication)
|
||||
tag_path = f"/releases/tags/{quote(tag, safe='')}"
|
||||
try:
|
||||
release = self._call("GET", tag_path)
|
||||
except HTTPError as error:
|
||||
if error.code != 404:
|
||||
raise
|
||||
error.close()
|
||||
release = self._call("POST", "/releases", {
|
||||
"tag_name": tag, "target_commitish": self.repository.branch,
|
||||
"name": f"{publication.product} {publication.version_name}",
|
||||
"body": publication.notes, "draft": False, "prerelease": False,
|
||||
})
|
||||
asset_name = urlsplit(entry["imageUrl"]).path.rsplit("/", 1)[1]
|
||||
assets_path = f"/releases/{int(release['id'])}/assets"
|
||||
# Gitea assets are paginated; do not silently miss an existing image.
|
||||
found = False
|
||||
page = 1
|
||||
while True:
|
||||
assets = self._call("GET", f"{assets_path}?limit=50&page={page}")
|
||||
if any(asset["name"] == unquote(asset_name) for asset in assets):
|
||||
found = True
|
||||
break
|
||||
if len(assets) < 50:
|
||||
break
|
||||
page += 1
|
||||
if not found:
|
||||
self._call("POST", f"{assets_path}?name={asset_name}", image)
|
||||
# Always verify against the server, never against a previous local cache.
|
||||
with tempfile.TemporaryDirectory(prefix="firmware-verify-") as cache:
|
||||
reader = FirmwareDatabase(self.repository.manifest_url, Path(cache), client=self.client)
|
||||
parsed = parse_firmware_catalog(
|
||||
json.dumps({"firmware": [entry]}).encode(), self.repository.manifest_url)[0]
|
||||
reader.download(parsed)
|
||||
|
||||
path = f"/contents/{quote(self.repository.manifest_path, safe='/')}"
|
||||
source = self._call("GET", f"{path}?ref={quote(self.repository.branch, safe='')}")
|
||||
manifest = json.loads(base64.b64decode(source["content"]))
|
||||
if not isinstance(manifest, dict):
|
||||
raise ValueError("Manifest must be a JSON object")
|
||||
updated = update_firmware_manifest(manifest, entry)
|
||||
if updated != manifest:
|
||||
self._call("PUT", path, {
|
||||
"branch": self.repository.branch, "sha": source["sha"],
|
||||
"message": f"Publish firmware {publication.product} {publication.version_name}",
|
||||
"content": base64.b64encode(json.dumps(updated, ensure_ascii=False).encode()).decode(),
|
||||
})
|
||||
# Use the contents API to avoid a stale raw-file CDN cache on readback.
|
||||
check = self._call("GET", f"{path}?ref={quote(self.repository.branch, safe='')}")
|
||||
rows = parse_firmware_catalog(base64.b64decode(check["content"]), self.repository.manifest_url)
|
||||
if parsed not in rows:
|
||||
raise RuntimeError("Published catalog readback does not match the release")
|
||||
return entry
|
||||
Reference in New Issue
Block a user