"""Stream digital CSV into DSView v3, preserving intervals between transitions.""" from contextlib import contextmanager import csv from decimal import Decimal, InvalidOperation import json from math import gcd import os from pathlib import Path import re import tempfile import zipfile from .files import ImportCancelled TICKS = 10**12 # exact decimal picoseconds, independent of float rounding BLOCK_SAMPLES = 8 * 1024 * 1024 MAX_PACKED_BYTES = 2 * 1024**3 TIME_HEADER = re.compile(r'(?:time|timestamp)\s*(?:[\[(](s|ms|us|µs|ns)[\])])?', re.I) def _check(cancel): if cancel(): raise ImportCancelled() @contextmanager def _reader(path, progress, cancel): size = max(1, path.stat().st_size) with path.open(encoding='utf-8-sig', newline='') as stream: consumed = 0 def lines(): nonlocal consumed for index, line in enumerate(stream): consumed += len(line) if index % 4096 == 0: _check(cancel) progress(min(99, consumed * 100 // size)) if line.strip() and not line.lstrip().startswith(('#', ';')): yield line source = lines() header = next(source, '') if not header: raise ValueError('CSV не содержит данных.') try: delimiter = csv.Sniffer().sniff(header, delimiters=',;\t').delimiter except csv.Error as error: raise ValueError('Не удалось определить разделитель CSV: используйте запятую, ; или табуляцию.') from error names = [value.strip() for value in next(csv.reader([header], delimiter=delimiter))] match = TIME_HEADER.fullmatch(names[0]) if not match or not 2 <= len(names) <= 65: raise ValueError('Ожидается CSV: Time [s] (или ms/us/ns), затем цифровые каналы 0/1.') channels = names[1:] if len(set(names)) != len(names) or any(not n or any(c in n for c in '\r\n=[]') for n in channels): raise ValueError('Имена каналов должны быть уникальными и не содержать =, [, ] или переносы строк.') unit = (match[1] or 's').lower() scale = {'s': TICKS, 'ms': 10**9, 'us': 10**6, 'µs': 10**6, 'ns': 1000}[unit] def rows(): previous = None for index, row in enumerate(csv.reader(source, delimiter=delimiter), 2): try: value = Decimal(row[0].strip().replace(',', '.')) * scale if not value.is_finite() or value != value.to_integral_value(): raise ValueError('время должно быть конечным, с точностью не выше 1 пс') timestamp = int(value) if previous is not None and timestamp <= previous: raise ValueError('время должно строго возрастать') levels = tuple(item.strip() for item in row[1:]) if len(levels) != len(channels) or any(item not in ('0', '1') for item in levels): raise ValueError('ожидаются цифровые уровни 0 или 1 для каждого канала') previous = timestamp yield timestamp, levels except (ValueError, InvalidOperation, IndexError) as error: raise ValueError('CSV, строка данных %d: %s' % (index, error)) from error yield channels, rows() def _ones(block, start, end): """Set a half-open run without iterating over every sample.""" if end <= start: return first, bit = divmod(start, 8) last, stop = divmod(end, 8) if first == last: block[first] |= ((1 << (end - start)) - 1) << bit return if bit: block[first] |= (255 << bit) & 255 first += 1 block[first:last] = b'\xff' * (last - first) if stop: block[last] |= (1 << stop) - 1 def convert_csv(source, destination, progress=lambda value: None, cancel=lambda: False, *, max_packed_bytes=MAX_PACKED_BYTES, block_samples=BLOCK_SAMPLES): """Two bounded-memory passes; input is untouched, output created exclusively. The first row establishes initial levels, not an edge. Its timestamp may precede the clock grid by a fraction of one period. All subsequent times and transition intervals are represented exactly; the origin rounds down at most one period. The final level is held to the 64-sample DSL boundary. """ source, destination = Path(source), Path(destination) if source.resolve() == destination.resolve() or destination.exists(): raise ValueError('Для импорта нужен новый выходной файл.') if block_samples < 64 or block_samples % 64: raise ValueError('Размер блока должен быть кратен 64 отсчётам.') stamp = (source.stat().st_size, source.stat().st_mtime_ns) first = previous = anchor = None step = count = 0 with _reader(source, lambda p: progress(p * 40 // 100), cancel) as (channels, rows): for time, _ in rows: if count == 0: first = time elif count == 1: anchor = time else: step = gcd(step, time - previous) previous = time count += 1 if count < 2: raise ValueError('В CSV нужны минимум две строки с разным временем.') if count == 2: step = anchor - first period = gcd(step, TICKS) rate = TICKS // period origin = anchor - ((anchor - first + period - 1) // period) * period samples = (previous - origin) // period + 1 padded = (samples + 63) // 64 * 64 if padded // 8 * len(channels) > max_packed_bytes: raise ValueError('Запись слишком велика для DSView при сохранении точности времени. Выберите меньший интервал CSV.') blocks_count = (padded + block_samples - 1) // block_samples result = dict(channels=channels, samplerate=rate, samples=samples, padding=padded - samples, origin_seconds=str(Decimal(origin) / TICKS), source=str(source)) header = ('[version]\nversion = 3\n[header]\ndriver = virtual-session\n' 'device mode = 0\ncapturefile = data\n' f'total samples = {padded}\ntotal probes = {len(channels)}\ntotal blocks = {blocks_count}\n' f'samplerate = {rate} Hz\ntrigger time = 0\ntrigger pos = 0\n' + ''.join(f'probe{i} = {name}\n' for i, name in enumerate(channels))) session = dict(Device='virtual-session', DeviceMode=0, Version=3, Title='DSView v1.3.2', **{'Sample count': str(padded), 'Sample rate': str(rate), 'Max Height': '1X'}, decoder=[], channel=[dict(colour='default', enabled=True, index=i, name=name, strigger=0, type=10000, view_index=i) for i, name in enumerate(channels)]) fd, temporary = tempfile.mkstemp(prefix='.csv-', suffix='.dsl', dir=destination.parent) os.close(fd) try: with zipfile.ZipFile(temporary, 'w', zipfile.ZIP_DEFLATED, compresslevel=1) as archive: archive.writestr('header', header) archive.writestr('session', json.dumps(session, ensure_ascii=False)) archive.writestr('decoders', '[]') archive.writestr('csv_import.json', json.dumps(result, ensure_ascii=False)) if stamp != (source.stat().st_size, source.stat().st_mtime_ns): raise ValueError('CSV изменился во время чтения.') with _reader(source, lambda p: None, cancel) as (names, rows): if names != channels: raise ValueError('CSV изменился во время чтения.') current = next(rows) if current[0] != first: raise ValueError('CSV изменился во время чтения.') following = next(rows, None) block_start = 0 row_count = 1 for block in range(blocks_count): _check(cancel) block_end = min(padded, block_start + block_samples) buffers = [bytearray((block_end - block_start) // 8) for _ in channels] position = block_start while position < block_end: boundary = (following[0] - origin) // period if following else padded if boundary < position or (following and (following[0] - anchor) % period): raise ValueError('CSV изменился во время чтения.') end = min(boundary, block_end) for buffer, level in zip(buffers, current[1]): if level == '1': _ones(buffer, position - block_start, end - block_start) position = end if following and position == boundary: current, following = following, next(rows, None) row_count += 1 for index, buffer in enumerate(buffers): _check(cancel) archive.writestr(f'L-{index}/{block}', buffer) block_start = block_end progress(40 + (block + 1) * 59 // blocks_count) if following is not None or row_count != count: raise ValueError('CSV изменился во время чтения.') _check(cancel) if stamp != (source.stat().st_size, source.stat().st_mtime_ns): raise ValueError('CSV изменился во время чтения.') # Never overwrite an existing user file, including one created mid-import. created = False try: with destination.open('xb') as output: created = True with open(temporary, 'rb') as data: while chunk := data.read(1024 * 1024): _check(cancel) output.write(chunk) except BaseException: if created: destination.unlink(missing_ok=True) raise progress(100) return result finally: Path(temporary).unlink(missing_ok=True)