212 lines
11 KiB
Python
212 lines
11 KiB
Python
"""Stream digital CSV into DSView v3, preserving intervals between transitions."""
|
||
from contextlib import contextmanager
|
||
import csv
|
||
from decimal import Decimal, InvalidOperation
|
||
import json
|
||
from math import gcd
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
import tempfile
|
||
import zipfile
|
||
|
||
from .files import ImportCancelled
|
||
|
||
TICKS = 10**12 # exact decimal picoseconds, independent of float rounding
|
||
BLOCK_SAMPLES = 8 * 1024 * 1024
|
||
MAX_PACKED_BYTES = 2 * 1024**3
|
||
TIME_HEADER = re.compile(r'(?:time|timestamp)\s*(?:[\[(](s|ms|us|µs|ns)[\])])?', re.I)
|
||
|
||
|
||
def _check(cancel):
|
||
if cancel():
|
||
raise ImportCancelled()
|
||
|
||
|
||
@contextmanager
|
||
def _reader(path, progress, cancel):
|
||
size = max(1, path.stat().st_size)
|
||
with path.open(encoding='utf-8-sig', newline='') as stream:
|
||
consumed = 0
|
||
|
||
def lines():
|
||
nonlocal consumed
|
||
for index, line in enumerate(stream):
|
||
consumed += len(line)
|
||
if index % 4096 == 0:
|
||
_check(cancel)
|
||
progress(min(99, consumed * 100 // size))
|
||
if line.strip() and not line.lstrip().startswith(('#', ';')):
|
||
yield line
|
||
|
||
source = lines()
|
||
header = next(source, '')
|
||
if not header:
|
||
raise ValueError('CSV не содержит данных.')
|
||
try:
|
||
delimiter = csv.Sniffer().sniff(header, delimiters=',;\t').delimiter
|
||
except csv.Error as error:
|
||
raise ValueError('Не удалось определить разделитель CSV: используйте запятую, ; или табуляцию.') from error
|
||
names = [value.strip() for value in next(csv.reader([header], delimiter=delimiter))]
|
||
match = TIME_HEADER.fullmatch(names[0])
|
||
if not match or not 2 <= len(names) <= 65:
|
||
raise ValueError('Ожидается CSV: Time [s] (или ms/us/ns), затем цифровые каналы 0/1.')
|
||
channels = names[1:]
|
||
if len(set(names)) != len(names) or any(not n or any(c in n for c in '\r\n=[]') for n in channels):
|
||
raise ValueError('Имена каналов должны быть уникальными и не содержать =, [, ] или переносы строк.')
|
||
unit = (match[1] or 's').lower()
|
||
scale = {'s': TICKS, 'ms': 10**9, 'us': 10**6, 'µs': 10**6, 'ns': 1000}[unit]
|
||
|
||
def rows():
|
||
previous = None
|
||
for index, row in enumerate(csv.reader(source, delimiter=delimiter), 2):
|
||
try:
|
||
value = Decimal(row[0].strip().replace(',', '.')) * scale
|
||
if not value.is_finite() or value != value.to_integral_value():
|
||
raise ValueError('время должно быть конечным, с точностью не выше 1 пс')
|
||
timestamp = int(value)
|
||
if previous is not None and timestamp <= previous:
|
||
raise ValueError('время должно строго возрастать')
|
||
levels = tuple(item.strip() for item in row[1:])
|
||
if len(levels) != len(channels) or any(item not in ('0', '1') for item in levels):
|
||
raise ValueError('ожидаются цифровые уровни 0 или 1 для каждого канала')
|
||
previous = timestamp
|
||
yield timestamp, levels
|
||
except (ValueError, InvalidOperation, IndexError) as error:
|
||
raise ValueError('CSV, строка данных %d: %s' % (index, error)) from error
|
||
|
||
yield channels, rows()
|
||
|
||
|
||
def _ones(block, start, end):
|
||
"""Set a half-open run without iterating over every sample."""
|
||
if end <= start:
|
||
return
|
||
first, bit = divmod(start, 8)
|
||
last, stop = divmod(end, 8)
|
||
if first == last:
|
||
block[first] |= ((1 << (end - start)) - 1) << bit
|
||
return
|
||
if bit:
|
||
block[first] |= (255 << bit) & 255
|
||
first += 1
|
||
block[first:last] = b'\xff' * (last - first)
|
||
if stop:
|
||
block[last] |= (1 << stop) - 1
|
||
|
||
|
||
def convert_csv(source, destination, progress=lambda value: None, cancel=lambda: False,
|
||
*, max_packed_bytes=MAX_PACKED_BYTES, block_samples=BLOCK_SAMPLES):
|
||
"""Two bounded-memory passes; input is untouched, output created exclusively.
|
||
|
||
The first row establishes initial levels, not an edge. Its timestamp may
|
||
precede the clock grid by a fraction of one period. All subsequent times
|
||
and transition intervals are represented exactly; the origin rounds down
|
||
at most one period. The final level is held to the 64-sample DSL boundary.
|
||
"""
|
||
source, destination = Path(source), Path(destination)
|
||
if source.resolve() == destination.resolve() or destination.exists():
|
||
raise ValueError('Для импорта нужен новый выходной файл.')
|
||
if block_samples < 64 or block_samples % 64:
|
||
raise ValueError('Размер блока должен быть кратен 64 отсчётам.')
|
||
stamp = (source.stat().st_size, source.stat().st_mtime_ns)
|
||
first = previous = anchor = None
|
||
step = count = 0
|
||
with _reader(source, lambda p: progress(p * 40 // 100), cancel) as (channels, rows):
|
||
for time, _ in rows:
|
||
if count == 0:
|
||
first = time
|
||
elif count == 1:
|
||
anchor = time
|
||
else:
|
||
step = gcd(step, time - previous)
|
||
previous = time
|
||
count += 1
|
||
if count < 2:
|
||
raise ValueError('В CSV нужны минимум две строки с разным временем.')
|
||
if count == 2:
|
||
step = anchor - first
|
||
period = gcd(step, TICKS)
|
||
rate = TICKS // period
|
||
origin = anchor - ((anchor - first + period - 1) // period) * period
|
||
samples = (previous - origin) // period + 1
|
||
padded = (samples + 63) // 64 * 64
|
||
if padded // 8 * len(channels) > max_packed_bytes:
|
||
raise ValueError('Запись слишком велика для DSView при сохранении точности времени. Выберите меньший интервал CSV.')
|
||
blocks_count = (padded + block_samples - 1) // block_samples
|
||
result = dict(channels=channels, samplerate=rate, samples=samples, padding=padded - samples,
|
||
origin_seconds=str(Decimal(origin) / TICKS), source=str(source))
|
||
header = ('[version]\nversion = 3\n[header]\ndriver = virtual-session\n'
|
||
'device mode = 0\ncapturefile = data\n'
|
||
f'total samples = {padded}\ntotal probes = {len(channels)}\ntotal blocks = {blocks_count}\n'
|
||
f'samplerate = {rate} Hz\ntrigger time = 0\ntrigger pos = 0\n'
|
||
+ ''.join(f'probe{i} = {name}\n' for i, name in enumerate(channels)))
|
||
session = dict(Device='virtual-session', DeviceMode=0, Version=3, Title='DSView v1.3.2',
|
||
**{'Sample count': str(padded), 'Sample rate': str(rate), 'Max Height': '1X'},
|
||
decoder=[], channel=[dict(colour='default', enabled=True, index=i, name=name,
|
||
strigger=0, type=10000, view_index=i)
|
||
for i, name in enumerate(channels)])
|
||
fd, temporary = tempfile.mkstemp(prefix='.csv-', suffix='.dsl', dir=destination.parent)
|
||
os.close(fd)
|
||
try:
|
||
with zipfile.ZipFile(temporary, 'w', zipfile.ZIP_DEFLATED, compresslevel=1) as archive:
|
||
archive.writestr('header', header)
|
||
archive.writestr('session', json.dumps(session, ensure_ascii=False))
|
||
archive.writestr('decoders', '[]')
|
||
archive.writestr('csv_import.json', json.dumps(result, ensure_ascii=False))
|
||
if stamp != (source.stat().st_size, source.stat().st_mtime_ns):
|
||
raise ValueError('CSV изменился во время чтения.')
|
||
with _reader(source, lambda p: None, cancel) as (names, rows):
|
||
if names != channels:
|
||
raise ValueError('CSV изменился во время чтения.')
|
||
current = next(rows)
|
||
if current[0] != first:
|
||
raise ValueError('CSV изменился во время чтения.')
|
||
following = next(rows, None)
|
||
block_start = 0
|
||
row_count = 1
|
||
for block in range(blocks_count):
|
||
_check(cancel)
|
||
block_end = min(padded, block_start + block_samples)
|
||
buffers = [bytearray((block_end - block_start) // 8) for _ in channels]
|
||
position = block_start
|
||
while position < block_end:
|
||
boundary = (following[0] - origin) // period if following else padded
|
||
if boundary < position or (following and (following[0] - anchor) % period):
|
||
raise ValueError('CSV изменился во время чтения.')
|
||
end = min(boundary, block_end)
|
||
for buffer, level in zip(buffers, current[1]):
|
||
if level == '1':
|
||
_ones(buffer, position - block_start, end - block_start)
|
||
position = end
|
||
if following and position == boundary:
|
||
current, following = following, next(rows, None)
|
||
row_count += 1
|
||
for index, buffer in enumerate(buffers):
|
||
_check(cancel)
|
||
archive.writestr(f'L-{index}/{block}', buffer)
|
||
block_start = block_end
|
||
progress(40 + (block + 1) * 59 // blocks_count)
|
||
if following is not None or row_count != count:
|
||
raise ValueError('CSV изменился во время чтения.')
|
||
_check(cancel)
|
||
if stamp != (source.stat().st_size, source.stat().st_mtime_ns):
|
||
raise ValueError('CSV изменился во время чтения.')
|
||
# Never overwrite an existing user file, including one created mid-import.
|
||
created = False
|
||
try:
|
||
with destination.open('xb') as output:
|
||
created = True
|
||
with open(temporary, 'rb') as data:
|
||
while chunk := data.read(1024 * 1024):
|
||
_check(cancel)
|
||
output.write(chunk)
|
||
except BaseException:
|
||
if created:
|
||
destination.unlink(missing_ok=True)
|
||
raise
|
||
progress(100)
|
||
return result
|
||
finally:
|
||
Path(temporary).unlink(missing_ok=True)
|