256 lines
14 KiB
Python
256 lines
14 KiB
Python
"""Digital CSV/JSON/SAL/DSL conversion using the shared capture readers.
|
||
|
||
SAL v3 encoding follows DSLogic_Logic_2/dslogic_script/tests/generate_test_sal.py.
|
||
Only digital signals are exported; analyzer settings and analog data are not copied.
|
||
"""
|
||
import csv
|
||
import heapq
|
||
import itertools
|
||
import json
|
||
import math
|
||
from pathlib import Path
|
||
import struct
|
||
import tempfile
|
||
import uuid
|
||
import zipfile
|
||
|
||
from .files import read_capture, ImportCancelled
|
||
from .csv_import import convert_csv, _ones, BLOCK_SAMPLES, MAX_PACKED_BYTES
|
||
from .sal_metadata import metadata
|
||
|
||
FORMATS = {'csv': 'CSV — цифровые переходы', 'json': 'JSON — цифровые переходы', 'dsl': 'DSL — DSView',
|
||
'sal': 'SAL — Logic 2 (v3)'}
|
||
|
||
|
||
def _check(cancel):
|
||
if cancel():
|
||
raise ImportCancelled()
|
||
|
||
|
||
def _csv(capture, path, progress, cancel):
|
||
with path.open('w', encoding='utf-8', newline='') as stream:
|
||
writer = csv.writer(stream)
|
||
writer.writerow(['Time [s]', *[c.name for c in capture.channels]])
|
||
events = heapq.merge([capture.start, capture.end],
|
||
*(c.edges for c in capture.channels))
|
||
for index, (time, _) in enumerate(itertools.groupby(events)):
|
||
if index % 4096 == 0:
|
||
_check(cancel)
|
||
progress(int(99 * (time-capture.start) / max(capture.end-capture.start, 1e-30)))
|
||
writer.writerow([repr(time), *[c.level_at(time) for c in capture.channels]])
|
||
|
||
|
||
def _json(capture, path, progress, cancel):
|
||
names = ['Time [s]', *[c.name for c in capture.channels]]
|
||
if len(set(names)) != len(names):
|
||
raise ValueError('Для JSON имена каналов должны быть уникальны и отличаться от Time [s].')
|
||
with path.open('w', encoding='utf-8', newline='') as stream:
|
||
stream.write('[\n')
|
||
events = heapq.merge([capture.start, capture.end], *(c.edges for c in capture.channels))
|
||
for index, (time, _) in enumerate(itertools.groupby(events)):
|
||
if index % 4096 == 0:
|
||
_check(cancel)
|
||
progress(int(99 * (time-capture.start) / max(capture.end-capture.start, 1e-30)))
|
||
if index:
|
||
stream.write(',\n')
|
||
row = dict(zip(names, [time, *[c.level_at(time) for c in capture.channels]]))
|
||
stream.write(' ' + json.dumps(row, ensure_ascii=False, allow_nan=False))
|
||
stream.write('\n]\n')
|
||
|
||
|
||
# Времена переводятся в целые отсчёты одинаково для SAL и DSL.
|
||
# Округление допускается лишь в пределах погрешности, без ресемплинга.
|
||
def _grid(capture, rate, cancel):
|
||
rate = rate or capture.sample_rate
|
||
if not math.isfinite(rate) or rate <= 0:
|
||
raise ValueError('Для экспорта в SAL/DSL укажите частоту дискретизации в Гц.')
|
||
def sample(time):
|
||
value = (time - capture.start) * rate
|
||
rounded = round(value)
|
||
if not math.isclose(value, rounded, rel_tol=0, abs_tol=1e-5):
|
||
raise ValueError('Временные метки не попадают на сетку выбранной частоты. Увеличьте частоту.')
|
||
return rounded
|
||
count = sample(capture.end)
|
||
# CSV final row is a sample, binary capture end is exclusive.
|
||
if capture.format.endswith('CSV') or capture.format == 'JSON':
|
||
count += 1
|
||
if not 0 < count < 2**53:
|
||
raise ValueError('Недопустимая длительность записи.')
|
||
edges = []
|
||
for channel in capture.channels:
|
||
result = []
|
||
previous = 0
|
||
for i, time in enumerate(channel.edges):
|
||
if i % 8192 == 0:
|
||
_check(cancel)
|
||
value = sample(time)
|
||
if not previous < value < count:
|
||
raise ValueError('Частота не позволяет сохранить все фронты.')
|
||
result.append(value)
|
||
previous = value
|
||
edges.append(result)
|
||
return rate, count, edges
|
||
|
||
|
||
def _run(length):
|
||
value = length - 1
|
||
if value < 64:
|
||
return bytes([value])
|
||
shift = 7
|
||
while value >> shift >= 64:
|
||
shift += 7
|
||
out = bytearray([64 | (value >> shift)])
|
||
for bit in range(shift - 7, -1, -7):
|
||
out.append(((value >> bit) & 127) | (128 if bit else 0))
|
||
return out
|
||
|
||
|
||
# SAL хранит длины серий уровней. Метаданные описывают цифровые каналы;
|
||
# настройки анализаторов исходной записи сюда не переносятся.
|
||
def _sal(capture, path, rate, count, edges, progress, cancel):
|
||
if len(capture.channels) > 16:
|
||
raise ValueError('Экспорт SAL поддерживает до 16 цифровых каналов.')
|
||
meta = metadata()
|
||
data = meta['data']
|
||
data['name'] = path.stem
|
||
data['captureStartTime'] = dict(unixTimeMilliseconds=0, fractionalMilliseconds=0)
|
||
data['captureNotes'] = 'Converted digital signals; source time origin: %s s' % capture.start
|
||
data['renderViewState'] = dict(type='PanAndZoom', leftEdgeTimeSec=0,
|
||
timeScaleSeconds=count/rate/10)
|
||
data['captureSettings']['timerModeSettings']['stopAfterSeconds'] = count/rate
|
||
data['legacySettings']['sampleRate'] = {'digital': rate}
|
||
data['legacySettings']['enabledChannels'] = [dict(type='Digital', index=i)
|
||
for i in range(len(edges))]
|
||
data['rowsSettings'] = [dict(id=str(uuid.uuid4()), height=100, isMarkedHidden=False,
|
||
type='channel', name=c.name,
|
||
channel=dict(category='legacy', type='Digital', deviceChannel=i))
|
||
for i, c in enumerate(capture.channels)]
|
||
with zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED) as archive:
|
||
archive.writestr('meta.json', json.dumps(meta, ensure_ascii=False))
|
||
for i, (channel, transitions) in enumerate(zip(capture.channels, edges)):
|
||
encoded = bytearray()
|
||
previous = 0
|
||
for j, boundary in enumerate(itertools.chain(transitions, [count])):
|
||
if j % 8192 == 0:
|
||
_check(cancel)
|
||
encoded.extend(_run(boundary - previous))
|
||
previous = boundary
|
||
header = b'<SALEAE>' + struct.pack('<II', 3, 100)
|
||
header += struct.pack('<BdQdBBQ', 1, rate, 0, 0, 0, 0, 1)
|
||
header += struct.pack('<QQBBQ', 0, count, channel.initial, 0, len(encoded))
|
||
archive.writestr('digital-%d.bin' % i, header + encoded)
|
||
progress((i+1)*99//len(edges))
|
||
archive.writestr('trigger-store.bin', b'<SALEAE>' + struct.pack('<IIIIQ', 3, 103, 1, 0, 0))
|
||
|
||
|
||
# DSL упаковывает каждый канал по блокам. Дополнение до 64 отсчётов
|
||
# удерживает последний уровень, не создавая ложный фронт.
|
||
def _dsl(capture, path, rate, count, edges, progress, cancel):
|
||
names = [c.name for c in capture.channels]
|
||
if any(any(ch in name for ch in '\r\n=[]') for name in names):
|
||
raise ValueError('Для DSL имена каналов не должны содержать =, [, ] и переносы строк.')
|
||
padded = (count+63)//64*64
|
||
if padded//8*len(names) > MAX_PACKED_BYTES:
|
||
raise ValueError('Запись DSL превышает ограничение 2 ГиБ распакованных данных.')
|
||
blocks = (padded+BLOCK_SAMPLES-1)//BLOCK_SAMPLES
|
||
header = ('[version]\nversion = 3\n[header]\ndriver = virtual-session\n'
|
||
'device mode = 0\ncapturefile = data\n'
|
||
f'total samples = {padded}\ntotal probes = {len(names)}\ntotal blocks = {blocks}\n'
|
||
f'samplerate = {rate:.12f} Hz\ntrigger time = 0\ntrigger pos = 0\n'
|
||
+ ''.join(f'probe{i} = {name}\n' for i, name in enumerate(names)))
|
||
session = dict(Device='virtual-session', DeviceMode=0, Version=3, Title='DSView v1.3.2',
|
||
**{'Sample count': str(padded), 'Sample rate': str(rate), 'Max Height': '1X'},
|
||
decoder=[], channel=[dict(colour='default', enabled=True, index=i, name=name,
|
||
strigger=0, type=10000, view_index=i)
|
||
for i, name in enumerate(names)])
|
||
with zipfile.ZipFile(path, 'w', zipfile.ZIP_DEFLATED, compresslevel=1) as archive:
|
||
archive.writestr('header', header)
|
||
archive.writestr('session', json.dumps(session, ensure_ascii=False))
|
||
archive.writestr('decoders', '[]')
|
||
for i, (channel, transitions) in enumerate(zip(capture.channels, edges)):
|
||
boundaries = iter(itertools.chain(transitions, [padded]))
|
||
boundary = next(boundaries)
|
||
level = channel.initial
|
||
for block in range(blocks):
|
||
_check(cancel)
|
||
start = block*BLOCK_SAMPLES
|
||
end = min(padded, start+BLOCK_SAMPLES)
|
||
buffer = bytearray((end-start)//8)
|
||
position = start
|
||
while position < end:
|
||
stop = min(end, boundary)
|
||
if level:
|
||
_ones(buffer, position-start, stop-start)
|
||
position = stop
|
||
if position == boundary:
|
||
level ^= 1
|
||
boundary = next(boundaries, padded)
|
||
_check(cancel)
|
||
archive.writestr(f'L-{i}/{block}', buffer)
|
||
progress((i*blocks+block+1)*99//(len(names)*blocks))
|
||
return padded-count
|
||
|
||
|
||
def convert_capture(source, destination, progress=lambda p: None, cancel=lambda: False,
|
||
*, sample_rate=0, csv_options=None):
|
||
"""Export to a new file; cancel/failure never leaves a partial destination."""
|
||
source, destination = Path(source), Path(destination)
|
||
suffix = destination.suffix.lower().lstrip('.')
|
||
if suffix not in FORMATS or source.suffix.lower().lstrip('.') not in FORMATS:
|
||
raise ValueError('Поддерживаются цифровые CSV, JSON, SAL и DSL.')
|
||
if source.resolve() == destination.resolve() or destination.exists():
|
||
raise ValueError('Выберите новый выходной файл: существующие файлы не перезаписываются.')
|
||
_check(cancel)
|
||
# Нормализация расширенного входа получает первые 25% прогресса.
|
||
# В рекурсивный вызов csv_options не передаётся: повторного прохода нет.
|
||
if source.suffix.lower() == '.csv' and csv_options is not None:
|
||
from .csv_options import normalize_csv
|
||
with tempfile.TemporaryDirectory(prefix='.csv-options-', dir=destination.parent) as directory:
|
||
normalized = Path(directory)/'normalized.csv'
|
||
normalize_csv(source, normalized, csv_options, sample_rate,
|
||
lambda p: progress(p*25//100), cancel)
|
||
return convert_capture(normalized, destination, lambda p: progress(25+p*75//100),
|
||
cancel, sample_rate=sample_rate)
|
||
# Автоматическая сетка использует точный потоковый CSV-импортёр.
|
||
# Явно заданную частоту проверяет _grid, её нельзя молча заменить.
|
||
if source.suffix.lower() == '.csv' and suffix == 'dsl' and not sample_rate:
|
||
result = convert_csv(source, destination, progress, cancel)
|
||
return 'Готово: %s\nDSL: %s Гц; дополнение %s отсчётов; начало %s с.' % (
|
||
destination, result['samplerate'], result['padding'], result['origin_seconds'])
|
||
capture = read_capture(source, lambda p: progress(p*35//100), cancel)
|
||
padding = 0
|
||
# Сначала собираем временный архив рядом с назначением. Его владелец —
|
||
# этот контекст: ошибка и отмена тоже удаляют промежуточный результат.
|
||
with tempfile.TemporaryDirectory(prefix='.convert-', dir=destination.parent) as directory:
|
||
temporary = Path(directory)/destination.name
|
||
report = lambda p: progress(35+p*60//100)
|
||
if suffix == 'csv':
|
||
_csv(capture, temporary, report, cancel)
|
||
elif suffix == 'json':
|
||
_json(capture, temporary, report, cancel)
|
||
else:
|
||
rate, count, edges = _grid(capture, sample_rate, cancel)
|
||
if suffix == 'sal':
|
||
_sal(capture, temporary, rate, count, edges, report, cancel)
|
||
else:
|
||
padding = _dsl(capture, temporary, rate, count, edges, report, cancel)
|
||
_check(cancel)
|
||
# Режим xb защищает от гонки с другим процессом после проверки пути.
|
||
# При ошибке удаляем только файл, который успели создать сами.
|
||
created = False
|
||
try:
|
||
with destination.open('xb') as output:
|
||
created = True
|
||
with temporary.open('rb') as stream:
|
||
while chunk := stream.read(1024*1024):
|
||
_check(cancel)
|
||
output.write(chunk)
|
||
_check(cancel)
|
||
except BaseException:
|
||
if created:
|
||
destination.unlink(missing_ok=True)
|
||
raise
|
||
progress(100)
|
||
return 'Готово: %s\nКаналов: %d. Дополнение DSL: %d отсчётов.' % (
|
||
destination, len(capture.channels), padding)
|