"""Второй проход: тезисы, ключевые слова, описания (этап «analyze»).

IDF считается по всей библиотеке, поэтому проход выполняется после того,
как тексты всех документов уже лежат в базе.
"""
from __future__ import annotations

import json
import time
from collections import Counter

from . import db
from .summarize import Analyzer, describe
from .textutil import stem, tokens


def build_df(con) -> tuple[dict[str, int], int]:
    """Документная частота каждой основы слова по всей библиотеке."""
    df: Counter = Counter()
    n = 0
    for row in con.execute("SELECT terms FROM docs WHERE terms IS NOT NULL"):
        try:
            terms = json.loads(row["terms"])
        except (TypeError, ValueError):
            continue
        n += 1
        df.update(terms.keys())
    return dict(df), n


def analyze(con, *, force: bool = False, quiet: bool = False,
            progress=None) -> dict:
    df, n_docs = build_df(con)
    an = Analyzer(df, n_docs)

    where = "" if force else " WHERE analyzed = 0"
    ids = [r["id"] for r in con.execute("SELECT id FROM docs" + where)]
    total = len(ids)
    if not quiet:
        print("Документов в базе: %d, к анализу: %d" % (n_docs, total), flush=True)
    if progress:
        progress(0, total, "Анализ: %d документов" % total)

    started = time.time()
    for i, doc_id in enumerate(ids, 1):
        row = con.execute(
            "SELECT title, body, toc, terms, lang, kind, pages, scanned, filename"
            " FROM docs WHERE id = ?", (doc_id,)).fetchone()
        body = row["body"] or ""
        try:
            terms = Counter(json.loads(row["terms"] or "{}"))
            toc = json.loads(row["toc"] or "[]")
        except ValueError:
            terms, toc = Counter(), []

        keywords = an.keywords(terms, body, row["title"] or "", toc)
        theses = an.theses(body, terms) if not row["scanned"] else []
        if not theses and toc:
            # Для сканов и справок без связного текста тезисы берём из оглавления.
            theses = [t for t in toc[:8] if len(t) > 8]
        description = describe(row["title"] or row["filename"], row["kind"] or "",
                               row["lang"] or "?", row["pages"], keywords,
                               theses, not row["scanned"])
        stems = " ".join(stem(t) for t in tokens(body[:20_000]))
        db.save_summary(con, doc_id, description, theses, keywords, stems)

        if i % 100 == 0:
            con.commit()
            if not quiet:
                print("  %d/%d" % (i, total), flush=True)
            if progress:
                progress(i, total, "Анализ %d/%d" % (i, total))
    con.commit()
    return {"analyzed": total, "corpus": n_docs,
            "seconds": round(time.time() - started, 1)}