import csv
import json
import threading
import queue
import sys
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor

CHUNK_SIZE = 1000
MAX_WORKERS = 8

task_queue = queue.Queue(maxsize=50)
result_queue = queue.Queue()
write_lock = threading.Lock()

def extract_field(row, field_name):
    value = row.get(field_name, '')
    if value is None:
        return ''
    if isinstance(value, dict):
        return json.dumps(value, ensure_ascii=False)
    return str(value)

def process_row(row):
    if row.get("di") and isinstance(row["di"], str):
        try:
            data = json.loads(row["di"])
            if isinstance(data, dict) and "nameDi" not in data:
                data["nameDi"] = ""
                row["di"] = data
        except json.JSONDecodeError:
            pass

    if row.get("prevDi") and isinstance(row["prevDi"], str):
        try:
            data = json.loads(row["prevDi"])
            if isinstance(data, dict) and "nameDi" not in data:
                data["nameDi"] = ""
                row["prevDi"] = data
        except json.JSONDecodeError:
            pass

    return {
        "id":            row.get("id"),
        "timestamp":     row.get('timestamp'),
        "code":          row.get('code'),
        "level":         row.get('level'),
        "controller":    extract_field(row, 'controller'),
        "location":      extract_field(row, 'location'),
        "user":          extract_field(row, 'user'),
        "relay":         extract_field(row, 'relay'),
        "di":            extract_field(row, 'di'),
        "camera":        extract_field(row, 'camera'),
        "cameraStream":  extract_field(row, 'cameraStream'),
        "card":          extract_field(row, 'card'),
        "car":           extract_field(row, 'car'),
        "worker":        extract_field(row, 'worker'),
        "guest":         extract_field(row, 'guest'),
        "prevCard":      extract_field(row, 'prevCard'),
        "prevWorker":    extract_field(row, 'prevWorker'),
        "prevGuest":     extract_field(row, 'prevGuest'),
        "prevDi":        extract_field(row, 'prevDi'),
        "service":       extract_field(row, 'service'),
        "updateTask":    extract_field(row, 'updateTask'),
        "faceList":      extract_field(row, 'faceList'),
        "carList":       extract_field(row, 'carList'),
        "face":          extract_field(row, 'face'),
        "object":        extract_field(row, 'object'),
        "directionZone": extract_field(row, 'directionZone'),
        "crossLine":     extract_field(row, 'crossLine'),
        # entity, отсутствовавшие в 1.5 — для старых событий всегда пустые
        "targetUser":    "",
        "role":          "",
    }

def process_chunk(chunk, chunk_id):
    return [process_row(row) for row in chunk]

def file_reader_thread(filename, chunk_size):
    current_chunk = []
    chunk_id = 0

    with open(filename, 'r', encoding='utf-8') as f:
        reader = csv.DictReader(f)
        for row in reader:
            current_chunk.append(row)
            if len(current_chunk) >= chunk_size:
                task_queue.put((chunk_id, current_chunk))
                current_chunk = []
                chunk_id += 1
        if current_chunk:
            task_queue.put((chunk_id, current_chunk))

def worker_task():
    while True:
        item = task_queue.get()
        if item is None:
            task_queue.task_done()
            break
        chunk_id, chunk = item
        try:
            processed_data = process_chunk(chunk, chunk_id)
            result_queue.put(processed_data)
        except Exception as e:
            print(f"Error processing chunk {chunk_id}: {e}")
        finally:
            task_queue.task_done()

def writer_thread(filename, fieldnames):
    is_first_chunk = True
    with open(filename, 'w', encoding='utf-8', newline='') as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        while True:
            data = result_queue.get()
            if data is None:
                break
            if is_first_chunk:
                writer.writeheader()
                is_first_chunk = False
            with write_lock:
                writer.writerows(data)
            result_queue.task_done()

def main():

    input_file = sys.argv[1]
    output_file = sys.argv[2]

    fieldnames = [
        "id", "timestamp", "code", "level", "controller", "location", "user",
        "relay", "di", "camera", "cameraStream", "card", "car", "worker",
        "guest", "prevCard", "prevWorker", "prevGuest", "prevDi", "service",
        "updateTask", "faceList", "carList", "face", "object", "directionZone",
        "crossLine", "targetUser", "role"
    ]

    reader_thread = threading.Thread(target=file_reader_thread, args=(input_file, CHUNK_SIZE))
    reader_thread.start()

    writer_th = threading.Thread(target=writer_thread, args=(output_file, fieldnames))
    writer_th.start()

    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        futures = [executor.submit(worker_task) for _ in range(MAX_WORKERS)]
        reader_thread.join()
        task_queue.join()
        for _ in range(MAX_WORKERS):
            task_queue.put(None)
        for future in futures:
            future.result()

    result_queue.put(None)
    writer_th.join()

if __name__ == '__main__':
    main()