<?php

/**
 * JAZ1 → JAZ2 Migration Verification Script
 *
 * This script compares data from all tables of JAZ1 → JAZ2.
 * Includes:
 *  - Control Tables: configuration and metadata
 *  - Initial Tables: static/reference data
 *  - Run Tables: job runtime and results data
 *
 * Features:
 *  - Supports PostgreSQL, MySQL, and MariaDB
 *  - Compare and output log and csv files for the comparison result
 */

/**
 * ==========================================
 * GLOBAL CONST
 * ==========================================
 */
define("CHECK_LOG_TEXT", "Check log for data");

$tableMap = [
    'ja_calendar_control_table' => 'ja_2_calendar_control_table',
    'ja_calendar_detail_table' => 'ja_2_calendar_detail_table',
    'ja_define_extjob_table' => 'ja_2_define_extjob_table',
    'ja_define_run_log_message_table' => 'ja_2_define_run_log_message_table',
    'ja_define_value_jobcon_table' => 'ja_2_define_value_jobcon_table',
    'ja_filter_control_table' => 'ja_2_filter_control_table',
    'ja_flow_control_table' => 'ja_2_flow_control_table',
    'ja_host_lock_table' => 'ja_2_host_lock_table',
    'ja_icon_agentless_table' => 'ja_2_icon_agentless_table',
    'ja_icon_calc_table' => 'ja_2_icon_calc_table',
    'ja_icon_end_table' => 'ja_2_icon_end_table',
    'ja_icon_extjob_table' => 'ja_2_icon_extjob_table',
    'ja_icon_fcopy_table' => 'ja_2_icon_fcopy_table',
    'ja_icon_fwait_table' => 'ja_2_icon_fwait_table',
    'ja_icon_if_table' => 'ja_2_icon_if_table',
    'ja_icon_info_table' => 'ja_2_icon_info_table',
    'ja_icon_job_table' => 'ja_2_icon_job_table',
    'ja_icon_jobnet_table' => 'ja_2_icon_jobnet_table',
    'ja_icon_reboot_table' => 'ja_2_icon_reboot_table',
    'ja_icon_release_table' => 'ja_2_icon_release_table',
    'ja_icon_task_table' => 'ja_2_icon_task_table',
    'ja_icon_value_table' => 'ja_2_icon_value_table',
    'ja_icon_zabbix_link_table' => 'ja_2_icon_zabbix_link_table',
    'ja_index_table' => 'ja_2_index_table',
    'ja_job_command_table' => 'ja_2_job_command_table',
    'ja_job_control_table' => 'ja_2_job_control_table',
    'ja_jobnet_control_table' => 'ja_2_jobnet_control_table',
    'ja_object_lock_table' => 'ja_2_object_lock_table',
    'ja_parameter_table' => 'ja_2_parameter_table',
    'ja_run_flow_table' => 'ja_2_run_flow_table',
    'ja_run_job_table' => 'ja_2_run_job_table',
    'ja_run_jobnet_summary_table' => 'ja_2_run_jobnet_summary_table',
    'ja_run_jobnet_table' => 'ja_2_run_jobnet_table',
    'ja_run_log_table' => 'ja_2_run_log_table',
    'ja_run_value_job_table' => 'ja_2_run_value_job_table',
    'ja_run_value_jobcon_table' => 'ja_2_run_value_jobcon_table',
    'ja_schedule_control_table' => 'ja_2_schedule_control_table',
    'ja_schedule_detail_table' => 'ja_2_schedule_detail_table',
    'ja_schedule_jobnet_table' => 'ja_2_schedule_jobnet_table',
    'ja_send_message_table' => 'ja_2_send_message_table',
    'ja_session_table' => 'ja_2_session_table',
    'ja_value_job_table' => 'ja_2_value_job_table',
    'ja_value_jobcon_table' => 'ja_2_value_jobcon_table',
];

$variableTableMap = [
    'ja_run_value_after_table' => 'ja_2_run_job_variable_table',
    'ja_run_value_before_table' => 'ja_2_run_job_variable_table',
    'ja_value_after_jobnet_table' => 'ja_2_run_jobnet_variable_table',
    'ja_value_before_jobnet_table' => 'ja_2_run_jobnet_variable_table',
];

$runDataMap = [
    'ja_run_icon_agentless_table' => ["host_flag", "connection_method", "session_flag", "auth_method", "run_mode", "line_feed_code", "timeout", "session_id", "login_user", "login_password", "public_key", "private_key", "passphrase", "host_name", "stop_code", "terminal_type", "character_code", "prompt_string", "command"],
    'ja_run_icon_calc_table' => ["hand_flag", "formula", "value_name", "timezone"],
    'ja_run_icon_end_table' => ["jobnet_stop_flag", "jobnet_stop_code"],
    'ja_run_icon_extjob_table' => ["command_id", "value", "pid", "timezone"],
    'ja_run_icon_fcopy_table' => ["from_host_flag", "to_host_flag", "overwrite_flag", "from_host_name", "from_directory", "from_file_name", "to_host_name", "to_directory"],
    'ja_run_icon_fwait_table' => ["host_flag", "fwait_mode_flag", "file_delete_flag", "file_wait_time", "host_name", "file_name"],
    'ja_run_icon_if_table' => ["hand_flag", "value_name", "comparison_value"],
    'ja_run_icon_info_table' => ["info_flag", "get_job_id", "get_calendar_id"],
    'ja_run_icon_job_table' => ["host_flag", "stop_flag", "command_type", "timeout", "host_name", "stop_code", "timeout_run_type", "command", "stop_command"],
    'ja_run_icon_jobnet_table' => ["link_jobnet_id", "link_inner_jobnet_id"],
    'ja_run_icon_reboot_table' => ["host_flag", "reboot_mode_flag", "reboot_wait_time", "host_name", "timeout"],
    'ja_run_icon_release_table' => ["release_job_id"],
    'ja_run_icon_task_table' => ["submit_inner_jobnet_id", "submit_jobnet_id"],
    'ja_run_icon_value_table' => ["variables"],
    'ja_run_icon_zabbix_link_table' => ["link_target", "link_operation", "groupid", "hostid", "itemid", "triggerid"],
    'ja_run_job_command_table' => ["host_flag", "stop_flag", "command_type", "timeout", "host_name", "stop_code", "timeout_run_type", "command", "stop_command"],
];

$zbxHostTable = [
    'ja_parameter_table' => 'ja_2_zbx_hosts_table'
];

$zbxHostCompMap = [
    "ZBXSND_ZABBIX_HOST" => "message_destination_hostname",
    "ZBXSND_ZABBIX_IP" => "zabbix_server_ip_address",
    "ZBXSND_ZABBIX_PORT" => "zabbix_server_port_number",
    "ZBXSND_ITEM_KEY" => "message_destination_item_key",
];

$runLogPKs = ['log_date', 'inner_job_id', 'inner_jobnet_id', 'message_id'];
$intKeys = ['inner_job_id', 'inner_jobnet_id'];

$timeColumns = ["created_date", "update_date", "message_date", "send_date", "send_error_date", "start_time", "end_time", "last_active_time", "log_date", "scheduled_time", "initial_scheduled_time"];
$removedCols = ["virtual_time", "virtual_start_time", "virtual_end_time"];
$newlyAddedCols = ["CURRENT_TIME_TZ", "JOBNET_BOOT_TIME_TZ", "JOBNET_SCHEDULED_TIME_TZ", "TIMEZONE"];

class DiffStatus
{
    public const YES = 'YES';
    public const NO = 'NO';
    public const ROW_MISSING = 'ROW_MISSING';
    public const EXTRA_ROW = 'EXTRA_ROW';
    public const NEWLY_ADDED = 'NEWLY_ADDED';
    public const REMOVED = 'REMOVED';
    public const EXTRA_FIELDS = 'EXTRA_FIELDS';
}

/**
 * ==========================================
 * GLOBAL VARs
 * ==========================================
 */
$csvRows = [];
$csvFp = null;
$csvFilePath = __DIR__ . '/migrate_compare_' . date('Ymd_His') . '.csv';
$logFp = fopen(__DIR__ . '/migrate_compare_' . date('Ymd_His') . '.log', 'a');

/**
 * ==========================================
 * LOGGING
 * ==========================================
 */
function writeLog($msg)
{
    $line = "[" . date('Y-m-d H:i:s') . "] " . $msg . PHP_EOL;
    fwrite($GLOBALS['logFp'], $line);
}


/**
 * ==========================================
 * CSV ROW WRITER (ALL COLUMNS)
 * ==========================================
 */
// Compare result export to csv
function toCsvString($v): string
{
    if ($v === null)
        return '';
    if (is_bool($v))
        return $v ? '1' : '0';

    if (is_array($v)) {
        return json_encode($v, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
    }

    return "'" . (string) $v;
}

function initCsv()
{
    global $csvFp;

    $csvFp = fopen($GLOBALS['csvFilePath'], 'w');

    fputcsv($csvFp, [
        'primary_keys',
        'column_name',
        'JAZ1',
        'JAZ2',
        'is_different',
        'jaz1_table',
        'jaz2_table',
    ]);
}

function closeCsv()
{
    global $csvFp;

    if ($csvFp) {
        fclose($csvFp);
    }
}

function writeCsvRow($pk, $column, $jaz1, $jaz2, $diffStatus, $table1, $table2)
{
    global $csvFp;

    fputcsv($csvFp, [
        "'$pk",
        $column,
        toCsvString($jaz1),
        toCsvString($jaz2),
        $diffStatus,
        $table1,
        $table2,
    ]);

    fflush($csvFp);
}

/**
 * ==========================================
 * UTILITY
 * ==========================================
 */
function toUnixTime(string $datetime, ?string $tz = null): int
{

    try {
        if ($datetime == 0) {
            return 0;
        }

        // Keep only first 14 characters (YYYYMMDDHHMMSS)
        if (ctype_digit($datetime) && strlen($datetime) > 14) {
            $datetime = substr($datetime, 0, 14);
        }

        if ($tz) {
            $timezone = new DateTimeZone($tz);
            $dt = new DateTime($datetime, $timezone);
        } else {
            // Use PHP default timezone
            $dt = new DateTime($datetime);
        }
        return $dt->getTimestamp();
    } catch (Exception $e) {
        writeLog("Error while converting date time ($datetime) to unix time: " . $e->getMessage());
        throw $e;
    }
}

/**
 * ==========================================
 * DB HELPERS
 * ==========================================
 */
function buildDsn($dbType, $cfg)
{
    switch (strtolower($dbType)) {

        case 'mysql':
        case 'mariadb':
            return sprintf(
                "mysql:host=%s;port=%d;dbname=%s;charset=%s",
                $cfg['host'],
                $cfg['port'] ?? 3306,
                $cfg['dbname'],
                $cfg['charset'] ?? 'utf8mb4'
            );

        case 'pgsql':
        case 'postgres':
        case 'postgresql':
            return sprintf(
                "pgsql:host=%s;port=%d;dbname=%s",
                $cfg['host'],
                $cfg['port'] ?? 5432,
                $cfg['dbname']
            );

        default:
            throw new Exception("Unsupported DB type: $dbType");
    }
}

function getPK(PDO $pdo, string $table)
{
    $dbType = $pdo->getAttribute(PDO::ATTR_DRIVER_NAME);

    if ($dbType === 'mysql') {
        $sql = "
            SELECT COLUMN_NAME
            FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE
            WHERE TABLE_SCHEMA = DATABASE()
              AND TABLE_NAME = ?
              AND CONSTRAINT_NAME = 'PRIMARY'
            ORDER BY ORDINAL_POSITION
        ";
        $st = $pdo->prepare($sql);
        $st->execute([$table]);
        return $st->fetchAll(PDO::FETCH_COLUMN);
    }

    if ($dbType === 'pgsql') {
        $sql = "
            SELECT a.attname
            FROM pg_index i
            JOIN pg_attribute a ON a.attrelid = i.indrelid
                              AND a.attnum = ANY(i.indkey)
            WHERE i.indrelid = ?::regclass
              AND i.indisprimary
            ORDER BY array_position(i.indkey, a.attnum)
        ";
        $st = $pdo->prepare($sql);
        $st->execute([$table]);
        return $st->fetchAll(PDO::FETCH_COLUMN);
    }

    throw new Exception("Unsupported DB type: $dbType");
}

function fetchRowsByPK(PDO $pdo, $table, $pks, $pkValues, $condition = "")
{
    if (!$pkValues)
        return [];

    $conds = [];

    foreach ($pkValues as $pkVal) {
        $vals = explode("\x1F", $pkVal);
        $sub = [];
        foreach ($pks as $i => $pk) {
            $val = $vals[$i];

            // If empty string, convert to NULL (safe for numeric columns)
            if ($val === '' && in_array($pk, $GLOBALS['intKeys'])) {
                $sub[] = "$pk IS NULL";
            } else {
                $sub[] = "$pk=" . $pdo->quote($val);
            }
        }
        $conds[] = "(" . implode(" AND ", $sub) . ")";
    }

    $sql = "SELECT * FROM $table WHERE " . implode(" OR ", $conds) . $condition;

    $rows = $pdo->query($sql)->fetchAll(PDO::FETCH_ASSOC);

    return $rows;
}

function fetchRowsByPKBatch(PDO $pdo, $table, $pks, $pkValues)
{
    $rows = fetchRowsByPK($pdo, $table, $pks, $pkValues);

    return getBatchByPKs($rows, $pks);
}

function getBatchByPKs(array $rows, $pks)
{
    $out = [];
    foreach ($rows as $r) {
        $key = [];
        foreach ($pks as $pk) {
            $key[] = $r[$pk];
        }
        $out[implode("\x1F", $key)] = $r;
    }

    return $out;
}

function fetchPKs(PDO $pdo, $table, $pks, $condition = "")
{
    $cols = implode(",", $pks);
    $rows = $pdo->query("SELECT $cols FROM $table $condition")->fetchAll(PDO::FETCH_ASSOC);

    $keys = [];
    foreach ($rows as $row) {
        $key = [];
        foreach ($pks as $pk) {
            $key[] = $row[$pk];
        }
        $keys[] = implode("\x1F", $key);
    }

    return $keys;
}

// This function would fetch primary keys as the associated arrays, where the keys are primary keys and the values are converted form to unix time for time columns. 
// For other columns, keys and values should be the same.
function fetchAssociatedPKs(PDO $pdo, $table, $pks, $condition = "")
{
    $cols = implode(",", $pks);
    $rows = $pdo->query("SELECT $cols FROM $table $condition")->fetchAll(PDO::FETCH_ASSOC);

    $assoc = [];

    foreach ($rows as $row) {
        $rawKeyParts = [];
        $normKeyParts = [];

        foreach ($pks as $pk) {
            $rawVal = $row[$pk];
            $normVal = $rawVal;

            // Convert time columns to unix for JAZ1 only
            if (strpos($table, "ja_2") === false && in_array($pk, $GLOBALS['timeColumns'])) {
                $normVal = toUnixTime($rawVal, VM_TIMEZONE);
            }

            $rawKeyParts[] = (string) $rawVal;
            $normKeyParts[] = (string) $normVal;
        }

        $rawKey = implode("\x1F", $rawKeyParts);
        $normKey = implode("\x1F", $normKeyParts);

        // key = normalized PK, value = primary PK
        $assoc[$normKey] = $rawKey;
    }

    return $assoc;
}

function fetchByPK(PDO $pdo, $table, $pks, $condition = "")
{
    $rows = $pdo->query("SELECT * FROM $table $condition")->fetchAll(PDO::FETCH_ASSOC);
    $out = [];

    foreach ($rows as $r) {
        $key = [];
        foreach ($pks as $pk) {
            $pkValue = $r[$pk];

            if (strpos($table, "ja_2") === false) {
                if (in_array($pk, $GLOBALS['timeColumns'])) {
                    $pkValue = toUnixTime($pkValue, VM_TIMEZONE);
                }

            }

            $key[] = $pkValue;
        }
        $out[implode("\x1F", $key)] = $r;
    }
    return $out;
}

function transformRunValueToJson(array $rows, string $jsonCol, string $valueCol, bool $isJobnetVar): array
{
    $out = [];

    foreach ($rows as $r) {

        // Build logical PK (exclude seq_no!)
        if ($isJobnetVar) {
            $pk = $r['inner_jobnet_id'];
        } else {
            $pk = $r['inner_job_id'];
        }

        if (!isset($out[$pk])) {
            $out[$pk] = [
                'inner_jobnet_id' => $r['inner_jobnet_id'],
                $jsonCol => [],
            ];

            if (!$isJobnetVar) {
                $out[$pk]['inner_job_id'] = $r['inner_job_id'];
            }
        }

        $out[$pk][$jsonCol][$r['value_name']] = $r[$valueCol];
    }

    return $out;
}

function stripLastPkPart(array $d2): array
{
    $out = [];

    foreach ($d2 as $pk => $row) {
        $parts = explode("\x1F", $pk);

        // remove last element (e.g., seq_no)
        array_pop($parts);

        // rebuild PK
        $newPk = implode("\x1F", $parts);

        $out[$newPk] = $row;
    }

    return $out;
}

function normalizeArray($arr): array
{
    if (!is_array($arr))
        return $arr;

    foreach ($arr as $k => $v) {
        if (is_array($v)) {
            $arr[$k] = normalizeArray($v);
        }
    }

    ksort($arr); // ensure stable ordering
    return $arr;
}

function arraysEqual($a, $b): bool
{
    $a = normalizeArray($a);
    $b = normalizeArray($b);
    return $a === $b;
}

function arraysDiffKeys($a, $b): array
{
    $a = normalizeArray($a);
    $b = normalizeArray($b);

    $allKeys = array_unique(array_merge(array_keys($a), array_keys($b)));
    $diffKeys = [];

    foreach ($allKeys as $k) {
        $va = $a[$k] ?? null;
        $vb = $b[$k] ?? null;

        if ($va !== $vb) {
            $diffKeys[] = $k;
        }
    }

    return $diffKeys;
}

function compareVarTable(PDO $pdo1, PDO $pdo2, $t1, $t2)
{

    writeLog("--- Start comparing variable tables $t1 (JAZ1), $t2 (JAZ2) ---");
    $isJobnetVar = false;
    if (strpos($t1, "jobnet") !== false) {
        $isJobnetVar = true;
    }

    $pks = [];
    if ($isJobnetVar) {
        $pks = ["inner_jobnet_id"];
    } else {
        $pks = ["inner_job_id"];
    }

    $jsonCol = "";
    $valueCol = "";
    if (strpos($t1, "after") !== false) {
        $jsonCol = "after_variable";
        $valueCol = "after_value";
    } else {
        $jsonCol = "before_variable";
        $valueCol = "before_value";
    }

    $keys1 = fetchPKs($pdo1, $t1, $pks);

    $dbType = $pdo2->getAttribute(PDO::ATTR_DRIVER_NAME);
    $condition = "";
    switch ($dbType) {
        case 'mysql': // MySQL + MariaDB
            $condition = "WHERE TRIM($jsonCol) <> '[]'";
            break;
        case 'pgsql':
            $condition = "WHERE $jsonCol <> '[]'::jsonb AND $jsonCol <> '{}';";
            break;
        default:
            throw new Exception("Unsupported DB type: $dbType");
    }

    $keys2 = fetchPKs($pdo2, $t2, $pks, $condition);

    $commonKeys = array_intersect($keys2, $keys1);
    $missingInD2 = array_diff($keys1, $keys2);
    $extraInD2 = array_diff($keys2, $keys1);

    foreach ($missingInD2 as $pk) {
        $pkVal = str_replace("\x1F", ",", $pk);
        $diffStatus = DiffStatus::ROW_MISSING;
        writeLog("jaz 1: $t1, jaz 2: $t2, col: all, pk: $pkVal, diff_status: $diffStatus, data: \nJAZ1:" . toCsvString($pkVal) . "\nJAZ2:" . null . "\n===");
        writeCsvRow(
            str_replace("\x1F", ",", $pk),
            "all",
            CHECK_LOG_TEXT,
            null,
            $diffStatus,
            $t1,
            $t2
        );
        continue;
    }

    foreach ($extraInD2 as $pk) {
        $pkVal = str_replace("\x1F", ",", $pk);
        $diffStatus = DiffStatus::EXTRA_ROW;
        writeLog("jaz 1: $t1, jaz 2: $t2, col: all, pk: $pkVal, diff_status: $diffStatus, data: \nJAZ1:" . null . "\nJAZ2:" . toCsvString($pkVal) . "\n===");

        writeCsvRow(
            str_replace("\x1F", ",", $pk),
            "all",
            null,
            CHECK_LOG_TEXT,
            $diffStatus,
            $t1,
            $t2
        );
        continue;
    }

    $batchSize = 50;

    foreach (array_chunk($commonKeys, $batchSize) as $batchKeys) {
        // Fetch full rows for this batch
        $d1 = fetchRowsByPK($pdo1, $t1, $pks, $batchKeys, "ORDER BY seq_no ASC");
        $d1 = transformRunValueToJson($d1, $jsonCol, $valueCol, $isJobnetVar);
        $d2 = fetchRowsByPKBatch($pdo2, $t2, $pks, $batchKeys);

        foreach ($batchKeys as $pkValue) {

            foreach ($d1[$pkValue] as $col => $val1) {
                $after_variable = $d2[$pkValue][$col] ?? null;

                // columns are datetime, convert to unix first
                if ($col === "created_date" || $col === "update_date") {
                    $val1 = toUnixTime($val1, VM_TIMEZONE);
                }

                // JSON string → array if needed
                if (is_string($after_variable) && ($after_variable[0] ?? '') === '{') {
                    $decoded = json_decode($after_variable, true);
                    if (json_last_error() === JSON_ERROR_NONE) {
                        $after_variable = $decoded;
                    }
                }

                $diffTitle = $col;

                // Compare arrays vs scalars
                if (is_array($val1) || is_array($after_variable)) {
                    $val1["TIMEZONE"] = VM_TIMEZONE;
                    $diffKeys = arraysDiffKeys((array) $val1, (array) $after_variable);
                    $isDiff = !empty($diffKeys);
                    if ($isDiff) {
                        $diffTitle = implode(',', $diffKeys);
                    }
                } else {
                    $isDiff = ($val1 !== $after_variable);
                }

                $diffStatus = $isDiff ? DiffStatus::YES : DiffStatus::NO;
                writeLog("jaz 1: $t1, jaz 2: $t2, col: $diffTitle, pk: $pkValue, diff_status: $diffStatus, data: \nJAZ1:" . toCsvString($val1) . "\nJAZ2:" . toCsvString($after_variable) . "\n===");

                // Write ALL columns
                writeCsvRow(
                    str_replace("\x1F", ",", $pkValue),
                    $diffTitle,
                    CHECK_LOG_TEXT,
                    CHECK_LOG_TEXT,
                    // $timeConverted ? $orgValue : $val1,
                    // $after_variable,
                    $diffStatus,
                    $t1,
                    $t2
                );
            }
        }

        unset($d1, $d2); // free memory
    }


    writeLog("--- Finished variable tables $t1 (JAZ1), $t2 (JAZ2) ---");
}

/**
 * ==========================================
 * TABLE COMPARISON (CORE LOGIC)
 * ==========================================
 */
function compareTable(PDO $pdo1, PDO $pdo2, $t1, $t2)
{
    writeLog("--- Start comparing table $t1 (JAZ1), $t2 (JAZ2) ---");

    if ($t1 == "ja_run_log_table") {
        $pks = $GLOBALS['runLogPKs'];
    } else {
        $pks = getPK($pdo1, $t1);
    }

    if (!$pks) {
        echo "[WARN] No primary key is found. Skipped. $t1 (JAZ1), $t2 (JAZ2)";
        writeLog("[WARN] No primary key is found. Skipped. $t1 (JAZ1), $t2 (JAZ2)");
        return;
    }

    $associatedPKs = fetchAssociatedPKs($pdo1, $t1, $pks);
    $keys2 = fetchPKs($pdo2, $t2, $pks);

    $commonKeys = array_intersect(array_keys($associatedPKs), $keys2);
    $missingInD2 = array_diff(array_keys($associatedPKs), $keys2);
    $extraInD2 = array_diff($keys2, array_keys($associatedPKs));

    foreach ($missingInD2 as $pk) {
        $pkVal = str_replace("\x1F", ",", $pk);
        if ($t1 == "ja_parameter_table" && key_exists($pkVal, $GLOBALS['zbxHostCompMap'])) {
            continue;
        }

        $diffStatus = DiffStatus::ROW_MISSING;
        writeLog("jaz 1: $t1, jaz 2: $t2, col: all, pk: $pkVal, diff_status: $diffStatus, data: \nJAZ1:" . toCsvString($pkVal) . "\nJAZ2:" . null . "\n===");
        writeCsvRow(
            $pkVal,
            "all",
            CHECK_LOG_TEXT,
            null,
            $diffStatus,
            $t1,
            $t2
        );
        continue;
    }

    foreach ($extraInD2 as $pk) {
        $pkVal = str_replace("\x1F", ",", $pk);
        $diffStatus = DiffStatus::EXTRA_ROW;

        if (in_array($pk, $GLOBALS['newlyAddedCols'])) {
            $diffStatus = DiffStatus::NEWLY_ADDED;
            writeLog("jaz 1: $t1, jaz 2: $t2, col: all, pk: $pkVal, diff_status: $diffStatus, data: \nJAZ1:" . null . "\nJAZ2:" . toCsvString($pkVal) . "\n===");

            writeCsvRow(
                $pkVal,
                "all",
                null,
                CHECK_LOG_TEXT,
                $diffStatus,
                $t1,
                $t2
            );
            continue;
        }

        writeLog("jaz 1: $t1, jaz 2: $t2, col: all, pk: $pkVal, diff_status: $diffStatus, data: \nJAZ1:" . null . "\nJAZ2:" . toCsvString($pkVal) . "\n===");
        writeCsvRow(
            str_replace("\x1F", ",", $pk),
            "all",
            null,
            CHECK_LOG_TEXT,
            $diffStatus,
            $t1,
            $t2
        );
        continue;
    }

    $batchSize = 50;
    foreach (array_chunk($commonKeys, $batchSize) as $batchKeys) {

        $fetchKeys = [];
        foreach ($batchKeys as $key) {
            $fetchKeys[] = $associatedPKs[$key];
        }

        // Fetch full rows for this batch
        $d1 = fetchRowsByPKBatch($pdo1, $t1, $pks, $fetchKeys);
        $d2 = fetchRowsByPKBatch($pdo2, $t2, $pks, $batchKeys);

        // Now reuse your existing compare logic unchanged
        foreach ($batchKeys as $pkValue2) {
            $pkValue1 = $associatedPKs[$pkValue2];

            if (empty($d1[$pkValue1])) {
                writeLog("[WARN] Missing value for pk: $pkValue1");
                continue;
            }

            foreach ($d1[$pkValue1] as $col => $val1) {
                $pkVal = str_replace("\x1F", ",", $pkValue1);

                $timeConverted = false;
                $val2 = $d2[$pkValue2][$col] ?? null;

                // removed columns in jaz2; they are ignored.
                if (in_array($col, $GLOBALS['removedCols'])) {
                    $diffStatus = DiffStatus::REMOVED;
                    writeLog("jaz 1: $t1, jaz 2: $t2, col: all, pk: $pkVal, diff_status: $diffStatus, data: \nJAZ1:" . toCsvString($val1) . "\nJAZ2:" . toCsvString($val2) . "\n===");

                    writeCsvRow(
                        $pkVal,
                        $col,
                        $val1,
                        $val2,
                        $diffStatus,
                        $t1,
                        $t2
                    );
                    continue;
                }

                // these rows should not exist in jaz2 table
                if ($t1 == "ja_parameter_table" && key_exists($val1, $GLOBALS['zbxHostCompMap'])) {
                    writeLog("jaz 1: $t1, jaz 2: $t2, col: all, pk: $pkVal, diff_status: $diffStatus, data: \nJAZ1:" . toCsvString($val1) . "\nJAZ2:" . toCsvString($val2) . "\n===");
                    writeCsvRow(
                        $pkVal,
                        $col,
                        $val1,
                        $val2,
                        DiffStatus::EXTRA_ROW,
                        $t1,
                        $t2
                    );
                    continue;
                }

                // columns are datetime, convert to unix first
                if (in_array($col, $GLOBALS['timeColumns'])) {
                    $orgValue = $val1;
                    $val1 = toUnixTime($val1, VM_TIMEZONE);
                    $timeConverted = true;
                }

                if ($t1 == "ja_run_job_table" && ($col == "start_time" || $col == "end_time")) {
                    $val1 *= 1000000000;
                }

                $diffStatus = $val1 == $val2 ? DiffStatus::NO : DiffStatus::YES;

                writeLog("jaz 1: $t1, jaz 2: $t2, col: $col, pk: $pkVal, diff_status: $diffStatus, data: \nJAZ1:" . toCsvString($timeConverted ? $orgValue : $val1) . "\nJAZ2:" . toCsvString($val2) . "\n===");

                // Write ALL columns
                writeCsvRow(
                    $pkVal,
                    $col,
                    $timeConverted ? $orgValue : $val1,
                    $val2,
                    $diffStatus,
                    $t1,
                    $t2
                );
            }
        }

        unset($d1, $d2); // free memory
    }

    writeLog("--- Finished comparing table $t1 (JAZ1), $t2 (JAZ2) ---");
}

function compareRunData(PDO $pdo1, PDO $pdo2, $runIconTable, $jsonFields)
{
    $t2 = "ja_2_run_job_table"; // compare data column from this table

    writeLog("--- Start comparing run data $runIconTable -> $t2 for json fields: " . toCsvString($jsonFields) . " ---");

    $pks = ["inner_job_id"];

    $keys1 = fetchPKs($pdo1, $runIconTable, $pks);
    $keys2 = fetchPKs($pdo2, $t2, $pks);

    $commonKeys = array_intersect($keys1, $keys2);
    $missingInD2 = array_diff($keys1, $keys2);
    // Skip the check for extras as there will be extra rows in ja_2_run_job_table

    foreach ($missingInD2 as $pk) {
        $pkVal = str_replace("\x1F", ",", $pk);
        $diffStatus = DiffStatus::ROW_MISSING;
        writeLog("jaz 1: $runIconTable, jaz 2: $t2, col: all, pk: $pkVal, diff_status: $diffStatus, data: \nJAZ1:" . toCsvString($pkVal) . "\nJAZ2:" . null . "\n===");

        writeCsvRow(
            $pkVal,
            "all",
            CHECK_LOG_TEXT,
            null,
            $diffStatus,
            $runIconTable,
            $t2
        );
        continue;
    }

    $batchSize = 50;
    foreach (array_chunk($commonKeys, $batchSize) as $batchKeys) {
        // Fetch full rows for this batch
        $d1 = fetchRowsByPK($pdo1, $runIconTable, $pks, $batchKeys);

        // Special handling for command and stop_command
        if ($runIconTable === "ja_run_job_command_table") {
            $grouped = [];

            foreach ($d1 as $row) {
                $inner_job_id = $row['inner_job_id'] ?? null;
                $command_cls = $row['command_cls'] ?? null;
                $inner_jobnet_id = $row['inner_jobnet_id'] ?? null;

                $newPk = $inner_job_id;

                if (!isset($grouped[$newPk])) {
                    $grouped[$newPk] = [
                        'inner_job_id' => $inner_job_id,
                        'inner_jobnet_id' => $inner_jobnet_id,
                        'command' => "",
                        'stop_command' => "",
                    ];
                }

                if ($command_cls == 0) {
                    $grouped[$newPk]['command'] = $row['command'];
                } elseif ($command_cls == 2) {
                    $grouped[$newPk]['stop_command'] = $row['command'];
                }
            }

            $d1 = $grouped;
        } else if ($runIconTable == "ja_run_icon_value_table") {
            // create variables column
            $d1 = transformRunValueToJson($d1, "variables", "value", false);
        } else {
            $d1 = getBatchByPKs($d1, $pks);
        }

        $d2 = fetchRowsByPKBatch($pdo2, $t2, $pks, $batchKeys);

        foreach ($batchKeys as $pkValue) {
            $jsonData = json_decode($d2[$pkValue]['data'] ?? '{}', true); // true = associative array
            $extraKeys = array_diff(array_keys($jsonData), $jsonFields);
            if (!empty($extraKeys)) {
                $extraKeysStr = implode(",", $extraKeys);
                // writeLog("Extra keys found in $pkValue of $runIconTable -> $t2: $extraKeysStr");
                $diffStatus = DiffStatus::EXTRA_FIELDS;
                writeLog("jaz 1: $runIconTable, jaz 2: $t2, col: $extraKeysStr, pk: $pkVal, diff_status: $diffStatus, data: \nJAZ1:" . null . "\nJAZ2:" . toCsvString($jsonData) . "\n===");
                writeCsvRow(
                    str_replace("\x1F", ",", $pkValue),
                    $extraKeysStr,
                    null,
                    $jsonData,
                    $diffStatus,
                    $runIconTable,
                    $t2
                );
                continue;
            }

            foreach ($d1[$pkValue] as $col => $val1) {

                if ($col === "inner_job_id" || $col == "inner_jobnet_id") {
                    continue;
                }

                // If the columns other than specified json fields to compare, skip them
                if (!in_array($col, $jsonFields)) {
                    continue;
                }

                $isDiff = false;

                // If this column exists inside JSON, compare
                if (isset($jsonData[$col])) {
                    $val2 = $jsonData[$col];

                    // handle arrays vs scalars
                    if (is_array($val1) || is_array($val2)) {
                        $isDiff = !arraysEqual((array) $val1, (array) $val2); // recursive array compare
                    } else {
                        $isDiff = ($val1 != $val2);
                    }
                } else {
                    // If JSON does not have this key, consider it a difference
                    $val2 = null;
                    $isDiff = true;
                }

                $diffStatus = $isDiff ? DiffStatus::YES : DiffStatus::NO;
                writeLog("jaz 1: $runIconTable, jaz 2: $t2, col: $col, pk: $pkValue, diff_status: $diffStatus, data: \nJAZ1:" . toCsvString($val1) . "\nJAZ2:" . toCsvString($jsonData) . "\n===");

                // Write ALL columns
                writeCsvRow(
                    str_replace("\x1F", ",", $pkValue),
                    $col,
                    CHECK_LOG_TEXT,
                    CHECK_LOG_TEXT,
                    $diffStatus,
                    $runIconTable,
                    $t2
                );
            }
        }

        unset($d1, $d2);
    }

    writeLog("--- Finished comparing run data $runIconTable -> $t2 for json fields: " . toCsvString($jsonFields) . " ---");
}

function compareZbxHost(PDO $pdo1, PDO $pdo2)
{
    // fetch from ja_2_zbx_host_table from jaz2
    $zbx_host_table = "ja_2_zbx_hosts_table";
    $ja_parameter_table = "ja_parameter_table";
    writeLog("--- Start comparing zbx host $ja_parameter_table (JAZ1), $zbx_host_table (JAZ2) ---");

    $zbx_host_rows = $pdo2->query("SELECT * FROM $zbx_host_table")->fetchAll(PDO::FETCH_ASSOC);

    // there must be a single line
    if (sizeof($zbx_host_rows) > 1) {
        writeLog("ERROR $zbx_host_table contains more than one row");
        throw new Exception("ERROR $zbx_host_table contains more than one row");
    }

    $dataCol = $zbx_host_rows[0]["data"];
    $data = json_decode($dataCol, true);

    // fetch from ja_parameter_table from jaz1
    $param_rows = $pdo1->query("SELECT * FROM $ja_parameter_table")->fetchAll(PDO::FETCH_ASSOC);

    foreach ($param_rows as $param_row) {
        $param_name = $param_row["parameter_name"];
        $param_value = $param_row["value"];
        if (!key_exists($param_name, $GLOBALS['zbxHostCompMap'])) {
            continue;
        }

        $comp_key = $GLOBALS['zbxHostCompMap'][$param_name];

        $diffStatus = $data[$comp_key] == $param_value ? DiffStatus::NO : DiffStatus::YES;

        writeLog("jaz 1: $ja_parameter_table, jaz 2: $zbx_host_table, col: $param_name, pk: $param_name, diff_status: $diffStatus, data: \nJAZ1:" . toCsvString($param_value) . "\nJAZ2:" . toCsvString($data[$comp_key]) . "\n===");

        writeCsvRow(
            $param_name,
            $param_name,
            $param_value,
            $data[$comp_key],
            $diffStatus,
            $ja_parameter_table,
            $zbx_host_table
        );
    }

    writeLog("--- Finished comparing zbx host $ja_parameter_table (JAZ1), $zbx_host_table (JAZ2) ---");
}

/**
 * ==========================================
 * MAIN
 * ==========================================
 */

$config = require __DIR__ . '/config.php';

$dbType = $config['dbType'];
$jaz1 = $config['jaz1'];
$jaz2 = $config['jaz2'];
if (!empty($config['timezone'])) {
    $timezone = $config['timezone'];
}

/**
 * Detects VM timezone
 */
define('VM_TIMEZONE', (function () {
    if (!empty($GLOBALS['timezone'])) {
        return $GLOBALS['timezone'];
    }

    $tz = null;
    if (function_exists('shell_exec')) {
        $tz = trim(@shell_exec("timedatectl show -p Timezone --value 2>/dev/null"));
    }
    if (!$tz && is_readable('/etc/timezone')) {
        $tz = trim(file_get_contents('/etc/timezone'));
    }
    return $tz ?: date_default_timezone_get();
})());

try {

    writeLog("TIMEZONE used: " . VM_TIMEZONE);

    $pdo1 = new PDO(buildDsn($dbType, $jaz1), $jaz1['user'], $jaz1['pass'], [
        PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION
    ]);

    writeLog("Connected to jaz1 database: host={$jaz1['host']}, dbname={$jaz1['dbname']}");

    $pdo2 = new PDO(buildDsn($dbType, $jaz2), $jaz2['user'], $jaz2['pass'], [
        PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION
    ]);

    writeLog("Connected to jaz2 database: host={$jaz2['host']}, dbname={$jaz2['dbname']}");

    initCsv();

    foreach ($tableMap as $oldTable => $newTable) {
        compareTable($pdo1, $pdo2, $oldTable, $newTable);
    }

    foreach ($variableTableMap as $oldTable => $newTable) {
        compareVarTable($pdo1, $pdo2, $oldTable, $newTable);
    }

    foreach ($runDataMap as $iconTable => $jsonFields) {
        compareRunData($pdo1, $pdo2, $iconTable, $jsonFields);
    }

    compareZbxHost($pdo1, $pdo2);

    writeLog("Comparison result has been written to {$GLOBALS['csvFilePath']}");

} catch (Exception $e) {
    echo "[ERROR] encountered while comparing migrated data: " . $e->getMessage();
    writeLog("[ERROR] encountered while comparing migrated data: " . $e->getMessage());
} finally {
    closeCsv();
    fclose($logFp);
}