<?php
/**
 * JAZ1 → JAZ2 Full Migration Script
 *
 * This script migrates all tables from JAZ1 → JAZ2.
 * Includes:
 *  - Control Tables: configuration and metadata
 *  - Initial Tables: static/reference data
 *  - Run Tables: job runtime and results data
 *
 * Features:
 *  - Handles PostgreSQL and MySQL
 *  - UPSERT / ON CONFLICT for Control/Initial tables
 *  - Truncates Run tables before migration
 *  - Filters extra columns to prevent undefined column errors
 *  - DRY_RUN option to test without modifying target DB
 */

ini_set('display_errors', 1);
error_reporting(E_ALL);

/********************
 * CONFIGURATION
 ********************/
$config = require __DIR__ . '/config.php';

$dbType = $config['dbType'];
$DRY_RUN = $config['dryRun'];
$jaz1 = $config['jaz1'];
$jaz2 = $config['jaz2'];
if (!empty($config['timezone'])) {
    $timezone = $config['timezone'];
}

/********************
 * FUNCTIONS
 ********************/

/**
 * Build DSN(Data Source Name) string for PDO
 */
function buildDsn($type, $cfg)
{
    // Postgres was default charset=utf8mb4
    if ($type === 'pgsql') {
        return "pgsql:host={$cfg['host']};port={$cfg['port']};dbname={$cfg['dbname']}";
    }
    // For MySQL/MariaDB: add charset=utf8mb4
    return "mysql:host={$cfg['host']};port={$cfg['port']};dbname={$cfg['dbname']};charset=utf8mb4";
}

/**
 * Simple logger with timestamp
 */
function logMsg($msg)
{
    echo date('[Y-m-d H:i:s] ') . $msg . PHP_EOL;
}

/**
 * Detects VM timezone
 * If user don't set timezone at config, will set VM timezone
 */
define('VM_TIMEZONE', (function () {
    if (!empty($GLOBALS['timezone'])) {
        return $GLOBALS['timezone'];
    }

    $tz = null;
    if (function_exists('shell_exec')) {
        $tz = trim(@shell_exec("timedatectl show -p Timezone --value 2>/dev/null"));
    }
    if (!$tz && is_readable('/etc/timezone')) {
        $tz = trim(file_get_contents('/etc/timezone'));
    }
    return $tz ?: date_default_timezone_get();
})());

/**
 * Convert YYYYMMDDHHMMSS in a given timezone to Unix timestamp (UTC)
 *
 * @param string|int $val  YYYYMMDDHHMMSS
 * @param string     $tz   Timezone like "Asia/Yangon", "UTC", "Asia/Tokyo"
 * @return int|string      Unix timestamp or original value on failure
 */
function jazToUnix($val, string $tz = VM_TIMEZONE)
{
    if ($val === null || $val === '')
        return $val;

    
    // YYYYMMDDHHMMSS → YYYY-MM-DD HH:MM:SS
    if (is_numeric($val)) {

        if (strlen((string) $val) > 14){
            $val = substr($val,0,14);
        }

        // If only 12 digits, add "00" seconds
        if (strlen($val) === 12) {
            $val .= '00';
        }

        $val =
            substr($val, 0, 4) . '-' .
            substr($val, 4, 2) . '-' .
            substr($val, 6, 2) . ' ' .
            substr($val, 8, 2) . ':' .
            substr($val, 10, 2) . ':' .
            substr($val, 12, 2);
    }

    try {
        $tzObj = new DateTimeZone($tz);
        $dt = new DateTime($val, $tzObj);
        return $dt->getTimestamp();
    } catch (Exception $e) {
        return $val;
    }
}

/**
 * Convert datetime to Unix nanoseconds (int as string)
 */
function jazToUnixNano($val, string $tz = VM_TIMEZONE)
{
    if ($val === null || $val === '')
        return $val;

    // Convert YYYYMMDDHHMMSS format
    if (is_numeric($val) && strlen((string) $val) === 14) {
        $val =
            substr($val, 0, 4) . '-' .
            substr($val, 4, 2) . '-' .
            substr($val, 6, 2) . ' ' .
            substr($val, 8, 2) . ':' .
            substr($val, 10, 2) . ':' .
            substr($val, 12, 2);
    }

    try {
        $tzObj = new DateTimeZone($tz);
        $dt = new DateTime($val, $tzObj);

        $seconds = $dt->getTimestamp();

        // Convert seconds → nanoseconds
        return (string) ($seconds * 1000000000);

    } catch (Exception $e) {
        return $val;
    }
}

/**
 * Converts all date/time columns in a row into Unix timestamps.
 */
function normalizeRow($row)
{
    $timestampCols = [
        'scheduled_time',
        'start_time',
        'end_time',
        'initial_scheduled_time',
        'update_date',
        'last_active_time',
        'created_date',
        'log_date'
    ];

    foreach ($row as $col => $val) {
        if ($val === null)
            continue;

        if (substr($col, -5) === '_date' || in_array($col, $timestampCols)) {
            if (!is_numeric($val) || strlen((string) $val) >= 12) {
                $row[$col] = jazToUnix($val);
            }
        }
    }
    return $row;
}

/**
 * Converts timestamps, JSON-encodes arrays/objects, and protects big integers in a run-job row.
 */
function normalizeRunJobRow(array $row)
{
    $timestampCols = ['created_date', 'start_time', 'end_time', 'update_date'];

    foreach ($row as $col => &$val) {
        if ($val === null)
            continue;

        if (in_array($col, $timestampCols)) {
            if ($col === 'start_time' || $col === 'end_time') {
                $val = jazToUnixNano($val);   // nano
            } else {
                $val = jazToUnix($val);       // normal seconds
            }
        }

        if (is_array($val) || is_object($val)) {
            $val = json_encode($val);
        }

        if (is_numeric($val) && (int) $val > 9007199254740991) {
            $val = (string) $val;
        }
    }
    return $row;
}

/**
 * Inject VM timezone into variable JSON for Job/Jobnet variables
 */
function addVmTimezone(array $vars): array
{
    // Add "TIMEZONE" key at top level using global constant
    $vars['TIMEZONE'] = VM_TIMEZONE;
    return $vars;
}

/**
 * Build UPSERT SQL for Control/Initial tables
 */
function buildUpsertSQL($dbType, $table, $cols, $pk)
{
    $placeholders = array();
    foreach ($cols as $c)
        $placeholders[] = ':' . $c;

    $sql = "INSERT INTO {$table} (" . implode(',', $cols) . ") VALUES (" . implode(',', $placeholders) . ")";

    $updates = array();
    $pkCols = array_map('trim', explode(',', $pk));

    foreach ($cols as $c) {
        if (!in_array($c, $pkCols)) {
            if ($dbType === 'pgsql')
                $updates[] = "{$c} = EXCLUDED.{$c}";
            else
                $updates[] = "{$c} = VALUES({$c})";
        }
    }

    if ($dbType === 'pgsql') {
        if (!empty($updates))
            $sql .= " ON CONFLICT ({$pk}) DO UPDATE SET " . implode(', ', $updates);
        else
            $sql .= " ON CONFLICT ({$pk}) DO NOTHING";
    } else {
        if (!empty($updates))
            $sql .= " ON DUPLICATE KEY UPDATE " . implode(', ', $updates);
    }

    return $sql;
}

/**
 * Fetch all columns of target table
 */
function getTargetColumns(PDO $jaz2, $table)
{
    $stmt = $jaz2->prepare("
    SELECT column_name 
    FROM information_schema.columns 
    WHERE table_name = :table
    ");
    $stmt->execute([':table' => strtolower($table)]);
    return $stmt->fetchAll(PDO::FETCH_COLUMN);
}

/**
 * Check if a job exists in the target run_job_table
 */
function jobExists(PDO $jaz2, $inner_job_id)
{
    $stmt = $jaz2->prepare("SELECT 1 FROM ja_2_run_job_table WHERE inner_job_id = ?");
    $stmt->execute(array($inner_job_id));
    return $stmt->fetchColumn() !== false;
}

/**
 * Proper global getJobData() function to avoid undefined function error
 */
function getJobData(PDO $pdo, array $job)
{
    static $jobTypeTables = [
    1 => ['ja_run_icon_end_table'],
    2 => ['ja_run_icon_if_table'],
    3 => ['ja_run_icon_value_table'],
    4 => ['ja_run_icon_job_table', 'ja_run_job_command_table'],
    5 => ['ja_run_icon_jobnet_table'],
    9 => ['ja_run_icon_extjob_table'],
    10 => ['ja_run_icon_calc_table'],
    11 => ['ja_run_icon_task_table'],
    12 => ['ja_run_icon_info_table'],
    14 => ['ja_run_icon_fcopy_table'],
    15 => ['ja_run_icon_fwait_table'],
    16 => ['ja_run_icon_reboot_table'],
    17 => ['ja_run_icon_release_table'],
    18 => ['ja_run_icon_agentless_table'],
    19 => ['ja_run_icon_zabbix_link_table']
    ];

    static $forceIntColumns = [
    'host_flag',
    'connection_method',
    'session_flag',
    'auth_method',
    'run_mode',
    'line_feed_code',
    'timeout',
    'jobnet_stop_flag',
    'stop_flag',
    'command_type',
    'timeout_run_type'
    ];

    static $iconTableStats = [];

    $data = [];
    $jobType = (int) $job['job_type'];

    if (!isset($jobTypeTables[$jobType])) {
        return json_encode(new stdClass());
    }

    foreach ($jobTypeTables[$jobType] as $tbl) {
        $stmt = $pdo->prepare("SELECT * FROM $tbl WHERE inner_job_id = ? AND inner_jobnet_id = ?");
        $stmt->execute([$job['inner_job_id'], $job['inner_jobnet_id']]);
        $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);

        if ($rows) {
            $iconTableStats[$tbl] = ($iconTableStats[$tbl] ?? 0) + count($rows);
        }

        foreach ($rows as $r) {
            unset($r['inner_job_id'], $r['inner_jobnet_id']);

            // VALUE TABLE → variables
            if ($tbl === 'ja_run_icon_value_table') {
                $data['variables'] = $data['variables'] ?? [];
                if (isset($r['value_name'])) {
                    $data['variables'][$r['value_name']] = $r['value'] ?? null;
                }
                continue;
            }

            // JOB COMMAND TABLE → command/stop_command
            if ($tbl === 'ja_run_job_command_table') {
                $cls = (int) ($r['command_cls'] ?? -1);
                $cmd = $r['command'] ?? null;
                if ($cls === 0 && $cmd !== null) {
                    $data['command'] = $cmd;
                } elseif ($cls === 2 && $cmd !== null) {
                    $data['stop_command'] = $cmd;
                }
                continue;
            }

            unset($r['command_cls']);

            // EXTJOB
            if ($tbl === 'ja_run_icon_extjob_table') {
                unset($r['wait_time'], $r['wait_count']);
            }

            // INFO ICON
            if ($tbl === 'ja_run_icon_info_table') {
                unset($r['item_id'], $r['host_name'], $r['host_group'], $r['trigger_id']);
                foreach (['get_calendar_id', 'get_job_id'] as $f) {
                    if (array_key_exists($f, $r) && $r[$f] === null)
                        $r[$f] = "";
                }
            }

            // AGENTLESS ICON
            if ($tbl === 'ja_run_icon_agentless_table') {
                if (array_key_exists('login_password', $r) && $r['login_password'] === null)
                    $r['login_password'] = "";
            }

            // Force integer casting
            foreach ($r as $key => $value) {
                if (in_array($key, $forceIntColumns, true) && $value !== null && $value !== '') {
                    $r[$key] = (int) $value;
                }
            }

            $data = array_merge($data, $r);

            if ($jobType === 4 && !isset($data['stop_command'])) {
                $data['stop_command'] = "";
            }
        }
    }

    return $data ? json_encode($data) : json_encode(new stdClass());
}


/********************
 * TABLES TO MIGRATE
 ********************/
$controlTables = array(
    'ja_calendar_control_table' => array('dst' => 'ja_2_calendar_control_table', 'pk' => 'calendar_id,update_date'),
    'ja_calendar_detail_table' => array('dst' => 'ja_2_calendar_detail_table', 'pk' => 'calendar_id,update_date,operating_date'),
    'ja_filter_control_table' => array('dst' => 'ja_2_filter_control_table', 'pk' => 'filter_id,update_date'),
    'ja_schedule_control_table' => array('dst' => 'ja_2_schedule_control_table', 'pk' => 'schedule_id,update_date'),
    'ja_schedule_detail_table' => array('dst' => 'ja_2_schedule_detail_table', 'pk' => 'schedule_id,calendar_id,update_date,boot_time,object_flag'),
    'ja_schedule_jobnet_table' => array('dst' => 'ja_2_schedule_jobnet_table', 'pk' => 'schedule_id,jobnet_id,update_date'),
    'ja_jobnet_control_table' => array('dst' => 'ja_2_jobnet_control_table', 'pk' => 'jobnet_id,update_date'),
    'ja_job_control_table' => array('dst' => 'ja_2_job_control_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_flow_control_table' => array('dst' => 'ja_2_flow_control_table', 'pk' => 'jobnet_id,start_job_id,end_job_id,update_date'),
    'ja_icon_agentless_table' => array('dst' => 'ja_2_icon_agentless_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_icon_calc_table' => array('dst' => 'ja_2_icon_calc_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_icon_end_table' => array('dst' => 'ja_2_icon_end_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_icon_extjob_table' => array('dst' => 'ja_2_icon_extjob_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_icon_fcopy_table' => array('dst' => 'ja_2_icon_fcopy_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_icon_fwait_table' => array('dst' => 'ja_2_icon_fwait_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_icon_if_table' => array('dst' => 'ja_2_icon_if_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_icon_info_table' => array('dst' => 'ja_2_icon_info_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_icon_jobnet_table' => array('dst' => 'ja_2_icon_jobnet_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_icon_job_table' => array('dst' => 'ja_2_icon_job_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_job_command_table' => array('dst' => 'ja_2_job_command_table', 'pk' => 'jobnet_id,job_id,update_date,command_cls'),
    'ja_value_job_table' => array('dst' => 'ja_2_value_job_table', 'pk' => 'jobnet_id,job_id,update_date,value_name'),
    'ja_value_jobcon_table' => array('dst' => 'ja_2_value_jobcon_table', 'pk' => 'jobnet_id,job_id,value_name,update_date'),
    'ja_icon_reboot_table' => array('dst' => 'ja_2_icon_reboot_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_icon_release_table' => array('dst' => 'ja_2_icon_release_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_icon_task_table' => array('dst' => 'ja_2_icon_task_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_icon_value_table' => array('dst' => 'ja_2_icon_value_table', 'pk' => 'jobnet_id,job_id,update_date,value_name'),
    'ja_icon_zabbix_link_table' => array('dst' => 'ja_2_icon_zabbix_link_table', 'pk' => 'jobnet_id,job_id,update_date'),
    'ja_object_lock_table' => array('dst' => 'ja_2_object_lock_table', 'pk' => 'object_id,object_type')
);

$initialTables = array(
    'ja_define_value_jobcon_table' => array('dst' => 'ja_2_define_value_jobcon_table', 'pk' => 'value_name'),
    'ja_define_run_log_message_table' => array('dst' => 'ja_2_define_run_log_message_table', 'pk' => 'message_id,lang'),
    'ja_parameter_table' => array('dst' => 'ja_2_parameter_table', 'pk' => 'parameter_name'),
    'ja_index_table' => array('dst' => 'ja_2_index_table', 'pk' => 'count_id'),
    'ja_define_extjob_table' => array('dst' => 'ja_2_define_extjob_table', 'pk' => 'command_id,lang'),
    'ja_host_lock_table' => array('dst' => 'ja_2_host_lock_table', 'pk' => 'lock_host_name')
);

$excludeParameters = [
    'ZBXSND_ZABBIX_IP',
    'ZBXSND_ZABBIX_PORT',
    'ZBXSND_ZABBIX_HOST',
    'ZBXSND_ITEM_KEY'
];

function truncateTables(PDO $pdo2)
{
    logMsg("Truncating JAZ2 run tables + control tables");

    $driver = $pdo2->getAttribute(PDO::ATTR_DRIVER_NAME);

    $runTables = [
        'ja_2_send_message_table',
        'ja_2_run_log_table',
        'ja_2_session_table',
        'ja_2_run_job_variable_table',
        'ja_2_run_jobnet_variable_table',
        'ja_2_run_value_jobcon_table',
        'ja_2_run_value_job_table',
        'ja_2_run_flow_table',
        'ja_2_run_job_table',
        'ja_2_run_jobnet_summary_table',
        'ja_2_run_jobnet_table',


        'ja_2_run_timeout_table',
        'ja_2_host_lock_table',
        'ja_2_zbx_hosts_table',

        'ja_2_ran_log_table',
        'ja_2_ran_job_variable_table',
        'ja_2_ran_jobnet_variable_table',
        'ja_2_ran_value_jobcon_table',
        'ja_2_ran_value_job_table',
        'ja_2_ran_flow_table',
        'ja_2_ran_job_table',
        'ja_2_ran_jobnet_summary_table',
        'ja_2_ran_jobnet_table',
    ];

    $controlTables = array_map(function ($cfg) {
        return $cfg['dst'];
    }, $GLOBALS['controlTables']);

    $tablesToTruncate = array_merge($runTables, $controlTables);

    try {

        switch ($driver) {

            case 'mysql':
                $pdo2->exec("SET FOREIGN_KEY_CHECKS = 0");
                foreach ($tablesToTruncate as $table) {
                    $pdo2->exec("TRUNCATE TABLE {$table}");
                }
                $pdo2->exec("SET FOREIGN_KEY_CHECKS = 1");
                break;

            case 'pgsql':
                foreach ($tablesToTruncate as $table) {
                    $pdo2->exec("TRUNCATE TABLE {$table} CASCADE");
                }
                break;

            default:
                throw new RuntimeException("Unsupported database driver: {$driver}");
        }

        logMsg("Truncate completed successfully.\n");

    } catch (Exception $e) {
        logMsg("Truncate FAILED: " . $e->getMessage());
        throw $e;
    }
}

/********************
 * MIGRATION FUNCTION
 ********************/

/**
 * Helper batch insert
 */
function executeMultiInsert(PDO $pdo, $table, $cols, array $rows)
{
    if (empty($rows)) return 0;

    $placeholders = [];
    $values = [];

    foreach ($rows as $row) {
        $placeholders[] = '(' . implode(',', array_fill(0, count($cols), '?')) . ')';
        foreach ($cols as $col) {
            $values[] = $row[$col] ?? null;
        }
    }

    $sql = "INSERT INTO {$table} (" . implode(',', $cols) . ")
            VALUES " . implode(',', $placeholders);

    $stmt = $pdo->prepare($sql);

    try {
        $stmt->execute($values);
    } catch (Exception $e) {
        logMsg("FAILED batch insert into {$table}: " . $e->getMessage());
        throw $e; // propagate to outer transaction
    }

    return count($rows);
}

/**
 * Migrate Zabbix host settings
 * ja_parameter_table → ja_2_zbx_hosts_table
 */
function migrateZbxHosts(PDO $pdo1, PDO $pdo2, $DRY_RUN)
{
    logMsg("Migrating ja_parameter_table => ja_2_zbx_hosts_table");

    $stmt = $pdo1->query("
        SELECT parameter_name, value
        FROM ja_parameter_table
        WHERE parameter_name IN (
            'ZBXSND_ZABBIX_IP',
            'ZBXSND_ZABBIX_PORT',
            'ZBXSND_ZABBIX_HOST',
            'ZBXSND_ITEM_KEY'
        )
    ");

    $params = [];

    while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
        $params[$row['parameter_name']] = $row['value'];
    }

    $totalRows = 0;
    $migrated  = 0;

    // If parameters exist → one logical row to migrate
    if (!empty($params)) {

        $data = [
            'zabbix_server_ip_address'     => $params['ZBXSND_ZABBIX_IP'] ?? null,
            'zabbix_server_port_number'    => $params['ZBXSND_ZABBIX_PORT'] ?? null,
            'message_destination_hostname' => $params['ZBXSND_ZABBIX_HOST'] ?? null,
            'message_destination_item_key' => $params['ZBXSND_ITEM_KEY'] ?? null,
        ];

        $totalRows = 1;

        if (!$DRY_RUN) {
            try {
                $stmt = $pdo2->prepare("
                    INSERT INTO ja_2_zbx_hosts_table (data)
                    VALUES (:data)
                ");

                $stmt->execute([
                    ':data' => json_encode($data)
                ]);

                $migrated = $stmt->rowCount();

            } catch (Exception $e) {
                logMsg("FAILED migrating ja_2_zbx_hosts_table: " . $e->getMessage());
                throw $e; // Let global transaction handler manage rollback
            }
        }
    }

    // If nothing to migrate
    if ($totalRows === 0) {
        logMsg("  - No data to migrate");
        return;
    }

    // Unified logging format
    if ($DRY_RUN) {
        logMsg("Total - {$totalRows} rows | DRY RUN (no insert)");
    } else {
        logMsg("Total - {$totalRows} rows | {$migrated} rows migrated successfully");
    }
}

/**
 * Migrate control + initial tables
 * Fully optimized: streaming fetch + prepare-once + batch insert/upsert
 */
function migrateTables($pdo1, $pdo2, $tables, $dbType, $DRY_RUN, $title, $useUpsert = false, $batchSize = 500)
{
    logMsg("Starting {$title} Migration (" . count($tables) . " tables)");

    foreach ($tables as $src => $cfg) {
        
        $dst = $cfg['dst'];
        $pk  = isset($cfg['pk']) ? $cfg['pk'] : null;

        logMsg("Migrating {$src} => {$dst}");

        try {
            // =====================================
            // Streaming fetch
            // =====================================
            $stmt1 = $pdo1->query("SELECT * FROM {$src}");
            if (!$stmt1) {
                logMsg("  - No data to migrate");
                continue;
            }

            $targetColumns = getTargetColumns($pdo2, $dst);
            $targetMap     = array_flip($targetColumns);

            $sampleRow = $stmt1->fetch(PDO::FETCH_ASSOC);
            if (!$sampleRow) {
                logMsg("  - No data to migrate");
                continue;
            }

            // Inject extra column
            if ($dst === 'ja_2_calendar_control_table') {
                $sampleRow['time_zone'] = VM_TIMEZONE;
            }

            $sampleRow = array_intersect_key(normalizeRow($sampleRow), $targetMap);
            $cols = array_keys($sampleRow);

            // =====================================
            // Prepare UPSERT statement ONLY if needed
            // =====================================
            $stmtUpsert = null;

            if ($useUpsert && $pk) {

                $sqlUpsert = buildUpsertSQL($dbType, $dst, $cols, $pk);
                $stmtUpsert = !$DRY_RUN ? $pdo2->prepare($sqlUpsert) : null;
            }

            // Reset cursor
            $stmt1->execute();

            // =====================================
            // Start streaming
            // =====================================
            $totalRows = 0;
            $inserted  = 0;
            $batchRows = [];

            while ($row = $stmt1->fetch(PDO::FETCH_ASSOC)) {

                $totalRows++;
                $row = normalizeRow($row);

                // Skip Zabbix-related parameters
                if ($src === 'ja_parameter_table'
                    && in_array($row['parameter_name'] ?? '', $GLOBALS['excludeParameters'], true)) {
                    continue;
                }

                if ($dst === 'ja_2_calendar_control_table') {
                    $row['time_zone'] = VM_TIMEZONE;
                }

                $row = array_intersect_key($row, $targetMap);

                // =====================================
                // UPSERT MODE (row-by-row safe)
                // =====================================
                if ($useUpsert && $pk) {

                    if (!$DRY_RUN) {
                        $stmtUpsert->execute($row);
                    }

                    $inserted++;
                    continue;
                }

                // =====================================
                // NORMAL INSERT MODE (REAL BATCH)
                // =====================================
                $batchRows[] = $row;

                if (count($batchRows) >= $batchSize) {

                    if (!$DRY_RUN) {
                        $inserted += executeMultiInsert($pdo2, $dst, $cols, $batchRows);
                    }

                    $batchRows = [];
                }
            }

            // Final batch flush (only for non-upsert)
            if (!$useUpsert && $batchRows) {
                if (!$DRY_RUN) {
                    $inserted += executeMultiInsert($pdo2, $dst, $cols, $batchRows);
                }
            }

            if ($DRY_RUN) {
                logMsg("Total - {$totalRows} rows | DRY RUN (no insert)");
            } else {
                logMsg("Total - {$totalRows} rows | {$inserted} rows migrated successfully");
            }

        } catch (Exception $e) {

            logMsg("FAILED migrating {$src}: " . $e->getMessage());
            throw $e;
        }
    }

    logMsg("{$title} migration finished\n");
}

/**
 * Migrate Run Jobs + related icon_* tables → JSONB (PHP 7.2 safe)
 * Optimized + Business logic preserved
 */
function migrateRunTables(PDO $pdo1, PDO $pdo2, $DRY_RUN)
{
    logMsg("Starting Run Tables Migration (11 tables)");

    /*
     * SAFETY: ensure getJobData exists
     */
    if (!function_exists('getJobData')) {
        throw new Exception("FATAL: getJobData() function is not defined. Check include order.");
    }

    /* =====================================================
     * Parent jobnet tables
     * ===================================================== */
    $parentTables = [
        'ja_run_jobnet_table' => 'ja_2_run_jobnet_table',
        'ja_run_jobnet_summary_table' => 'ja_2_run_jobnet_summary_table'
    ];

        foreach ($parentTables as $src => $dst) {

            logMsg("Migrating $src => $dst");

            $stmt1 = $pdo1->query("SELECT * FROM $src");
            if (!$stmt1) {
                logMsg("  - No data to migrate");
                continue;
            }

            $targetColumns = getTargetColumns($pdo2, $dst);
            $targetMap = array_flip($targetColumns);

            $sampleRow = $stmt1->fetch(PDO::FETCH_ASSOC);
            if (!$sampleRow) {
                logMsg("  - No data to migrate");
                continue;
            }

            $stmt1->execute(); // reset for streaming

            $sampleRow = normalizeRow($sampleRow);
            $sampleRow = array_intersect_key($sampleRow, $targetMap);

            $cols = array_keys($sampleRow);

            $totalRows = 0;
            $migrated = 0;

            $batchSize = 500;
            $batchRows = [];

            while ($r = $stmt1->fetch(PDO::FETCH_ASSOC)) {

                $totalRows++;

                $row = normalizeRow($r);
                $row = array_intersect_key($row, $targetMap);

                if ($dst === 'ja_2_run_jobnet_summary_table'
                    && in_array('time_zone', $targetColumns, true)) {
                    $row['time_zone'] = VM_TIMEZONE;
                }

                $batchRows[] = $row;

                if (count($batchRows) >= $batchSize) {
                    if (!$DRY_RUN) {
                        $migrated += executeMultiInsert($pdo2, $dst, $cols, $batchRows);
                    }
                    $batchRows = [];
                }
            }

            if ($batchRows) {
                 if (!$DRY_RUN) {
                    $migrated += executeMultiInsert($pdo2, $dst, $cols, $batchRows);
                }
            }

            if ($DRY_RUN) {
                logMsg("Total - {$totalRows} rows | DRY RUN (no insert)");
            } else {
                logMsg("Total - {$totalRows} rows | {$migrated} rows migrated successfully");
            }
        }

        /* =====================================================
        * ja_run_job_table
        * ===================================================== */
        logMsg("Migrating ja_run_job_table => ja_2_run_job_table");

            $jobStmt = $pdo1->query("SELECT * FROM ja_run_job_table");
            if (!$jobStmt)
                return;

            $targetColumns = getTargetColumns($pdo2, 'ja_2_run_job_table');
            $targetMap = array_flip($targetColumns);

            $sampleJob = $jobStmt->fetch(PDO::FETCH_ASSOC);
            if (!$sampleJob)
                return;

            $jobStmt->execute(); // reset streaming

            $sampleJob = normalizeRunJobRow($sampleJob);

            /* FORCE include data column if exists in target */
            if (isset($targetMap['data'])) {
                $sampleJob['data'] = null;
            }

            $sampleJob = array_intersect_key($sampleJob, $targetMap);

            $cols = array_keys($sampleJob);

            // PREPARE parent check ONCE (performance fix)
            $parentCheckStmt = $pdo2->prepare(
                "SELECT 1 FROM ja_2_run_jobnet_table WHERE inner_jobnet_id = ?"
            );

            $totalRows = 0;
            $migrated = 0;

            $batchSize = 300;
            $batchRows = [];

            while ($job = $jobStmt->fetch(PDO::FETCH_ASSOC)) {

                $totalRows++;

                try {
                    
                    if(!$DRY_RUN){
                        // Parent existence check
                        $parentCheckStmt->execute([$job['inner_jobnet_id']]);
                        if (!$parentCheckStmt->fetchColumn()) {
                            throw new Exception("Missing parent jobnet");
                        }
                    }

                    $insertRow = $job;

                    unset($insertRow['data']);

                    // ===== BUSINESS LOGIC PRESERVED =====
                    $insertRow['data'] = getJobData($pdo1, $job);
                    // ====================================

                    $insertRow = normalizeRunJobRow($insertRow);

                    /* FORCE include data column if exists */
                    if (isset($targetMap['data']) && !isset($insertRow['data'])) {
                        $insertRow['data'] = null;
                    }

                    $insertRow = array_intersect_key($insertRow, $targetMap);

                    $batchRows[] = $insertRow;

                    if (count($batchRows) >= $batchSize) {
                        if (!$DRY_RUN) {
                            $migrated += executeMultiInsert($pdo2,'ja_2_run_job_table',$cols,$batchRows);
                        }
                        $batchRows = [];
                    }

                } catch (Exception $e) {

                    logMsg("FAILED inserting job_id={$job['inner_job_id']} / jobnet_id={$job['inner_jobnet_id']}");
                    logMsg("SQL Error: " . $e->getMessage());
                    throw $e;
                }
            }

            // FINAL FLUSH
            if (!$DRY_RUN && !empty($batchRows)) {
                $migrated += executeMultiInsert($pdo2,'ja_2_run_job_table',$cols,$batchRows);
            }

            if ($DRY_RUN) {
                logMsg("Total - {$totalRows} rows | DRY RUN (no insert)");
            } else {
                logMsg("Total - {$totalRows} rows | {$migrated} rows migrated successfully");
            }

    /* =====================================================
     * Extra run tables
     * ===================================================== */
    $extraTables = [
        'ja_run_value_job_table' => 'ja_2_run_value_job_table',
        'ja_run_value_jobcon_table' => 'ja_2_run_value_jobcon_table',
        'ja_run_flow_table' => 'ja_2_run_flow_table',
        'ja_session_table' => 'ja_2_session_table',
        'ja_run_log_table' => 'ja_2_run_log_table',
        'ja_send_message_table' => 'ja_2_send_message_table'
    ];

    foreach ($extraTables as $src => $dst) {

        logMsg("Migrating $src => $dst");

        $stmt1 = $pdo1->query("SELECT * FROM $src");
        if (!$stmt1)
            continue;

        $targetColumns = getTargetColumns($pdo2, $dst);
        $targetMap = array_flip($targetColumns);

        $sampleRow = $stmt1->fetch(PDO::FETCH_ASSOC);
        if (!$sampleRow) {
            logMsg("  - No data to migrate");
            continue;
        }

        $stmt1->execute();

        $sampleRow = array_intersect_key(
            normalizeRow($sampleRow),
            $targetMap
        );

        $cols = array_keys($sampleRow);
        $placeholders = array_map(function ($c) {
            return ":$c";
        }, $cols);

        $stmtInsert = $pdo2->prepare(
            "INSERT INTO $dst (" . implode(',', $cols) . ")
            VALUES (" . implode(',', $placeholders) . ")"
        );

        $totalRows = 0;
        $migrated = 0;

        $batchSize = 500;
        $batchRows = [];

        while ($r = $stmt1->fetch(PDO::FETCH_ASSOC)) {

            $totalRows++; // always count rows, even in DRY_RUN

            try {
                $row = normalizeRow($r);
                $row = array_intersect_key($row, $targetMap);

                // // ===== FIX log_date for normal Unix seconds =====
                // if ($dst === 'ja_2_run_log_table' && isset($row['log_date'])) {
                //     $row['log_date'] = jazToUnix($row['log_date']);
                // }

                // Only add to batch if not DRY_RUN
                if (!$DRY_RUN) {
                    $batchRows[] = $row;

                    if (count($batchRows) >= $batchSize) {
                        $migrated += executeMultiInsert($pdo2,$dst,$cols,$batchRows);
                        $batchRows = [];
                    }
                }

            } catch (Exception $e) {
                logMsg("FAILED migrating row in $src | " . $e->getMessage());
                throw $e;
            }
        }

        /* FINAL FLUSH */
        if (!$DRY_RUN && !empty($batchRows)) {
            $migrated += executeMultiInsert($pdo2,$dst,$cols,$batchRows);
        }

        // DRY_RUN aware logging
        if ($DRY_RUN) {
            logMsg("Total - {$totalRows} rows | DRY RUN (no insert)");
        } else {
            logMsg("Total - {$totalRows} rows | {$migrated} rows migrated successfully");
        }
    }
}


/**
 * Migrate jobnet variables with REAL batch insert
 */
function migrateJobnetVariables(PDO $pdo1, PDO $pdo2, $DRY_RUN, $batchSize = 500)
{
    logMsg("Migrating ja_value_before_jobnet_table + ja_value_after_jobnet_table => ja_2_run_jobnet_variable_table");

    $grouped = [];

    // ================================
    // STREAM FIX #1 — BEFORE variables
    // ================================
    $beforeStmt = $pdo1->query("
        SELECT inner_jobnet_id, value_name, before_value
        FROM ja_value_before_jobnet_table
        ORDER BY inner_jobnet_id, value_name, seq_no ASC
    ");
    while ($r = $beforeStmt->fetch(PDO::FETCH_ASSOC)) {
        $grouped[$r['inner_jobnet_id']]['before'][$r['value_name']] = $r['before_value'];
    }

    // ================================
    // STREAM FIX #2 — AFTER variables
    // ================================
    $afterStmt = $pdo1->query("
        SELECT inner_jobnet_id, value_name, after_value
        FROM ja_value_after_jobnet_table
        ORDER BY inner_jobnet_id, value_name, seq_no ASC
    ");
    while ($r = $afterStmt->fetch(PDO::FETCH_ASSOC)) {
        $grouped[$r['inner_jobnet_id']]['after'][$r['value_name']] = $r['after_value'];
    }

    if ($DRY_RUN) {
        logMsg("Total - " . count($grouped) . " rows | DRY RUN (no insert)");
        return;
    }

    try {

        $count = 0;
        $batchRowsWithAfter = [];
        $batchRowsWithoutAfter = [];

        foreach ($grouped as $innerJobnetId => $vars) {

            $beforeVars = $vars['before'] ?? [];
            $afterVars  = $vars['after'] ?? null;

            $beforeJson = json_encode(addVmTimezone($beforeVars));

            // ================================
            // WITH AFTER VARIABLE
            // ================================
            if ($afterVars !== null && !empty($afterVars)) {

                $afterJson = json_encode(addVmTimezone($afterVars));

                $batchRowsWithAfter[] = [
                    'inner_jobnet_id' => $innerJobnetId,
                    'before_variable' => $beforeJson,
                    'after_variable'  => $afterJson,
                ];

                if (count($batchRowsWithAfter) >= $batchSize) {

                    $count += executeMultiInsert(
                        $pdo2,
                        'ja_2_run_jobnet_variable_table',
                        ['inner_jobnet_id', 'before_variable', 'after_variable'],
                        $batchRowsWithAfter
                    );

                    $batchRowsWithAfter = [];
                }

            } else {

                // ================================
                // WITHOUT AFTER VARIABLE
                // ================================
                $batchRowsWithoutAfter[] = [
                    'inner_jobnet_id' => $innerJobnetId,
                    'before_variable' => $beforeJson,
                ];

                if (count($batchRowsWithoutAfter) >= $batchSize) {

                    $count += executeMultiInsert(
                        $pdo2,
                        'ja_2_run_jobnet_variable_table',
                        ['inner_jobnet_id', 'before_variable'],
                        $batchRowsWithoutAfter
                    );

                    $batchRowsWithoutAfter = [];
                }
            }
        }

        // ================================
        // FINAL FLUSH
        // ================================
        if (!empty($batchRowsWithAfter)) {
            $count += executeMultiInsert(
                $pdo2,
                'ja_2_run_jobnet_variable_table',
                ['inner_jobnet_id', 'before_variable', 'after_variable'],
                $batchRowsWithAfter
            );
        }

        if (!empty($batchRowsWithoutAfter)) {
            $count += executeMultiInsert(
                $pdo2,
                'ja_2_run_jobnet_variable_table',
                ['inner_jobnet_id', 'before_variable'],
                $batchRowsWithoutAfter
            );
        }

        $totalRows = count($grouped);
        logMsg("Total - {$totalRows} rows | {$count} rows migrated successfully");

    } catch (Exception $e) {
        logMsg("FAILED jobnet variable migration: " . $e->getMessage());
        throw $e;
    }
}

/**
 * Migrate job variables with REAL batch insert
 */
function migrateJobVariables(PDO $pdo1, PDO $pdo2, $DRY_RUN, $batchSize = 500)
{
    logMsg("Migrating ja_run_value_before_table + ja_run_value_after_table => ja_2_run_job_variable_table");

    try {

        $grouped = [];

        // ======================================
        // STREAM FIX #1 — BEFORE variables
        // ======================================
        $beforeStmt = $pdo1->query("
            SELECT inner_job_id, inner_jobnet_id, value_name, before_value
            FROM ja_run_value_before_table
            ORDER BY inner_job_id, inner_jobnet_id, value_name, seq_no ASC
        ");
        while ($r = $beforeStmt->fetch(PDO::FETCH_ASSOC)) {
            $grouped[$r['inner_job_id']][$r['inner_jobnet_id']]['before'][$r['value_name']] = $r['before_value'];
        }

        // ======================================
        // STREAM FIX #2 — AFTER variables
        // ======================================
        $afterStmt = $pdo1->query("
            SELECT inner_job_id, inner_jobnet_id, value_name, after_value
            FROM ja_run_value_after_table
            ORDER BY inner_job_id, inner_jobnet_id, value_name, seq_no ASC
        ");
        while ($r = $afterStmt->fetch(PDO::FETCH_ASSOC)) {
            $grouped[$r['inner_job_id']][$r['inner_jobnet_id']]['after'][$r['value_name']] = $r['after_value'];
        }

        if ($DRY_RUN) {
            logMsg("Total - " . count($grouped) . " rows | DRY RUN (no insert)");
            logMsg("Run Tables migration finished");
            return;
        }

        $count = 0;
        $batchRowsWithAfter = [];
        $batchRowsWithoutAfter = [];

        foreach ($grouped as $innerJobId => $jobnets) {

            foreach ($jobnets as $innerJobnetId => $vars) {

                $beforeVars = $vars['before'] ?? [];
                $afterVars  = $vars['after'] ?? null;

                $beforeJson = json_encode(addVmTimezone($beforeVars));

                // ======================================
                // WITH AFTER VARIABLE
                // ======================================
                if ($afterVars !== null && !empty($afterVars)) {

                    $afterJson = json_encode(addVmTimezone($afterVars));

                    $batchRowsWithAfter[] = [
                        'inner_job_id'     => $innerJobId,
                        'inner_jobnet_id'  => $innerJobnetId,
                        'before_variable'  => $beforeJson,
                        'after_variable'   => $afterJson,
                    ];

                    if (count($batchRowsWithAfter) >= $batchSize) {

                        $count += executeMultiInsert(
                            $pdo2,
                            'ja_2_run_job_variable_table',
                            ['inner_job_id', 'inner_jobnet_id', 'before_variable', 'after_variable'],
                            $batchRowsWithAfter
                        );

                        $batchRowsWithAfter = [];
                    }

                } else {

                    // ======================================
                    // WITHOUT AFTER VARIABLE
                    // ======================================
                    $batchRowsWithoutAfter[] = [
                        'inner_job_id'     => $innerJobId,
                        'inner_jobnet_id'  => $innerJobnetId,
                        'before_variable'  => $beforeJson,
                    ];

                    if (count($batchRowsWithoutAfter) >= $batchSize) {

                        $count += executeMultiInsert(
                            $pdo2,
                            'ja_2_run_job_variable_table',
                            ['inner_job_id', 'inner_jobnet_id', 'before_variable'],
                            $batchRowsWithoutAfter
                        );

                        $batchRowsWithoutAfter = [];
                    }
                }
            }
        }

        // ======================================
        // FINAL FLUSH
        // ======================================
        if (!empty($batchRowsWithAfter)) {
            $count += executeMultiInsert(
                $pdo2,
                'ja_2_run_job_variable_table',
                ['inner_job_id', 'inner_jobnet_id', 'before_variable', 'after_variable'],
                $batchRowsWithAfter
            );
        }

        if (!empty($batchRowsWithoutAfter)) {
            $count += executeMultiInsert(
                $pdo2,
                'ja_2_run_job_variable_table',
                ['inner_job_id', 'inner_jobnet_id', 'before_variable'],
                $batchRowsWithoutAfter
            );
        }

        $totalRows = count($grouped);
        logMsg("Total - {$totalRows} rows | {$count} rows migrated successfully");
        logMsg("Run Tables migration finished");

    } catch (Exception $e) {
        logMsg("FAILED job variable migration: " . $e->getMessage());
        throw $e;
    }
}


/********************
 * CONNECT TO DATABASES
 ********************/
try {
    $pdo1 = new PDO(buildDsn($dbType, $jaz1), $jaz1['user'], $jaz1['pass'], array(PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION));
    $pdo2 = new PDO(buildDsn($dbType, $jaz2), $jaz2['user'], $jaz2['pass'], array(PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION));
    logMsg("Connected to JAZ1 ({$jaz1['dbname']}) and JAZ2 ({$jaz2['dbname']}) databases");
} catch (Exception $e) {
    logMsg("Database connection failed: " . $e->getMessage());
    exit(1);
}

/********************
 * EXECUTE MIGRATION (ALL-IN-ONE TRANSACTION)
 ********************/
try {

    logMsg("TIMEZONE used: " . VM_TIMEZONE . "\n");

    if (!$DRY_RUN) {
        truncateTables($pdo2);
        $pdo2->beginTransaction();
    }

    migrateZbxHosts($pdo1, $pdo2, $DRY_RUN);

    // Initial Tables → UPSERT
    migrateTables($pdo1, $pdo2, $initialTables, $dbType, $DRY_RUN, "Initial Tables", true);

    // Control Tables
    migrateTables($pdo1, $pdo2, $controlTables, $dbType, $DRY_RUN, "Control Tables", false);

    // Run Jobs
    migrateRunTables($pdo1, $pdo2, $DRY_RUN);

    // Jobnet Variables
    migrateJobnetVariables($pdo1, $pdo2, $DRY_RUN);

    // Job Variables
    migrateJobVariables($pdo1, $pdo2, $DRY_RUN);

    if (!$DRY_RUN) {
        $pdo2->commit();
    }

    logMsg("All migrations finished successfully.");

} catch (Exception $e) {

    if (!$DRY_RUN && $pdo2->inTransaction()) {
        $pdo2->rollBack();
    }

    logMsg("MIGRATION FAILED — ALL CHANGES ROLLED BACK");
    logMsg("Error: " . $e->getMessage());

    exit(1);
}


