#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
EPB Export Tracker REX/SOO Submission Automation (Commercial RPA Edition)
Author: Izaz Ahamed
Platform: Commercial RPA
---------------------------------------------------------------------------------
Workflow:
1️⃣ Login with EPB credentials (manual login through browser)
2️⃣ Add SOO → Confirm popup
3️⃣ Open first record
4️⃣ Fill all fields from CSV
5️⃣ Upload Commercial Invoice + Bill of Lading PDFs from extracted ZIP
6️⃣ Save + Back
7️⃣ Repeat for each CSV row
8️⃣ Generate results CSV with success/failure status
---------------------------------------------------------------------------------
Usage: python rex_submission.py <csv_file> <output_dir> <job_id> <pdf_dir>
"""

import os, sys, csv, time, logging, datetime
import re
import shutil
import subprocess
# Use Electron browser instead of Selenium
try:
    from shared.electron_browser import ElectronBrowser, By, ElectronWait, EC
except ImportError:
    # Fallback for direct execution
    import sys
    import os
    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
    from automation_scripts.shared.electron_browser import ElectronBrowser, By, ElectronWait, EC

# Compatibility exceptions
class TimeoutException(Exception):
    pass

class NoSuchElementException(Exception):
    pass

# Compatibility - Select is not yet implemented in ElectronBrowser
class Select:
    def __init__(self, element):
        self.element = element
    def select_by_value(self, value):
        # Use JavaScript to set select value
        script = f"arguments[0].value = '{value}'; arguments[0].dispatchEvent(new Event('change', {{bubbles: true}}));"
        return self.element.browser.execute_script(script)

# Windows registry access for Chrome version detection
try:
    import winreg
    HAS_WINREG = True
except ImportError:
    HAS_WINREG = False

try:
    import undetected_chromedriver as uc
except ImportError:
    uc = None
    print("⚠️ Warning: 'undetected_chromedriver' library not found.")
    print("   Install with: pip install undetected-chromedriver")

try:
    import requests
except ImportError:
    requests = None
    print("⚠️ Warning: 'requests' library not found. Server communication will be disabled.")
    print("   Install with: pip install requests")


# -----------------------------------------------------------------------------
# SERVER CLIENT
# -----------------------------------------------------------------------------
class ServerClient:
    """Handles all server communication for the automation script"""
    
    def __init__(self, api_base_url: str, user_id: str, job_id: str, logger=None):
        self.api_base_url = api_base_url.rstrip('/')
        self.user_id = user_id
        self.job_id = job_id
        self.logger = logger or logging.getLogger("ServerClient")
        
        if requests is None:
            self.logger.warning("⚠️ 'requests' library not available. Server communication disabled.")
            self.enabled = False
        else:
            self.enabled = True
    
    def upload_log_file(self, log_file_path: str, job_data: dict) -> str:
        """Upload log file to server via POST /api/local-exe/jobs/:jobId/upload-log"""
        if not self.enabled:
            self.logger.warning("⚠️ Server communication disabled. Skipping log file upload.")
            return None
        
        if not os.path.exists(log_file_path):
            self.logger.warning(f"⚠️ Log file not found: {log_file_path}")
            return None
        
        try:
            url = f"{self.api_base_url}/api/local-exe/jobs/{self.job_id}/upload-log"
            
            with open(log_file_path, 'rb') as f:
                files = {'logFile': (os.path.basename(log_file_path), f, 'text/plain')}
                data = {
                    'userId': job_data.get('userId', self.user_id),
                    'serviceId': job_data.get('serviceId', 'rex-soo-submission'),
                    'serviceName': job_data.get('serviceName', 'REX/SOO Submission'),
                    'inputFileName': job_data.get('inputFileName', 'input.csv'),
                    'inputFilePath': job_data.get('inputFilePath', ''),
                    'creditsUsed': str(job_data.get('creditsUsed', 0))
                }
                
                response = requests.post(url, files=files, data=data, timeout=30)
                response.raise_for_status()
                result = response.json()
                
                if result.get('success') and result.get('logFilePath'):
                    self.logger.info(f"✅ Log file uploaded: {result['logFilePath']}")
                    return result['logFilePath']
                else:
                    self.logger.warning(f"⚠️ Log upload failed: {result.get('error', 'Unknown error')}")
                    return None
        except requests.exceptions.RequestException as e:
            self.logger.warning(f"⚠️ Failed to upload log file: {e}")
            return None
        except Exception as e:
            self.logger.warning(f"⚠️ Unexpected error uploading log file: {e}")
            return None
    
    def upload_results_csv(self, csv_file_path: str, job_data: dict) -> str:
        """Upload results CSV file to server via POST /api/local-exe/jobs/:jobId/upload-log"""
        if not self.enabled:
            self.logger.warning("⚠️ Server communication disabled. Skipping CSV upload.")
            return None
        
        if not os.path.exists(csv_file_path):
            self.logger.warning(f"⚠️ CSV file not found: {csv_file_path}")
            return None
        
        try:
            url = f"{self.api_base_url}/api/local-exe/jobs/{self.job_id}/upload-log"
            
            with open(csv_file_path, 'rb') as f:
                files = {'logFile': (os.path.basename(csv_file_path), f, 'text/csv')}
                data = {
                    'userId': job_data.get('userId', self.user_id),
                    'serviceId': job_data.get('serviceId', 'rex-soo-submission'),
                    'serviceName': job_data.get('serviceName', 'REX/SOO Submission'),
                    'inputFileName': job_data.get('inputFileName', 'input.csv'),
                    'inputFilePath': job_data.get('inputFilePath', ''),
                    'creditsUsed': str(job_data.get('creditsUsed', 0))
                }
                
                response = requests.post(url, files=files, data=data, timeout=30)
                response.raise_for_status()
                result = response.json()
                
                if result.get('success') and result.get('logFilePath'):
                    self.logger.info(f"✅ Results CSV uploaded: {result['logFilePath']}")
                    return result['logFilePath']
                else:
                    self.logger.warning(f"⚠️ CSV upload failed: {result.get('error', 'Unknown error')}")
                    return None
        except requests.exceptions.RequestException as e:
            self.logger.warning(f"⚠️ Failed to upload CSV file: {e}")
            return None
        except Exception as e:
            self.logger.warning(f"⚠️ Unexpected error uploading CSV file: {e}")
            return None
    
    def report_job_result(self, result_files: list, output_directory: str, successful_count: int = None) -> bool:
        """Report job completion to server via POST /api/local-exe/jobs/:jobId/complete"""
        if not self.enabled:
            self.logger.warning("⚠️ Server communication disabled. Skipping job result reporting.")
            return False
        
        try:
            url = f"{self.api_base_url}/api/local-exe/jobs/{self.job_id}/complete"
            
            payload = {
                'success': True,
                'resultFiles': result_files,
                'outputFiles': [],
                'outputDirectory': output_directory
            }
            if successful_count is not None:
                payload['successfulCount'] = successful_count
            
            response = requests.post(url, json=payload, timeout=30)
            response.raise_for_status()
            result = response.json()
            
            if result.get('success'):
                self.logger.info("✅ Job result reported successfully to server")
                return True
            else:
                self.logger.warning(f"⚠️ Job result reporting failed: {result.get('error', 'Unknown error')}")
                return False
        except requests.exceptions.RequestException as e:
            self.logger.warning(f"⚠️ Failed to report job result: {e}")
            return False
        except Exception as e:
            self.logger.warning(f"⚠️ Unexpected error reporting job result: {e}")
            return False

# ------------------- Argument validation -------------------
if len(sys.argv) < 5:
    print("❌ Usage: python rex_submission.py <csv_file> <output_dir> <job_id> <pdf_dir> [api_base_url] [user_id] [credits_used]")
    sys.exit(1)

CSV_FILE = sys.argv[1]
BASE_OUTPUT_DIR = sys.argv[2]
JOB_ID = sys.argv[3]
PDF_DIR = sys.argv[4]

# Optional server communication parameters
API_BASE_URL = sys.argv[5] if len(sys.argv) > 5 else None
USER_ID = sys.argv[6] if len(sys.argv) > 6 else None
CREDITS_USED = float(sys.argv[7]) if len(sys.argv) > 7 and sys.argv[7] else 0.0

# Use isolated service-specific output directory provided by the server
OUTPUT_DIR = os.path.abspath(BASE_OUTPUT_DIR)
os.makedirs(OUTPUT_DIR, exist_ok=True)

URL = "https://epb-exporttracker.gov.bd/#/login"
RESULT_LOG = os.path.join(OUTPUT_DIR, f"rex_submission_results_{JOB_ID}.csv")
TIMEOUT = 30

# ------------------- Logging setup -------------------
SESSION_DIR = os.path.join(OUTPUT_DIR, 'sessions', JOB_ID)
os.makedirs(SESSION_DIR, exist_ok=True)

log_file_path = ''

console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(logging.INFO)
console_formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
console_handler.setFormatter(console_formatter)

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    handlers=[console_handler]
)

logging.info("=" * 80)
logging.info("🚀 EPB REX/SOO Submission Automation Started")
logging.info("=" * 80)
logging.info(f"📁 Input CSV: {CSV_FILE}")
logging.info(f"📂 Output Directory: {OUTPUT_DIR}")
logging.info(f"🆔 Job ID: {JOB_ID}")
logging.info(f"📦 PDF Directory: {PDF_DIR}")
logging.info(f"💾 Session Directory: {SESSION_DIR}")
logging.info("=" * 80)

# -----------------------------------------------------------------------------
# Browser Setup Functions
# -----------------------------------------------------------------------------

def _detect_chrome_binary_paths():
    candidates = []
    for path in [
        os.environ.get('GOOGLE_CHROME_BIN'),
        '/usr/bin/google-chrome',
        '/usr/bin/google-chrome-stable',
        '/usr/bin/chromium',
        '/usr/bin/chromium-browser',
        '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
        'C:/Program Files/Google/Chrome/Application/chrome.exe',
        'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe'
    ]:
        if path and os.path.exists(path):
            candidates.append(path)
    return candidates


def get_installed_chrome_major_version():
    """Detect the major version of installed Chrome/Chromium browser.
    
    Returns:
        int: Major version number (e.g., 141) or None if not detected
    """
    # Try detected binary paths first
    for binary in _detect_chrome_binary_paths():
        try:
            out = subprocess.check_output([binary, '--version'], stderr=subprocess.STDOUT, timeout=5)
            text = out.decode('utf-8', errors='ignore')
            # Match patterns like "Chrome 141.0.7390.123" or "Chromium 141.0.7390.123"
            m = re.search(r'(?:Chrome|Chromium)\s+([0-9]+)\.', text)
            if m:
                version = int(m.group(1))
                return version
        except (subprocess.TimeoutExpired, subprocess.CalledProcessError, FileNotFoundError, ValueError):
            continue
        except Exception:
            continue
    
    # Fallback: try plain `google-chrome --version` commands
    for cmd in ['google-chrome', 'google-chrome-stable', 'chromium', 'chromium-browser']:
        try:
            out = subprocess.check_output([cmd, '--version'], stderr=subprocess.STDOUT, timeout=5)
            text = out.decode('utf-8', errors='ignore')
            m = re.search(r'([0-9]+)\.', text)
            if m:
                version = int(m.group(1))
                return version
        except (subprocess.TimeoutExpired, subprocess.CalledProcessError, FileNotFoundError, ValueError):
            pass
        except Exception:
            pass
    
    # Windows-specific: Try reading from registry
    if HAS_WINREG and os.name == 'nt':
        try:
            # Try multiple registry paths where Chrome version might be stored
            registry_paths = [
                (winreg.HKEY_CURRENT_USER, r'Software\Google\Chrome\BLBeacon'),
                (winreg.HKEY_LOCAL_MACHINE, r'Software\Google\Chrome\BLBeacon'),
                (winreg.HKEY_CURRENT_USER, r'Software\Google\Update\ClientState\{8A69D345-D564-463C-AFF1-A69D9E530F96}'),
                (winreg.HKEY_LOCAL_MACHINE, r'Software\Google\Update\ClientState\{8A69D345-D564-463C-AFF1-A69D9E530F96}'),
            ]
            
            for hkey, path in registry_paths:
                try:
                    key = winreg.OpenKey(hkey, path)
                    try:
                        # Try to get version from 'version' value
                        version_str, _ = winreg.QueryValueEx(key, 'version')
                        m = re.search(r'([0-9]+)\.', str(version_str))
                        if m:
                            version = int(m.group(1))
                            winreg.CloseKey(key)
                            return version
                    except FileNotFoundError:
                        pass
                    finally:
                        winreg.CloseKey(key)
                except (FileNotFoundError, OSError):
                    continue
        except Exception:
            pass
    
    # Windows-specific: Try reading from Chrome's version file
    if os.name == 'nt':
        version_file_paths = [
            os.path.expanduser(r'~\AppData\Local\Google\Chrome\Application\version'),
            r'C:\Program Files\Google\Chrome\Application\version',
            r'C:\Program Files (x86)\Google\Chrome\Application\version',
        ]
        for version_file in version_file_paths:
            try:
                if os.path.exists(version_file):
                    with open(version_file, 'r', encoding='utf-8', errors='ignore') as f:
                        content = f.read()
                        m = re.search(r'([0-9]+)\.', content)
                        if m:
                            version = int(m.group(1))
                            return version
            except Exception:
                continue
    
    return None


def ensure_uc_cache_directory():
    """Ensure undetected_chromedriver cache directory exists with proper permissions.
    
    Returns:
        str: Path to the cache directory that was ensured, or None if failed
    """
    cache_dirs = [
        os.path.expanduser('~/.local/share/undetected_chromedriver'),  # Linux
        os.path.expanduser('~/AppData/Roaming/undetected_chromedriver'),  # Windows
        os.path.expanduser('~/Library/Application Support/undetected_chromedriver'),  # macOS
    ]
    
    for cache_dir in cache_dirs:
        try:
            # Create directory if it doesn't exist
            os.makedirs(cache_dir, mode=0o755, exist_ok=True)
            # Verify it's actually a directory and we can write to it
            if os.path.isdir(cache_dir) and os.access(cache_dir, os.W_OK):
                return cache_dir
        except Exception:
            continue
    
    return None


def clear_uc_cache():
    """Clear undetected_chromedriver cache from common locations"""
    cache_dirs = [
        os.path.expanduser('~/.local/share/undetected_chromedriver'),  # Linux
        os.path.expanduser('~/AppData/Roaming/undetected_chromedriver'),  # Windows
        os.path.expanduser('~/Library/Application Support/undetected_chromedriver'),  # macOS
    ]
    for cache_dir in cache_dirs:
        try:
            if os.path.isdir(cache_dir):
                shutil.rmtree(cache_dir, ignore_errors=True)
        except Exception:
            pass


def validate_driver_file(cache_dir=None, logger=None):
    """Validate that ChromeDriver executable exists and is accessible.
    
    Args:
        cache_dir: Optional cache directory path to check
        logger: Optional logger instance for logging
        
    Returns:
        bool: True if driver file appears to exist, False otherwise
    """
    if not cache_dir:
        cache_dir = ensure_uc_cache_directory()
    
    if not cache_dir:
        return False
    
    # Check common driver file locations
    driver_paths = [
        os.path.join(cache_dir, 'undetected_chromedriver'),
        os.path.join(cache_dir, 'undetected', 'chromedriver-linux64', 'chromedriver'),
        os.path.join(cache_dir, 'undetected', 'chromedriver-win64', 'chromedriver.exe'),
        os.path.join(cache_dir, 'undetected', 'chromedriver-mac-x64', 'chromedriver'),
    ]
    
    for driver_path in driver_paths:
        if os.path.exists(driver_path):
            # Check if it's executable (on Unix-like systems)
            if os.name != 'nt' and not os.access(driver_path, os.X_OK):
                try:
                    os.chmod(driver_path, 0o755)
                except Exception:
                    pass
            if logger:
                logger.debug(f"✅ Found driver file: {driver_path}")
            return True
    
    return False


def validate_driver_session(driver, logger=None):
    """Validate that the ChromeDriver session is actually working.
    
    Args:
        driver: ChromeDriver instance to validate
        logger: Optional logger instance for logging
        
    Returns:
        bool: True if session is working, False otherwise
    """
    try:
        # Try a simple command to verify the session is alive
        _ = driver.current_url
        return True
    except Exception as e:
        error_msg = str(e)
        if logger:
            # Check for connection errors
            if "Connection refused" in error_msg or "Failed to establish" in error_msg:
                logger.warning(f"⚠️ Driver session connection error: {error_msg}")
            else:
                logger.warning(f"⚠️ Driver session validation failed: {error_msg}")
        return False


def build_default_options(headless=True, user_data_dir=None, window_size='1366,768', incognito=True):
    """Build ChromeOptions with proper headless configuration.
    
    Args:
        headless: If True, run Chrome in headless mode (no visible window)
        user_data_dir: Optional user data directory path (if provided, cache will be preserved for session persistence)
        window_size: Window size string like '1920,1080'
        incognito: If True, run Chrome in incognito/private mode (default True for backward compatibility)
                   Note: When user_data_dir is provided, incognito is automatically disabled for session persistence
    
    Returns:
        uc.ChromeOptions: Configured ChromeOptions object
    """
    if uc is None:
        raise ImportError("undetected_chromedriver is not installed")
    
    opts = uc.ChromeOptions()
    
    # When user_data_dir is provided, we need to preserve cache and disable incognito for session persistence
    # Session persistence requires non-incognito mode and cache to be enabled
    if user_data_dir:
        incognito = False  # Override incognito when using user_data_dir for session persistence
    
    # Headless mode configuration - must be set first
    if headless:
        # Primary headless flag (--headless=new is the modern headless mode)
        opts.add_argument('--headless=new')
        opts.add_argument('--disable-gpu')  # Required for headless on Windows
        opts.add_argument('--remote-debugging-port=0')  # Disable remote debugging in headless
        opts.add_argument('--disable-software-rasterizer')
        # Window positioning to move off-screen if window appears
        opts.add_argument('--window-position=-2000,-2000')
        # Additional flags to prevent window visibility
        opts.add_argument('--hide-scrollbars')
        opts.add_argument('--mute-audio')
        opts.add_argument('--disable-background-timer-throttling')
        opts.add_argument('--disable-backgrounding-occluded-windows')
        opts.add_argument('--disable-renderer-backgrounding')
        opts.add_argument('--disable-features=TranslateUI')
        opts.add_argument('--disable-ipc-flooding-protection')
        # Additional aggressive flags to prevent any window visibility
        opts.add_argument('--disable-infobars')
        opts.add_argument('--disable-notifications')
        opts.add_argument('--disable-popup-blocking')
        opts.add_argument('--disable-default-apps')
        opts.add_argument('--disable-session-crashed-bubble')
        opts.add_argument('--disable-component-update')
        opts.add_argument('--disable-sync')
        opts.add_argument('--no-pings')
        opts.add_argument('--disable-hang-monitor')
        opts.add_argument('--disable-prompt-on-repost')
        opts.add_argument('--disable-domain-reliability')
        opts.add_argument('--disable-client-side-phishing-detection')
        opts.add_argument('--disable-component-extensions-with-background-pages')
        opts.add_argument('--force-color-profile=srgb')
        opts.add_argument('--metrics-recording-only')
        opts.add_argument('--disable-breakpad')
    else:
        # Even in non-headless mode, disable GPU can help with stability
        opts.add_argument('--disable-gpu')
    
    # Common options for both headless and non-headless
    opts.add_argument('--no-sandbox')
    opts.add_argument('--disable-dev-shm-usage')
    opts.add_argument('--disable-blink-features=AutomationControlled')
    # Only add --incognito flag when incognito=True and user_data_dir is not provided
    if incognito and not user_data_dir:
        opts.add_argument('--incognito')
    opts.add_argument(f'--window-size={window_size}')
    
    # Only disable cache when user_data_dir is NOT provided (to allow session persistence)
    # When user_data_dir is provided, cache must be enabled for cookies and session storage to persist
    if not user_data_dir:
        opts.add_argument('--disable-cache')
        opts.add_argument('--disable-application-cache')
        opts.add_argument('--disk-cache-size=0')
    
    opts.add_argument('--no-first-run')
    opts.add_argument('--no-default-browser-check')
    opts.add_argument('--disable-extensions')
    opts.add_argument('--disable-plugins')

    # Prefer an existing Chrome binary if present
    for path in _detect_chrome_binary_paths():
        opts.binary_location = path
        break

    if user_data_dir:
        opts.add_argument('--user-data-dir=' + user_data_dir)

    return opts


def get_uc_driver(headless=True, user_data_dir=None, prefs=None, window_size='1366,768', retries=2, logger=None, incognito=True):
    """Get undetected Chrome driver with automatic version matching and retry logic.
    
    Creates fresh ChromeOptions for each attempt to avoid reuse errors.
    Automatically matches ChromeDriver version to installed Chrome version.
    
    Args:
        headless: If True, run Chrome in headless mode (no visible window)
        user_data_dir: Optional user data directory path
        prefs: Dictionary of Chrome preferences (download settings, etc.)
        window_size: Window size string like '1920,1080'
        retries: Number of retry attempts (unused, kept for backward compatibility)
        logger: Logger instance for logging messages
        incognito: If True, run Chrome in incognito/private mode (default True for backward compatibility)
    """
    if uc is None:
        raise ImportError("undetected_chromedriver is not installed")
    
    # Set environment variable to force headless mode
    if headless:
        os.environ['CHROME_HEADLESS'] = '1'
        os.environ['DISPLAY'] = ':99'  # For Linux systems
    
    major = get_installed_chrome_major_version()
    attempt = 0
    last_err = None
    connection_error_detected = False
    
    if logger:
        if major:
            logger.info(f"🔍 Detected Chrome major version: {major}")
        logger.info(f"🔧 Headless mode: {'ENABLED' if headless else 'DISABLED'}")
    
    if logger and headless:
        logger.info("📌 Chrome will run in headless mode (no visible browser window)")

    # Ensure cache directory exists before attempting driver creation
    cache_dir = ensure_uc_cache_directory()
    if logger and cache_dir:
        logger.debug(f"✅ Cache directory ensured: {cache_dir}")
    elif logger:
        logger.warning("⚠️ Could not ensure cache directory exists, continuing anyway")

    # Build version ring: prefer exact match, then try -1, never try +1
    ring = []
    if major:
        # Try exact version first (most reliable)
        ring.append(major)
        # Try -1 as fallback (older compatible version)
        if major > 1:
            ring.append(major - 1)
    else:
        # If version detection fails, let uc auto-manage
        ring.append(None)

    # Track if we've seen version mismatch errors (driver too new for Chrome)
    version_mismatch_detected = False
    detected_chrome_version = None
    
    for vm in ring:
        attempt += 1
        try:
            if logger:
                logger.info(f"🧩 Starting Chrome (attempt {attempt}) with version_main={vm}")
            
            # Create fresh ChromeOptions for each attempt to avoid reuse errors
            opts = build_default_options(headless=headless, user_data_dir=user_data_dir, window_size=window_size, incognito=incognito)
            if prefs:
                opts.add_experimental_option('prefs', prefs)
            
            # Launch with specified version
            if vm:
                driver = uc.Chrome(options=opts, use_subprocess=True, version_main=vm)
            else:
                driver = uc.Chrome(options=opts, use_subprocess=True)
            
            # Validate driver file exists (if we can check)
            if cache_dir and not validate_driver_file(cache_dir, logger):
                if logger:
                    logger.warning("⚠️ Driver file validation failed, but driver was created. Continuing...")
            
            # Validate session is working
            if not validate_driver_session(driver, logger):
                connection_error_detected = True
                try:
                    driver.quit()
                except Exception:
                    pass
                raise Exception("Driver session validation failed: connection refused or session lost")
            
            if logger:
                logger.info("✅ Chrome driver created and session validated successfully")
            return driver
            
        except Exception as e:
            last_err = e
            error_msg = str(e)
            
            # Check for connection errors
            if "Connection refused" in error_msg or "Failed to establish" in error_msg or "Connection refused" in str(type(e).__name__):
                connection_error_detected = True
                if logger:
                    logger.warning(f"⚠️ Connection error detected (attempt {attempt}): {error_msg}")
            
            # Check if error indicates version mismatch (driver too new for Chrome)
            if "only supports Chrome version" in error_msg and "Current browser version" in error_msg:
                version_mismatch_detected = True
                # Extract the actual Chrome version from error message
                # Error format: "This version of ChromeDriver only supports Chrome version 142\nCurrent browser version is 141.0.7390.123"
                browser_version_match = re.search(r'Current browser version is (\d+)\.', error_msg)
                if browser_version_match:
                    detected_chrome_version = int(browser_version_match.group(1))
                    if logger:
                        logger.warning(f"⚠️ Version mismatch detected: Driver too new. Chrome is version {detected_chrome_version}")
                
                if logger:
                    logger.warning(f"⚠️ Version mismatch error: {error_msg}")
                
                # If we detected the Chrome version from error, use it immediately
                if detected_chrome_version and detected_chrome_version != vm:
                    if logger:
                        logger.info(f"🔁 Breaking out of version ring to try detected Chrome version: {detected_chrome_version}")
                    break  # Exit the loop to try the detected version
            
            # Check for driver download/file errors
            if "No such file or directory" in error_msg or "Unable to obtain driver" in error_msg:
                if logger:
                    logger.warning(f"⚠️ Driver download/file error (attempt {attempt}): {error_msg}")
            
            if logger:
                logger.warning(f"Chrome launch failed (version_main={vm}): {e}")
            
            # Only clear cache if we detected a version mismatch or connection error
            # Don't clear cache on every retry as it may interrupt downloads
            if version_mismatch_detected or connection_error_detected:
                if logger:
                    logger.info("🧹 Clearing cache due to version mismatch or connection error...")
                clear_uc_cache()
                time.sleep(5)  # Increased wait time to allow cache clearing and downloads to complete
            else:
                # Small delay before retry without clearing cache
                time.sleep(2)
    
    # If we detected version mismatch, try the detected Chrome version immediately
    if version_mismatch_detected and detected_chrome_version:
        try:
            if logger:
                logger.info(f"🔁 Trying detected Chrome version (version_main={detected_chrome_version}) due to version mismatch")
            
            # Create fresh ChromeOptions
            opts = build_default_options(headless=headless, user_data_dir=user_data_dir, window_size=window_size, incognito=incognito)
            if prefs:
                opts.add_experimental_option('prefs', prefs)
            
            # Clear cache before trying detected version
            clear_uc_cache()
            time.sleep(5)  # Increased wait time
            
            driver = uc.Chrome(options=opts, use_subprocess=True, version_main=detected_chrome_version)
            
            # Validate session
            if not validate_driver_session(driver, logger):
                connection_error_detected = True
                try:
                    driver.quit()
                except Exception:
                    pass
                raise Exception("Driver session validation failed after version match")
            
            if logger:
                logger.info("✅ Chrome driver created with detected version and session validated")
            return driver
            
        except Exception as e:
            if logger:
                logger.warning(f"Chrome launch failed with version_main={detected_chrome_version}: {e}")
            last_err = e
            error_msg = str(e)
            if "Connection refused" in error_msg or "Failed to establish" in error_msg:
                connection_error_detected = True
    
    # If we detected version mismatch but couldn't extract version, try major-1 and major-2
    if version_mismatch_detected and major and major > 2:
        for fallback_version in [major - 1, major - 2]:
            if fallback_version <= 0:
                continue
            try:
                if logger:
                    logger.info(f"🔁 Trying older ChromeDriver version (version_main={fallback_version}) due to version mismatch")
                
                # Create fresh ChromeOptions
                opts = build_default_options(headless=headless, user_data_dir=user_data_dir, window_size=window_size, incognito=incognito)
                if prefs:
                    opts.add_experimental_option('prefs', prefs)
                
                # Clear cache before fallback attempt
                clear_uc_cache()
                time.sleep(5)  # Increased wait time
                
                driver = uc.Chrome(options=opts, use_subprocess=True, version_main=fallback_version)
                
                # Validate session
                if not validate_driver_session(driver, logger):
                    connection_error_detected = True
                    try:
                        driver.quit()
                    except Exception:
                        pass
                    raise Exception("Driver session validation failed with fallback version")
                
                if logger:
                    logger.info("✅ Chrome driver created with fallback version and session validated")
                return driver
                
            except Exception as e:
                if logger:
                    logger.warning(f"Chrome launch failed with version_main={fallback_version}: {e}")
                last_err = e
                error_msg = str(e)
                if "Connection refused" in error_msg or "Failed to establish" in error_msg:
                    connection_error_detected = True

    # Final fallback: only use auto-detection if we haven't detected a version mismatch
    # If version mismatch was detected, we should have tried the correct version already
    if not version_mismatch_detected:
        try:
            if logger:
                logger.info('🔁 Final fallback: launching uc.Chrome() with auto-version detection')
            
            # Create fresh ChromeOptions for final fallback
            opts = build_default_options(headless=headless, user_data_dir=user_data_dir, window_size=window_size, incognito=incognito)
            if prefs:
                opts.add_experimental_option('prefs', prefs)
            
            # Clear cache before final attempt if we had connection errors
            if connection_error_detected:
                if logger:
                    logger.info("🧹 Clearing cache before final fallback attempt...")
                clear_uc_cache()
                time.sleep(5)  # Increased wait time
            
            # Let undetected_chromedriver auto-detect and download the correct version
            driver = uc.Chrome(options=opts, use_subprocess=True)
            
            # Validate session
            if not validate_driver_session(driver, logger):
                connection_error_detected = True
                try:
                    driver.quit()
                except Exception:
                    pass
                raise Exception("Driver session validation failed in final fallback")
            
            if logger:
                logger.info("✅ Chrome driver created with auto-detection and session validated")
            return driver
            
        except Exception as e:
            if logger:
                logger.error(f"❌ Final Chrome launch failed: {e}")
            raise last_err or e
    else:
        # If we detected version mismatch but all attempts failed, raise the last error
        if logger:
            logger.error(f"❌ All Chrome launch attempts failed after version mismatch detection")
        raise last_err or Exception("Failed to launch Chrome after version mismatch detection")


# ------------------- Setup Chrome Driver -------------------
def setup_driver(headless=True, user_data_dir=None):
    """Setup Chrome driver with session persistence support
    
    Args:
        headless: If True, run in headless mode
        user_data_dir: Optional user data directory for session persistence
    """
    try:
        # Get job_id from sys.argv if available (typically 3rd argument), otherwise use default
        job_id = sys.argv[3] if len(sys.argv) > 3 else os.environ.get('JOB_ID', 'rex_submission')
        driver = ElectronBrowser.create(
            job_id=job_id,
            headless=headless,
            logger=logging.getLogger()
        )
        logging.info("✅ Chrome driver initialized successfully")
        return driver
    except Exception as e:
        logging.error(f"❌ Chrome driver initialization failed: {e}")
        raise

# ------------------- Utility Functions -------------------
def read_csv(file_path):
    with open(file_path, newline='', encoding='utf-8') as f:
        return list(csv.DictReader(f))

def write_result(data, success, message):
    return

def find_pdf_file(invoice_no, pdf_type):
    """Find PDF file:
       Invoice → I<InvoiceNo>.pdf
       Bill of Lading → <InvoiceNo>.pdf
    """
    # Search directly in PDF_DIR (no longer need extracted_pdfs subdirectory)
    search_root = PDF_DIR
    target_name = f"I{invoice_no}.pdf" if pdf_type == "invoice" else f"{invoice_no}.pdf"

    for root, _, files in os.walk(search_root):
        for f in files:
            if f.lower() == target_name.lower():
                full_path = os.path.abspath(os.path.join(root, f))
                logging.info(f"📄 Found {pdf_type} file for {invoice_no}: {f}")
                return full_path

    logging.warning(f"⚠️ No {pdf_type} PDF found for {invoice_no} under {search_root}")
    return None

# ------------------- Login -------------------
def check_session_valid(driver):
    """Check if existing session is still valid
    
    Args:
        driver: Selenium WebDriver instance
        
    Returns:
        bool: True if session is valid, False if login is required
    """
    try:
        logging.info("Checking if existing session is valid...")
        # Navigate to EPB portal (try accessing a protected page)
        driver.get("https://epb-exporttracker.gov.bd/#/sooList")
        time.sleep(3)
        
        # Check if we're redirected to login page
        current_url = driver.current_url.lower()
        if "login" in current_url:
            logging.info("Session invalid - redirected to login page")
            return False
        
        # Check if login form is present
        try:
            driver.find_element(By.ID, "inputUserName")
            logging.info("Session invalid - login form detected")
            return False
        except NoSuchElementException:
            # Check if we can see SOO list or dashboard elements (indicating we're logged in)
            try:
                WebDriverWait(driver, 5).until(
                    EC.presence_of_element_located((By.CSS_SELECTOR, "body"))
                )
                logging.info("✓ Session is valid")
                return True
            except:
                logging.info("Session invalid - cannot verify login state")
                return False
    except Exception as e:
        logging.warning(f"Error checking session validity: {e}")
        return False

def wait_for_manual_login(job_id, output_dir, driver):
    """Wait for user to manually log in through browser
    
    Creates a signal file to notify Electron that login is needed,
    then waits for Electron to signal that login is complete.
    
    Args:
        job_id: Job ID for signal file naming
        output_dir: Output directory where signal files will be created
        driver: Selenium WebDriver instance (should be in visible mode)
        
    Returns:
        bool: True if login successful, False otherwise
    """
    signal_file = os.path.join(output_dir, f"waiting_for_login_{job_id}.txt")
    complete_file = os.path.join(output_dir, f"login_complete_{job_id}.txt")
    
    # Create signal file to notify Electron
    logging.info("Creating login signal file for Electron...")
    try:
        with open(signal_file, 'w', encoding='utf-8') as f:
            f.write(f"job_id={job_id}\n")
            f.write(f"login_url={URL}\n")
            f.write(f"timestamp={datetime.datetime.now().isoformat()}\n")
        logging.info(f"✓ Signal file created: {signal_file}")
    except Exception as e:
        logging.error(f"Failed to create signal file: {e}")
        raise
    
    # Navigate to login URL
    logging.info("Navigating to login page...")
    logging.info("⚠️ Please log in through the browser window")
    driver.get(URL)
    time.sleep(3)
    
    # Wait for login completion signal (poll every 1 second, timeout 10 minutes)
    logging.info("Waiting for login completion signal from Electron...")
    timeout = 600  # 10 minutes
    start_time = time.time()
    check_interval = 1  # Check every second
    
    while time.time() - start_time < timeout:
        # Check for stop signal
        stop_file = os.path.join(output_dir, f"stop_{job_id}.txt")
        if os.path.exists(stop_file):
            logging.info("Stop signal received during login wait")
            raise Exception("Automation stopped by user")
        
        # Check for completion signal
        if os.path.exists(complete_file):
            logging.info("✓ Login completion signal received")
            # Wait a bit for login to complete in browser
            time.sleep(3)
            break
        
        time.sleep(check_interval)
    
    # Check if timeout occurred
    if not os.path.exists(complete_file):
        # Clean up signal file
        try:
            if os.path.exists(signal_file):
                os.remove(signal_file)
        except:
            pass
        raise Exception("Login timeout - user did not complete login within 10 minutes")
    
    # Verify login was successful
    time.sleep(2)
    if not check_session_valid(driver):
        # Clean up signal files
        try:
            if os.path.exists(signal_file):
                os.remove(signal_file)
            if os.path.exists(complete_file):
                os.remove(complete_file)
        except:
            pass
        raise Exception("Login verification failed - session may not be valid")
    
    # Clean up signal files
    try:
        if os.path.exists(signal_file):
            os.remove(signal_file)
        if os.path.exists(complete_file):
            os.remove(complete_file)
    except:
        pass
    
    logging.info("✓ Manual login completed successfully")
    return True

def login(driver, wait, job_id, output_dir):
    """Perform login with session management
    
    First checks if existing session is valid. If not, waits for manual login.
    
    Args:
        driver: Selenium WebDriver instance
        wait: WebDriverWait instance
        job_id: Job ID for signal file naming
        output_dir: Output directory for signal files
        
    Returns:
        bool: True if login successful, False otherwise
    """
    # Check if existing session is valid
    if check_session_valid(driver):
        logging.info("✓ Using existing valid session")
        
        # Duplicate tab for stable session (existing behavior)
        try:
            current_url = driver.current_url
            driver.execute_script("window.open(arguments[0], '_blank');", current_url)
            time.sleep(2)
            driver.switch_to.window(driver.window_handles[-1])
            logging.info("🆕 Switched to duplicated tab")
            driver.switch_to.window(driver.window_handles[0])
            driver.close()
            driver.switch_to.window(driver.window_handles[-1])
            logging.info("✅ Ready on new tab session")
        except Exception as e:
            logging.warning(f"Could not duplicate tab: {e}")
        
        return True
    
    # Session invalid or not found - wait for manual login
    logging.info("Session invalid or not found - waiting for manual login...")
    if not wait_for_manual_login(job_id, output_dir, driver):
        return False
    
    # After manual login, duplicate tab for stable session
    try:
        current_url = driver.current_url
        driver.execute_script("window.open(arguments[0], '_blank');", current_url)
        time.sleep(2)
        driver.switch_to.window(driver.window_handles[-1])
        logging.info("🆕 Switched to duplicated tab")
        driver.switch_to.window(driver.window_handles[0])
        driver.close()
        driver.switch_to.window(driver.window_handles[-1])
        logging.info("✅ Ready on new tab session")
    except Exception as e:
        logging.warning(f"Could not duplicate tab: {e}")
    
    return True

# ------------------- SOO Processing -------------------
def process_soo_record(driver, wait, row, index, total):
    invoice_no = row.get('InvoiceNo', 'Unknown')
    try:
        logging.info(f"\n{'=' * 60}")
        logging.info(f"🚀 Processing record {index}/{total} → Invoice: {invoice_no}")
        logging.info(f"{'=' * 60}")

        driver.find_element(By.CSS_SELECTOR, "div.tile a[href*='sooList']").click()
        time.sleep(2)

        wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[ng-click='checkSooFormEligibility()']"))).click()
        time.sleep(1)

        wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button.btn.btn.btn-primary[ng-click=\"close('yes')\"]"))).click()
        logging.info("☑️  Add SOO confirmed")
        time.sleep(3)

        first_row = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.k-grid-content tbody tr:first-child a")))
        driver.execute_script("arguments[0].click();", first_row)
        logging.info("📋 Opened SOO record")
        time.sleep(2)

        wait.until(EC.element_to_be_clickable((By.LINK_TEXT, "SoO Form Details"))).click()
        time.sleep(2)

        logging.info("📝 Filling form fields...")
        rex_dropdown = Select(wait.until(EC.presence_of_element_located((By.ID, "RexImporterId"))))
        rex_value = row.get("RexImporterId", "").strip()
        try:
            rex_dropdown.select_by_visible_text(rex_value)
        except Exception:
            for option in rex_dropdown.options:
                if rex_value.lower() in option.text.lower():
                    option.click()
                    break

        dest_dropdown = Select(wait.until(EC.presence_of_element_located((By.ID, "DestinationCountryId"))))
        dest_country = row.get("DestinationCountryId", "").strip()
        try:
            dest_dropdown.select_by_visible_text(dest_country)
        except Exception:
            for option in dest_dropdown.options:
                if dest_country.lower() in option.text.lower():
                    option.click()
                    break

        Select(driver.find_element(By.ID, "inputFreightRoute")).select_by_visible_text(row.get("FreightRoute", ""))
        driver.find_element(By.ID, "inputBLNo").send_keys(row.get("BLNo", ""))
        driver.find_element(By.ID, "inputBLDate").send_keys(row.get("BLDate", ""))
        driver.find_element(By.ID, "inputContainerNo").send_keys(row.get("ContainerNo", ""))
        driver.find_element(By.ID, "adCode").send_keys(row.get("AdCode", ""))
        driver.find_element(By.ID, "serial").send_keys(row.get("Serial", ""))
        Select(driver.find_element(By.ID, "year")).select_by_visible_text(row.get("Year", ""))
        driver.find_element(By.ID, "inputEXPDate").send_keys(row.get("EXPDate", ""))
        driver.find_element(By.ID, "inputBillOfExportNo").send_keys(row.get("BillOfExportNo", ""))
        driver.find_element(By.ID, "inputBillOfExportDate").send_keys(row.get("BillOfExportDate", ""))

        hs_code = row.get("HSCode", "").strip()
        hs_dropdown = Select(wait.until(EC.presence_of_element_located((By.ID, "inputHSCode"))))
        try:
            hs_dropdown.select_by_visible_text(hs_code)
        except Exception:
            for option in hs_dropdown.options:
                if hs_code.lower() in option.text.lower():
                    option.click()
                    break

        driver.find_element(By.ID, "inputQnty").send_keys(row.get("Quantity", ""))
        Select(driver.find_element(By.ID, "inputUnitType")).select_by_visible_text(row.get("UnitType", ""))
        driver.find_element(By.CSS_SELECTOR, "a[ng-click^='addHsCodeInfo']").click()
        time.sleep(1)

        driver.find_element(By.ID, "inputInvoiceNo").send_keys(invoice_no)
        driver.find_element(By.ID, "inputInvoiceDate").send_keys(row.get("InvoiceDate", ""))
        Select(driver.find_element(By.ID, "currency")).select_by_visible_text(row.get("Currency", ""))
        driver.find_element(By.ID, "inputInvoiceValue").send_keys(row.get("InvoiceValue", ""))
        driver.find_element(By.ID, "inputDate").send_keys(row.get("DeclarationDate", ""))
        logging.info("✅ Form fields filled successfully")

        # Upload Commercial Invoice
        logging.info("📤 Uploading Commercial Invoice...")
        wait.until(EC.element_to_be_clickable((By.LINK_TEXT, "Commercial Invoice"))).click()
        time.sleep(1)
        ci_file = find_pdf_file(invoice_no, "invoice")
        if not ci_file:
            raise FileNotFoundError(f"Commercial Invoice PDF not found for {invoice_no}")
        wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "input[type='file'][accept*='.pdf']"))).send_keys(ci_file)
        logging.info(f"✅ Uploaded Invoice: {os.path.basename(ci_file)}")
        time.sleep(2)

        # Upload Bill of Lading
        logging.info("📦 Uploading Bill of Lading...")
        wait.until(EC.element_to_be_clickable((By.LINK_TEXT, "Bill of Lading"))).click()
        time.sleep(1)
        bol_file = find_pdf_file(invoice_no, "bol")
        if not bol_file:
            raise FileNotFoundError(f"Bill of Lading PDF not found for {invoice_no}")
        wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "input[type='file'][accept*='.pdf']"))).send_keys(bol_file)
        logging.info(f"✅ Uploaded B/L: {os.path.basename(bol_file)}")
        time.sleep(2)

        # Save
        logging.info("💾 Saving SOO form...")
        wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a.control-bar-save-btn[ng-click*='save()']"))).click()
        time.sleep(4)

        wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a[href*='sooList'].navigate-link"))).click()
        logging.info("⬅️ Returned to SOO list")
        time.sleep(2)

        write_result(row, True, "SOO submitted successfully")
        logging.info(f"✅ Record {index} completed successfully")
        return True

    except Exception as e:
        logging.error(f"❌ Error on record {index} ({invoice_no}): {e}")
        write_result(row, False, str(e))
        return False

# ------------------- Main -------------------
def main():
    driver = None
    success, failed = 0, 0
    try:
        if not os.path.exists(CSV_FILE):
            logging.error(f"❌ CSV file not found: {CSV_FILE}")
            return 1
        if not os.path.exists(PDF_DIR):
            logging.error(f"❌ PDF directory not found: {PDF_DIR}")
            return 1

        rows = read_csv(CSV_FILE)
        if not rows:
            logging.warning("⚠️ No records found in CSV.")
            return 1

        total = len(rows)
        
        # Launch Chrome in visible mode for login (with session directory)
        logging.info("Launching Chrome for login (visible mode)...")
        driver = setup_driver(headless=False, user_data_dir=SESSION_DIR)
        wait = WebDriverWait(driver, TIMEOUT)
        
        try:
            driver.maximize_window()
        except:
            pass  # Ignore if maximize fails

        if not login(driver, wait, JOB_ID, OUTPUT_DIR):
            logging.error("❌ Login failed")
            return 1

        for i, row in enumerate(rows, start=1):
            # Check for stop signal
            stop_file = os.path.join(OUTPUT_DIR, f"stop_{JOB_ID}.txt")
            if os.path.exists(stop_file):
                logging.info("Stop signal received, stopping automation...")
                if driver:
                    try:
                        driver.quit()
                    except:
                        pass
                sys.exit(130)
            
            # Log progress: items remaining
            items_remaining = total - i + 1
            logging.info(f"📊 Progress: {i}/{total} (Remaining: {items_remaining})")
            
            if process_soo_record(driver, wait, row, i, total):
                success += 1
                logging.info(f"✓ Successfully processed entry {i}/{total}")
            else:
                failed += 1
                logging.error(f"✗ Failed to process entry {i}/{total}")
            
            logging.info(f"📥 Submissions complete: {success} successful, {failed} failed")
            time.sleep(2)

        logging.info("\n" + "=" * 80)
        logging.info("🎉 Processing Complete!")
        logging.info("=" * 80)
        logging.info(f"Total entries: {total}")
        logging.info(f"✅ Successful submissions: {success}")
        logging.info(f"❌ Failed submissions: {failed}")
        logging.info(f"📥 Total submissions completed: {success + failed}/{total}")
        logging.info(f"📄 Results: {RESULT_LOG}")
        logging.info(f"\n📋 Detailed log saved to: {log_file_path}")
        logging.info("=" * 80)

        # Upload files to server if server parameters provided
        uploaded_log_path = None
        uploaded_csv_path = None
        
        if API_BASE_URL and USER_ID:
            try:
                server_client = ServerClient(API_BASE_URL, USER_ID, JOB_ID, logging.getLogger())
                
                if server_client.enabled:
                    # Prepare job data for uploads
                    job_data = {
                        'userId': USER_ID,
                        'serviceId': 'rex-soo-submission',
                        'serviceName': 'REX/SOO Submission',
                        'inputFileName': os.path.basename(CSV_FILE),
                        'inputFilePath': CSV_FILE,
                        'creditsUsed': CREDITS_USED
                    }
                    
                    # Upload log file
                    if os.path.exists(log_file_path):
                        uploaded_log_path = server_client.upload_log_file(log_file_path, job_data)
                    
                    # Upload results CSV
                    if os.path.exists(RESULT_LOG):
                        uploaded_csv_path = server_client.upload_results_csv(RESULT_LOG, job_data)
                    
                    # Report job result
                    result_files = []
                    if uploaded_csv_path:
                        result_files.append(uploaded_csv_path)
                    elif uploaded_log_path:
                        result_files.append(uploaded_log_path)
                    
                    if result_files:
                        server_client.report_job_result(
                            result_files,
                            OUTPUT_DIR,
                            successful_count=success
                        )
            except Exception as e:
                logging.warning(f"⚠️ Error uploading files to server: {e}")
                # Continue even if upload fails

        return 0

    except Exception as e:
        logging.error(f"🚨 Fatal error: {e}")
        return 1
    finally:
        if driver:
            driver.quit()
            logging.info("🔒 Browser closed")

if __name__ == "__main__":
    sys.exit(main())