#!/usr/bin/env python3
"""
imapsync 1.0 - Email tool for syncing two IMAP accounts, one way, and without duplicates.

Copyright (C) 2026. Steven Vertigan

This program comes with ABSOLUTELY NO WARRANTY.
This is free software, and you are welcome to redistribute it
under certain conditions (GPLv3).

https://opensource.org/license/gpl-3.0

This is a python3 port of the original Perl imapsync by Gilles LAMIRAL.
Original: https://imapsync.lamiral.info/

CGI functionality removed; full CLI functionality retained.
"""

import argparse
import base64
import email.utils
import hashlib
import imaplib
import logging
import os
import platform
import re
import signal
import socket
import ssl
import subprocess
import sys
import time
from datetime import datetime
from pathlib import Path
from typing import Optional

# ── Version ────────────────────────────────────────────────────────────────────
VERSION = "1.0"

# ── Exit codes (mirrors sysexits.h + imapsync custom codes) ───────────────────
EX_OK                              = 0
EX_USAGE                           = 64
EX_NOINPUT                         = 66
EX_UNAVAILABLE                     = 69
EX_SOFTWARE                        = 70
EXIT_CATCH_ALL                     = 1
EXIT_BY_SIGNAL                     = 6
EXIT_BY_FILE                       = 7
EXIT_PID_FILE_ERROR                = 8
EXIT_CONNECTION_FAILURE            = 10
EXIT_TLS_FAILURE                   = 12
EXIT_AUTHENTICATION_FAILURE        = 16
EXIT_SUBFOLDER1_NO_EXISTS          = 21
EXIT_WITH_ERRORS                   = 111
EXIT_WITH_ERRORS_MAX               = 112
EXIT_OVERQUOTA                     = 113
EXIT_ERR_APPEND                    = 114
EXIT_ERR_FETCH                     = 115
EXIT_ERR_CREATE                    = 116
EXIT_ERR_SELECT                    = 117
EXIT_TRANSFER_EXCEEDED             = 118
EXIT_ERR_APPEND_VIRUS              = 119
EXIT_ERR_FLAGS                     = 120
EXIT_ERR_SEARCH                    = 121
EXIT_TESTS_FAILED                  = 254
EXIT_CONNECTION_FAILURE_HOST1      = 101
EXIT_CONNECTION_FAILURE_HOST2      = 102
EXIT_AUTHENTICATION_FAILURE_USER1  = 161
EXIT_AUTHENTICATION_FAILURE_USER2  = 162

EXIT_TXT = {
    EX_OK:                             "EX_OK: successful termination",
    EX_USAGE:                          "EX_USAGE: command line usage error",
    EX_NOINPUT:                        "EX_NOINPUT: cannot open input",
    EX_UNAVAILABLE:                    "EX_UNAVAILABLE: service unavailable",
    EX_SOFTWARE:                       "EX_SOFTWARE: internal software error",
    EXIT_CATCH_ALL:                    "EXIT_CATCH_ALL",
    EXIT_BY_SIGNAL:                    "EXIT_BY_SIGNAL",
    EXIT_BY_FILE:                      "EXIT_BY_FILE",
    EXIT_PID_FILE_ERROR:               "EXIT_PID_FILE_ERROR",
    EXIT_CONNECTION_FAILURE:           "EXIT_CONNECTION_FAILURE",
    EXIT_TLS_FAILURE:                  "EXIT_TLS_FAILURE",
    EXIT_AUTHENTICATION_FAILURE:       "EXIT_AUTHENTICATION_FAILURE",
    EXIT_SUBFOLDER1_NO_EXISTS:         "EXIT_SUBFOLDER1_NO_EXISTS",
    EXIT_WITH_ERRORS:                  "EXIT_WITH_ERRORS",
    EXIT_WITH_ERRORS_MAX:              "EXIT_WITH_ERRORS_MAX",
    EXIT_OVERQUOTA:                    "EXIT_OVERQUOTA",
    EXIT_ERR_APPEND:                   "EXIT_ERR_APPEND",
    EXIT_ERR_APPEND_VIRUS:             "EXIT_ERR_APPEND_VIRUS",
    EXIT_ERR_FETCH:                    "EXIT_ERR_FETCH",
    EXIT_ERR_FLAGS:                    "EXIT_ERR_FLAGS",
    EXIT_ERR_SEARCH:                   "EXIT_ERR_SEARCH",
    EXIT_ERR_CREATE:                   "EXIT_ERR_CREATE",
    EXIT_ERR_SELECT:                   "EXIT_ERR_SELECT",
    EXIT_TESTS_FAILED:                 "EXIT_TESTS_FAILED",
    EXIT_TRANSFER_EXCEEDED:            "EXIT_TRANSFER_EXCEEDED",
    EXIT_CONNECTION_FAILURE_HOST1:     "EXIT_CONNECTION_FAILURE_HOST1",
    EXIT_CONNECTION_FAILURE_HOST2:     "EXIT_CONNECTION_FAILURE_HOST2",
    EXIT_AUTHENTICATION_FAILURE_USER1: "EXIT_AUTHENTICATION_FAILURE_USER1",
    EXIT_AUTHENTICATION_FAILURE_USER2: "EXIT_AUTHENTICATION_FAILURE_USER2",
}

# ── Constants ──────────────────────────────────────────────────────────────────
DEFAULT_LOGDIR      = "LOG_imapsync"
ERRORS_MAX          = 50
IMAP_PORT           = 143
IMAP_SSL_PORT       = 993
DEFAULT_TIMEOUT     = 120
MAX_SLEEP           = 2
KIBI                = 1024
GMAIL_MAXSIZE       = 35_651_584
DEFAULT_BUFFER_SIZE = 4096
NB_SECONDS_IN_A_DAY = 86_400

FAILEDMESSAGES_HEADER = "Failed Message Transfers\r\n"
FAILEDMESSAGES        = FAILEDMESSAGES_HEADER

# ── Logging setup ──────────────────────────────────────────────────────────────
logger = logging.getLogger("imapsync")


# ── Custom exceptions ──────────────────────────────────────────────────────────

class ImapAuthExpired(Exception):
    """Raised when a reconnect attempt succeeds at the TCP/SSL level but
    authentication fails — indicating an expired OAuth token or revoked
    credentials rather than a transient network error.

    Carries the host number (1 or 2) so the caller can exit with the
    correct code (EXIT_AUTHENTICATION_FAILURE_USER1/2).
    """
    def __init__(self, host: int, msg: str = ""):
        super().__init__(msg or f"Authentication expired on host{host}")
        self.host = host


# ══════════════════════════════════════════════════════════════════════════════
# Utility helpers
# ══════════════════════════════════════════════════════════════════════════════

def bytes_display_bin(n) -> str:
    """Human-readable binary size string."""
    if n is None:
        return "NA"
    try:
        n = float(n)
    except (TypeError, ValueError):
        return "NA"
    if abs(n) < 1000 * KIBI:
        return f"{n / KIBI:.3f} KiB"
    if abs(n) < 1000 * KIBI**2:
        return f"{n / KIBI**2:.3f} MiB"
    if abs(n) < 1000 * KIBI**3:
        return f"{n / KIBI**3:.3f} GiB"
    if abs(n) < 1000 * KIBI**4:
        return f"{n / KIBI**4:.3f} TiB"
    return f"{n / KIBI**5:.3f} PiB"


def bytes_display_dec(n) -> str:
    """Human-readable decimal size string."""
    if n is None:
        return "NA"
    try:
        n = float(n)
    except (TypeError, ValueError):
        return "NA"
    if abs(n) < 1000:
        return f"{n:.0f} bytes"
    if abs(n) < 1_000**2:
        return f"{n / 1000:.3f} KB"
    if abs(n) < 999_999_500:
        return f"{n / 1_000**2:.3f} MB"
    if abs(n) < 999_999_500_000:
        return f"{n / 1_000**3:.3f} GB"
    if abs(n) < 999_999_500_000_000:
        return f"{n / 1_000**4:.3f} TB"
    return f"{n / 1_000**5:.3f} PB"


def filter_forbidden_chars(s: str) -> str:
    """Replace filesystem-unsafe characters with underscores."""
    if s is None:
        return ""
    out = re.sub(r'[*|?:"<>\'\\ &\t\r\n]', '_', s)
    out = re.sub(r'[^\x00-\x7F]', '_', out)
    return out


def slash_to_underscore(s: str) -> str:
    return s.replace("/", "_") if s else s


def first_line(path: str) -> str:
    """Return first line of a file, stripped; empty string on error."""
    try:
        with open(path, "r", encoding="utf-8", errors="replace") as fh:
            return fh.readline().rstrip("\r\n")
    except OSError:
        return ""


def first_line_or_string(value: str) -> str:
    """If value looks like a readable file, return its first line; else return value."""
    if value and os.path.isfile(value) and os.access(value, os.R_OK):
        logger.info("Reading first line of file %s", value)
        return first_line(value)
    return value or "please, give a value"


def logfile_name(timestart: float, suffix: str = "", logdir: str = "") -> str:
    dt = datetime.fromtimestamp(timestart)
    ms = int((timestart - int(timestart)) * 1000)
    date_str   = dt.strftime("%Y_%m_%d_%H_%M_%S") + f"_{ms:03d}"
    sep_suffix = f"_{suffix}" if suffix else ""
    sep_dir    = f"{logdir}/" if logdir else ""
    return f"{sep_dir}{date_str}{sep_suffix}.txt"


def setup_logging(logfile: Optional[str] = None, debug: bool = False) -> logging.Logger:
    level = logging.DEBUG if debug else logging.INFO
    fmt   = logging.Formatter("%(message)s")
    root  = logging.getLogger("imapsync")
    root.setLevel(level)
    root.handlers.clear()
    ch = logging.StreamHandler(sys.stdout)
    ch.setFormatter(fmt)
    root.addHandler(ch)
    if logfile:
        try:
            Path(logfile).parent.mkdir(parents=True, exist_ok=True)
            fh = logging.FileHandler(logfile, encoding="utf-8")
            fh.setFormatter(fmt)
            root.addHandler(fh)
        except OSError as exc:
            root.warning("Cannot open log file %s: %s", logfile, exc)
    return root


def write_pid_file(path: str, pid: int, logfile: str = "") -> bool:
    try:
        with open(path, "w") as fh:
            fh.write(f"{pid}\n")
            if logfile:
                fh.write(f"{logfile}\n")
        return True
    except OSError as exc:
        logger.error("Could not write pidfile %s: %s", path, exc)
        return False


def remove_pid_file(path: str) -> None:
    try:
        if path and os.path.exists(path):
            logger.info("Removing pidfile %s", path)
            os.unlink(path)
    except OSError:
        pass


def sanitize_host(host: str) -> str:
    """Strip spaces and slashes from hostname."""
    if not host:
        return host
    return re.sub(r"[ /]", "", host)


def imap_utf7_decode(s: str) -> str:
    """Decode IMAP-modified UTF-7 folder name to a Unicode string."""
    # Manual IMAP modified UTF-7 decode (RFC 3501 §5.1.3)
    try:
        result = []
        i = 0
        while i < len(s):
            if s[i] == "&":
                end = s.find("-", i + 1)
                if end == -1:
                    result.append(s[i])
                    i += 1
                    continue
                b64 = s[i + 1:end]
                if b64 == "":
                    result.append("&")
                else:
                    b64_std = b64.replace(",", "/")
                    padding = "=" * (-len(b64_std) % 4)
                    decoded = base64.b64decode(b64_std + padding).decode("utf-16-be")
                    result.append(decoded)
                i = end + 1
            else:
                result.append(s[i])
                i += 1
        return "".join(result)
    except Exception:
        return s


def jux_utf8(s: str) -> str:
    decoded = imap_utf7_decode(s)
    if s == decoded:
        return f"[{s}]"
    return f"[{s}] = [{decoded}]"


def error_type(error: str) -> str:
    """Classify an error string into a named error type."""
    if not error:
        return "ERR_NOTHING_REPORTED"
    checks = [
        (r"Host1 failure: Error login on",                                      "ERR_AUTHENTICATION_FAILURE_USER1"),
        (r"Host2 failure: Error login on",                                      "ERR_AUTHENTICATION_FAILURE_USER2"),
        (r"Host. failure: Can not go to tls encryption on host.",                "ERR_EXIT_TLS_FAILURE"),
        (r"could not be fetched:",                                               "ERR_Host1_FETCH"),
        (r"could not append .*BAD maximum message size exceeded",                "ERR_APPEND_SIZE"),
        (r"could not append.*Maximum size of appendable message has been exceeded", "ERR_APPEND_SIZE"),
        (r"OVERQUOTA",                                                           "ERR_OVERQUOTA"),
        (r"Quota limit will be exceeded",                                        "ERR_OVERQUOTA"),
        (r"full: it is time to find a bigger place",                             "ERR_OVERQUOTA"),
        (r"could not append.*virus",                                             "ERR_APPEND_VIRUS"),
        (r"could not append",                                                    "ERR_APPEND"),
        (r"could not add flags",                                                 "ERR_FLAGS"),
        (r"BAD .* SEARCH: Unknown argument",                                     "ERR_SEARCH"),
        (r"Could not create folder",                                             "ERR_CREATE"),
        (r"Could not select:",                                                   "ERR_SELECT"),
        (r"Maximum bytes transferred reached",                                   "ERR_TRANSFER_EXCEEDED"),
        (r"can not open imap connection on host1",                               "ERR_CONNECTION_FAILURE_HOST1"),
        (r"can not open imap connection on host2",                               "ERR_CONNECTION_FAILURE_HOST2"),
    ]
    for pattern, label in checks:
        if re.search(pattern, error):
            return label
    return "ERR_UNCLASSIFIED"


EXIT_VALUE_OF_ERR_TYPE = {
    "ERR_APPEND_SIZE":                  EXIT_ERR_APPEND,
    "ERR_OVERQUOTA":                    EXIT_OVERQUOTA,
    "ERR_APPEND":                       EXIT_ERR_APPEND,
    "ERR_APPEND_VIRUS":                 EXIT_ERR_APPEND_VIRUS,
    "ERR_CREATE":                       EXIT_ERR_CREATE,
    "ERR_SELECT":                       EXIT_ERR_SELECT,
    "ERR_Host1_FETCH":                  EXIT_ERR_FETCH,
    "ERR_FLAGS":                        EXIT_ERR_FLAGS,
    "ERR_SEARCH":                       EXIT_ERR_SEARCH,
    "ERR_UNCLASSIFIED":                 EXIT_WITH_ERRORS,
    "ERR_NOTHING_REPORTED":             EXIT_WITH_ERRORS,
    "ERR_TRANSFER_EXCEEDED":            EXIT_TRANSFER_EXCEEDED,
    "ERR_CONNECTION_FAILURE_HOST1":     EXIT_CONNECTION_FAILURE_HOST1,
    "ERR_CONNECTION_FAILURE_HOST2":     EXIT_CONNECTION_FAILURE_HOST2,
    "ERR_AUTHENTICATION_FAILURE_USER1": EXIT_AUTHENTICATION_FAILURE_USER1,
    "ERR_AUTHENTICATION_FAILURE_USER2": EXIT_AUTHENTICATION_FAILURE_USER2,
    "ERR_EXIT_TLS_FAILURE":             EXIT_TLS_FAILURE,
}


def most_common_error(errors: list) -> str:
    """Return the most frequent error type from a list of error strings."""
    if not errors:
        return "ERR_NOTHING_REPORTED"
    counts: dict = {}
    for e in errors:
        et = error_type(e)
        counts[et] = counts.get(et, 0) + 1
    return sorted(counts.keys(), key=lambda k: (-counts[k], k))[0]


# ══════════════════════════════════════════════════════════════════════════════
# IMAP connection helper
# ══════════════════════════════════════════════════════════════════════════════

class ImapConnection:
    """Thin wrapper around imaplib.IMAP4 / IMAP4_SSL."""

    def __init__(self, host, port, user, password,
                 ssl_on=False, starttls=False,
                 timeout=DEFAULT_TIMEOUT,
                 debug_imap=False,
                 authmech="LOGIN",
                 authuser=None,
                 domain=None,
                 oauthdirect=None,
                 oauthaccesstoken=None,
                 oauthrefreshcmd=None,
                 ssl_context=None,
                 side="Host?"):
        self.host             = host
        self.port             = port
        self.user             = user
        self.password         = password
        self.ssl_on           = ssl_on
        self.starttls         = starttls
        self.timeout          = timeout
        self.authmech         = authmech.upper() if authmech else "LOGIN"
        self.authuser         = authuser
        self.domain           = domain
        self.oauthdirect      = oauthdirect
        self.oauthaccesstoken = oauthaccesstoken
        self.oauthrefreshcmd  = oauthrefreshcmd
        self.ssl_context      = ssl_context
        self.side             = side
        self.debug_imap       = debug_imap
        self.imap: Optional[imaplib.IMAP4] = None
        self.reconnect_count  = 0
        self.proxyauth        = False
        self.debugssl         = False
        self.compress         = False  # set by _make_conn from --compress / --compress1/2

    # ── connect + login ────────────────────────────────────────────────────────

    def connect(self) -> bool:
        """Open the IMAP connection.

        If the caller has explicitly chosen a transport (ssl_on, starttls, or
        a non-default port) we use exactly that.  Otherwise we auto-negotiate:

          1. SSL on port 993
          2. STARTTLS on port 143
          3. Plain on port 143  (accepted only if server does not reject it)

        This mirrors what Thunderbird and the Perl imapsync do and handles
        servers that listen on 143 but require encryption.
        """
        explicit_transport = (self.ssl_on or self.starttls
                              or self.port not in (IMAP_PORT, IMAP_SSL_PORT, None))
        if explicit_transport:
            return self._connect_once(self.host, self.port, self.ssl_on, self.starttls)

        # Auto-negotiate: try each transport in preference order
        ctx = self.ssl_context or ssl.create_default_context()
        ctx.check_hostname = False
        ctx.verify_mode    = ssl.CERT_NONE

        attempts = [
            (IMAP_SSL_PORT, True,  False, "SSL"),
            (IMAP_PORT,     False, True,  "STARTTLS"),
            (IMAP_PORT,     False, False, "plain"),
        ]
        for port, ssl_on, starttls, label in attempts:
            logger.info("%s: trying %s on [%s] port [%s]",
                        self.side, label, self.host, port)
            try:
                if ssl_on:
                    imap = imaplib.IMAP4_SSL(
                        self.host, port, ssl_context=ctx, timeout=self.timeout)
                else:
                    imap = imaplib.IMAP4(self.host, port, timeout=self.timeout)
                    if starttls:
                        imap.starttls(ssl_context=ctx)
                if self.debug_imap:
                    imap.debug = 4
                self.imap    = imap
                self.port    = port
                self.ssl_on  = ssl_on
                self.starttls = starttls
                logger.info("%s: connected via %s on port %d", self.side, label, port)
                return True
            except Exception as exc:
                logger.info("%s: %s on port %d failed: %s", self.side, label, port, exc)

        logger.error(
            "%s failure: can not open imap connection on %s [%s] "
            "(tried SSL/993, STARTTLS/143, plain/143)",
            self.side, self.side.lower(), self.host)
        return False

    def _connect_once(self, host: str, port: int, ssl_on: bool,
                      starttls: bool) -> bool:
        """Connect with exactly the specified transport — no fallback."""
        logger.info("%s: connecting on %s [%s] port [%s]",
                    self.side, self.side.lower(), host, port)
        try:
            ctx = self.ssl_context or ssl.create_default_context()
            ctx.check_hostname = False
            ctx.verify_mode    = ssl.CERT_NONE
            if ssl_on:
                self.imap = imaplib.IMAP4_SSL(
                    host, port, ssl_context=ctx, timeout=self.timeout)
            else:
                self.imap = imaplib.IMAP4(host, port, timeout=self.timeout)
                if starttls:
                    self.imap.starttls(ssl_context=ctx)
            if self.debug_imap:
                self.imap.debug = 4
            if self.debugssl:
                logging.getLogger("ssl").setLevel(logging.DEBUG)
            return True
        except Exception as exc:
            logger.error(
                "%s failure: can not open imap connection on %s [%s] port [%s]: %s",
                self.side, self.side.lower(), host, port, exc)
            return False

    def login(self) -> bool:
        if self.imap is None:
            return False
        try:
            if self.oauthdirect is not None:
                return self._login_xoauth2_direct()
            if self.oauthaccesstoken is not None:
                return self._login_xoauth2_token()
            if self.authmech == "XOAUTH2":
                return self._login_xoauth2_direct()
            if self.authmech == "CRAM-MD5":
                return self._login_cram_md5()
            if self.authmech == "PLAIN":
                return self._login_plain()

            user     = self.authuser or self.user
            password = self.password or ""
            try:
                self.imap.login(user, password)
            except imaplib.IMAP4.error as exc:
                exc_str = str(exc)
                # Server requires encryption — retry with CRAM-MD5 if available,
                # otherwise PLAIN (both carry the password differently but work
                # over plain connections on servers that permit it, and work fine
                # over SSL/TLS where PRIVACYREQUIRED is never raised).
                if re.search(r"PRIVACYREQUIRED|Plaintext authentication disallowed",
                             exc_str, re.I):
                    logger.warning(
                        "%s: LOGIN rejected (%s) — trying AUTHENTICATE PLAIN",
                        self.side, exc_str.strip())
                    # Check server capabilities for preferred mechanism
                    try:
                        _, cap_data = self.imap.capability()
                        cap_str = " ".join(
                            c.decode("utf-8", errors="replace")
                            if isinstance(c, bytes) else str(c)
                            for c in (cap_data or []))
                        if "AUTH=CRAM-MD5" in cap_str.upper():
                            return self._login_cram_md5()
                    except Exception:
                        pass
                    return self._login_plain()
                raise

            # PROXYAUTH: impersonate target user after admin login
            if self.proxyauth and self.user != user:
                try:
                    self.imap.proxyauth(self.user)
                    logger.info("%s: PROXYAUTH as [%s]", self.side, self.user)
                except Exception as exc:
                    logger.warning("%s: PROXYAUTH failed: %s", self.side, exc)

            logger.info("%s: success login on [%s] with user [%s] auth [%s]",
                        self.side, self.host, self.user, self.authmech)
            return True
        except imaplib.IMAP4.error as exc:
            logger.error(
                "%s failure: Error login on [%s] with user [%s] auth [%s]: %s",
                self.side, self.host, self.user, self.authmech, exc)
            return False

    def _oauth2_bearer_string(self, token: str) -> bytes:
        """Build the XOAUTH2 bearer token as raw bytes.

        imaplib.authenticate base64-encodes the callback return value itself,
        so we must return the raw (un-encoded) string as bytes.
        """
        raw = f"user={self.user}\x01auth=Bearer {token}\x01\x01"
        return raw.encode()

    def _refresh_token(self, token_or_file: str) -> str:
        if self.oauthrefreshcmd:
            try:
                subprocess.run(self.oauthrefreshcmd, shell=True, check=False)
            except Exception:
                pass
        return first_line_or_string(token_or_file)

    def _login_xoauth2_direct(self) -> bool:
        token  = self._refresh_token(self.oauthdirect or "")
        bearer = self._oauth2_bearer_string(token)
        try:
            self.imap.authenticate("XOAUTH2", lambda challenge: bearer)
            logger.info("%s: success login on [%s] with user [%s] auth [XOAUTH2]",
                        self.side, self.host, self.user)
            return True
        except imaplib.IMAP4.error as exc:
            logger.error("%s XOAUTH2 direct login failed: %s", self.side, exc)
            return False

    def _login_xoauth2_token(self) -> bool:
        token  = self._refresh_token(self.oauthaccesstoken or "")
        bearer = self._oauth2_bearer_string(token)
        try:
            self.imap.authenticate("XOAUTH2", lambda challenge: bearer)
            logger.info("%s: success login on [%s] with user [%s] auth [XOAUTH2]",
                        self.side, self.host, self.user)
            return True
        except imaplib.IMAP4.error as exc:
            logger.error("%s XOAUTH2 accesstoken login failed: %s", self.side, exc)
            return False

    def _login_cram_md5(self) -> bool:
        """Authenticate using CRAM-MD5 (RFC 2195).

        The server sends a challenge; the client responds with
        username + HMAC-MD5(challenge, password) in base64.
        Works over plain connections — the password is never sent in clear.
        """
        import hmac
        user     = self.authuser or self.user
        password = (self.password or "").encode("utf-8")

        def _cram_md5_response(challenge: bytes) -> bytes:
            # imaplib base64-decodes the challenge before passing it here
            digest = hmac.new(password, challenge, "md5").hexdigest()
            return f"{user} {digest}".encode()

        try:
            self.imap.authenticate("CRAM-MD5", _cram_md5_response)
            logger.info("%s: success login on [%s] with user [%s] auth [CRAM-MD5]",
                        self.side, self.host, self.user)
            return True
        except imaplib.IMAP4.error as exc:
            logger.error(
                "%s failure: Error login on [%s] with user [%s] auth [CRAM-MD5]: %s",
                self.side, self.host, self.user, exc)
            return False

    def _login_plain(self) -> bool:
        """Authenticate using SASL PLAIN (RFC 4616).

        Sends: base64(authzid NUL authcid NUL password)
        Only safe over SSL/TLS but used as a last resort when the server
        explicitly advertises AUTH=PLAIN and the connection is encrypted.
        """
        user     = self.authuser or self.user
        password = self.password or ""
        # PLAIN: NUL + authcid + NUL + password (authzid left empty)
        plain    = f"\x00{user}\x00{password}".encode("utf-8")

        def _plain_response(challenge: bytes) -> bytes:
            return plain

        try:
            self.imap.authenticate("PLAIN", _plain_response)
            logger.info("%s: success login on [%s] with user [%s] auth [PLAIN]",
                        self.side, self.host, self.user)
            return True
        except imaplib.IMAP4.error as exc:
            logger.error(
                "%s failure: Error login on [%s] with user [%s] auth [PLAIN]: %s",
                self.side, self.host, self.user, exc)
            return False

    def list_folders(self) -> list:
        try:
            typ, data = self.imap.list()
            if typ != "OK":
                return []
            folders = []
            for item in data:
                if isinstance(item, bytes):
                    item = item.decode("utf-8", errors="replace")
                m = re.search(r'"([^"]+)"\s*$|(\S+)\s*$', item)
                if m:
                    folders.append(m.group(1) or m.group(2))
            return sorted(folders)
        except Exception as exc:
            logger.error("%s list folders error: %s", self.side, exc)
            return []

    def select_folder(self, folder: str, readonly: bool = False) -> bool:
        try:
            fn = self.imap.examine if readonly else self.imap.select
            typ, _ = fn(f'"{folder}"')
            return typ == "OK"
        except Exception:
            return False

    def create_folder(self, folder: str) -> bool:
        try:
            typ, _ = self.imap.create(f'"{folder}"')
            return typ == "OK"
        except Exception:
            return False

    def create_folder_with_parents(self, folder: str, sep: str = "/") -> bool:
        """Create *folder* on the server, first creating any missing ancestors.

        Dovecot (and some other servers) reject CREATE for a path whose parent
        does not yet exist.  We walk the hierarchy from the root down, creating
        each missing level in turn.

        If the server returns an "already exists" error (e.g. because
        folder_exists() gave a false-negative for a namespace root), we treat
        that as success and continue rather than aborting the whole chain.
        """
        parts = folder.split(sep)
        paths = [sep.join(parts[:i]) for i in range(1, len(parts) + 1)]
        for path in paths:
            if not path:
                continue
            # INBOX is RFC-mandatory and permanently exists on every server;
            # attempting to CREATE it causes Gmail to return [ALREADYEXISTS]
            # which then aborts the whole ancestor chain.  Skip it explicitly.
            if path.upper() == "INBOX":
                logger.debug("%s: skipping CREATE for [%s] (RFC mandatory)",
                             self.side, path)
                continue
            if self.folder_exists(path):
                logger.debug("%s: folder [%s] already exists, skipping",
                             self.side, path)
                continue
            try:
                typ, data = self.imap.create(f'"{path}"')
                if typ == "OK":
                    logger.debug("%s: created folder [%s]", self.side, path)
                else:
                    # Some servers return NO/[ALREADYEXISTS] — treat as
                    # non-fatal so we can continue creating child folders.
                    resp = str(data)
                    if re.search(r"already.?exist|\[ALREADYEXISTS\]", resp, re.I):
                        logger.debug(
                            "%s: folder [%s] already exists (server said so)",
                            self.side, path)
                        continue
                    logger.warning("%s: could not create folder [%s]: %s",
                                   self.side, path, resp)
                    return False
            except Exception as exc:
                exc_str = str(exc)
                if re.search(r"already.?exist|\[ALREADYEXISTS\]", exc_str, re.I):
                    logger.debug(
                        "%s: folder [%s] already exists (exception)",
                        self.side, path)
                    continue
                # SSL EOF or "Too many protocol errors" — Gmail drops the
                # connection when too many commands are sent in quick
                # succession.  Attempt one reconnect and retry the CREATE.
                # "illegal in state LOGOUT" means the server sent BYE and
                # terminated the session — treat this as auth expiry.
                if re.search(r"illegal in state LOGOUT", exc_str, re.I):
                    raise ImapAuthExpired(
                        int(self.side[-1]) if self.side[-1].isdigit() else 2,
                        f"{self.side}: session terminated by server (BYE) — "
                        f"authentication may have expired: {exc}")
                if re.search(r"EOF|protocol error|SSL|socket|Connection closed",
                             exc_str, re.I):
                    logger.warning("%s: connection error creating [%s]: %s — "
                                   "reconnecting and retrying", self.side, path, exc)
                    rc = self.reconnect()
                    if rc == "auth_failed":
                        raise ImapAuthExpired(
                            int(self.side[-1]) if self.side[-1].isdigit() else 2,
                            f"{self.side}: re-authentication failed after reconnect — "
                            f"token may have expired")
                    if rc == "ok":
                        try:
                            typ2, data2 = self.imap.create(f'"{path}"')
                            if typ2 == "OK":
                                logger.debug("%s: created folder [%s] after reconnect",
                                             self.side, path)
                                continue
                            resp2 = str(data2)
                            if re.search(r"already.?exist|\[ALREADYEXISTS\]", resp2, re.I):
                                continue
                            logger.warning("%s: could not create folder [%s] after "
                                           "reconnect: %s", self.side, path, resp2)
                        except Exception as exc2:
                            logger.warning("%s: error creating folder [%s] after "
                                           "reconnect: %s", self.side, path, exc2)
                    return False
                logger.warning("%s: error creating folder [%s]: %s",
                               self.side, path, exc)
                return False
        return True

    def delete_folder(self, folder: str) -> bool:
        try:
            self.imap.unsubscribe(f'"{folder}"')
        except Exception:
            pass
        try:
            typ, _ = self.imap.delete(f'"{folder}"')
            return typ == "OK"
        except Exception:
            return False

    def folder_exists(self, folder: str) -> bool:
        """Return True if *folder* exists on the server.

        The original check treated any non-None list entry as proof of
        existence, but Dovecot can return ``[None]`` or ``[b'']`` for a
        namespace root folder (e.g. the parent created by --subfolder2) which
        caused a false-negative and triggered repeated, failing CREATE
        attempts.  We now require at least one non-empty response line.
        """
        try:
            typ, data = self.imap.list("", f'"{folder}"')
            if typ != "OK":
                return False
            for item in (data or []):
                if item is None:
                    continue
                if isinstance(item, bytes):
                    item = item.decode("utf-8", errors="replace")
                if item.strip():
                    return True
            return False
        except Exception:
            return False

    def ensure_selectable(self, folder: str, sep: str = "/") -> bool:
        """Ensure *folder* exists and is selectable (not \\NoSelect) on the server.

        On Dovecot (and some other servers) a freshly CREATEd folder is not
        guaranteed to have its on-disk maildir structure initialised until at
        least one message has been APPENDed to it.  This method:

          1. Creates the folder (and any missing ancestors) if it does not exist.
          2. SUBSCRIBEs it so Dovecot fully registers it in the mailbox index.
          3. If the folder is already non-empty, there is nothing more to do.
          4. Otherwise APPENDs a tiny dummy message flagged \\Deleted, then
             re-SELECTs and EXPUNGEs it so the folder ends up empty but with
             its physical storage initialised and its \\NoSelect flag cleared.
        """
        # ── 1. Create if absent ───────────────────────────────────────────────
        if not self.folder_exists(folder):
            ok = self.create_folder_with_parents(folder, sep)
            if not ok and not self.folder_exists(folder):
                logger.error("%s: ensure_selectable: failed to create [%s]",
                             self.side, folder)
                return False

        # ── 2. Subscribe ──────────────────────────────────────────────────────
        # Dovecot requires a folder to be subscribed (or at least referenced
        # via LSUB) before it is fully reflected in the mailbox list index.
        try:
            self.imap.subscribe(f'"{folder}"')
        except Exception as exc:
            logger.warning("%s: ensure_selectable: SUBSCRIBE [%s] failed: %s",
                           self.side, folder, exc)

        # ── 3. If already non-empty, nothing more to do ───────────────────────
        try:
            typ, data = self.imap.select(f'"{folder}"')
            if typ == "OK":
                exists = int(data[0]) if data and data[0] else 0
                if exists > 0:
                    logger.info("%s: folder [%s] already selectable (%d messages)",
                                self.side, folder, exists)
                    return True
        except Exception as exc:
            logger.warning("%s: ensure_selectable: SELECT [%s] failed: %s",
                           self.side, folder, exc)

        # ── 4. Append a dummy \Deleted message then expunge it ────────────────
        # This forces Dovecot to create the physical cur/new/tmp maildir
        # directories on disk and clears any \NoSelect attribute.
        try:
            now_str = email.utils.formatdate(localtime=True)
            dummy = (
                f"From: imapsync@localhost\r\n"
                f"To: imapsync@localhost\r\n"
                f"Subject: folder initialisation\r\n"
                f"Date: {now_str}\r\n"
                f"\r\n"
                f"This message was created by imapsync to initialise the folder.\r\n"
            ).encode()
            typ, resp = self.imap.append(
                f'"{folder}"', r"(\Deleted)",
                imaplib.Time2Internaldate(time.time()), dummy)
            if typ != "OK":
                logger.error(
                    "%s: ensure_selectable: APPEND to [%s] failed: %s",
                    self.side, folder, resp)
                return False
            # Re-SELECT so the server lets us EXPUNGE the \Deleted message.
            # Do NOT do UID STORE — the message was already appended \Deleted.
            self.imap.select(f'"{folder}"')
            self.imap.expunge()
            logger.info("%s: folder [%s] initialised as selectable "
                        "(dummy message appended and expunged)", self.side, folder)
            return True
        except Exception as exc:
            logger.error("%s: ensure_selectable: failed to initialise [%s]: %s",
                         self.side, folder, exc)
        return False

    def search_messages(self, search_cmd: str = "ALL") -> list:
        try:
            typ, data = self.imap.uid("search", None, search_cmd)
            if typ != "OK" or not data or not data[0]:
                return []
            return data[0].decode().split()
        except Exception as exc:
            logger.error("%s search error: %s", self.side, exc)
            return []

    def fetch_message(self, uid: str) -> Optional[bytes]:
        try:
            typ, data = self.imap.uid("fetch", uid, "(RFC822)")
            if typ != "OK":
                return None
            for part in data:
                if isinstance(part, tuple):
                    return part[1]
            return None
        except Exception:
            return None

    def fetch_flags(self, uid: str) -> str:
        try:
            typ, data = self.imap.uid("fetch", uid, "(FLAGS)")
            if typ != "OK":
                return ""
            raw = data[0].decode() if data and data[0] else ""
            m = re.search(r"FLAGS \(([^)]*)\)", raw)
            return m.group(1) if m else ""
        except Exception:
            return ""

    def fetch_internaldate(self, uid: str) -> str:
        try:
            typ, data = self.imap.uid("fetch", uid, "(INTERNALDATE)")
            if typ != "OK":
                return ""
            raw = data[0].decode() if data and data[0] else ""
            m = re.search(r'INTERNALDATE "([^"]+)"', raw)
            return m.group(1) if m else ""
        except Exception:
            return ""

    def fetch_size(self, uid: str) -> int:
        try:
            typ, data = self.imap.uid("fetch", uid, "(RFC822.SIZE)")
            if typ != "OK":
                return 0
            raw = data[0].decode() if data and data[0] else ""
            m = re.search(r"RFC822\.SIZE (\d+)", raw)
            return int(m.group(1)) if m else 0
        except Exception:
            return 0

    def fetch_headers(self, uid: str, headers: list) -> dict:
        """Fetch specific header fields for a message."""
        header_str = " ".join(headers)
        try:
            typ, data = self.imap.uid(
                "fetch", uid,
                f"(BODY.PEEK[HEADER.FIELDS ({header_str})])")
            if typ != "OK":
                return {}
            raw = b""
            for part in data:
                if isinstance(part, tuple):
                    raw = part[1]
                    break
            return self._parse_headers(raw.decode("utf-8", errors="replace"))
        except Exception:
            return {}

    @staticmethod
    def _parse_headers(raw: str) -> dict:
        result: dict = {}
        current_key = None
        for line in raw.splitlines():
            if re.match(r"^\s", line) and current_key:
                result[current_key][-1] += " " + line.strip()
            elif ":" in line:
                k, _, v = line.partition(":")
                current_key = k.strip().upper()
                result.setdefault(current_key, []).append(v.strip())
        return result

    def append_message(self, folder: str, msg_bytes: bytes,
                       flags: str = "", idate: str = "") -> tuple:
        """Append a message to a folder.

        Returns (uid_string, None) on success, or (None, reason_string) on
        failure where reason_string is the server's rejection text (e.g.
        "BAD maximum message size exceeded").  This lets the caller both
        detect failure and record the server's reason in FAILEDMESSAGES.
        """
        # Pass flags as a plain string like r"(\Seen \Answered)" — imaplib
        # imap.append() accepts either a tuple (from ParseFlags) or a string.
        # ParseFlags requires a full IMAP FETCH response line as input and
        # silently returns () for anything else, dropping all flags.  Passing
        # the string directly is the correct approach.
        flag_arg  = flags.strip() if flags and flags.strip() else None
        date_time = imaplib.Internaldate2tuple(
            f'"{idate}"'.encode()) if idate else None
        try:
            typ, data = self.imap.append(
                f'"{folder}"', flag_arg, date_time, msg_bytes)
            if typ != "OK":
                reason = ""
                if data:
                    try:
                        reason = (data[0].decode("utf-8", errors="replace")
                                  if isinstance(data[0], bytes) else str(data[0]))
                    except Exception:
                        reason = str(data)
                reason = reason.strip() or "unknown server error"
                logger.error("could not append to folder %s: %s", folder, reason)
                return None, reason
            # APPENDUID response: [APPENDUID validity uid]
            resp = data[0].decode() if data and data[0] else ""
            m = re.search(r"\[APPENDUID \d+ (\d+)\]", resp)
            return (m.group(1) if m else "OK"), None
        except Exception as exc:
            reason = str(exc).strip() or "unknown error"
            logger.error("could not append to folder %s: %s", folder, reason)
            return None, reason

    def store_flags(self, uid: str, flags: str) -> bool:
        try:
            typ, _ = self.imap.uid("store", uid, "FLAGS.SILENT", f"({flags})")
            return typ == "OK"
        except Exception:
            return False

    def expunge(self) -> None:
        try:
            self.imap.expunge()
        except Exception:
            pass

    def delete_message(self, uid: str) -> bool:
        try:
            typ, _ = self.imap.uid("store", uid, "+FLAGS.SILENT", r"(\Deleted)")
            return typ == "OK"
        except Exception:
            return False

    def get_quota(self) -> Optional[dict]:
        """Return dict with limit_bytes and current_bytes, or None."""
        try:
            typ, data = self.imap.getquotaroot("INBOX")
            if typ != "OK":
                return None
            combined = " ".join(
                (d.decode() if isinstance(d, bytes) else str(d)) for d in data)
            m_limit   = re.search(r"STORAGE\s+\d+\s+(\d+)", combined, re.I)
            m_current = re.search(r"STORAGE\s+(\d+)\s+\d+", combined, re.I)
            return {
                "limit_bytes":   int(m_limit.group(1))   * KIBI if m_limit   else 0,
                "current_bytes": int(m_current.group(1)) * KIBI if m_current else 0,
            }
        except Exception:
            return None

    def logout(self) -> None:
        try:
            if self.imap:
                self.imap.logout()
        except Exception:
            pass

    def noop(self) -> bool:
        try:
            typ, _ = self.imap.noop()
            return typ == "OK"
        except Exception:
            return False

    def reconnect(self) -> str:
        """Attempt to reconnect and re-authenticate.

        Returns:
          "ok"           — reconnected and logged in successfully
          "auth_failed"  — TCP/SSL connected but login was rejected
                           (expired token, revoked credentials)
          "connect_failed" — could not open the TCP/SSL connection at all
        """
        self.reconnect_count += 1
        logger.info("%s: reconnecting (attempt %d)", self.side, self.reconnect_count)
        try:
            self.logout()
        except Exception:
            pass
        if not self.connect():
            logger.warning("%s: reconnect failed — could not open connection",
                           self.side)
            return "connect_failed"
        if not self.login():
            logger.warning("%s: reconnect failed — authentication rejected "
                           "(token may have expired)", self.side)
            return "auth_failed"
        if self.compress:
            self.enable_compression()
        logger.info("%s: reconnected", self.side)
        return "ok"

    def enable_compression(self) -> bool:
        """Negotiate DEFLATE compression per RFC 4978 (IMAP COMPRESS extension).

        Must be called after a successful LOGIN / AUTHENTICATE and before any
        further IMAP commands.  Wraps the underlying socket with a transparent
        zlib compressor/decompressor so that all subsequent traffic is
        compressed without any changes elsewhere in the code.

        Returns True if compression was successfully enabled, False otherwise
        (non-fatal — the connection continues uncompressed).
        """
        if self.imap is None:
            return False
        try:
            import zlib

            # ── Check server capability ───────────────────────────────────────
            typ, cap_data = self.imap.capability()
            cap_str = " ".join(
                (c.decode("utf-8", errors="replace") if isinstance(c, bytes) else str(c))
                for c in (cap_data or []))
            if "COMPRESS=DEFLATE" not in cap_str.upper():
                logger.info("%s: server does not advertise COMPRESS=DEFLATE — "
                            "compression not enabled", self.side)
                return False

            # ── Send COMPRESS DEFLATE ─────────────────────────────────────────
            # imaplib.Commands is a whitelist — COMPRESS is not in it, so
            # _command() raises an error before even sending anything.
            # We register the command for the states where it is valid (AUTH
            # and SELECTED), call the proper tagged-command pair, then remove
            # our temporary entry so we don't pollute the global state.
            _injected = "COMPRESS" not in imaplib.Commands
            if _injected:
                imaplib.Commands["COMPRESS"] = ("AUTH", "SELECTED")
            try:
                tag = self.imap._command("COMPRESS", "DEFLATE")
                typ, resp = self.imap._command_complete("COMPRESS", tag)
            finally:
                if _injected:
                    del imaplib.Commands["COMPRESS"]
            if typ != "OK":
                resp_str = str(resp)
                logger.info("%s: COMPRESS DEFLATE declined by server: %s",
                            self.side, resp_str)
                return False

            # ── Wrap socket ───────────────────────────────────────────────────
            # imaplib uses self.imap.sock for writes (via sendall) and
            # self.imap.file (a makefile() wrapper) for reads.  We replace
            # both with a single _ZlibSocket instance that compresses outgoing
            # data and decompresses incoming data transparently.
            raw_sock = self.imap.sock

            class _ZlibSocket:
                """Transparent RFC 4978 zlib wrapper around a raw socket."""

                def __init__(self, s):
                    self._s     = s
                    # raw deflate (wbits=-15) as required by RFC 4978
                    self._comp   = zlib.compressobj(6, zlib.DEFLATED, -15)
                    self._decomp = zlib.decompressobj(-15)
                    self._rbuf   = b""

                # ── read-side (used by imaplib via self.file) ─────────────────

                def read(self, size: int = -1) -> bytes:
                    if size == -1:
                        raw = self._s.recv(65536)
                        return self._decomp.decompress(raw) if raw else b""
                    while len(self._rbuf) < size:
                        raw = self._s.recv(65536)
                        if not raw:
                            break
                        self._rbuf += self._decomp.decompress(raw)
                    out, self._rbuf = self._rbuf[:size], self._rbuf[size:]
                    return out

                def readline(self, size: int = -1) -> bytes:
                    while b"\n" not in self._rbuf:
                        raw = self._s.recv(65536)
                        if not raw:
                            break
                        self._rbuf += self._decomp.decompress(raw)
                    if b"\n" in self._rbuf:
                        nl = self._rbuf.index(b"\n")
                        out, self._rbuf = self._rbuf[:nl + 1], self._rbuf[nl + 1:]
                        return out
                    out, self._rbuf = self._rbuf, b""
                    return out

                # ── write-side (used by imaplib via self.sock.sendall) ────────

                def sendall(self, data: bytes) -> None:
                    compressed = (self._comp.compress(data)
                                  + self._comp.flush(zlib.Z_SYNC_FLUSH))
                    self._s.sendall(compressed)

                # imaplib calls sock.makefile('rb') during __init__ but we
                # replace file afterwards, so this is only needed for reconnect.
                def makefile(self, mode: str = "rb"):
                    return self

                def __getattr__(self, name):
                    return getattr(self._s, name)

            zs = _ZlibSocket(raw_sock)
            self.imap.sock = zs
            self.imap.file = zs
            logger.info("%s: COMPRESS DEFLATE enabled", self.side)
            return True

        except Exception as exc:
            logger.warning("%s: could not enable compression: %s", self.side, exc)
            return False


# ══════════════════════════════════════════════════════════════════════════════
# Core sync engine
# ══════════════════════════════════════════════════════════════════════════════

class ImapSync:
    """Orchestrates the one-way IMAP synchronisation."""

    def __init__(self, args):
        self.args                        = args
        self.nb_errors                   = 0
        self.errors_log: list            = []
        self.errors_max                  = args.errorsmax
        self.nb_msg_transferred          = 0
        self.nb_msg_skipped              = 0
        self.nb_msg_skipped_dry          = 0
        self.total_bytes_transferred     = 0
        self.total_bytes_skipped         = 0
        self.biggest_msg_transferred     = 0
        self.h1_nb_msg_noheader          = 0
        self.h2_nb_msg_noheader          = 0
        self.deleted_folders_h2          = 0
        self.deleted_folders_h2_fail     = 0
        self.timestart                   = time.time()
        self.conn1: Optional[ImapConnection] = None
        self.conn2: Optional[ImapConnection] = None
        self.h1_sep                      = "/"
        self.h2_sep                      = "/"
        self.h1_prefix                   = ""
        self.h2_prefix                   = ""
        self.logfile: Optional[str]      = None
        self.pidfile: Optional[str]      = None
        self.use_header                  = ["MESSAGE-ID", "RECEIVED"]
        self.permanentflags2             = ""
        self.h2_folders_of_md5: dict     = {}
        self.h1_folders_of_md5: dict     = {}
        self.begin_transfer_time         = time.time()
        self._abort_requested            = False
        self._total_h1_msgs              = 0
        self._global_msg_idx             = 0
        self.h1_subscribed: set          = set()
        self._sizes_before_text: str     = ""
        self._sizes_after_text: str      = ""

    # ── signal handling ────────────────────────────────────────────────────────

    def install_signals(self):
        def _handle_sigint(signum, frame):
            logger.info("\nReceived SIGINT – aborting.")
            self._abort_requested = True

        def _handle_sigterm(signum, frame):
            logger.info("\nReceived SIGTERM – aborting.")
            self._abort_requested = True

        try:
            signal.signal(signal.SIGINT,  _handle_sigint)
            signal.signal(signal.SIGTERM, _handle_sigterm)
        except (OSError, ValueError):
            pass  # not in main thread

    # ── error tracking ─────────────────────────────────────────────────────────

    def errors_incr(self, msg: str = ""):
        self.nb_errors += 1
        if msg:
            self.errors_log.append(msg)
            logger.error("%s", msg.rstrip())
        if self.nb_errors >= self.errors_max:
            logger.error("Maximum number of errors %d reached. Exiting.", self.errors_max)
            # Populate the after-sizes report before printing stats so it
            # appears in both the console output and the email report even
            # when the run is cut short by hitting errors_max.
            if getattr(self.args, "foldersizesatend", True) and not self._sizes_after_text:
                try:
                    wanted = self._build_wanted_list(self.conn1.list_folders())
                    self._sizes_after_text = self._print_folder_sizes(wanted, "after")
                except Exception:
                    pass
            self.do_and_print_stats()
            self.exit_clean(EXIT_WITH_ERRORS_MAX)

    # ── logging helper ─────────────────────────────────────────────────────────

    def pr(self, *args):
        logger.info(" ".join(str(a) for a in args))

    # ── credentials ───────────────────────────────────────────────────────────

    def get_password(self, which: int) -> str:
        """Return password, prompting interactively if needed and OAuth is not in use."""
        a = self.args
        n = str(which)
        using_oauth = (
            getattr(a, f"oauthaccesstoken{n}", None)
            or getattr(a, f"oauthdirect{n}", None)
            or getattr(a, f"oauthrefreshcmd{n}", None)
            or (getattr(a, f"authmech{n}", "") or "").upper() == "XOAUTH2"
        )
        if which == 1:
            pw = a.password1 or os.environ.get("IMAPSYNC_PASSWORD1", "")
            if not pw and a.passfile1:
                pw = first_line(a.passfile1)
            if not pw and not using_oauth:
                import getpass
                pw = getpass.getpass(f"Password for {a.user1}@{a.host1}: ")
            return pw
        pw = a.password2 or os.environ.get("IMAPSYNC_PASSWORD2", "")
        if not pw and a.passfile2:
            pw = first_line(a.passfile2)
        if not pw and not using_oauth:
            import getpass
            pw = getpass.getpass(f"Password for {a.user2}@{a.host2}: ")
        return pw

    # ── connection factory ─────────────────────────────────────────────────────

    def _make_conn(self, which: int) -> ImapConnection:
        a  = self.args
        n  = str(which)
        host     = sanitize_host(getattr(a, f"host{n}"))
        port     = getattr(a, f"port{n}", None)
        user     = getattr(a, f"user{n}", "")
        password = self.get_password(which)
        ssl_on   = getattr(a, f"ssl{n}", None) is True
        tls_on   = getattr(a, f"tls{n}", None) is True
        authmech = getattr(a, f"authmech{n}", "LOGIN") or "LOGIN"
        authuser = getattr(a, f"authuser{n}", None)
        domain   = getattr(a, f"domain{n}", None)
        oauth_d  = getattr(a, f"oauthdirect{n}", None)
        oauth_t  = getattr(a, f"oauthaccesstoken{n}", None)
        oauth_rc = getattr(a, f"oauthrefreshcmd{n}", None)
        debug_im = getattr(a, "debugimap", False) or getattr(a, f"debugimap{n}", False)
        if port is None:
            port = IMAP_SSL_PORT if ssl_on else IMAP_PORT
        conn = ImapConnection(
            host=host, port=port, user=user, password=password,
            ssl_on=ssl_on, starttls=tls_on,
            timeout=getattr(a, f"timeout{n}", a.timeout),
            debug_imap=debug_im,
            authmech=authmech, authuser=authuser, domain=domain,
            oauthdirect=oauth_d, oauthaccesstoken=oauth_t,
            oauthrefreshcmd=oauth_rc,
            side=f"Host{which}")
        conn.proxyauth = bool(getattr(a, f"proxyauth{which}", False))
        conn.debugssl  = bool(getattr(a, "debugssl", False))
        conn.compress  = bool(getattr(a, "compress", False)
                              or getattr(a, f"compress{which}", False))
        return conn

    # ── folder name translation ────────────────────────────────────────────────

    def imap2_folder_name(self, h1_fold: str) -> str:
        a   = self.args
        sf1 = getattr(a, "subfolder1", None)
        sf2 = getattr(a, "subfolder2", None)

        # Explicit --f1f2 mapping
        if hasattr(a, "f1f2_map") and h1_fold in a.f1f2_map:
            return a.f1f2_map[h1_fold]

        # subfolder1 stripping: remove sf1 prefix to get relative path
        if sf1:
            esc = re.escape(sf1.rstrip(self.h1_sep))
            h1_fold = re.sub(rf"^{esc}{re.escape(self.h1_sep)}?", "", h1_fold)
        if not h1_fold or h1_fold == self.h1_prefix:
            h1_fold = "INBOX"

        # Remove h1 prefix
        if self.h1_prefix and h1_fold.startswith(self.h1_prefix):
            h1_fold = h1_fold[len(self.h1_prefix):]

        # Swap separators
        h2_fold = self._separator_invert(h1_fold, self.h1_sep, self.h2_sep)

        # Apply regextrans2
        if hasattr(a, "regextrans2") and a.regextrans2:
            for rx in a.regextrans2:
                h2_fold = self._apply_regex(h2_fold, rx)

        # Add h2 prefix (unless already present)
        if self.h2_prefix and not h2_fold.startswith(self.h2_prefix):
            if h2_fold.upper() != "INBOX":
                h2_fold = self.h2_prefix + h2_fold

        # subfolder2 mapping
        if sf2:
            sf2 = sf2.rstrip(self.h2_sep)
            # Gmail and Outlook use labels rather than real folders — to make
            # sub-folders visible in IMAP clients they must be nested under
            # INBOX.  This is activated by --gmail2, --office2, or the
            # explicit --force-folders2 flag.  Do NOT infer it from the
            # separator alone, which would incorrectly prepend "INBOX/" on
            # Dovecot servers that happen to use "/" as separator.
            dest_force_folders = getattr(a, "force_folders2", False)
            if dest_force_folders and not sf2.upper().startswith("INBOX"):
                sf2 = "INBOX" + self.h2_sep + sf2

            sf1_base = sf1.rstrip(self.h1_sep) if sf1 else None
            if sf1 and sf1_base == sf2.split(self.h2_sep)[-1]:
                # sf1 name == sf2 name: contents go directly into sf2
                h2_fold = sf2 if h2_fold == "INBOX" else sf2 + self.h2_sep + h2_fold
            elif sf1:
                # sf1 != sf2: nest sf1 tree under sf2/sf1
                h2_fold = (sf2 + self.h2_sep + sf1_base
                           if h2_fold == "INBOX"
                           else sf2 + self.h2_sep + sf1_base + self.h2_sep + h2_fold)
            else:
                # subfolder2 only: INBOX maps directly to sf2; all other
                # folders nest under sf2.  This mirrors what the sf1==sf2
                # branch does and is what Gmail and Dovecot both expect —
                # the subfolder2 root IS the destination inbox.
                h2_fold = sf2 if h2_fold == "INBOX" else sf2 + self.h2_sep + h2_fold

        return h2_fold

    def _separator_invert(self, folder: str, sep1: str, sep2: str) -> str:
        if sep1 == sep2:
            if getattr(self.args, "fixslash2", True) and sep2 != "/" and sep1 == "/":
                folder = folder.replace("/", "_")
            return folder
        placeholder = "\x00"
        folder = folder.replace(sep2, placeholder)
        folder = folder.replace(sep1, sep2)
        folder = folder.replace(placeholder, sep1)
        if getattr(self.args, "fixslash2", True) and sep2 != "/" and "/" in folder:
            folder = folder.replace("/", "_")
        return folder

    @staticmethod
    def _apply_regex(s: str, pattern: str) -> str:
        try:
            m = re.match(r"^s([/,|!#@])(.+?)\1(.*?)\1([gimsxe]*)$", pattern)
            if m:
                old, new, flags_str = m.group(2), m.group(3), m.group(4)
                new = re.sub(r"\$([1-9])", r"\\\1", new)
                re_flags = 0
                if "i" in flags_str: re_flags |= re.I
                if "m" in flags_str: re_flags |= re.M
                if "s" in flags_str: re_flags |= re.S
                count = 0 if "g" in flags_str else 1
                return re.sub(old, new, s, count=count, flags=re_flags)
        except Exception:
            pass
        return s

    # ── message fingerprint ────────────────────────────────────────────────────

    def message_fingerprint(self, headers: dict, size: int) -> str:
        """Build a deduplication key from chosen headers (+ optionally size)."""
        parts = []
        for h in self.use_header:
            for v in headers.get(h, []):
                v = re.sub(r"[\x80-\xff]", "X", v)
                v = re.sub(r"\t", " ", v)
                v = re.sub(r"\s+", " ", v.strip())
                parts.append(f"{h}: {v}".upper())
        key = "".join(sorted(parts))
        md5 = hashlib.md5(key.encode("utf-8", errors="replace")).hexdigest()
        if getattr(self.args, "skipsize", True):
            return md5
        return f"{md5}:{size}"

    # ── flags helpers ──────────────────────────────────────────────────────────

    def flags_for_host2(self, flags: str) -> str:
        flags = re.sub(r"\\Recent\s?", "", flags, flags=re.I)
        if hasattr(self.args, "regexflag") and self.args.regexflag:
            for rx in self.args.regexflag:
                flags = self._apply_regex(flags, rx)
        if getattr(self.args, "flagscase", True):
            flags = self._flags_case(flags)
        if self.permanentflags2 and getattr(self.args, "filterflags", True):
            allowed = set(self.permanentflags2.split())
            flags = " ".join(f for f in flags.split() if f in allowed or not allowed)
        return flags

    @staticmethod
    def _flags_case(flags: str) -> str:
        rfc_flags = {"\\answered", "\\flagged", "\\deleted", "\\seen", "\\draft"}
        result = []
        for f in flags.split():
            if f.lower() in rfc_flags:
                result.append(f[0] + f[1].upper() + f[2:].lower())
            else:
                result.append(f)
        return " ".join(result)

    # ── stats ──────────────────────────────────────────────────────────────────

    def do_and_print_stats(self):
        timeend    = time.time()
        elapsed    = timeend - self.timestart
        elapsed_s  = max(elapsed, 1e-9)
        ts_start   = datetime.fromtimestamp(self.timestart).strftime(
            "%A %d %B %Y-%m-%d %H:%M:%S")
        ts_end     = datetime.fromtimestamp(timeend).strftime(
            "%A %d %B %Y-%m-%d %H:%M:%S")
        if FAILEDMESSAGES != FAILEDMESSAGES_HEADER:
            sys.stdout.write(FAILEDMESSAGES)
            sys.stdout.flush()
        self.pr("++++ Statistics")
        self.pr(f"Transfer started on  : {ts_start}")
        self.pr(f"Transfer ended on    : {ts_end}")
        self.pr(f"Transfer time        : {elapsed:.1f} sec")
        self.pr(f"Messages transferred : {self.nb_msg_transferred}")
        self.pr(f"Messages skipped     : {self.nb_msg_skipped}")
        self.pr(f"Messages void (h1)   : {self.h1_nb_msg_noheader}")
        self.pr(f"Messages void (h2)   : {self.h2_nb_msg_noheader}")
        self.pr(f"Folders deleted h2   : {self.deleted_folders_h2}")
        self.pr("Total bytes transferred: %d (%s)" % (
            self.total_bytes_transferred,
            bytes_display_bin(self.total_bytes_transferred)))
        self.pr("Total bytes skipped    : %d (%s)" % (
            self.total_bytes_skipped,
            bytes_display_bin(self.total_bytes_skipped)))
        self.pr("Message rate           : %.1f msg/s" % (
            self.nb_msg_transferred / elapsed_s))
        self.pr("Average bandwidth      : %.1f KiB/s" % (
            self.total_bytes_transferred / KIBI / elapsed_s))
        self.pr("Detected %d errors" % self.nb_errors)

    # ── email report ───────────────────────────────────────────────────────────

    def _send_email_report(self, conn: "ImapConnection", side: int) -> None:
        """Append a post-transfer statistics email to the INBOX of the given connection."""
        try:
            timeend   = time.time()
            elapsed   = timeend - self.timestart
            elapsed_s = max(elapsed, 1e-9)
            ts_start  = datetime.fromtimestamp(self.timestart).strftime(
                "%A %d %B %Y-%m-%d %H:%M:%S")
            ts_end    = datetime.fromtimestamp(timeend).strftime(
                "%A %d %B %Y-%m-%d %H:%M:%S")
            NL   = "\n"
            body = (
                "++++ Statistics" + NL
                + f"Transfer started on  : {ts_start}" + NL
                + f"Transfer ended on    : {ts_end}" + NL
                + f"Transfer time        : {elapsed:.1f} sec" + NL
                + f"Messages transferred : {self.nb_msg_transferred}" + NL
                + f"Messages skipped     : {self.nb_msg_skipped}" + NL
                + f"Messages void (h1)   : {self.h1_nb_msg_noheader}" + NL
                + f"Messages void (h2)   : {self.h2_nb_msg_noheader}" + NL
                + f"Folders deleted h2   : {self.deleted_folders_h2}" + NL
                + f"Total bytes transferred: {self.total_bytes_transferred}"
                + f" ({bytes_display_bin(self.total_bytes_transferred)})" + NL
                + f"Total bytes skipped    : {self.total_bytes_skipped}"
                + f" ({bytes_display_bin(self.total_bytes_skipped)})" + NL
                + f"Message rate           : {self.nb_msg_transferred / elapsed_s:.1f} msg/s" + NL
                + f"Average bandwidth      : "
                + f"{self.total_bytes_transferred / KIBI / elapsed_s:.1f} KiB/s" + NL
                + f"Detected {self.nb_errors} errors" + NL
                + (NL + self._sizes_after_text + NL if self._sizes_after_text else "")
                + (NL + FAILEDMESSAGES + NL if FAILEDMESSAGES != FAILEDMESSAGES_HEADER else "")
                + NL
                + "--" + NL
                + "Mail Transfer by MailFerry" + NL
                + "https://software.vertigan.au/MailFerry/" + NL
            )
            a         = self.args
            to_addr   = getattr(a, f"user{side}", None) or f"imapsync@host{side}"
            now_str   = email.utils.formatdate(localtime=True)
            msg_id    = email.utils.make_msgid(domain="software.vertigan.au")
            CRLF      = "\r\n"
            msg_bytes = (
                f"From: mailferry@software.vertigan.au{CRLF}"
                f"To: {to_addr}{CRLF}"
                f"Subject: Mail Transfer Report{CRLF}"
                f"Date: {now_str}{CRLF}"
                f"Message-Id: {msg_id}{CRLF}"
                f"MIME-Version: 1.0{CRLF}"
                f"Content-Type: text/plain; charset=UTF-8{CRLF}"
                f"{CRLF}"
                + body
            ).encode("utf-8", errors="replace")

            conn.select_folder("INBOX")
            try:
                conn.imap.append(
                    "INBOX",
                    None,
                    imaplib.Time2Internaldate(time.time()),
                    msg_bytes)
            except Exception as exc:
                # SSL EOF is common when the connection has been idle since
                # the end of the transfer.  Reconnect once and retry.
                logger.warning("Host%d: email report append failed (%s) — "
                               "reconnecting and retrying", side, exc)
                if conn.reconnect() == "ok":
                    conn.select_folder("INBOX")
                    conn.imap.append(
                        "INBOX",
                        None,
                        imaplib.Time2Internaldate(time.time()),
                        msg_bytes)
                else:
                    raise
            logger.info("Host%d: email report appended to INBOX", side)
        except Exception as exc:
            logger.warning("Host%d: could not append email report: %s", side, exc)

    # ── clean exit ─────────────────────────────────────────────────────────────

    def exit_clean(self, status: int, msg: str = ""):
        if msg:
            logger.info("%s", msg)
        status_txt = EXIT_TXT.get(status, str(status))
        logger.info(
            "Exiting with return value %d (%s) %d/%d nb_errors/max_errors",
            status, status_txt, self.nb_errors, self.errors_max)
        if getattr(self.args, "emailreport1", False) and self.conn1:
            self._send_email_report(self.conn1, 1)
        if getattr(self.args, "emailreport2", False) and self.conn2:
            self._send_email_report(self.conn2, 2)
        if self.conn1:
            self.conn1.logout()
        if self.conn2:
            self.conn2.logout()
        remove_pid_file(self.pidfile or "")
        if self.logfile:
            logger.info("Log file is %s", self.logfile)
        sys.exit(status)

    # ── ETA ───────────────────────────────────────────────────────────────────

    def eta(self, msg_duration: float = 0.0) -> str:
        elapsed     = max(time.time() - self.begin_transfer_time, 1e-9)
        processed   = self._global_msg_idx
        total       = self._total_h1_msgs
        if total == 0:
            return ""
        remaining_n = max(total - processed, 0)
        remaining_t = (elapsed / processed * remaining_n
                       if processed > 0 else 0.0)
        eta_dt  = datetime.fromtimestamp(time.time() + remaining_t).astimezone()
        eta_str = eta_dt.strftime("%A %d %B %Y-%m-%d %H:%M:%S").strip()
        return (f"ETA: {eta_str}  {msg_duration:.0f} s  "
                f"{remaining_n}/{total} msgs left")

    # ── folder sizes ──────────────────────────────────────────────────────────

    def folder_size(self, conn: ImapConnection, folder: str) -> tuple:
        """Return (total_bytes, nb_msgs, biggest_bytes).

        Uses a single bulk UID FETCH 1:* RFC822.SIZE command rather than one
        fetch per message — vastly faster and avoids per-message round-trip
        failures when compression is active.

        Uses SELECT (read-write) rather than EXAMINE (readonly) because on
        some Dovecot configurations with compression active, EXAMINE fails
        silently while SELECT works correctly — consistent with how the
        pre-count loop and _sync_folder work.
        """
        if not conn.select_folder(folder, readonly=False):
            logger.info("folder_size: select failed for [%s] on %s", folder, conn.side)
            return 0, 0, 0
        try:
            typ, data = conn.imap.uid("fetch", "1:*", "(RFC822.SIZE)")
            if typ != "OK" or not data or data == [None]:
                return 0, 0, 0
            sizes = []
            for item in data:
                if not item:
                    continue
                raw = item.decode("utf-8", errors="replace") if isinstance(item, bytes) else str(item)
                m = re.search(r"RFC822\.SIZE\s+(\d+)", raw)
                if m:
                    sizes.append(int(m.group(1)))
            total   = sum(sizes)
            biggest = max(sizes) if sizes else 0
            return total, len(sizes), biggest
        except Exception as exc:
            logger.info("folder_size: %s [%s] exception: %s", conn.side, folder, exc)
            return 0, 0, 0

    def _print_folder_sizes(self, wanted: list, label: str) -> str:
        """Print paired host1/host2 folder size lines and return the text block.

        *label* is either "before" or "after" — used in the header line.
        Returns the full output as a string so it can be included in email
        reports.
        """
        a      = self.args
        lines: list = []

        def emit(s: str) -> None:
            self.pr(s)
            lines.append(s)

        if label == "before":
            emit("Folders sizes before the synchronization. "
                 "It can take some time. Be patient.")
            emit('You can remove foldersizes listings by using '
                 '"--nofoldersizes" and "--nofoldersizesatend"')
            emit("but then you will also lose the ETA (Estimation Time of "
                 "Arrival) given after each message copy.")
        else:
            emit("Folders sizes after the synchronization.")
            emit('You can remove this foldersizes listing by using '
                 ' "--nofoldersizesatend"')

        t_start = time.time()
        nb      = len(wanted)

        h1_total = h1_msgs = h1_big = 0
        h2_total = h2_msgs = h2_big = 0

        for idx, h1_fold in enumerate(wanted, 1):
            h2_fold = self.imap2_folder_name(h1_fold) if self.conn2 else None

            s1, n1, b1 = self.folder_size(self.conn1, h1_fold)
            h1_total += s1; h1_msgs += n1; h1_big = max(h1_big, b1)

            h1_label = jux_utf8(h1_fold)
            emit(f"Host1 folder {idx:5}/{nb} {h1_label:40} "
                 f"Size: {s1:11} Messages: {n1:5} Biggest: {b1:11}")

            if self.conn2 and h2_fold:
                if self.conn2.select_folder(h2_fold):
                    h2_exists = True
                    s2, n2, b2 = self.folder_size(self.conn2, h2_fold)
                else:
                    h2_exists = False
                    s2, n2, b2 = 0, 0, 0
                h2_total += s2; h2_msgs += n2; h2_big = max(h2_big, b2)

                h2_label = jux_utf8(h2_fold)
                if h2_exists:
                    emit(f"Host2 folder {idx:5}/{nb} {h2_label:40} "
                         f"Size: {s2:11} Messages: {n2:5} Biggest: {b2:11}")
                else:
                    emit(f"Host2 folder {idx:5}/{nb} {h2_label:40}  does not exist yet")

                diff_s = s2 - s1
                diff_n = n2 - n1
                diff_b = b2 - b1
                emit(f"Host2-Host1 {'':<43} {diff_s:16} {diff_n:15} {diff_b:18}")

        # ── totals ────────────────────────────────────────────────────────────
        emit(f"Host1 Nb folders: {nb:>24} folders")
        if self.conn2:
            emit(f"Host2 Nb folders: {nb:>24} folders")
        emit(f"Host1 Nb messages: {h1_msgs:>23} messages")
        if self.conn2:
            emit(f"Host2 Nb messages: {h2_msgs:>23} messages")
        emit(f"Host1 Total size: {h1_total:>19} bytes ({bytes_display_bin(h1_total)})")
        if self.conn2:
            emit(f"Host2 Total size: {h2_total:>19} bytes ({bytes_display_bin(h2_total)})")
        emit(f"Host1 Biggest message: {h1_big:>14} bytes ({bytes_display_bin(h1_big)})")
        if self.conn2:
            emit(f"Host2 Biggest message: {h2_big:>14} bytes ({bytes_display_bin(h2_big)})")
        emit(f"Time spent on sizing: {time.time() - t_start:>10.1f} seconds")

        # Send NOOP on both connections to reset the server's idle timer.
        # Sizing can take many minutes; without this the connection that was
        # not being actively used during sizing (typically conn2 while conn1
        # folders are measured, or vice-versa) will be dropped by the server.
        self.conn1.noop()
        if self.conn2:
            self.conn2.noop()

        return "\n".join(lines)

    # ── separator detection ───────────────────────────────────────────────────

    def _detect_separator(self, conn: ImapConnection) -> str:
        """Detect the hierarchy separator from the IMAP LIST response.

        RFC 3501 §7.2.2: each LIST response line contains the hierarchy
        delimiter as the second field after the flags, e.g.:

            * LIST (\\HasNoChildren) "." INBOX
            * LIST (\\HasNoChildren) "." ""

        imaplib strips the leading "* LIST " so each data item looks like:
            b'(\\HasNoChildren) "." INBOX'

        We use re.search (not re.match) to tolerate minor whitespace
        variations, and try three LIST patterns in order so we always get at
        least one response line even on a completely empty mailbox.
        """
        for pattern in ("%", "INBOX", "*"):
            try:
                typ, data = conn.imap.list("", pattern)
                if typ != "OK":
                    continue
                for item in (data or []):
                    if item is None:
                        continue
                    if isinstance(item, bytes):
                        item = item.decode("utf-8", errors="replace")
                    if not item.strip():
                        continue
                    # Quoted separator:   ) "SEP" name
                    m = re.search(r'\)\s+"(.)"\s', item)
                    if m:
                        sep = m.group(1)
                        if sep and sep not in ('"', ' '):
                            logger.debug("%s: detected separator [%s] from LIST (%s)",
                                         conn.side, sep, pattern)
                            return sep
                    # NIL means flat namespace — skip this item
                    if re.search(r'\)\s+NIL\s', item, re.I):
                        continue
                    # Unquoted single-char separator (unusual but valid)
                    m = re.search(r'\)\s+([^"\s])\s', item)
                    if m:
                        sep = m.group(1)
                        if sep and sep not in ('"', ' '):
                            logger.debug("%s: detected separator [%s] (unquoted) from LIST (%s)",
                                         conn.side, sep, pattern)
                            return sep
            except Exception:
                pass
        # Fallback: heuristic count — prefer "." over "/" on a tie so that
        # Dovecot-style servers (which reject "/" in mailbox names) are handled
        # correctly when the mailbox list is too sparse to count reliably.
        folders = conn.list_folders()
        sep = self._guess_separator(folders)
        logger.debug("%s: guessed separator [%s] from folder names", conn.side, sep)
        return sep

    # ══════════════════════════════════════════════════════════════════════════
    # Main sync entry point
    # ══════════════════════════════════════════════════════════════════════════

    def run(self) -> int:
        a = self.args
        self.install_signals()

        if getattr(a, "usecache", False):
            logger.info(
                "Note: --usecache disk caching is accepted but not fully implemented; "
                "duplicate detection via fingerprint is used.")

        # ── use_header ──────────────────────────────────────────────────────
        if a.useheader:
            self.use_header = [h.upper() for h in a.useheader]

        # ── log setup ───────────────────────────────────────────────────────
        do_log = a.log if a.log is not None else True
        if do_log:
            logdir = a.logdir or DEFAULT_LOGDIR
            Path(logdir).mkdir(parents=True, exist_ok=True)
            suffix = (filter_forbidden_chars(slash_to_underscore(a.user1 or ""))
                      + "_"
                      + filter_forbidden_chars(slash_to_underscore(a.user2 or "")))
            self.logfile = a.logfile or os.path.join(
                logdir, logfile_name(self.timestart, suffix))
        setup_logging(self.logfile, debug=a.debug)

        # ── banner ──────────────────────────────────────────────────────────
        self.pr(f"MailFerry-imapsync version {VERSION} on {platform.node()} ({platform.system()})")
        self.pr("Command: " + " ".join(self._safe_argv()))
        self.pr(f"Transfer started at "
                f"{datetime.fromtimestamp(self.timestart).strftime('%A %d %B %Y-%m-%d %H:%M:%S')}")

        # ── tmpdir ──────────────────────────────────────────────────────────
        tmpdir = a.tmpdir or os.path.join(os.environ.get("HOME", "."), "tmp")
        Path(tmpdir).mkdir(parents=True, exist_ok=True)

        # ── pidfile ─────────────────────────────────────────────────────────
        self.pidfile = a.pidfile or os.path.join(tmpdir, "imapsync.pid")
        if a.pidfilelocking and os.path.exists(self.pidfile):
            logger.error("pidfile %s already exists – aborting.", self.pidfile)
            return EXIT_PID_FILE_ERROR
        write_pid_file(self.pidfile, os.getpid(), self.logfile or "")

        # ── Gmail / Office / Exchange convenience presets ────────────────────
        self._apply_easy_presets()

        # ── connect host1 ────────────────────────────────────────────────────
        self.conn1 = self._make_conn(1)
        if not self.conn1.connect():
            self.errors_incr(
                f"Host1 failure: can not open imap connection on host1 "
                f"[{a.host1}] with user [{a.user1 or ''}]")
            self.exit_clean(EXIT_CONNECTION_FAILURE_HOST1)

        # --justconnect: report connectivity and exit without logging in
        if a.justconnect:
            self.pr(f"Host1: connected to [{a.host1}]")
            if a.host2:
                self.conn2 = self._make_conn(2)
                if not self.conn2.connect():
                    self.errors_incr(
                        f"Host2 failure: can not open imap connection on host2 "
                        f"[{a.host2}] with user [{a.user2 or ''}]")
                    self.exit_clean(EXIT_CONNECTION_FAILURE_HOST2)
                self.pr(f"Host2: connected to [{a.host2}]")
            self.exit_clean(EX_OK, "Exiting because of --justconnect")

        if not self.conn1.login():
            self.errors_incr(
                f"Host1 failure: Error login on [{a.host1}] "
                f"with user [{a.user1 or ''}]")
            self.exit_clean(EXIT_AUTHENTICATION_FAILURE_USER1)
        self.pr("Host1: state Authenticated")
        if self.conn1.compress:
            self.conn1.enable_compression()

        # --justlogin with host1 only — exit here
        if a.justlogin and not a.host2:
            self.conn1.logout()
            self.exit_clean(EX_OK, "Exiting because of --justlogin")

        # --justfoldersizes with host1 only — connect, print sizes, exit
        if a.justfoldersizes and not a.host2:
            h1_all = self.conn1.list_folders()
            self.h1_sep    = self._detect_separator(self.conn1)
            self.h1_prefix = self._guess_prefix(h1_all)
            self.pr(f"Host1: separator [{self.h1_sep}] prefix [{self.h1_prefix}]")
            wanted = self._build_wanted_list(h1_all)
            self._print_folder_sizes(wanted, "before")
            self.exit_clean(EX_OK)

        if not a.host2:
            logger.error("--host2 is required")
            return EX_USAGE

        # ── connect host2 ────────────────────────────────────────────────────
        self.conn2 = self._make_conn(2)
        if not self.conn2.connect():
            self.errors_incr(
                f"Host2 failure: can not open imap connection on host2 "
                f"[{a.host2}] with user [{a.user2 or ''}]")
            self.exit_clean(EXIT_CONNECTION_FAILURE_HOST2)
        if not self.conn2.login():
            self.errors_incr(
                f"Host2 failure: Error login on [{a.host2}] "
                f"with user [{a.user2 or ''}]")
            self.exit_clean(EXIT_AUTHENTICATION_FAILURE_USER2)
        self.pr("Host2: state Authenticated")
        if self.conn2.compress:
            self.conn2.enable_compression()

        if a.justlogin:
            self.conn1.logout()
            self.conn2.logout()
            self.exit_clean(EX_OK, "Exiting because of --justlogin")

        # ── separators & prefixes ─────────────────────────────────────────────
        # Use _detect_separator (reads the RFC 3501 LIST delimiter field)
        # rather than the old heuristic _guess_separator so that Dovecot
        # servers with "." as the hierarchy separator are handled correctly
        # even when host2 has very few folders.
        h1_all = self.conn1.list_folders()
        h2_all = self.conn2.list_folders()
        self.h1_sep    = self._detect_separator(self.conn1)
        self.h2_sep    = self._detect_separator(self.conn2)
        self.h1_prefix = self._guess_prefix(h1_all)
        self.h2_prefix = self._guess_prefix(h2_all)
        if a.sep1:               self.h1_sep    = a.sep1
        if a.sep2:               self.h2_sep    = a.sep2
        if a.prefix1 is not None: self.h1_prefix = a.prefix1
        if a.prefix2 is not None: self.h2_prefix = a.prefix2
        self.pr(f"Host1: separator [{self.h1_sep}] prefix [{self.h1_prefix}]")
        self.pr(f"Host2: separator [{self.h2_sep}] prefix [{self.h2_prefix}]")

        # Ensure subfolder2 root is selectable on host2.
        # Use subfolder2 directly (stripped of trailing separator) as the root
        # folder that must exist and be selectable before the sync loop begins.
        # Gmail is the only case where an INBOX/ prefix is needed.
        if getattr(a, "subfolder2", None) and not getattr(a, "dry", False):
            sf2_root = a.subfolder2.rstrip(self.h2_sep)
            if getattr(a, "force_folders2", False) and not sf2_root.upper().startswith("INBOX"):
                sf2_root = "INBOX" + self.h2_sep + sf2_root
            self.pr(f"Host2: ensuring folder [{sf2_root}] is selectable")
            self.conn2.ensure_selectable(sf2_root, self.h2_sep)

        # ── quota check ───────────────────────────────────────────────────────
        q = self.conn2.get_quota()
        if q:
            pct = 100 * q["current_bytes"] / max(q["limit_bytes"], 1)
            self.pr(f"Host2: Quota {q['current_bytes']} / {q['limit_bytes']} bytes "
                    f"({pct:.1f}%)")
            if pct > 90:
                self.errors_incr(
                    f"Host2: {pct:.1f}% full: it is time to find a bigger place!")

        # ── fetch h1 subscriptions ────────────────────────────────────────────
        # Try LSUB with unquoted wildcard first (RFC 3501), then quoted, then
        # "%".  If every attempt fails (some Dovecot configs reject all of
        # them) fall back to treating the full LIST result as subscribed so
        # that --subscribed and per-folder subscribe logic still work.
        _lsub_patterns = ["*", '"*"', "%"]
        _lsub_ok = False
        for _pat in _lsub_patterns:
            try:
                typ, data = self.conn1.imap.lsub("", _pat)
                if typ == "OK":
                    for item in (data or []):
                        if isinstance(item, bytes):
                            item = item.decode("utf-8", errors="replace")
                        m = re.search(r'"([^"]+)"\s*$|(\S+)\s*$', item)
                        if m:
                            self.h1_subscribed.add(m.group(1) or m.group(2))
                    _lsub_ok = True
                    break
            except Exception:
                pass
        if not _lsub_ok:
            logger.warning(
                "Host1: LSUB failed for all patterns; treating all LIST folders "
                "as subscribed.")
            self.h1_subscribed = set(h1_all)

        if getattr(a, "subscribed", False):
            h1_all = [f for f in h1_all if f in self.h1_subscribed]

        # ── folder listing ────────────────────────────────────────────────────
        self.pr("++++ Listing folders")
        self.pr(f"Host1: found {len(h1_all)} folders.")
        self.pr(f"Host2: found {len(h2_all)} folders.")
        for f in h1_all:
            self.pr(f"  {jux_utf8(f)}")

        # ── wanted folders ────────────────────────────────────────────────────
        wanted = self._build_wanted_list(h1_all)

        # --justfoldersizes / --foldersizes: paired per-folder size report
        if getattr(a, "foldersizes", True) or getattr(a, "justfoldersizes", False):
            self._sizes_before_text = self._print_folder_sizes(wanted, "before")
        else:
            self._sizes_before_text = ""

        if getattr(a, "justfoldersizes", False):
            self.exit_clean(EX_OK)

        # --justfolders
        if a.justfolders:
            self.pr("++++ End (--justfolders)")
            self.do_and_print_stats()
            self.exit_clean(EX_OK)

        # ── folder loop ───────────────────────────────────────────────────────
        h1_set = set(h1_all)
        # Re-fetch host2 folder list: ensure_selectable (called above for
        # --subfolder2) may have created folders that are not yet in h2_all.
        # After a long foldersizes run the connection may have gone idle and
        # been dropped — reconnect transparently before listing.
        h2_all = self.conn2.list_folders()
        if not h2_all and self.conn2.imap:
            # Empty result on a non-empty server almost certainly means the
            # connection dropped.  Attempt one reconnect and retry.
            rc = self.conn2.reconnect()
            if rc == "auth_failed":
                self.errors_incr("Host2: re-authentication failed after idle timeout")
                self.exit_clean(EXIT_AUTHENTICATION_FAILURE_USER2)
            h2_all = self.conn2.list_folders()
        h2_set = set(h2_all)
        nb_fold = len(wanted)
        self.pr(f"++++ Looping on each one of {nb_fold} folders to sync")

        # Pre-count messages across all wanted folders for accurate eta().
        # Use SELECT (read-write) — EXAMINE + UID SEARCH is unreliable on
        # some Dovecot configurations and returns empty results.
        # _sync_folder does its own SELECT + SEARCH; the double-SELECT is
        # harmless and keeps the two code paths independent.
        search1_count = a.search1 or a.search or "ALL"
        self._total_h1_msgs = 0
        self.pr("Counting messages in all folders ...")
        for fold in wanted:
            if self.conn1.select_folder(fold):
                self._total_h1_msgs += len(
                    self.conn1.search_messages(search1_count))
        self.pr(f"Total messages to consider: {self._total_h1_msgs}")

        self._global_msg_idx     = 0
        self.begin_transfer_time = time.time()

        for fold_idx, h1_fold in enumerate(wanted, 1):
            if self._abort_requested:
                break
            h2_fold = self.imap2_folder_name(h1_fold)
            self.pr(f"Folder {fold_idx}/{nb_fold}  "
                    f"{jux_utf8(h1_fold):35} -> {jux_utf8(h2_fold)}")

            if not self.conn1.select_folder(h1_fold):
                self.errors_incr(f"Could not select: Host1 [{h1_fold}]")
                continue

            h2_folder_is_new = h2_fold not in h2_set
            if h2_folder_is_new:
                try:
                    created = self.conn2.create_folder_with_parents(h2_fold, self.h2_sep)
                except ImapAuthExpired as exc:
                    logger.error("%s", exc)
                    self.exit_clean(EXIT_AUTHENTICATION_FAILURE_USER2
                                    if exc.host == 2
                                    else EXIT_AUTHENTICATION_FAILURE_USER1)
                if not created:
                    if not self.conn2.folder_exists(h2_fold):
                        self.errors_incr(f"Could not create folder [{h2_fold}]")
                        continue
                h2_set.add(h2_fold)
                self.pr(f"Created folder [{h2_fold}] on host2")

            h1_was_subscribed = h1_fold in self.h1_subscribed
            should_subscribe  = (getattr(a, "subscribeall", False)
                                 or (getattr(a, "subscribe", True) and h1_was_subscribed))
            if should_subscribe and not getattr(a, "dry", False):
                try:
                    self.conn2.imap.subscribe(f'"{h2_fold}"')
                    if h2_folder_is_new:
                        self.pr(f"Subscribed to folder [{h2_fold}] on host2")
                except Exception as exc:
                    logger.warning("Could not subscribe to [%s]: %s", h2_fold, exc)

            if not self.conn2.select_folder(h2_fold):
                self.errors_incr(f"Could not select: Host2 [{h2_fold}]")
                continue

            self.permanentflags2 = ""
            self._sync_folder(h1_fold, h2_fold)

            if a.expunge1 and not a.dry:
                self.conn1.expunge()
            if a.expunge2 and not a.dry:
                self.conn2.expunge()

        # ── delete folders on host2 not on host1 ─────────────────────────────
        if a.delete2folders:
            h2_not_in_h1 = [f for f in h2_all if f not in h1_set]
            self._delete_folders_in_2(h2_not_in_h1)

        self.pr("++++ End looping on each folder")

        # --foldersizesatend: paired per-folder size report after transfers
        if getattr(a, "foldersizesatend", True):
            self._sizes_after_text = self._print_folder_sizes(wanted, "after")
        else:
            self._sizes_after_text = ""

        self.do_and_print_stats()

        if self.nb_errors:
            mc = most_common_error(self.errors_log)
            ev = EXIT_VALUE_OF_ERR_TYPE.get(mc, EXIT_CATCH_ALL)
            self.exit_clean(ev)
        else:
            self.exit_clean(EX_OK)
        return EX_OK  # unreachable; satisfies type checkers

    # ── folder sync ───────────────────────────────────────────────────────────

    def _sync_folder(self, h1_fold: str, h2_fold: str):
        a = self.args
        search1 = a.search1 or a.search or "ALL"
        search2 = a.search2 or a.search or "ALL"

        h1_uids = self.conn1.search_messages(search1)
        h2_uids = self.conn2.search_messages(search2)
        self.pr(f"Host1: folder [{h1_fold}] considering {len(h1_uids)} messages")
        self.pr(f"Host2: folder [{h2_fold}] considering {len(h2_uids)} messages")

        if getattr(a, "debugfolders", False):
            logger.debug("Host1 folder [%s] -> Host2 folder [%s]", h1_fold, h2_fold)
        if a.skipemptyfolders and not h1_uids:
            self.pr(f"Host1: skipping empty folder [{h1_fold}]")
            return

        # Build h2 fingerprint index
        use_hdrs = a.useheader or ["Message-Id", "Received"]
        h2_hash: dict = {}
        for uid in h2_uids:
            hdrs = self.conn2.fetch_headers(uid, use_hdrs)
            size = self.conn2.fetch_size(uid)
            fp   = self.message_fingerprint(hdrs, size)
            if fp in h2_hash and getattr(a, "delete2duplicates", False) and not a.dry:
                self.pr(f"Host2: deleting duplicate msg {h2_fold}/{uid}")
                self.conn2.delete_message(uid)
            else:
                h2_hash[fp] = uid

        h1_msgs_to_delete = []

        for uid in h1_uids:
            if self._abort_requested:
                return

            self._global_msg_idx += 1
            hdrs  = self.conn1.fetch_headers(uid, use_hdrs)
            size  = self.conn1.fetch_size(uid)
            flags = self.conn1.fetch_flags(uid)
            idate = self.conn1.fetch_internaldate(uid)

            if getattr(a, "debugflags", False):
                logger.debug("Host1 uid=%s flags=%s", uid, flags)

            fp = self.message_fingerprint(hdrs, size)

            # size filtering
            if a.maxsize and size >= a.maxsize:
                self.pr(f"msg {h1_fold}/{uid} skipped (size {size} >= maxsize {a.maxsize})")
                self.total_bytes_skipped += size
                self.nb_msg_skipped      += 1
                continue
            if a.minsize and size <= a.minsize:
                self.pr(f"msg {h1_fold}/{uid} skipped (size {size} <= minsize {a.minsize})")
                self.total_bytes_skipped += size
                self.nb_msg_skipped      += 1
                continue

            # age filtering
            if a.maxage or a.minage:
                idate_str = self.conn1.fetch_internaldate(uid)
                try:
                    msg_ts   = email.utils.parsedate_to_datetime(
                        idate_str).timestamp() if idate_str else 0.0
                    age_days = (time.time() - msg_ts) / NB_SECONDS_IN_A_DAY
                    if a.maxage and age_days > a.maxage:
                        self.pr(f"msg {h1_fold}/{uid} skipped "
                                f"(age {age_days:.1f}d > maxage {a.maxage}d)")
                        self.nb_msg_skipped += 1
                        continue
                    if a.minage and age_days < a.minage:
                        self.pr(f"msg {h1_fold}/{uid} skipped "
                                f"(age {age_days:.1f}d < minage {a.minage}d)")
                        self.nb_msg_skipped += 1
                        continue
                except Exception:
                    pass

            # cross-duplicate check
            if a.skipcrossduplicates and fp in self.h2_folders_of_md5:
                self.pr(f"msg {h1_fold}/{uid} skipped (cross-duplicate)")
                self.total_bytes_skipped += size
                self.nb_msg_skipped      += 1
                continue

            if fp in h2_hash and not getattr(a, "syncduplicates", False):
                h2_uid = h2_hash[fp]
                if a.resyncflags and not a.dry:
                    target_flags = self.flags_for_host2(flags)
                    h2_flags     = self.conn2.fetch_flags(h2_uid)
                    if set(target_flags.split()) != set(h2_flags.split()):
                        self.conn2.store_flags(h2_uid, target_flags)
                self.total_bytes_skipped += size
                self.nb_msg_skipped      += 1
                self.h2_folders_of_md5.setdefault(fp, {})[h2_fold] = 1
                continue

            # transfer the message
            if not a.dry:
                msg_start = time.time()
                raw = self.conn1.fetch_message(uid)
                if getattr(a, "debugcontent", False) and raw:
                    logger.debug("Host1 uid=%s content preview: %s",
                                 uid, raw[:200].decode("utf-8", errors="replace"))
                if raw is None:
                    self.errors_incr(f"- msg {h1_fold}/{uid} could not be fetched")
                    continue
                raw = self._transform_message(raw, uid, h1_fold)
                if raw is None:
                    continue
                if a.truncmess:
                    raw = raw[:a.truncmess]
                target_flags = self.flags_for_host2(flags)
                target_idate = idate if a.syncinternaldates else ""
                new_uid, append_err = self.conn2.append_message(
                    h2_fold, raw, target_flags, target_idate)
                if new_uid is None:
                    global FAILEDMESSAGES
                    # Fetch From/Subject/Date for the failure record
                    meta_hdrs = self.conn1.fetch_headers(
                        uid, ["From", "Subject", "Date"])
                    from_h    = (meta_hdrs.get("FROM",    [""])[0]).strip()
                    subj_h    = (meta_hdrs.get("SUBJECT", [""])[0]).strip()
                    date_h    = (meta_hdrs.get("DATE",    [""])[0]).strip()
                    real_size = len(raw)
                    FAILEDMESSAGES += (
                        f"msg {h1_fold}/{uid} {{{real_size}}} to {h2_fold}"
                        f" (From: {from_h}; Subject: {subj_h}; Date: {date_h})\r\n"
                        f"{append_err}\r\n"
                    )
                    self.errors_incr(
                        f"could not append msg {h1_fold}/{uid} "
                        f"({bytes_display_dec(real_size)}) to {h2_fold}: {append_err}")
                    continue
                real_size = len(raw)
                self.total_bytes_transferred += real_size
                self.nb_msg_transferred      += 1
                self.biggest_msg_transferred  = max(
                    self.biggest_msg_transferred, real_size)
                h2_hash[fp] = new_uid
                self.h2_folders_of_md5.setdefault(fp, {})[h2_fold] = 1
                elapsed      = max(time.time() - self.begin_transfer_time, 1e-9)
                msg_duration = time.time() - msg_start
                msgs_per_s   = self.nb_msg_transferred / elapsed
                kib_per_s    = self.total_bytes_transferred / 1024 / elapsed
                kib_copied   = self.total_bytes_transferred / 1024
                self.pr(
                    f"msg {h1_fold}/{uid} {{{real_size}}}"
                    f"\t to {h2_fold}/{new_uid}"
                    f"\t {msgs_per_s:.2f} msgs/s"
                    f"  {kib_per_s:.3f} KiB/s"
                    f"  {kib_copied:.3f} KiB copied"
                    f"  {self.eta(msg_duration)}"
                )
                self._sleep_if_needed()
                if a.delete1:
                    h1_msgs_to_delete.append(uid)
                if a.exitwhenover and self.total_bytes_transferred >= a.exitwhenover:
                    self.errors_incr(
                        f"Maximum bytes transferred reached, "
                        f"{self.total_bytes_transferred} >= {a.exitwhenover}, ending sync")
                    return
            else:
                self.pr(f"msg {h1_fold}/{uid} copying to {h2_fold} "
                        f"(not really – dry mode)")
                self.nb_msg_skipped_dry += 1

        # delete2: remove host2 messages absent from host1
        if a.delete2:
            h1_fps = set()
            for uid in h1_uids:
                hdrs = self.conn1.fetch_headers(uid, use_hdrs)
                size = self.conn1.fetch_size(uid)
                h1_fps.add(self.message_fingerprint(hdrs, size))
            for fp, h2_uid in list(h2_hash.items()):
                if fp not in h1_fps:
                    self.pr(f"Host2: msg {h2_fold}/{h2_uid} marked \\Deleted")
                    if not a.dry:
                        self.conn2.delete_message(h2_uid)
            if not a.dry:
                self.conn2.expunge()

        # delete1 after successful transfer
        if a.delete1 and h1_msgs_to_delete and not a.dry:
            for uid in h1_msgs_to_delete:
                self.conn1.delete_message(uid)
            if a.expunge1:
                self.conn1.expunge()

        # delete1emptyfolders
        if getattr(a, "delete1emptyfolders", False) and not a.dry:
            remaining = self.conn1.search_messages("ALL")
            if not remaining:
                try:
                    self.conn1.imap.delete(f'"{h1_fold}"')
                    self.pr(f"Host1: deleted empty folder [{h1_fold}]")
                except Exception as exc:
                    logger.warning(
                        "Host1: could not delete folder [%s]: %s", h1_fold, exc)

    # ── message transformation ────────────────────────────────────────────────

    def _transform_message(self, raw: bytes, uid: str, folder: str) -> Optional[bytes]:
        a = self.args

        # skipmess patterns
        if a.skipmess:
            text = raw.decode("utf-8", errors="replace")
            for pattern in a.skipmess:
                try:
                    if re.search(pattern, text):
                        self.pr(f"msg {folder}/{uid} skipped by --skipmess")
                        self.nb_msg_skipped += 1
                        return None
                except re.error:
                    pass

        # regexmess patterns
        if a.regexmess:
            text = raw.decode("utf-8", errors="replace")
            for pattern in a.regexmess:
                text = self._apply_regex(text, pattern)
                if text is None:
                    return None
            raw = text.encode("utf-8", errors="replace")

        # pipemess
        if a.pipemess:
            for cmd in a.pipemess:
                try:
                    result = subprocess.run(
                        cmd, shell=True, input=raw,
                        capture_output=True, timeout=60)
                    if result.returncode != 0 or not result.stdout:
                        logger.error("pipemess command failed: %s", cmd)
                        return None
                    raw = result.stdout
                except Exception as exc:
                    logger.error("pipemess error %s: %s", cmd, exc)
                    return None

        # maxlinelength: wrap excessively long lines
        if getattr(a, "maxlinelength", None):
            try:
                import textwrap as _tw
                text  = raw.decode("utf-8", errors="replace")
                lines = text.splitlines(keepends=True)
                out   = []
                for line in lines:
                    stripped = line.rstrip("\r\n")
                    ending   = line[len(stripped):]
                    if len(stripped) > a.maxlinelength:
                        wrapped = _tw.wrap(stripped, a.maxlinelength) or [""]
                        out.append("\n".join(wrapped) + ending)
                    else:
                        out.append(line)
                raw = "".join(out).encode("utf-8", errors="replace")
            except Exception:
                pass

        return raw

    # ── rate limiting ─────────────────────────────────────────────────────────

    def _sleep_if_needed(self):
        a        = self.args
        maxsleep = getattr(a, "maxsleep", MAX_SLEEP) or MAX_SLEEP
        elapsed  = max(time.time() - self.begin_transfer_time, 1e-9)
        to_sleep = 0.0
        if a.maxmessagespersecond:
            to_sleep = max(
                to_sleep,
                (self.nb_msg_transferred / a.maxmessagespersecond) - elapsed)
        if a.maxbytespersecond:
            transfer_above = max(
                0, self.total_bytes_transferred - (a.maxbytesafter or 0))
            to_sleep = max(
                to_sleep,
                (transfer_above / a.maxbytespersecond) - elapsed)
        to_sleep = min(to_sleep, maxsleep)
        if to_sleep > 0:
            logger.info("sleeping %.2fs", to_sleep)
            time.sleep(to_sleep)

    # ── wanted folder list ────────────────────────────────────────────────────

    def _build_wanted_list(self, h1_all: list) -> list:
        a      = self.args
        h1_set = set(h1_all)
        requested: set = set()

        if a.folder:
            requested.update(f for f in a.folder if f in h1_set)

        if a.folderrec:
            for base in a.folderrec:
                requested.update(
                    f for f in h1_all
                    if f == base or f.startswith(base + self.h1_sep))

        if not a.folder and not a.folderrec:
            requested = h1_set.copy()

        if a.include:
            extra: set = set()
            for pattern in a.include:
                extra.update(f for f in h1_all if re.search(pattern, f))
            requested |= extra

        if a.exclude:
            for pattern in a.exclude:
                requested = {f for f in requested if not re.search(pattern, f)}

        if getattr(a, "checkfoldersexist", True):
            requested = {f for f in requested if f in h1_set}

        if a.subfolder1:
            sf      = a.subfolder1.rstrip(self.h1_sep)
            prefix  = self.h1_prefix + sf
            requested = {
                f for f in requested
                if f == prefix or f.startswith(prefix + self.h1_sep)
            }
            if not requested:
                logger.error("subfolder1 %s does not exist on host1", a.subfolder1)
                self.exit_clean(EXIT_SUBFOLDER1_NO_EXISTS)

        first  = [f for f in (a.folderfirst or []) if f in requested]
        last   = [f for f in (a.folderlast  or []) if f in requested]
        mid    = sorted(f for f in requested if f not in first and f not in last)
        wanted = first + mid + last

        self.pr(f"++++ {len(wanted)} folders to sync")
        return wanted

    # ── delete host2 folders not in host1 ─────────────────────────────────────

    def _delete_folders_in_2(self, folders: list):
        a = self.args
        self.pr("Entering delete_folders_in_2_not_in_1()")
        for folder in folders:
            if folder.upper() == "INBOX":
                self.pr(f"Not deleting {folder} (RFC mandatory)")
                continue
            only = getattr(a, "delete2foldersonly", None)
            if only:
                try:
                    if not re.search(only.strip("/"), folder):
                        self.pr(f"Not deleting {folder} (--delete2foldersonly)")
                        continue
                except re.error:
                    pass
            butnot = getattr(a, "delete2foldersbutnot", None)
            if butnot:
                try:
                    if re.search(butnot.strip("/"), folder):
                        self.pr(f"Not deleting {folder} (--delete2foldersbutnot)")
                        continue
                except re.error:
                    pass
            self.pr(f"Deleting folder [{folder}] on host2")
            if not a.dry:
                if self.conn2.delete_folder(folder):
                    self.deleted_folders_h2 += 1
                else:
                    self.deleted_folders_h2_fail += 1

    # ── separator / prefix guessing ───────────────────────────────────────────

    @staticmethod
    def _guess_separator(folders: list) -> str:
        """Heuristic fallback: count separator characters across folder names.

        On a tie (e.g. empty or single-folder mailbox) we prefer "." over "/"
        because Dovecot — the most common self-hosted IMAP server — uses "."
        and will reject mailbox names that contain "/" characters.
        """
        dot   = sum(f.count(".") for f in folders)
        slash = sum(f.count("/") for f in folders)
        # Strict greater-than so ties go to "."
        return "/" if slash > dot else "."

    @staticmethod
    def _guess_prefix(folders: list) -> str:
        for f in folders:
            if f.upper() == "INBOX":
                continue
            if not f.upper().startswith("INBOX"):
                return ""
            m = re.match(r"(?i)(INBOX[./])", f)
            if m:
                return m.group(1)
        return ""

    # ── Gmail / Office / Exchange presets ─────────────────────────────────────

    def _apply_easy_presets(self):
        a = self.args
        if a.gmail1 or a.gmail2:
            if not a.host1 and a.gmail1: a.host1 = "imap.gmail.com"
            if not a.host2 and a.gmail2: a.host2 = "imap.gmail.com"
            if a.ssl1 is None and a.gmail1: a.ssl1 = True
            if a.ssl2 is None and a.gmail2: a.ssl2 = True
            if not a.maxbytespersecond:   a.maxbytespersecond = 300_000
            if not a.maxbytesafter:       a.maxbytesafter     = 3_000_000_000
            if a.automap is None:         a.automap           = True
            if a.idatefromheader is None: a.idatefromheader   = True
            if not a.useheader:           a.useheader = ["X-Gmail-Received", "Message-Id"]
            if not a.regextrans2:         a.regextrans2 = []
            a.regextrans2.append(r"s,\[Gmail\]\.,,")
            if not a.exclude:             a.exclude = []
            a.exclude.append(r"\[Gmail\]$")
            if not a.folderlast:          a.folderlast = []
            a.folderlast.extend([
                "[Gmail]/Sent Mail", "[Gmail]/Important", "[Gmail]/Starred",
                "[Gmail]/Drafts", "[Gmail]/Trash", "[Gmail]/Spam",
                "[Gmail]/Chats", "[Gmail]/All Mail"])

        if a.office1:
            if not a.host1: a.host1 = "outlook.office365.com"
            if a.ssl1 is None: a.ssl1 = True
            if not a.exclude: a.exclude = []
            a.exclude.append("^Files$")

        if a.office2:
            if not a.host2: a.host2 = "outlook.office365.com"
            if a.ssl2 is None: a.ssl2 = True
            if not a.maxsize: a.maxsize = 45_000_000
            if not a.maxmessagespersecond: a.maxmessagespersecond = 4
            if a.disarmreadreceipts is None: a.disarmreadreceipts = True

        # OAuth always requires SSL
        for n in (1, 2):
            using_oauth = (
                getattr(a, f"oauthaccesstoken{n}", None)
                or getattr(a, f"oauthdirect{n}", None)
                or getattr(a, f"oauthrefreshcmd{n}", None)
                or (getattr(a, f"authmech{n}", "") or "").upper() == "XOAUTH2"
            )
            if using_oauth and getattr(a, f"ssl{n}") is None:
                setattr(a, f"ssl{n}", True)
                logger.debug("Host%d: OAuth detected, defaulting to SSL (port %d)",
                             n, IMAP_SSL_PORT)

    # ── safe argv (mask passwords) ─────────────────────────────────────────────

    def _safe_argv(self) -> list:
        argv = sys.argv[:]
        for flag in ("--password1", "--password2"):
            try:
                i = argv.index(flag)
                if i + 1 < len(argv):
                    argv[i + 1] = "MASKED"
            except ValueError:
                pass
        return argv


# ══════════════════════════════════════════════════════════════════════════════
# CLI argument parser
# ══════════════════════════════════════════════════════════════════════════════

def build_parser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(
        prog="imapsync",
        description="Email tool for syncing two IMAP accounts, one way, without duplicates.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=f"imapsync version 2.316 (compatible; MailFerry {VERSION}  https://software.vertigan.au/MailFerry/")

    # ── credentials ────────────────────────────────────────────────────────────
    cred = p.add_argument_group("Credentials")
    cred.add_argument("--host1",      metavar="str")
    cred.add_argument("--port1",      metavar="int", type=int)
    cred.add_argument("--user1",      metavar="str")
    cred.add_argument("--password1",  metavar="str")
    cred.add_argument("--passfile1",  metavar="str")
    cred.add_argument("--host2",      metavar="str")
    cred.add_argument("--port2",      metavar="int", type=int)
    cred.add_argument("--user2",      metavar="str")
    cred.add_argument("--password2",  metavar="str")
    cred.add_argument("--passfile2",  metavar="str")
    cred.add_argument("--showpasswords", action="store_true")

    # ── encryption ─────────────────────────────────────────────────────────────
    enc  = p.add_argument_group("Encryption")
    ssl1 = enc.add_mutually_exclusive_group()
    ssl1.add_argument("--ssl1",   dest="ssl1", action="store_true",  default=None)
    ssl1.add_argument("--nossl1", dest="ssl1", action="store_false")
    ssl2 = enc.add_mutually_exclusive_group()
    ssl2.add_argument("--ssl2",   dest="ssl2", action="store_true",  default=None)
    ssl2.add_argument("--nossl2", dest="ssl2", action="store_false")
    tls1 = enc.add_mutually_exclusive_group()
    tls1.add_argument("--tls1",   dest="tls1", action="store_true",  default=None)
    tls1.add_argument("--notls1", dest="tls1", action="store_false")
    tls2 = enc.add_mutually_exclusive_group()
    tls2.add_argument("--tls2",   dest="tls2", action="store_true",  default=None)
    tls2.add_argument("--notls2", dest="tls2", action="store_false")
    enc.add_argument("--debugssl", metavar="int", type=int, default=1)

    # ── authentication ─────────────────────────────────────────────────────────
    auth = p.add_argument_group("Authentication")
    auth.add_argument("--authmech1",         metavar="str", default="LOGIN",
                      help="Auth mechanism for host1: LOGIN, PLAIN, CRAM-MD5, XOAUTH2")
    auth.add_argument("--authmech2",         metavar="str", default="LOGIN",
                      help="Auth mechanism for host2: LOGIN, PLAIN, CRAM-MD5, XOAUTH2")
    auth.add_argument("--authuser1",         metavar="str")
    auth.add_argument("--authuser2",         metavar="str")
    auth.add_argument("--proxyauth1",        action="store_true")
    auth.add_argument("--proxyauth2",        action="store_true")
    auth.add_argument("--domain1",           metavar="str")
    auth.add_argument("--domain2",           metavar="str")
    auth.add_argument("--oauthdirect1",      metavar="str")
    auth.add_argument("--oauthdirect2",      metavar="str")
    auth.add_argument("--oauthaccesstoken1", metavar="str")
    auth.add_argument("--oauthaccesstoken2", metavar="str")
    auth.add_argument("--oauthrefreshcmd1",  metavar="cmd")
    auth.add_argument("--oauthrefreshcmd2",  metavar="cmd")

    # ── folders ────────────────────────────────────────────────────────────────
    fld = p.add_argument_group("Folders")
    fld.add_argument("--folder",        metavar="str", action="append")
    fld.add_argument("--folderrec",     metavar="str", action="append")
    fld.add_argument("--folderfirst",   metavar="str", action="append")
    fld.add_argument("--folderlast",    metavar="str", action="append")
    fld.add_argument("--include",       metavar="reg", action="append")
    fld.add_argument("--exclude",       metavar="reg", action="append")
    fld.add_argument("--noexclude",     action="store_true")
    fld.add_argument("--f1f2",          metavar="s1=s2", action="append",
                     help="Map host1 folder to host2 folder. Repeatable.")
    fld.add_argument("--automap",       action="store_true",  default=None)
    fld.add_argument("--noautomap",     dest="automap",        action="store_false")
    fld.add_argument("--subfolder1",    metavar="str")
    fld.add_argument("--subfolder2",    metavar="str")
    fld.add_argument("--force-folders",  dest="force_folders",  action="store_true",
                     default=False,
                     help="Nest all folders under INBOX on host2 (for Gmail/Outlook labels).")
    fld.add_argument("--force-folders2", dest="force_folders2", action="store_true",
                     default=False,
                     help="Alias for --force-folders.")
    fld.add_argument("--subscribed",    action="store_true")
    fld.add_argument("--subscribe",     action="store_true",  default=True)
    fld.add_argument("--nosubscribe",   dest="subscribe",     action="store_false")
    fld.add_argument("--subscribeall",  action="store_true")
    fld.add_argument("--sep1",          metavar="str")
    fld.add_argument("--sep2",          metavar="str")
    fld.add_argument("--prefix1",       metavar="str")
    fld.add_argument("--prefix2",       metavar="str")
    fld.add_argument("--regextrans2",   metavar="reg", action="append")
    fld.add_argument("--skipemptyfolders",    action="store_true")
    fld.add_argument("--noskipemptyfolders",  dest="skipemptyfolders", action="store_false")
    fld.add_argument("--nomixfolders",        action="store_true")
    fld.add_argument("--checkfoldersexist",   action="store_true",  default=True)
    fld.add_argument("--nocheckfoldersexist", dest="checkfoldersexist", action="store_false")

    # ── folder sizes ───────────────────────────────────────────────────────────
    fsz = p.add_argument_group("Folder sizes")
    fsz.add_argument("--foldersizes",          action="store_true",  default=True)
    fsz.add_argument("--nofoldersizes",        dest="foldersizes",      action="store_false")
    fsz.add_argument("--foldersizesatend",     action="store_true",  default=True)
    fsz.add_argument("--nofoldersizesatend",   dest="foldersizesatend", action="store_false")
    fsz.add_argument("--justfoldersizes",      action="store_true")

    # ── messages ───────────────────────────────────────────────────────────────
    msg = p.add_argument_group("Messages")
    msg.add_argument("--skipmess",      metavar="reg", action="append")
    msg.add_argument("--regexmess",     metavar="reg", action="append")
    msg.add_argument("--pipemess",      metavar="cmd", action="append")
    msg.add_argument("--disarmreadreceipts",    action="store_true",  default=None)
    msg.add_argument("--nodisarmreadreceipts",  dest="disarmreadreceipts",
                     action="store_false")
    msg.add_argument("--truncmess",     metavar="int", type=int)
    msg.add_argument("--addheader",     action="store_true",  default=False)
    msg.add_argument("--noaddheader",   dest="addheader",     action="store_false")
    msg.add_argument("--nodry1",        action="store_true")
    msg.add_argument("--maxlinelength", metavar="int", type=int)

    # ── flags ──────────────────────────────────────────────────────────────────
    flg = p.add_argument_group("Flags")
    flg.add_argument("--regexflag",     metavar="reg", action="append")
    flg.add_argument("--resyncflags",     action="store_true",  default=True)
    flg.add_argument("--noresyncflags",   dest="resyncflags",   action="store_false")
    flg.add_argument("--filterbuggyflags",  action="store_true")
    flg.add_argument("--flagscase",       action="store_true",  default=True)
    flg.add_argument("--noflagscase",     dest="flagscase",     action="store_false")
    flg.add_argument("--filterflags",     action="store_true",  default=True)
    flg.add_argument("--nofilterflags",   dest="filterflags",   action="store_false")

    # ── deletions ──────────────────────────────────────────────────────────────
    dl = p.add_argument_group("Deletions")
    dl.add_argument("--delete1",                action="store_true")
    dl.add_argument("--expunge1",               action="store_true")
    dl.add_argument("--noexpunge1",             dest="expunge1",  action="store_false")
    dl.add_argument("--delete1emptyfolders",    action="store_true")
    dl.add_argument("--delete2",                action="store_true")
    dl.add_argument("--delete2duplicates",      action="store_true")
    dl.add_argument("--delete2folders",         action="store_true")
    dl.add_argument("--delete2foldersonly",     metavar="reg")
    dl.add_argument("--delete2foldersbutnot",   metavar="reg")
    dl.add_argument("--expunge2",               action="store_true")
    dl.add_argument("--noexpunge2",             dest="expunge2",  action="store_false")
    dl.add_argument("--nouidexpunge2",          action="store_true")

    # ── dates ──────────────────────────────────────────────────────────────────
    dt = p.add_argument_group("Dates")
    dt.add_argument("--syncinternaldates",    action="store_true",  default=True)
    dt.add_argument("--nosyncinternaldates",  dest="syncinternaldates",
                    action="store_false")
    dt.add_argument("--idatefromheader",      action="store_true",  default=None)

    # ── message selection ──────────────────────────────────────────────────────
    sel = p.add_argument_group("Message selection")
    sel.add_argument("--maxsize",    metavar="int", type=int)
    sel.add_argument("--minsize",    metavar="int", type=int)
    sel.add_argument("--maxage",     metavar="int", type=float)
    sel.add_argument("--minage",     metavar="int", type=float)
    sel.add_argument("--search",     metavar="str")
    sel.add_argument("--search1",    metavar="str")
    sel.add_argument("--search2",    metavar="str")
    sel.add_argument("--useheader",           metavar="str", action="append")
    sel.add_argument("--syncduplicates",      action="store_true")
    sel.add_argument("--skipcrossduplicates", action="store_true")
    sel.add_argument("--debugcrossduplicates",action="store_true")
    sel.add_argument("--skipsize",    action="store_true",  default=True)
    sel.add_argument("--noskipsize",  dest="skipsize",      action="store_false")
    sel.add_argument("--usecache",    action="store_true")
    sel.add_argument("--useuid",      action="store_true")

    # ── specific server presets ────────────────────────────────────────────────
    sp = p.add_argument_group("Server presets")
    sp.add_argument("--gmail1",    action="store_true")
    sp.add_argument("--gmail2",    action="store_true")
    sp.add_argument("--office1",   action="store_true")
    sp.add_argument("--office2",   action="store_true")
    sp.add_argument("--exchange1", action="store_true")
    sp.add_argument("--exchange2", action="store_true")
    sp.add_argument("--domino1",   action="store_true")
    sp.add_argument("--domino2",   action="store_true")

    # ── behaviour ─────────────────────────────────────────────────────────────
    beh = p.add_argument_group("Behaviour")
    beh.add_argument("--timeout",  metavar="flo", type=float, default=DEFAULT_TIMEOUT)
    beh.add_argument("--timeout1", metavar="flo", type=float)
    beh.add_argument("--timeout2", metavar="flo", type=float)
    beh.add_argument("--keepalive1",     action="store_true",  default=True)
    beh.add_argument("--nokeepalive1",   dest="keepalive1",    action="store_false")
    beh.add_argument("--keepalive2",     action="store_true",  default=True)
    beh.add_argument("--nokeepalive2",   dest="keepalive2",    action="store_false")
    beh.add_argument("--maxmessagespersecond", metavar="flo", type=float, default=0)
    beh.add_argument("--maxbytespersecond",    metavar="int", type=int,   default=0)
    beh.add_argument("--maxbytesafter",        metavar="int", type=int,   default=0)
    beh.add_argument("--maxsleep",             metavar="flo", type=float, default=MAX_SLEEP)
    beh.add_argument("--exitwhenover",         metavar="int", type=int)
    beh.add_argument("--fixslash2",      action="store_true",  default=True)
    beh.add_argument("--nofixslash2",    dest="fixslash2",     action="store_false")
    beh.add_argument("--fixInboxINBOX",  action="store_true",  default=True)
    beh.add_argument("--nofixInboxINBOX",dest="fixInboxINBOX", action="store_false")
    beh.add_argument("--allowsizemismatch", action="store_true")
    beh.add_argument("--syncacls",   action="store_true")
    beh.add_argument("--nosyncacls", dest="syncacls", action="store_false")
    beh.add_argument("--compress",  "--compression",  action="store_true", default=False,
                     help="Enable IMAP COMPRESS DEFLATE (RFC 4978) on both hosts.")
    beh.add_argument("--compress1", "--compression1", action="store_true", default=False,
                     help="Enable IMAP COMPRESS DEFLATE (RFC 4978) on host1 only.")
    beh.add_argument("--compress2", "--compression2", action="store_true", default=False,
                     help="Enable IMAP COMPRESS DEFLATE (RFC 4978) on host2 only.")

    # ── debug ──────────────────────────────────────────────────────────────────
    dbg = p.add_argument_group("Debugging")
    dbg.add_argument("--debug",        action="store_true")
    dbg.add_argument("--debugfolders", action="store_true")
    dbg.add_argument("--debugcontent", action="store_true")
    dbg.add_argument("--debugflags",   action="store_true")
    dbg.add_argument("--debugimap",    action="store_true")
    dbg.add_argument("--debugimap1",   action="store_true")
    dbg.add_argument("--debugimap2",   action="store_true")
    dbg.add_argument("--debugmemory",  action="store_true")
    dbg.add_argument("--errorsmax",    metavar="int", type=int, default=ERRORS_MAX)

    # ── logging ────────────────────────────────────────────────────────────────
    lg = p.add_argument_group("Logging")
    lg.add_argument("--log",    dest="log", action="store_true",  default=None)
    lg.add_argument("--nolog",  dest="log", action="store_false")
    lg.add_argument("--logfile",  metavar="str")
    lg.add_argument("--logdir",   metavar="str")

    # ── tmpdir / pidfile ───────────────────────────────────────────────────────
    tmp = p.add_argument_group("Temp / PID")
    tmp.add_argument("--tmpdir",         metavar="str")
    tmp.add_argument("--pidfile",        metavar="str")
    tmp.add_argument("--pidfilelocking", action="store_true")

    # ── misc ───────────────────────────────────────────────────────────────────
    msc = p.add_argument_group("Miscellaneous")
    msc.add_argument("--dry",          action="store_true",
                     help="Simulate only; do not transfer or delete anything.")
    msc.add_argument("--justfolders",  action="store_true")
    msc.add_argument("--justlogin",    action="store_true")
    msc.add_argument("--justconnect",  action="store_true")
    msc.add_argument("--emailreport1", action="store_true",
                     help="Append the sync report as an email to host1 INBOX after the run.")
    msc.add_argument("--emailreport2", action="store_true",
                     help="Append the sync report as an email to host2 INBOX after the run.")
    msc.add_argument("--version",      action="store_true")
    msc.add_argument("--releasecheck",            action="store_true",  default=False)
    msc.add_argument("--noreleasecheck",          dest="releasecheck",  action="store_false")
    msc.add_argument("--noid",                    action="store_true")
    msc.add_argument("--noreleasecheck_already_set", dest="releasecheck",
                     action="store_false", default=argparse.SUPPRESS)

    return p


def post_process_args(args: argparse.Namespace) -> argparse.Namespace:
    """Derive secondary fields from raw parsed args."""

    # --f1f2 list -> dict
    args.f1f2_map: dict = {}
    if args.f1f2:
        for item in args.f1f2:
            if "=" in item:
                k, _, v = item.partition("=")
                args.f1f2_map[k.strip()] = v.strip()

    # --force-folders sets force_folders2 (destination only)
    if getattr(args, "force_folders", False):
        args.force_folders2 = True

    # filterbuggyflags -> prepend regexflag entries
    if getattr(args, "filterbuggyflags", False):
        buggy_rx = (
            r"s/\\RECEIPTCHECKED|\\JUNK|\\Indexed|\\X-EON-HAS-ATTACHMENT"
            r"|\\UNSEEN|\\ATTACHED|\\X-HAS-ATTACH|\\FORWARDED|\\FORWARD"
            r"|\\X-FORWARDED|\\\$FORWARDED|\\PRIORITY|\\READRCPT//gi"
        )
        if not args.regexflag:
            args.regexflag = []
        args.regexflag.insert(0, buggy_rx)

    # disarmreadreceipts -> prepend regexmess
    if args.disarmreadreceipts:
        rx = (r"s{\A((?:[^\n]+\r\n)+|)"
              r"(^Disposition-Notification-To:[^\n]*\n)"
              r"(\r?\n|.*\n\r?\n)}{$1X-$2$3}ims")
        if not args.regexmess:
            args.regexmess = []
        args.regexmess.insert(0, rx)

    return args


# ══════════════════════════════════════════════════════════════════════════════
# Entry point
# ══════════════════════════════════════════════════════════════════════════════

def main() -> int:
    parser = build_parser()
    args   = parser.parse_args()
    args   = post_process_args(args)

    if args.version:
        print(f"imapsync {VERSION}")
        return EX_OK

    # --releasecheck / --noreleasecheck: stub notice (not implemented)
    if getattr(args, "releasecheck", True):
        print("Release checking not implemented on this version")

    if args.justconnect or args.justlogin or args.justfoldersizes:
        if not args.host1:
            logger.error("--host1 is required")
            return EX_USAGE
    elif not args.host1 or not args.host2:
        parser.print_help()
        return EX_USAGE

    syncer = ImapSync(args)
    return syncer.run()


if __name__ == "__main__":
    sys.exit(main())
