canti/scratch/deploy_ftp.py

import os
import sys
import threading
from ftplib import FTP, error_perm
from concurrent.futures import ThreadPoolExecutor, as_completed

thread_local = threading.local()

def load_env():
    env = {}
    if os.path.exists('.env'):
        with open('.env', 'r', encoding='utf-8') as f:
            for line in f:
                line = line.strip()
                if line and not line.startswith('#'):
                    if '=' in line:
                        key, val = line.split('=', 1)
                        env[key.strip()] = val.strip()
    return env

def get_ftp_connection(ftp_host, ftp_user, ftp_pass):
    if not hasattr(thread_local, "ftp") or thread_local.ftp is None:
        ftp = FTP(ftp_host, timeout=30)
        ftp.login(ftp_user, ftp_pass)
        ftp.passive = True
        thread_local.ftp = ftp
    return thread_local.ftp

def close_thread_connection():
    if hasattr(thread_local, "ftp") and thread_local.ftp is not None:
        try:
            thread_local.ftp.quit()
        except:
            try:
                thread_local.ftp.close()
            except:
                pass
        thread_local.ftp = None

def ensure_remote_dir(ftp, path):
    parts = path.replace('\\', '/').strip('/').split('/')
    current = ""
    for part in parts:
        if not part:
            continue
        current += "/" + part
        try:
            ftp.mkd(current)
            print(f"\nCreated remote dir: {current}")
        except error_perm:
            # Directory already exists or permission error (which is normal if it exists)
            pass

def upload_file_task(local_path, rel_path, ftp_host, ftp_user, ftp_pass, created_dirs, created_dirs_lock):
    ftp_path = rel_path.replace('\\', '/')
    remote_dir = os.path.dirname(ftp_path)
    
    # Ensure remote directory exists
    if remote_dir:
        with created_dirs_lock:
            if remote_dir not in created_dirs:
                try:
                    ftp = get_ftp_connection(ftp_host, ftp_user, ftp_pass)
                    ensure_remote_dir(ftp, remote_dir)
                    created_dirs.add(remote_dir)
                except Exception as e:
                    thread_local.ftp = None
                    try:
                        ftp = get_ftp_connection(ftp_host, ftp_user, ftp_pass)
                        ensure_remote_dir(ftp, remote_dir)
                        created_dirs.add(remote_dir)
                    except Exception as err2:
                        return False, ftp_path, f"Failed directory creation: {err2}"
                        
    # Upload the file
    for attempt in range(3):
        try:
            ftp = get_ftp_connection(ftp_host, ftp_user, ftp_pass)
            with open(local_path, 'rb') as f:
                ftp.storbinary(f"STOR {ftp_path}", f)
            return True, ftp_path, None
        except Exception as e:
            thread_local.ftp = None  # Force reconnect on next attempt
            if attempt == 2:
                return False, ftp_path, str(e)
    return False, ftp_path, "Unknown error"

def main():
    env = load_env()
    ftp_host = "ftp.canticristiani.it"
    ftp_user = "canticristiani.it"
    ftp_pass = env.get("FTP_PASSWORD")
    
    if not ftp_pass:
        print("❌ Error: FTP_PASSWORD not found in .env")
        sys.exit(1)
        
    local_dir = "www"
    if not os.path.isdir(local_dir):
        print(f"❌ Error: Local dir '{local_dir}' not found.")
        sys.exit(1)
        
    files_to_upload = []
    for root, dirs, files in os.walk(local_dir):
        for file in files:
            local_path = os.path.join(root, file)
            rel_path = os.path.relpath(local_path, local_dir)
            files_to_upload.append((local_path, rel_path))
            
    total = len(files_to_upload)
    uploaded = 0
    errors = []
    
    created_dirs = set()
    created_dirs_lock = threading.Lock()
    progress_lock = threading.Lock()
    
    # Thread limit: default to 100 as requested
    num_threads = int(os.environ.get("FTP_THREADS", env.get("FTP_THREADS", "100")))
    print(f"🚀 Starting parallel upload of {total} files using {num_threads} threads...")
    
    def run_task(item):
        nonlocal uploaded
        local_path, rel_path = item
        success, ftp_path, err_msg = upload_file_task(
            local_path, rel_path, ftp_host, ftp_user, ftp_pass, created_dirs, created_dirs_lock
        )
        with progress_lock:
            uploaded += 1
            print(f"\rUploading [{uploaded}/{total}] {ftp_path}...", end="", flush=True)
            if not success:
                errors.append((ftp_path, err_msg))

    with ThreadPoolExecutor(max_workers=num_threads) as executor:
        futures = [executor.submit(run_task, item) for item in files_to_upload]
        for future in as_completed(futures):
            pass

    # Cleanup connections
    with ThreadPoolExecutor(max_workers=num_threads) as cleanup_executor:
        cleanups = [cleanup_executor.submit(close_thread_connection) for _ in range(num_threads)]
        for future in as_completed(cleanups):
            pass
            
    if errors:
        print(f"\n⚠️ Deploy completed with {len(errors)} errors:")
        for path, err in errors:
            print(f"  {path}: {err}")
        sys.exit(1)
    else:
        print("\n\n✅ Deploy completed successfully with 100% files uploaded without errors!")

if __name__ == '__main__':
    main()