import os import sys import threading from ftplib import FTP, error_perm from concurrent.futures import ThreadPoolExecutor, as_completed thread_local = threading.local() def load_env(): env = {} if os.path.exists('.env'): with open('.env', 'r', encoding='utf-8') as f: for line in f: line = line.strip() if line and not line.startswith('#'): if '=' in line: key, val = line.split('=', 1) env[key.strip()] = val.strip() return env def get_ftp_connection(ftp_host, ftp_user, ftp_pass): if not hasattr(thread_local, "ftp") or thread_local.ftp is None: ftp = FTP(ftp_host, timeout=30) ftp.login(ftp_user, ftp_pass) ftp.passive = True thread_local.ftp = ftp return thread_local.ftp def close_thread_connection(): if hasattr(thread_local, "ftp") and thread_local.ftp is not None: try: thread_local.ftp.quit() except: try: thread_local.ftp.close() except: pass thread_local.ftp = None def ensure_remote_dir(ftp, path): parts = path.replace('\\', '/').strip('/').split('/') current = "" for part in parts: if not part: continue current += "/" + part try: ftp.mkd(current) print(f"\nCreated remote dir: {current}") except error_perm: # Directory already exists or permission error (which is normal if it exists) pass def upload_file_task(local_path, rel_path, ftp_host, ftp_user, ftp_pass, created_dirs, created_dirs_lock): ftp_path = rel_path.replace('\\', '/') remote_dir = os.path.dirname(ftp_path) # Ensure remote directory exists if remote_dir: with created_dirs_lock: if remote_dir not in created_dirs: try: ftp = get_ftp_connection(ftp_host, ftp_user, ftp_pass) ensure_remote_dir(ftp, remote_dir) created_dirs.add(remote_dir) except Exception as e: thread_local.ftp = None try: ftp = get_ftp_connection(ftp_host, ftp_user, ftp_pass) ensure_remote_dir(ftp, remote_dir) created_dirs.add(remote_dir) except Exception as err2: return False, ftp_path, f"Failed directory creation: {err2}" # Upload the file for attempt in range(3): try: ftp = get_ftp_connection(ftp_host, ftp_user, ftp_pass) with open(local_path, 'rb') as f: ftp.storbinary(f"STOR {ftp_path}", f) return True, ftp_path, None except Exception as e: thread_local.ftp = None # Force reconnect on next attempt if attempt == 2: return False, ftp_path, str(e) return False, ftp_path, "Unknown error" def main(): env = load_env() ftp_host = "ftp.canticristiani.it" ftp_user = "canticristiani.it" ftp_pass = env.get("FTP_PASSWORD") if not ftp_pass: print("āŒ Error: FTP_PASSWORD not found in .env") sys.exit(1) local_dir = "www" if not os.path.isdir(local_dir): print(f"āŒ Error: Local dir '{local_dir}' not found.") sys.exit(1) files_to_upload = [] for root, dirs, files in os.walk(local_dir): for file in files: local_path = os.path.join(root, file) rel_path = os.path.relpath(local_path, local_dir) files_to_upload.append((local_path, rel_path)) total = len(files_to_upload) uploaded = 0 errors = [] created_dirs = set() created_dirs_lock = threading.Lock() progress_lock = threading.Lock() # Thread limit: default to 100 as requested num_threads = int(os.environ.get("FTP_THREADS", env.get("FTP_THREADS", "100"))) print(f"šŸš€ Starting parallel upload of {total} files using {num_threads} threads...") def run_task(item): nonlocal uploaded local_path, rel_path = item success, ftp_path, err_msg = upload_file_task( local_path, rel_path, ftp_host, ftp_user, ftp_pass, created_dirs, created_dirs_lock ) with progress_lock: uploaded += 1 print(f"\rUploading [{uploaded}/{total}] {ftp_path}...", end="", flush=True) if not success: errors.append((ftp_path, err_msg)) with ThreadPoolExecutor(max_workers=num_threads) as executor: futures = [executor.submit(run_task, item) for item in files_to_upload] for future in as_completed(futures): pass # Cleanup connections with ThreadPoolExecutor(max_workers=num_threads) as cleanup_executor: cleanups = [cleanup_executor.submit(close_thread_connection) for _ in range(num_threads)] for future in as_completed(cleanups): pass if errors: print(f"\nāš ļø Deploy completed with {len(errors)} errors:") for path, err in errors: print(f" {path}: {err}") sys.exit(1) else: print("\n\nāœ… Deploy completed successfully with 100% files uploaded without errors!") if __name__ == '__main__': main()