Initial commit
This commit is contained in:
6
sftp/__init__.py
Normal file
6
sftp/__init__.py
Normal file
@@ -0,0 +1,6 @@
|
||||
"""SFTP module for ACH file processing."""
|
||||
|
||||
from .sftp_client import SFTPClient
|
||||
from .file_monitor import FileMonitor
|
||||
|
||||
__all__ = ['SFTPClient', 'FileMonitor']
|
||||
108
sftp/file_monitor.py
Normal file
108
sftp/file_monitor.py
Normal file
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
File monitoring and discovery for NEFT Inward files.
|
||||
Scans SFTP directories for new files across multiple banks.
|
||||
|
||||
Filename convention handled:
|
||||
DDMMYYYY_HH_NEFT_INWARD.TXT
|
||||
Example:
|
||||
06032026_14_NEFT_INWARD.TXT
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import List, Tuple, Dict
|
||||
from logging_config import get_logger
|
||||
from config import get_config
|
||||
from .sftp_client import SFTPClient
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class FileMonitor:
|
||||
"""Monitors SFTP for new NEFT Outward files."""
|
||||
|
||||
def __init__(self, sftp_client: SFTPClient = None):
|
||||
"""
|
||||
Initialize file monitor.
|
||||
|
||||
Args:
|
||||
sftp_client: SFTPClient instance (optional)
|
||||
"""
|
||||
self.config = get_config()
|
||||
self.sftp_client = sftp_client or SFTPClient()
|
||||
|
||||
def scan_for_new_files(self, processed_filenames: List[str]) -> List[Tuple[str, str, str]]:
|
||||
"""
|
||||
Scan all bank directories for new NEFT files.
|
||||
|
||||
Args:
|
||||
processed_filenames: List of already processed filenames to skip
|
||||
|
||||
Returns:
|
||||
List of (filename, bankcode, full_remote_path) tuples
|
||||
"""
|
||||
new_files: List[Tuple[str, str, str]] = []
|
||||
|
||||
for bank_code in self.config.bank_codes:
|
||||
# Adjust subfolder name here if required (e.g., 'NEFT_INWARD' or other)
|
||||
remote_path = f"{self.config.sftp_base_path}/{bank_code}/NEFT"
|
||||
|
||||
# Match any NEFT inward file for any date/hour
|
||||
files = self.sftp_client.list_files(remote_path, pattern='*_NEFT_OUTWARD.TXT')
|
||||
|
||||
for filename in files:
|
||||
if filename not in processed_filenames:
|
||||
full_path = f"{remote_path}/{filename}"
|
||||
new_files.append((filename, bank_code, full_path))
|
||||
logger.info(f"Found new NEFT file: {filename} (bank: {bank_code})")
|
||||
else:
|
||||
logger.debug(f"Skipping already processed NEFT file: {filename}")
|
||||
|
||||
logger.info(f"NEFT scan complete: Found {len(new_files)} new files")
|
||||
return new_files
|
||||
|
||||
@staticmethod
|
||||
def parse_filename(filename: str) -> Dict[str, str]:
|
||||
"""
|
||||
Parse NEFT filename to extract metadata.
|
||||
|
||||
Expected format:
|
||||
DDMMYYYY_HH_NEFT_OUTWARD.TXT
|
||||
Example:
|
||||
06032026_14_NEFT_OUTWARD.TXT
|
||||
|
||||
Args:
|
||||
filename: Filename to parse
|
||||
|
||||
Returns:
|
||||
Dictionary with extracted metadata or empty dict if parse fails
|
||||
"""
|
||||
# Groups: DD, MM, YYYY, HH
|
||||
pattern = r'^(\d{2})(\d{2})(\d{4})_(\d{2})_NEFT_OUTWARD\.TXT$'
|
||||
match = re.match(pattern, filename, flags=re.IGNORECASE)
|
||||
|
||||
if not match:
|
||||
logger.warning(f"Could not parse NEFT filename: {filename}")
|
||||
return {}
|
||||
|
||||
day, month, year, hour = match.groups()
|
||||
|
||||
return {
|
||||
'filename': filename,
|
||||
'day': day,
|
||||
'month': month,
|
||||
'year': year,
|
||||
'hour': hour,
|
||||
'timestamp': f"{day}/{month}/{year} {hour}:00:00"
|
||||
}
|
||||
|
||||
def __enter__(self):
|
||||
"""Context manager entry."""
|
||||
if not self.sftp_client.sftp:
|
||||
self.sftp_client.connect()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Context manager exit."""
|
||||
self.sftp_client.disconnect()
|
||||
|
||||
157
sftp/sftp_client.py
Normal file
157
sftp/sftp_client.py
Normal file
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
SFTP client for file operations.
|
||||
Handles connection, file discovery, and download operations.
|
||||
"""
|
||||
|
||||
import paramiko
|
||||
import os
|
||||
from pathlib import Path
|
||||
from logging_config import get_logger
|
||||
from config import get_config
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class SFTPClient:
|
||||
"""SFTP operations for NEFT file processing."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize SFTP client."""
|
||||
self.config = get_config()
|
||||
self.sftp = None
|
||||
self.ssh = None
|
||||
|
||||
def connect(self) -> bool:
|
||||
"""
|
||||
Establish SFTP connection.
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
# Create SSH client
|
||||
self.ssh = paramiko.SSHClient()
|
||||
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
# Connect
|
||||
self.ssh.connect(
|
||||
self.config.sftp_host,
|
||||
port=self.config.sftp_port,
|
||||
username=self.config.sftp_username,
|
||||
password=self.config.sftp_password,
|
||||
timeout=10
|
||||
)
|
||||
|
||||
# Get SFTP channel
|
||||
self.sftp = self.ssh.open_sftp()
|
||||
logger.info(f"Connected to SFTP server: {self.config.sftp_host}:{self.config.sftp_port}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to connect to SFTP server: {e}", exc_info=True)
|
||||
return False
|
||||
|
||||
def disconnect(self):
|
||||
"""Close SFTP connection."""
|
||||
try:
|
||||
if self.sftp:
|
||||
self.sftp.close()
|
||||
if self.ssh:
|
||||
self.ssh.close()
|
||||
logger.info("SFTP connection closed")
|
||||
except Exception as e:
|
||||
logger.error(f"Error closing SFTP connection: {e}")
|
||||
|
||||
def list_files(self, remote_path: str, pattern: str) -> list:
|
||||
"""
|
||||
List files matching pattern in remote directory.
|
||||
|
||||
Args:
|
||||
remote_path: Path on SFTP server
|
||||
|
||||
|
||||
Returns:
|
||||
List of matching filenames
|
||||
"""
|
||||
if not self.sftp:
|
||||
logger.error("SFTP not connected")
|
||||
return []
|
||||
|
||||
try:
|
||||
files = []
|
||||
try:
|
||||
items = self.sftp.listdir_attr(remote_path)
|
||||
except FileNotFoundError:
|
||||
logger.warning(f"Directory not found: {remote_path}")
|
||||
return []
|
||||
|
||||
import fnmatch
|
||||
for item in items:
|
||||
if fnmatch.fnmatch(item.filename, pattern):
|
||||
files.append(item.filename)
|
||||
|
||||
logger.debug(f"Found {len(files)} files matching {pattern} in {remote_path}")
|
||||
return sorted(files)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing files in {remote_path}: {e}", exc_info=True)
|
||||
return []
|
||||
|
||||
def download_file(self, remote_path: str, local_path: str) -> bool:
|
||||
"""
|
||||
Download file from SFTP server.
|
||||
|
||||
Args:
|
||||
remote_path: Full path on SFTP server
|
||||
local_path: Local destination path
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
if not self.sftp:
|
||||
logger.error("SFTP not connected")
|
||||
return False
|
||||
|
||||
try:
|
||||
# Create local directory if needed
|
||||
Path(local_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Download file
|
||||
self.sftp.get(remote_path, local_path)
|
||||
logger.info(f"Downloaded file: {remote_path} -> {local_path}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error downloading file {remote_path}: {e}", exc_info=True)
|
||||
return False
|
||||
|
||||
def get_file_size(self, remote_path: str) -> int:
|
||||
"""
|
||||
Get size of remote file.
|
||||
|
||||
Args:
|
||||
remote_path: Full path on SFTP server
|
||||
|
||||
Returns:
|
||||
File size in bytes, or -1 if error
|
||||
"""
|
||||
if not self.sftp:
|
||||
logger.error("SFTP not connected")
|
||||
return -1
|
||||
|
||||
try:
|
||||
stat = self.sftp.stat(remote_path)
|
||||
return stat.st_size
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting file size {remote_path}: {e}")
|
||||
return -1
|
||||
|
||||
def __enter__(self):
|
||||
"""Context manager entry."""
|
||||
self.connect()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Context manager exit."""
|
||||
self.disconnect()
|
||||
Reference in New Issue
Block a user