diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8094aca..176ef88 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.4.2 +current_version = 0.4.3 commit = True tag = True diff --git a/CHANGELOG.md b/CHANGELOG.md index 88952f8..f76a3f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## [0.4.3] - 2025-06-08 +### Added +- Docs: installation from compose +- Custom access & blocked log +### Change +- Default console format + +## [0.4.2] - 2025-06-06 +### Added +- Custom 403 page for unauthorized IPs +### Change +- Rework code format (server.py, handlers/) + ## [0.4.1] - 2025-06-05 ### Added - Custom log format in config.ini diff --git a/README.md b/README.md index a72ec98..d37123b 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,12 @@ docker run -d ghcr.io/6c656c65/pyproxy:latest ``` You can use slim images by adding `-slim` to the end of the tags +### Install with Compose +```bash +wget https://raw.githubusercontent.com/6C656C65/pyproxy/main/docker-compose.yml +docker-compose up -d +``` + ## 🚀 **Usage** ### Start the proxy diff --git a/config.ini.example b/config.ini.example index 5b8bd72..aeedb10 100644 --- a/config.ini.example +++ b/config.ini.example @@ -8,8 +8,10 @@ access_log = ./logs/access.log block_log = ./logs/block.log no_logging_access = false no_logging_block = false -console_format = %(asctime)s - %(levelname)s - %(message)s -datefmt = %d/%m/%Y %H:%M:%S +console_format = date=%(asctime)s level=%(levelname)s file=%(filename)s function=%(funcName)s message=%(message)s +access_log_format = date=%(asctime)s ip_src=%(ip_src)s url=%(url)s method=%(method)s domain=%(domain)s port=%(port)s protocol=%(protocol)s bytes_sent=%(bytes_sent)s bytes_received=%(bytes_received)s tls_version=%(tls_version)s +block_log_format = date=%(asctime)s ip_src=%(ip_src)s url=%(url)s method=%(method)s domain=%(domain)s port=%(port)s protocol=%(protocol)s +datefmt = %Y-%m-%d %H:%M:%S [Files] html_403 = assets/403.html diff --git a/pyproxy/__init__.py b/pyproxy/__init__.py index dd3677d..d87385b 100644 --- a/pyproxy/__init__.py +++ b/pyproxy/__init__.py @@ -5,7 +5,7 @@ import os -__version__ = "0.4.2" +__version__ = "0.4.3" if os.path.isdir("pyproxy/monitoring"): __slim__ = False diff --git a/pyproxy/handlers/http.py b/pyproxy/handlers/http.py index b583b28..9451f43 100644 --- a/pyproxy/handlers/http.py +++ b/pyproxy/handlers/http.py @@ -109,8 +109,18 @@ def _send_403(self, client_socket, url, first_line): Sends an HTTP 403 Forbidden response to the client. """ if not self.logger_config.no_logging_block: + method, domain_port, protocol = first_line.split(" ") + domain, port = domain_port.split(":") self.logger_config.block_logger.info( - "%s - %s - %s", client_socket.getpeername()[0], url, first_line + "", + extra={ + "ip_src": client_socket.getpeername()[0], + "url": url, + "method": method, + "domain": domain, + "port": port, + "protocol": protocol, + }, ) with open(self.html_403, "r", encoding="utf-8") as f: custom_403_page = f.read() @@ -155,16 +165,6 @@ def handle_http_request(self, client_socket, request): self._send_403(client_socket, url, first_line) return - parsed_url = urlparse(url) - server_host = parsed_url.hostname - if not self.logger_config.no_logging_access: - self.logger_config.access_logger.info( - "%s - %s - %s", - client_socket.getpeername()[0], - f"http://{server_host}", - first_line, - ) - if self.config_custom_header and os.path.isfile(self.config_custom_header): request_text = request.decode(errors="ignore") request_lines = request_text.split("\r\n") @@ -177,12 +177,14 @@ def handle_http_request(self, client_socket, request): ) modified_request = self._rebuild_http_request(request_line, headers, body) - self.forward_request_to_server(client_socket, modified_request, url) + self.forward_request_to_server( + client_socket, modified_request, url, first_line + ) else: - self.forward_request_to_server(client_socket, request, url) + self.forward_request_to_server(client_socket, request, url, first_line) - def forward_request_to_server(self, client_socket, request, url): + def forward_request_to_server(self, client_socket, request, url, first_line): """ Forwards the HTTP request to the target server and sends the response back to the client. @@ -190,6 +192,7 @@ def forward_request_to_server(self, client_socket, request, url): client_socket (socket): The socket object for the client connection. request (bytes): The raw HTTP request sent by the client. url (str): The target URL from the HTTP request. + first_line (str): The first line of the HTTP request (e.g., "GET / HTTP/1.1"). """ if self.proxy_config.enable: server_host, server_port = self.proxy_config.host, self.proxy_config.port @@ -202,8 +205,12 @@ def forward_request_to_server(self, client_socket, request, url): thread_id = threading.get_ident() if thread_id in self.active_connections: - self.active_connections[thread_id]["target_ip"] = server_host - self.active_connections[thread_id]["target_port"] = server_port + self.active_connections[thread_id] = { + "target_ip": server_host, + "target_port": server_port, + "bytes_sent": 0, + "bytes_received": 0, + } try: server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -238,6 +245,23 @@ def forward_request_to_server(self, client_socket, request, url): client_socket.close() self.active_connections.pop(thread_id, None) finally: + if not self.logger_config.no_logging_access: + method, url, protocol = first_line.split(" ") + + conn_data = self.active_connections.get(thread_id, {}) + self.logger_config.access_logger.info( + "", + extra={ + "ip_src": client_socket.getpeername()[0], + "url": f"http://{server_host}", + "method": method, + "domain": parsed_url.hostname, + "port": parsed_url.port, + "protocol": protocol, + "bytes_sent": conn_data.get("bytes_sent", 0), + "bytes_received": conn_data.get("bytes_received", 0), + }, + ) client_socket.close() server_socket.close() self.active_connections.pop(thread_id, None) diff --git a/pyproxy/handlers/https.py b/pyproxy/handlers/https.py index ddabeeb..e2b3deb 100644 --- a/pyproxy/handlers/https.py +++ b/pyproxy/handlers/https.py @@ -80,8 +80,18 @@ def _send_403(self, client_socket, url, first_line): Sends an HTTP 403 Forbidden response to the client. """ if not self.logger_config.no_logging_block: + method, domain_port, protocol = first_line.split(" ") + domain, port = domain_port.split(":") self.logger_config.block_logger.info( - "%s - %s - %s", client_socket.getpeername()[0], url, first_line + "", + extra={ + "ip_src": client_socket.getpeername()[0], + "url": url, + "method": method, + "domain": domain, + "port": port, + "protocol": protocol, + }, ) with open(self.html_403, "r", encoding="utf-8") as f: custom_403_page = f.read() @@ -150,7 +160,15 @@ def _wrap_client_socket_with_ssl(self, client_socket, cert_path, key_path): ssl_client_socket = client_context.wrap_socket( client_socket, server_side=True, do_handshake_on_connect=False ) - ssl_client_socket.do_handshake() + try: + ssl_client_socket.do_handshake() + except ssl.SSLError as e: + if "TLSV1_ALERT_UNKNOWN_CA" in str(e): + self.console_logger.debug("Client refused cert: %s", e) + ssl_client_socket.close() + raise ConnectionAbortedError("Client refused SSL cert") + else: + raise return ssl_client_socket def _wrap_server_socket_with_ssl(self, server_socket, server_host): @@ -171,7 +189,7 @@ def _wrap_server_socket_with_ssl(self, server_socket, server_host): ) return ssl_server_socket - def _process_first_ssl_request(self, ssl_client_socket, server_host): + def _process_first_ssl_request(self, ssl_client_socket, server_host, first_line): """ Reads and processes the first SSL client request, extracts the method and full URL. """ @@ -188,15 +206,6 @@ def _process_first_ssl_request(self, ssl_client_socket, server_host): if self._is_blocked(f"{server_host}{path}"): return None, full_url, True - if not self.logger_config.no_logging_access: - self.logger_config.access_logger.info( - "%s - %s - %s %s", - ssl_client_socket.getpeername()[0], - f"https://{server_host}", - method, - full_url, - ) - return first_request, full_url, False except Exception as e: self.logger_config.error_logger.error(f"SSL request processing error : {e}") @@ -221,6 +230,12 @@ def handle_https_connection(self, client_socket, first_line): not_inspect = self._should_skip_inspection(server_host) + thread_id = threading.get_ident() + self.active_connections[thread_id] = { + "bytes_sent": 0, + "bytes_received": 0, + } + if self.ssl_config.ssl_inspect and not not_inspect: try: cert_path, key_path = generate_certificate( @@ -236,6 +251,8 @@ def handle_https_connection(self, client_socket, first_line): client_socket, cert_path, key_path ) + tls_version = ssl_client_socket.version() or "unknown" + server_socket = self._establish_server_connection( server_host, server_port ) @@ -244,7 +261,7 @@ def handle_https_connection(self, client_socket, first_line): ) first_request, full_url, is_blocked = self._process_first_ssl_request( - ssl_client_socket, server_host + ssl_client_socket, server_host, first_line ) if is_blocked: self._send_403(ssl_client_socket, target, first_line) @@ -254,9 +271,31 @@ def handle_https_connection(self, client_socket, first_line): return ssl_server_socket.sendall(first_request.encode()) + client_ip = ssl_client_socket.getpeername()[0] + bytes_sent, bytes_received = self.transfer_data_between_sockets( + ssl_client_socket, ssl_server_socket + ) - self.transfer_data_between_sockets(ssl_client_socket, ssl_server_socket) + if not self.logger_config.no_logging_access: + method, _, protocol = first_line.split(" ") + self.logger_config.access_logger.info( + "", + extra={ + "ip_src": client_ip, + "url": target, + "method": method, + "domain": server_host, + "port": server_port, + "protocol": protocol, + "bytes_sent": bytes_sent, + "bytes_received": bytes_received, + "tls_version": tls_version, + }, + ) + except ConnectionAbortedError: + self.active_connections.pop(threading.get_ident(), None) + return except ssl.SSLError as e: self.console_logger.error("SSL error: %s", str(e)) except socket.error as e: @@ -267,17 +306,31 @@ def handle_https_connection(self, client_socket, first_line): else: try: - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.connect((server_host, server_port)) + server_socket = self._establish_server_connection( + server_host, server_port + ) client_socket.sendall(b"HTTP/1.1 200 Connection Established\r\n\r\n") + + client_ip = client_socket.getpeername()[0] + bytes_sent, bytes_received = self.transfer_data_between_sockets( + client_socket, server_socket + ) + if not self.logger_config.no_logging_access: + _, _, protocol = first_line.split(" ") self.logger_config.access_logger.info( - "%s - %s - %s", - client_socket.getpeername()[0], - f"https://{server_host}", - first_line, + "", + extra={ + "ip_src": client_ip, + "url": target, + "method": "CONNECT", + "domain": server_host, + "port": server_port, + "protocol": protocol, + "bytes_sent": bytes_sent, + "bytes_received": bytes_received, + }, ) - self.transfer_data_between_sockets(client_socket, server_socket) except ( socket.timeout, socket.gaierror, @@ -295,6 +348,8 @@ def handle_https_connection(self, client_socket, first_line): ) client_socket.sendall(response.encode()) client_socket.close() + finally: + self.active_connections.pop(thread_id, None) def transfer_data_between_sockets(self, client_socket, server_socket): """ @@ -327,8 +382,12 @@ def transfer_data_between_sockets(self, client_socket, server_socket): self.console_logger.debug("Closing connection.") client_socket.close() server_socket.close() + bytes_sent = self.active_connections[thread_id]["bytes_sent"] + bytes_received = self.active_connections[thread_id][ + "bytes_received" + ] self.active_connections.pop(threading.get_ident(), None) - return + return bytes_sent, bytes_received if sock is client_socket: server_socket.sendall(data) self.active_connections[thread_id]["bytes_sent"] += len(data) diff --git a/pyproxy/pyproxy.py b/pyproxy/pyproxy.py index da4bdcb..9a3ef26 100644 --- a/pyproxy/pyproxy.py +++ b/pyproxy/pyproxy.py @@ -56,6 +56,8 @@ def main(): ) console_format = config.get("Logging", "console_format", fallback=None) + access_log_format = config.get("Logging", "access_log_format", fallback=None) + block_log_format = config.get("Logging", "block_log_format", fallback=None) datefmt = config.get("Logging", "datefmt", fallback=None) logger_config = ProxyConfigLogger( @@ -74,9 +76,44 @@ def main(): console_format=( console_format if console_format is not None - else "%(asctime)s - %(levelname)s - %(message)s" - ), - datefmt=datefmt if datefmt is not None else "%d/%m/%Y %H:%M:%S", + else ( + "date=%(asctime)s " + "level=%(levelname)s " + "file=%(filename)s " + "function=%(funcName)s " + "message=%(message)s" + ) + ), + access_log_format=( + access_log_format + if access_log_format is not None + else ( + "date=%(asctime)s " + "ip_src=%(ip_src)s " + "url=%(url)s " + "method=%(method)s " + "domain=%(domain)s " + "port=%(port)s " + "protocol=%(protocol)s " + "bytes_sent=%(bytes_sent)s " + "bytes_received=%(bytes_received)s " + "tls_version=%(tls_version)s" + ) + ), + block_log_format=( + block_log_format + if block_log_format is not None + else ( + "date=%(asctime)s " + "ip_src=%(ip_src)s " + "url=%(url)s " + "method=%(method)s " + "domain=%(domain)s " + "port=%(port)s " + "protocol=%(protocol)s" + ) + ), + datefmt=datefmt if datefmt is not None else "%Y-%m-%d %H:%M:%S", ) filter_config = ProxyConfigFilter( diff --git a/pyproxy/server.py b/pyproxy/server.py index 53c64e7..d3360e7 100644 --- a/pyproxy/server.py +++ b/pyproxy/server.py @@ -105,11 +105,17 @@ def __init__( self.console_logger = configure_console_logger(self.logger_config) if not self.logger_config.no_logging_access: self.logger_config.access_logger = configure_file_logger( - self.logger_config.access_log, "AccessLogger" + self.logger_config.access_log, + "AccessLogger", + self.logger_config.access_log_format, + self.logger_config.datefmt, ) if not self.logger_config.no_logging_block: self.logger_config.block_logger = configure_file_logger( - self.logger_config.block_log, "BlockLogger" + self.logger_config.block_log, + "BlockLogger", + self.logger_config.block_log_format, + self.logger_config.datefmt, ) # Configuration files diff --git a/pyproxy/utils/config.py b/pyproxy/utils/config.py index 11a97e7..85b7ae3 100644 --- a/pyproxy/utils/config.py +++ b/pyproxy/utils/config.py @@ -63,6 +63,8 @@ class ProxyConfigLogger: no_logging_access: bool no_logging_block: bool console_format: str + access_log_format: str + block_log_format: str datefmt: str def to_dict(self): diff --git a/pyproxy/utils/logger.py b/pyproxy/utils/logger.py index d2d425d..45c60de 100644 --- a/pyproxy/utils/logger.py +++ b/pyproxy/utils/logger.py @@ -9,6 +9,23 @@ import colorlog +class SafeFormatter(logging.Formatter): + def format(self, record): + if self._fmt: + for key in self._fmt.split("%("): + if ")" in key: + var = key.split(")")[0] + if not hasattr(record, var): + setattr(record, var, "") + else: + val = getattr(record, var) + if isinstance(val, str): + setattr( + record, var, val.replace("\n", "").replace("\r", "") + ) + return super().format(record) + + def configure_console_logger(logger_config) -> logging.Logger: """ Configures and returns a logger that outputs log messages to the console. @@ -41,13 +58,18 @@ def configure_console_logger(logger_config) -> logging.Logger: return console_logger -def configure_file_logger(log_path: str, name: str) -> logging.Logger: +def configure_file_logger( + log_path: str, name: str, log_format: str, datefmt: str +) -> logging.Logger: """ Configures and returns a logger that writes log messages to a specified file. Args: log_path (str): The path where the log file will be created or appended to. name (str): Logger's name. + log_format (str): The format string for log messages + (e.g., with fields like %(ip_src)s, %(method)s). + datefmt (str): The format string for timestamps (e.g., "%d/%m/%Y %H:%M:%S"). Returns: logging.Logger: A logger instance that writes to the specified log file. @@ -56,6 +78,6 @@ def configure_file_logger(log_path: str, name: str) -> logging.Logger: file_logger = logging.getLogger(name) file_logger.setLevel(logging.INFO) file_handler = logging.FileHandler(log_path) - file_handler.setFormatter(logging.Formatter("%(asctime)s - %(message)s")) + file_handler.setFormatter(SafeFormatter(log_format, datefmt=datefmt)) file_logger.addHandler(file_handler) return file_logger diff --git a/tests/utils/test_logger.py b/tests/utils/test_logger.py index 0f6ad7e..3b0d056 100644 --- a/tests/utils/test_logger.py +++ b/tests/utils/test_logger.py @@ -14,10 +14,15 @@ class DummyLoggerConfig: def __init__(self, console_format=None, datefmt=None): - self.console_format = ( - console_format or "%(log_color)s%(asctime)s - %(levelname)s - %(message)s" + self.console_format = console_format or ( + "%(log_color)s" + "date=%(asctime)s " + "level=%(levelname)s " + "file=%(filename)s " + "function=%(funcName)s " + "message=%(message)s" ) - self.datefmt = datefmt or "%d/%m/%Y %H:%M:%S" + self.datefmt = datefmt or "%Y-%m-%d %H:%M:%S" class TestLogger(unittest.TestCase): @@ -56,11 +61,15 @@ def test_configure_file_logger(self, mock_file_handler): mock_file_handler.return_value = mock_handler_instance log_path = "logs/test.log" - logger = configure_file_logger(log_path, "TestLogger") + log_name = "TestLogger" + log_format = "%(asctime)s - %(levelname)s - %(message)s" + datefmt = "%Y-%m-%d %H:%M:%S" + logger = configure_file_logger(log_path, log_name, log_format, datefmt) - self.assertTrue(logger.hasHandlers()) - self.assertEqual(logger.level, logging.INFO) + self.assertTrue(logger.hasHandlers(), "Logger should have handlers.") + self.assertEqual(logger.level, logging.INFO, "Logger level should be INFO.") mock_file_handler.assert_called_once_with(log_path) + mock_handler_instance.setFormatter.assert_called_once() def tearDown(self): """