From 768656433345395b50d7cbeb0fe829b5889c6c40 Mon Sep 17 00:00:00 2001 From: MuxZeroNet Date: Wed, 4 Jan 2017 05:16:37 +0000 Subject: [PATCH 1/5] "library" --- zeronet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zeronet.py b/zeronet.py index f56161ee1..46e5dcb36 100755 --- a/zeronet.py +++ b/zeronet.py @@ -12,7 +12,7 @@ def main(): try: app_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(app_dir) # Change working dir to zeronet.py dir - sys.path.insert(0, os.path.join(app_dir, "src/lib")) # External liblary directory + sys.path.insert(0, os.path.join(app_dir, "src/lib")) # External library directory sys.path.insert(0, os.path.join(app_dir, "src")) # Imports relative to src import main main.start() @@ -65,4 +65,4 @@ def main(): print "Bye." if __name__ == '__main__': - main() \ No newline at end of file + main() From 813e401cb3d05634ead8784b97b7a491b2fc4628 Mon Sep 17 00:00:00 2001 From: MuxZeroNet Date: Wed, 4 Jan 2017 05:18:49 +0000 Subject: [PATCH 2/5] "library" --- src/Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config.py b/src/Config.py index a8fbb3300..a664f08fc 100644 --- a/src/Config.py +++ b/src/Config.py @@ -158,7 +158,7 @@ def createArguments(self): self.parser.add_argument('--ip_external', help='Set reported external ip (tested on start if None)', metavar='ip') self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*') self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', default=False, metavar='path') - self.parser.add_argument('--use_openssl', help='Use OpenSSL liblary for speedup', + self.parser.add_argument('--use_openssl', help='Use OpenSSL library for speedup', type='bool', choices=[True, False], default=use_openssl) self.parser.add_argument('--disable_db', help='Disable database updating', action='store_true') self.parser.add_argument('--disable_encryption', help='Disable connection encryption', action='store_true') From bec92f05e99d05ad7aec6bd415bcdd244f9ff668 Mon Sep 17 00:00:00 2001 From: MuxZeroNet Date: Wed, 4 Jan 2017 08:23:30 +0000 Subject: [PATCH 3/5] Fixed encoding issues + basic IDNA support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I fixed many encoding issues of ZeroNet. `os.path` module sometimes returns a unicode string, and sometimes returns a string encoded with file system encoding. To fix that inconsistency, I wrote a `self.fixFsEncoding` method. I also made it support IDNA namecoin domain. Check out some namecoin domains starting with `xn--`: ``` Emoji domains from ZeroNet domain registry: πŸŒ”.bit πŸŒ•.bit 🌝.bit 🌌.bit Ι₯sıɯɐΙ₯.bit ΞΊΟƒ.bit ⛏.bit ``` --- src/Ui/UiRequest.py | 63 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 560486e84..5e7560b21 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -1,6 +1,7 @@ import time import re import os +import sys import mimetypes import json import cgi @@ -40,11 +41,17 @@ def __init__(self, server, get, env, start_response): self.start_response = start_response # Start response function self.user = None + self.FS_ENCODING = sys.getfilesystemencoding() + # Call the request handler function base on path def route(self, path): if config.ui_restrict and self.env['REMOTE_ADDR'] not in config.ui_restrict: # Restict Ui access by ip return self.error403(details=False) + self.log.debug("-- path repr: " + repr(path)) + path = path.decode('utf-8') + self.log.debug("-- path decoded repr: " + repr(path)) + path = re.sub("^http://zero[/]+", "/", path) # Remove begining http://zero/ for chrome extension path = re.sub("^http://", "/", path) # Remove begining http for chrome extension .bit access @@ -96,6 +103,19 @@ def isProxyRequest(self): def isAjaxRequest(self): return self.env.get("HTTP_X_REQUESTED_WITH") == "XMLHttpRequest" + def splitRawDomain(self, path): + if not path.startswith('/'): + return (None, None) + + stripped_path = path.lstrip('/') + index_2nd_slash = stripped_path.find('/') + str_upper_bound = index_2nd_slash if index_2nd_slash >= 0 else len(stripped_path) + + raw_domain = stripped_path[0:str_upper_bound] # get raw domain + the_rest = stripped_path[str_upper_bound] if str_upper_bound < len(stripped_path) else "" + + return (raw_domain, the_rest) + # Get mime by filename def getContentType(self, file_name): content_type = mimetypes.guess_type(file_name)[0] @@ -214,7 +234,18 @@ def actionWrapper(self, path, extra_headers=None): return iter([self.renderWrapper(site, path, inner_path, title, extra_headers)]) # Dont know why wrapping with iter necessary, but without it around 100x slower - else: # Bad url + else: # IDNA? + raw_domain, the_rest = self.splitRawDomain(path) + self.log.debug(".. raw_domain repr: " + repr(raw_domain)) + self.log.debug(".. the_rest repr: " + repr(the_rest)) + if raw_domain: # possible IDNA + idna_encoded = raw_domain.encode('idna') + if idna_encoded != raw_domain: + redirect_to = '/' + idna_encoded + the_rest + self.log.debug("IDNA domain: " + idna_encoded) + return self.actionRedirect(redirect_to) + + # Otherwise, Bad url return False def renderWrapper(self, site, path, inner_path, title, extra_headers): @@ -322,6 +353,13 @@ def parsePath(self, path): else: return None + def fixFsEncoding(self, path): + if isinstance(path, unicode): + return path + elif isinstance(path, str): + return path.decode(self.FS_ENCODING) + else: + raise Exception("Not a string!") # Serve a media for site def actionSiteMedia(self, path, header_length=True): @@ -343,12 +381,25 @@ def actionSiteMedia(self, path, header_length=True): if path_parts: # Looks like a valid path address = path_parts["address"] - file_path = "%s/%s/%s" % (config.data_dir, address, path_parts["inner_path"]) - allowed_dir = os.path.abspath("%s/%s" % (config.data_dir, address)) # Only files within data/sitehash allowed + file_path = u"%s/%s/%s" % (config.data_dir, address, path_parts["inner_path"]) + + self.log.debug("--- FS encoding: " + self.FS_ENCODING) + + allowed_dir = os.path.abspath(u"%s/%s" % (config.data_dir, address)) # Only files within data/sitehash allowed + allowed_dir = self.fixFsEncoding(allowed_dir) + self.log.debug("--- allowed_dir repr: " + repr(allowed_dir)) + data_dir = os.path.abspath(config.data_dir) # No files from data/ allowed + data_dir = self.fixFsEncoding(data_dir) + self.log.debug("--- data_dir repr: " + repr(data_dir)) + + fp = os.path.dirname(os.path.abspath(file_path)) + fp = self.fixFsEncoding(fp) + self.log.debug("--- fp repr: " + repr(fp)) + if ( ".." in file_path or - not os.path.dirname(os.path.abspath(file_path)).startswith(allowed_dir) or + not fp.startswith(allowed_dir) or allowed_dir == data_dir ): # File not in allowed path return self.error403() @@ -387,8 +438,8 @@ def actionUiMedia(self, path): match = re.match("/uimedia/(?P.*)", path) if match: # Looks like a valid path file_path = "src/Ui/media/%s" % match.group("inner_path") - allowed_dir = os.path.abspath("src/Ui/media") # Only files within data/sitehash allowed - if ".." in file_path or not os.path.dirname(os.path.abspath(file_path)).startswith(allowed_dir): + allowed_dir = self.fixFsEncoding(os.path.abspath("src/Ui/media")) # Only files within data/sitehash allowed + if ".." in file_path or not self.fixFsEncoding(os.path.dirname(os.path.abspath(file_path))).startswith(allowed_dir): # File not in allowed path return self.error403() else: From b7c98b6f695438dbc74c9473dc5e73876907e53a Mon Sep 17 00:00:00 2001 From: MuxZeroNet Date: Thu, 5 Jan 2017 05:33:26 +0000 Subject: [PATCH 4/5] Keep query string + prevent gevent bug --- src/Ui/UiRequest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 5e7560b21..50e1897fd 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -242,8 +242,14 @@ def actionWrapper(self, path, extra_headers=None): idna_encoded = raw_domain.encode('idna') if idna_encoded != raw_domain: redirect_to = '/' + idna_encoded + the_rest + if self.env.get('QUERY_STRING'): + redirect_to += '?' + self.env.get('QUERY_STRING') + self.log.debug("IDNA domain: " + idna_encoded) - return self.actionRedirect(redirect_to) + self.log.debug('Location repr: ' + repr(redirect_to)) + return self.actionRedirect(str(redirect_to)) # gevent bug + # https://github.com/socketio/socket.io/issues/2446 + # https://github.com/ajenti/ajenti/issues/838 # Otherwise, Bad url return False From 9273693ffde550566065373f648fae4c6c4bd20e Mon Sep 17 00:00:00 2001 From: MuxZeroNet Date: Thu, 5 Jan 2017 06:55:20 +0000 Subject: [PATCH 5/5] Added basic code outline for warning --- src/Ui/UiRequest.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 50e1897fd..72ef93c53 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -247,13 +247,17 @@ def actionWrapper(self, path, extra_headers=None): self.log.debug("IDNA domain: " + idna_encoded) self.log.debug('Location repr: ' + repr(redirect_to)) - return self.actionRedirect(str(redirect_to)) # gevent bug - # https://github.com/socketio/socket.io/issues/2446 - # https://github.com/ajenti/ajenti/issues/838 + return self.actionWarnSeeding(redirect_to, raw_domain, idna_encoded) # Otherwise, Bad url return False + def actionWarnSeeding(self, redirect_to, raw_domain, idna_encoded): + self.log.info("!!! TODO: show warning") + return self.actionRedirect(str(redirect_to)) # gevent bug + # https://github.com/socketio/socket.io/issues/2446 + # https://github.com/ajenti/ajenti/issues/838 + def renderWrapper(self, site, path, inner_path, title, extra_headers): file_inner_path = inner_path if not file_inner_path: