Implemented '--tor inside' option to run inside of the Tor-connected virtual machine

src/Config.py

@@ -167,9 +167,10 @@ def createArguments(self):
         self.parser.add_argument('--coffeescript_compiler', help='Coffeescript compiler for developing', default=coffeescript,
                                  metavar='executable_path')
 
-        self.parser.add_argument('--tor', help='enable: Use only for Tor peers, always: Use Tor for every connection', choices=["disable", "enable", "always"], default='enable')
+        self.parser.add_argument('--tor', help='enable: Use only for Tor peers, always: Use Tor for every connection', choices=["disable", "enable", "always", "inside"], default='enable')
         self.parser.add_argument('--tor_controller', help='Tor controller address', metavar='ip:port', default='127.0.0.1:9051')
         self.parser.add_argument('--tor_proxy', help='Tor proxy address', metavar='ip:port', default='127.0.0.1:9050')
+        self.parser.add_argument('--tor_hidden_services', help='Tor hidden services file', default='')
 
         self.parser.add_argument('--version', action='version', version='ZeroNet %s r%s' % (self.version, self.rev))
 

src/Connection/Connection.py

@@ -177,11 +177,11 @@ def getHandshakeInfo(self):
         # Setup peer lock from requested onion address
         if self.handshake and self.handshake.get("target_ip", "").endswith(".onion"):
             target_onion = self.handshake.get("target_ip").replace(".onion", "")  # My onion address
-            onion_sites = {v: k for k, v in self.server.tor_manager.site_onions.items()}  # Inverse, Onion: Site address
-            self.site_lock = onion_sites.get(target_onion)
-            if not self.site_lock:
-                self.server.log.error("Unknown target onion address: %s" % target_onion)
-                self.site_lock = "unknown"
+            self.site_lock = self.server.tor_manager.onion_sites.get(target_onion)
+            if not self.site_lock: # TODO Need to also verify that this is the same site connection was opened for
+                self.log("Unknown target onion address %s, closing connection" % target_onion)
+                self.close()
+                return
 
         handshake = {
             "version": config.version,

src/Connection/ConnectionServer.py

@@ -13,6 +13,7 @@
 from Crypt import CryptConnection
 from Crypt import CryptHash
 from Tor import TorManager
+from Tor import TorManagerInside
 
 
 class ConnectionServer:
@@ -23,8 +24,10 @@ def __init__(self, ip=None, port=None, request_handler=None):
         self.log = logging.getLogger("ConnServer")
         self.port_opened = None
 
-        if config.tor != "disabled":
+        if config.tor == "enable" or config.tor == "always":
             self.tor_manager = TorManager(self.ip, self.port)
+        elif config.tor == "inside":
+            self.tor_manager = TorManagerInside(config.tor_hidden_services)
         else:
             self.tor_manager = None
 

src/File/FileServer.py

@@ -64,7 +64,7 @@ def openport(self, port=None, check=True):
             if self.testOpenport(port, use_alternative=False)["result"] is True:
                 return True  # Port already opened
 
-        if config.tor == "always":  # Port opening won't work in Tor mode
+        if config.tor == "always" or config.tor == "inside":  # Port opening won't work in Tor mode
             return False
 
         self.log.info("Trying to open port using UpnpPunch...")
@@ -102,7 +102,7 @@ def testOpenportPortchecker(self, port=None):
             data = ""
 
         if "closed" in message or "Error" in message:
-            if config.tor != "always":
+            if config.tor != "always" and config.tor != "inside":
                 self.log.info("[BAD :(] Port closed: %s" % message)
             if port == self.port:
                 self.port_opened = False  # Self port, update port_opened status
@@ -135,7 +135,7 @@ def testOpenportCanyouseeme(self, port=None):
             message = "Error: %s" % Debug.formatException(err)
 
         if "Error" in message:
-            if config.tor != "always":
+            if config.tor != "always"  and config.tor != "inside":
                 self.log.info("[BAD :(] Port closed: %s" % message)
             if port == self.port:
                 self.port_opened = False  # Self port, update port_opened status
@@ -186,7 +186,7 @@ def checkSites(self, check_files=True, force_port_check=False):
                     gevent.spawn(self.checkSite, site, check_files)
 
             self.openport()
-            if self.port_opened is False:
+            if self.port_opened is False and config.tor != "inside":
                 self.tor_manager.startOnions()
 
         if not sites_checking:

src/Site/SiteManager.py

@@ -3,6 +3,7 @@
 import re
 import os
 import time
+import sys
 
 import gevent
 
@@ -24,15 +25,24 @@ def load(self):
             self.sites = {}
         address_found = []
         added = 0
+        serving = 0
         # Load new adresses
         for address in json.load(open("%s/sites.json" % config.data_dir)):
             if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
                 s = time.time()
                 self.sites[address] = Site(address)
                 logging.debug("Loaded site %s in %.3fs" % (address, time.time() - s))
                 added += 1
+                if self.sites[address].settings["serving"]:
+                    serving += 1
             address_found.append(address)
 
+        # check if there are enough onions available
+        if sys.modules.get("main") and sys.modules["main"].file_server:
+            tor_manager = sys.modules["main"].file_server.tor_manager
+            if tor_manager and tor_manager.numOnions() < serving+1:
+                sys.exit("Insufficient number of onions: supplied %u, need at least %u, recommended to have %u+ onions" % (tor_manager.numOnions(), serving+1, serving*3))
+
         # Remove deleted adresses
         for address in self.sites.keys():
             if address not in address_found:

src/Tor/TorManager.py

@@ -23,6 +23,7 @@ class TorManager:
     def __init__(self, fileserver_ip=None, fileserver_port=None):
         self.privatekeys = {}  # Onion: Privatekey
         self.site_onions = {}  # Site address: Onion
+        self.onion_sites = {}  # Onion: Site address
         self.tor_exe = "tools/tor/tor.exe"
         self.tor_process = None
         self.log = logging.getLogger("TorManager")
@@ -141,6 +142,7 @@ def connect(self):
         if not self.enabled:
             return False
         self.site_onions = {}
+        self.onion_sites = {}
         self.privatekeys = {}
 
         if "socket_noproxy" in dir(socket):  # Socket proxy-patched, use non-proxy one
@@ -193,6 +195,12 @@ def resetCircuits(self):
             self.status = u"Reset circuits error (%s)" % res
             self.log.error("Tor reset circuits error: %s" % res)
 
+    def haveOnionsAvailable(self):
+        return True # Can always create more onions
+
+    def numOnions(self):
+        return 100000 # Need the real limit here, but it is not very important to have am exact number
+
     def addOnion(self):
         res = self.request("ADD_ONION NEW:RSA1024 port=%s" % self.fileserver_port)
         match = re.search("ServiceID=([A-Za-z0-9]+).*PrivateKey=RSA1024:(.*?)[\r\n]", res, re.DOTALL)
@@ -207,18 +215,28 @@ def addOnion(self):
             self.log.error("Tor addOnion error: %s" % res)
             return False
 
-    def delOnion(self, address):
-        res = self.request("DEL_ONION %s" % address)
+    def delSiteOnion(self, site_address, onion):
+        res = self.request("DEL_ONION %s" % onion)
         if "250 OK" in res:
-            del self.privatekeys[address]
-            self.status = "OK (%s onion running)" % len(self.privatekeys)
+            del self.privatekeys[onion]
+            del self.site_onions[site_address]
+            del self.onion_sites[onion]
+            self.status = "OK (%s onion running)" % len(self.onion_sites)
             return True
         else:
             self.status = u"DelOnion error (%s)" % res
-            self.log.error("Tor delOnion error: %s" % res)
+            self.log.error("Tor delOnion %s for site %s error: %s" % (onion, site_address, res))
             self.disconnect()
             return False
 
+    def delOnion(self, onion):
+        site_address = self.onion_sites[onion]
+        return self.delSiteOnion(site_address, onion)
+
+    def delSite(self, site_address):
+        onion = self.site_onions[site_address]
+        return self.delSiteOnion(site_address, onion)
+
     def request(self, cmd):
         with self.lock:
             if not self.enabled:
@@ -255,6 +273,7 @@ def getOnion(self, site_address):
             if not onion:
                 self.site_onions[site_address] = self.addOnion()
                 onion = self.site_onions[site_address]
+                self.onion_sites[onion] = site_address
                 self.log.debug("Created new hidden service for %s: %s" % (site_address, onion))
             return onion
 

src/Tor/TorManagerInside.py

@@ -0,0 +1,126 @@
+import logging
+import socket
+import random
+import sys
+
+from Config import config
+from Crypt import CryptRsa
+from Site import SiteManager
+
+# TorManagerInside is the version of TorManager reduced to the needs of the Tor-connected VM.
+
+class TorManagerInside:
+    def __init__(self, tor_hidden_services_fname):
+        self.privatekeys = {}  # Onion: Privatekey
+        self.site_onions = {}  # Site address: Onion
+        self.onion_sites = {}  # Onion: Site address
+        self.log = logging.getLogger("TorManagerInside")
+
+        self.ip, self.port = config.tor_controller.split(":")
+        self.port = int(self.port)
+
+        self.hss_all = self.reshuffleList(self.readHiddenServices(tor_hidden_services_fname))
+        self.hss_unused = self.hss_all[:]
+
+        # Add our onions to the black list
+        for hs in self.hss_all:
+            SiteManager.peer_blacklist.append((hs[0], config.fileserver_port))
+
+        self.enabled = True
+        self.start_onions = True
+        self.updateStatus()
+
+    def readHiddenServices(self, fname):
+        with open(fname) as f:
+            lines = f.readlines()
+        hss = []
+        hs = []
+        phase = 'O'
+        onion = ""
+        key = ""
+        for line in lines:
+            line = line.strip('\n')
+            if line == "":
+                continue
+            if phase == 'O':
+                if not line.endswith(".onion"):
+                    sys.exit("Onion address is expected in the hidden services file, got the line >%s<" % line)
+                onion = line
+                phase = 'H'
+            elif phase == 'H':
+                if line != "-----BEGIN RSA PRIVATE KEY-----":
+                    sys.exit("Key header is expected in the hidden services file, got the line >%s<" % line)
+                phase = 'K'
+            elif phase == 'K':
+                if line != "-----END RSA PRIVATE KEY-----":
+                    key = key+line
+                else:
+                    hss.append([onion, key])
+                    key = ""
+                    phase = 'O'
+        if phase != 'O':
+            sys.exit("Unexpected end of the hidden services file")
+        return hss
+
+    def reshuffleList(self, lst):
+        idxs = range(0, len(lst))
+        shuf = []
+        while len(idxs) > 0:
+            n = random.randrange(0, len(idxs))
+            shuf.append(lst[idxs[n]])
+            del idxs[n]
+        return shuf
+
+    def updateStatus(self):
+        self.status = u"OK (%s onion used of %s available)" % (len(self.onion_sites), len(self.hss_all))
+
+    def getPrivatekey(self, address):
+        return self.privatekeys[address]
+
+    def getPublickey(self, address):
+        return CryptRsa.privatekeyToPublickey(self.privatekeys[address])
+
+    def haveOnionsAvailable(self):
+        return len(self.hss_unused) > 0
+
+    def numOnions(self):
+        return len(self.hss_all)
+
+    def getOnion(self, site_address):
+        onion = self.site_onions.get(site_address)
+        if onion:
+            return onion
+        if len(self.hss_unused) == 0:
+            sys.exit("TorManager ran out of onions (%u onions were supplied)" % len(self.hss_all))
+        hs_info = self.hss_unused[0]
+        del self.hss_unused[0]
+        onion = hs_info[0].replace(".onion", "")
+        self.site_onions[site_address] = onion
+        self.onion_sites[onion] = site_address
+        self.privatekeys[onion] = hs_info[1]
+        self.log.debug("Using the next onion for the site %s: %s.onion" % (site_address, onion))
+        self.updateStatus()
+        return onion
+
+    def delSiteOnion(self, site_address, onion):
+        self.log.debug("Deleting the site %s, recycling its onion %s.onion" % (site_address, onion))
+        self.hss_unused.append([onion+".onion", self.privatekeys[onion]])
+        del self.privatekeys[onion]
+        del self.site_onions[site_address]
+        del self.onion_sites[onion]
+        self.updateStatus()
+        return True
+
+    def delOnion(self, onion):
+        site_address = self.onion_sites[onion]
+        return self.delSiteOnion(site_address, onion)
+
+    def delSite(self, site_address):
+        onion = self.site_onions[site_address]
+        return self.delSiteOnion(site_address, onion)
+
+    def createSocket(self, onion, port):
+        self.log.debug("Creating new socket to %s:%s" % (onion, port))
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.connect((onion, int(port)))
+        return sock

src/Tor/__init__.py

@@ -1 +1,2 @@
-from TorManager import TorManager
+from TorManager import TorManager
+from TorManagerInside import TorManagerInside

src/Ui/UiWebsocket.py

@@ -56,6 +56,15 @@ def start(self):
                     """,
                     10000
                 ])
+            elif config.tor == "inside" and file_server.tor_manager.start_onions:
+                self.site.notifications.append([
+                    "done",
+                    """
+                    Tor mode active, every connection using Onion route.<br>
+                    Using externally supplied Tor onion hidden services.
+                    """,
+                    10000
+                ])
             elif config.tor == "always" and file_server.tor_manager.start_onions is not False:
                 self.site.notifications.append([
                     "error",
@@ -608,6 +617,8 @@ def actionSitePause(self, to, address):
             site.saveSettings()
             site.updateWebsocket()
             site.worker_manager.stopWorkers()
+            if sys.modules["main"].file_server.tor_manager:
+                sys.modules["main"].file_server.tor_manager.delSite(address)
             self.response(to, "Paused")
         else:
             self.response(to, {"error": "Unknown site: %s" % address})
@@ -616,6 +627,8 @@ def actionSitePause(self, to, address):
     def actionSiteResume(self, to, address):
         site = self.server.sites.get(address)
         if site:
+            if sys.modules["main"].file_server.tor_manager and not sys.modules["main"].file_server.tor_manager.haveOnionsAvailable():
+                return # Failed to resume
             site.settings["serving"] = True
             site.saveSettings()
             gevent.spawn(site.update, announce=True)
@@ -636,6 +649,8 @@ def actionSiteDelete(self, to, address):
             site.updateWebsocket()
             SiteManager.site_manager.delete(address)
             self.user.deleteSiteData(address)
+            if sys.modules["main"].file_server.tor_manager:
+                sys.modules["main"].file_server.tor_manager.delSite(address)
             self.response(to, "Deleted")
         else:
             self.response(to, {"error": "Unknown site: %s" % address})

src/main.py

@@ -26,6 +26,14 @@
 if not config.arguments:  # Config parse failed, show the help screen and exit
     config.parse()
 
+# Check config sanity
+if config.tor == "inside" and (config.tor_hidden_services == "" or not os.access(config.tor_hidden_services, os.R_OK)):
+    print "have to specify --tor_hidden_services <hs_file> with tor=inside"
+    sys.exit(1)
+if config.tor != "inside" and config.tor_hidden_services != "":
+    print "--tor_hidden_services <hs_file> can only be specified with tor=inside"
+    sys.exit(1)
+
 # Create necessary files and dirs
 if not os.path.isdir(config.log_dir):
     os.mkdir(config.log_dir)