Open Table Of Contents

Source code for bridgedb.https.distributor

# -*- coding: utf-8 ; test-case-name: bridgedb.test.test_https_distributor -*-
#
# This file is part of BridgeDB, a Tor bridge distribution system.
#
# :authors: Nick Mathewson
#           Isis Lovecruft 0xA3ADB67A2CDB8B35 <isis@torproject.org>
#           Matthew Finkel 0x017DD169EA793BE2 <sysrqb@torproject.org>
# :copyright: (c) 2013-2015, Isis Lovecruft
#             (c) 2013-2015, Matthew Finkel
#             (c) 2007-2015, The Tor Project, Inc.
# :license: see LICENSE for licensing information

"""
bridgedb.https.distributor
==========================

A Distributor that hands out bridges through a web interface.

.. inheritance-diagram:: HTTPSDistributor
    :parts: 1
"""

import ipaddr
import logging

import bridgedb.Storage

from bridgedb import proxy
from bridgedb.Bridges import BridgeRing
from bridgedb.Bridges import FilteredBridgeSplitter
from bridgedb.crypto import getHMAC
from bridgedb.crypto import getHMACFunc
from bridgedb.distribute import Distributor
from bridgedb.filters import byIPv4
from bridgedb.filters import byIPv6
from bridgedb.filters import byFilters
from bridgedb.filters import bySubring


[docs]class HTTPSDistributor(Distributor): """A Distributor that hands out bridges based on the IP address of an incoming request and the current time period. :type proxies: :class:`~bridgedb.proxies.ProxySet` :ivar proxies: All known proxies, which we treat differently. See :param:`proxies`. :type hashring: :class:`bridgedb.Bridges.FilteredBridgeSplitter` :ivar hashring: A hashring that assigns bridges to subrings with fixed proportions. Used to assign bridges into the subrings of this distributor. """ def __init__(self, totalSubrings, key, proxies=None, answerParameters=None): """Create a Distributor that decides which bridges to distribute based upon the client's IP address and the current time. :param int totalSubrings: The number of subhashrings to group clients into. Note that if ``PROXY_LIST_FILES`` is set in bridgedb.conf, then the actual number of clusters is one higher than ``totalSubrings``, because the set of all known open proxies is given its own subhashring. :param bytes key: The master HMAC key for this distributor. All added bridges are HMACed with this key in order to place them into the hashrings. :type proxies: :class:`~bridgedb.proxy.ProxySet` :param proxies: A :class:`bridgedb.proxy.ProxySet` containing known Tor Exit relays and other known proxies. These will constitute the extra cluster, and any client requesting bridges from one of these **proxies** will be distributed bridges from a separate subhashring that is specific to Tor/proxy users. :type answerParameters: :class:`bridgedb.Bridges.BridgeRingParameters` :param answerParameters: A mechanism for ensuring that the set of bridges that this distributor answers a client with fit certain parameters, i.e. that an answer has "at least two obfsproxy bridges" or "at least one bridge on port 443", etc. """ super(HTTPSDistributor, self).__init__(key) self.totalSubrings = totalSubrings self.answerParameters = answerParameters if proxies: logging.info("Added known proxies to HTTPS distributor...") self.proxies = proxies self.totalSubrings += 1 self.proxySubring = self.totalSubrings else: logging.warn("No known proxies were added to HTTPS distributor!") self.proxies = proxy.ProxySet() self.proxySubring = 0 self.ringCacheSize = self.totalSubrings * 3 key2 = getHMAC(key, "Assign-Bridges-To-Rings") key3 = getHMAC(key, "Order-Areas-In-Rings") key4 = getHMAC(key, "Assign-Areas-To-Rings") self._clientToPositionHMAC = getHMACFunc(key3, hex=False) self._subnetToSubringHMAC = getHMACFunc(key4, hex=True) self.hashring = FilteredBridgeSplitter(key2, self.ringCacheSize) self.name = 'HTTPS' logging.debug("Added %s to %s distributor." % (self.hashring.__class__.__name__, self.name))
[docs] def bridgesPerResponse(self, hashring=None): return super(HTTPSDistributor, self).bridgesPerResponse(hashring)
@classmethod
[docs] def getSubnet(cls, ip, usingProxy=False, proxySubnets=4): """Map all clients whose **ip**s are within the same subnet to the same arbitrary string. .. hint:: For non-proxy IP addresses, any two IPv4 addresses within the same ``/16`` subnet, or any two IPv6 addresses in the same ``/32`` subnet, will get the same string. Subnets for this distributor are grouped into the number of rings specified by the ``N_IP_CLUSTERS`` configuration option, such that Alice (with the address ``1.2.3.4`` and Bob (with the address ``1.2.178.234``) are placed within the same cluster, but Carol (with address ``1.3.11.33``) *might* end up in a different cluster. >>> from bridgedb.https.distributor import HTTPSDistributor >>> HTTPSDistributor.getSubnet('1.2.3.4') '1.2.0.0/16' >>> HTTPSDistributor.getSubnet('1.2.211.154') '1.2.0.0/16' >>> HTTPSDistributor.getSubnet('2001:f::bc1:b13:2808') '2001:f::/32' >>> HTTPSDistributor.getSubnet('2a00:c98:2030:a020:2::42') '2a00:c98::/32' :param str ip: A string representing an IPv4 or IPv6 address. :param bool usingProxy: Set to ``True`` if the client was using one of the known :data:`proxies`. :param int proxySubnets: Place Tor/proxy users into this number of "subnet" groups. This means that no matter how many different Tor Exits or proxies a client uses, the most they can ever get is **proxySubnets** different sets of bridge lines (per interval). This parameter only has any effect when **usingProxy** is ``True``. :rtype: str :returns: The appropriately sized CIDR subnet representation of the **ip**. """ if not usingProxy: # We aren't using bridgedb.parse.addr.isIPAddress(ip, # compressed=False) here because adding the string "False" into # the map would land any and all clients whose IP address appeared # to be invalid at the same position in a hashring. address = ipaddr.IPAddress(ip) if address.version == 6: truncated = ':'.join(address.exploded.split(':')[:2]) subnet = str(ipaddr.IPv6Network(truncated + "::/32")) else: truncated = '.'.join(address.exploded.split('.')[:2]) subnet = str(ipaddr.IPv4Network(truncated + '.0.0/16')) else: group = (int(ipaddr.IPAddress(ip)) % 4) + 1 subnet = "proxy-group-%d" % group logging.debug("Client IP was within area: %s" % subnet) return subnet
[docs] def mapSubnetToSubring(self, subnet, usingProxy=False): """Determine the correct subhashring for a client, based upon the **subnet**. :param str subnet: The subnet which contains the client's IP. See :staticmethod:`getSubnet`. :param bool usingProxy: Set to ``True`` if the client was using one of the known :data:`proxies`. """ # If the client wasn't using a proxy, select the client's subring # based upon the client's subnet (modulo the total subrings): if not usingProxy: mod = self.totalSubrings # If there is a proxy subring, don't count it for the modulus: if self.proxySubring: mod -= 1 return (int(self._subnetToSubringHMAC(subnet)[:8], 16) % mod) + 1 else: return self.proxySubring
[docs] def mapClientToHashringPosition(self, interval, subnet): """Map the client to a position on a (sub)hashring, based upon the **interval** which the client's request occurred within, as well as the **subnet** of the client's IP address. .. note:: For an explanation of how **subnet** is determined, see :staticmethod:`getSubnet`. :param str interval: The interval which this client's request for bridges took place within. :param str subnet: A string representing the subnet containing the client's IP address. :rtype: int :returns: The results of keyed HMAC, which should determine the client's position in a (sub)hashring of bridges (and thus determine which bridges they receive). """ position = "<%s>%s" % (interval, subnet) mapping = self._clientToPositionHMAC(position) return mapping
[docs] def prepopulateRings(self): """Prepopulate this distributor's hashrings and subhashrings with bridges. The hashring structure for this distributor is influenced by the ``N_IP_CLUSTERS`` configuration option, as well as the number of ``PROXY_LIST_FILES``. Essentially, :data:`totalSubrings` is set to the specified ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of Tor Exit relays (downloaded into memory with :script:`get-tor-exits`), are stored in :data:`proxies`, and the latter is added as an additional cluster (such that :data:`totalSubrings` becomes ``N_IP_CLUSTERS + 1``). The number of subhashrings which this :class:`Distributor` has active in its hashring is then :data:`totalSubrings`, where the last cluster is reserved for all :data:`proxies`. As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and ``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number of subhashrings is five — four for the "clusters", and one for the :data:`proxies`. Thus, the resulting hashring-subhashring structure would look like: +------------------+---------------------------------------------------+-------------+ | | Directly connecting users | Tor / known | | | | proxy users | +------------------+------------+------------+------------+------------+-------------+ | Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 | +==================+============+============+============+============+=============+ | Subhashrings | | | | | | | (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) | +------------------+------------+------------+------------+------------+-------------+ | Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 | | Subhashrings | | | | | | | bBy requested +------------+------------+------------+------------+-------------+ | bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 | | | | | | | | +------------------+------------+------------+------------+------------+-------------+ The "filtered subhashrings" are essentially filtered copies of their respective subhashring, such that they only contain bridges which support IPv4 or IPv6, respectively. Additionally, the contents of ``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint. Thus, in this example, we end up with **10 total subhashrings**. """ logging.info("Prepopulating %s distributor hashrings..." % self.name) for filterFn in [byIPv4, byIPv6]: for subring in range(1, self.totalSubrings + 1): filters = self._buildHashringFilters([filterFn,], subring) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = BridgeRing(key1, self.answerParameters) # For consistency with previous implementation of this method, # only set the "name" for "clusters" which are for this # distributor's proxies: if subring == self.proxySubring: ring.setName('{0} Proxy Ring'.format(self.name)) self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges)
[docs] def insert(self, bridge): """Assign a bridge to this distributor.""" self.hashring.insert(bridge)
def _buildHashringFilters(self, previousFilters, subring): f = bySubring(self.hashring.hmac, subring, self.totalSubrings) previousFilters.append(f) return frozenset(previousFilters)
[docs] def getBridges(self, bridgeRequest, interval): """Return a list of bridges to give to a user. :type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest` :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` attribute set to a string containing the client's IP address. :param str interval: The time period when we got this request. This can be any string, so long as it changes with every period. :rtype: list :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in the response. See :meth:`bridgedb.https.server.WebResourceBridges.getBridgeRequestAnswer` for an example of how this is used. """ logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) if not len(self.hashring): logging.warn("Bailing! Hashring has zero bridges!") return [] usingProxy = False # First, check if the client's IP is one of the known :data:`proxies`: if bridgeRequest.client in self.proxies: # The tag is a tag applied to a proxy IP address when it is added # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default # is 'exit_relay'. For other proxies loaded from the # PROXY_LIST_FILES config option, the default tag is the full # filename that the IP address originally came from. usingProxy = True tag = self.proxies.getTag(bridgeRequest.client) logging.info("Client was from known proxy (tag: %s): %s" % (tag, bridgeRequest.client)) subnet = self.getSubnet(bridgeRequest.client, usingProxy) subring = self.mapSubnetToSubring(subnet, usingProxy) position = self.mapClientToHashringPosition(interval, subnet) filters = self._buildHashringFilters(bridgeRequest.filters, subring) logging.debug("Client request within time interval: %s" % interval) logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) logging.debug("Total bridges: %d" % len(self.hashring)) logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) # Check wheth we have a cached copy of the hashring: if filters in self.hashring.filterRings.keys(): logging.debug("Cache hit %s" % filters) _, ring = self.hashring.filterRings[filters] # Otherwise, construct a new hashring and populate it: else: logging.debug("Cache miss %s" % filters) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = BridgeRing(key1, self.answerParameters) self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges) # Determine the appropriate number of bridges to give to the client: returnNum = self.bridgesPerResponse(ring) answer = ring.getBridges(position, returnNum) return answer