diff --git a/docs/source/conf.py b/docs/source/conf.py index aa8ffca2..eb57ebbe 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,6 +13,8 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # import os + + def find_version(): with open(os.path.join("..", "..", "sbws", "__init__.py")) as fp: for line in fp: @@ -197,5 +199,5 @@ def find_version(): todo_include_todos = True source_parsers = { - '.md': 'recommonmark.parser.CommonMarkParser', + '.md': 'recommonmark.parser.CommonMarkParser', } diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py index 9a04ec0e..ad94fe50 100644 --- a/sbws/core/cleanup.py +++ b/sbws/core/cleanup.py @@ -1,28 +1,27 @@ """Util functions to cleanup disk space.""" +import gzip +import logging +import os +import shutil +import time import types +from argparse import ArgumentDefaultsHelpFormatter +from datetime import datetime, timedelta -from sbws.util.filelock import DirectoryLock from sbws.globals import fail_hard +from sbws.util.filelock import DirectoryLock from sbws.util.timestamp import unixts_to_dt_obj -from argparse import ArgumentDefaultsHelpFormatter -from datetime import datetime -from datetime import timedelta -import os -import gzip -import shutil -import logging -import time log = logging.getLogger(__name__) def gen_parser(sub): - ''' + """ Helper function for the broader argument parser generating code that adds in all the possible command line arguments for the cleanup command. :param argparse._SubParsersAction sub: what to add a sub-parser to - ''' + """ d = 'Compress and delete results and/or v3bw files old files.' \ 'Configuration options are read to determine which are old files' p = sub.add_parser('cleanup', description=d, @@ -168,12 +167,12 @@ def _clean_result_files(args, conf): def main(args, conf): - ''' + """ Main entry point in to the cleanup command. :param argparse.Namespace args: command line arguments :param configparser.ConfigParser conf: parsed config files - ''' + """ datadir = conf.getpath('paths', 'datadir') if not os.path.isdir(datadir): fail_hard('%s does not exist', datadir) diff --git a/sbws/core/generate.py b/sbws/core/generate.py index 353bea04..34114aff 100644 --- a/sbws/core/generate.py +++ b/sbws/core/generate.py @@ -1,13 +1,13 @@ +import logging +import os +from argparse import ArgumentDefaultsHelpFormatter from math import ceil -from sbws.globals import (fail_hard, SBWS_SCALE_CONSTANT, TORFLOW_SCALING, - SBWS_SCALING, TORFLOW_BW_MARGIN, PROP276_ROUND_DIG, - DAY_SECS, NUM_MIN_RESULTS) -from sbws.lib.v3bwfile import V3BWFile +from sbws.globals import (DAY_SECS, NUM_MIN_RESULTS, PROP276_ROUND_DIG, + SBWS_SCALE_CONSTANT, SBWS_SCALING, TORFLOW_BW_MARGIN, + TORFLOW_SCALING, fail_hard) from sbws.lib.resultdump import load_recent_results_in_datadir -from argparse import ArgumentDefaultsHelpFormatter -import os -import logging +from sbws.lib.v3bwfile import V3BWFile from sbws.util.timestamp import now_fname log = logging.getLogger(__name__) diff --git a/sbws/core/scanner.py b/sbws/core/scanner.py index 7246f12f..26f00f5d 100644 --- a/sbws/core/scanner.py +++ b/sbws/core/scanner.py @@ -1,26 +1,27 @@ -''' Measure the relays. ''' +"""Measure the relays. """ -from ..lib.circuitbuilder import GapsCircuitBuilder as CB -from ..lib.resultdump import ResultDump -from ..lib.resultdump import ResultSuccess, ResultErrorCircuit -from ..lib.resultdump import ResultErrorStream -from ..lib.relaylist import RelayList -from ..lib.relayprioritizer import RelayPrioritizer -from ..lib.destination import DestinationList -from ..util.timestamp import now_isodt_str -from ..util.state import State -from sbws.globals import fail_hard -import sbws.util.stem as stem_utils -import sbws.util.requests as requests_utils +import logging +import os +import random +import time from argparse import ArgumentDefaultsHelpFormatter from multiprocessing.dummy import Pool from threading import Event -import time -import os -import logging + import requests -import random +import sbws.util.requests as requests_utils +import sbws.util.stem as stem_utils +from sbws.globals import fail_hard + +from ..lib.circuitbuilder import GapsCircuitBuilder as CB +from ..lib.destination import DestinationList +from ..lib.relaylist import RelayList +from ..lib.relayprioritizer import RelayPrioritizer +from ..lib.resultdump import (ResultDump, ResultErrorCircuit, + ResultErrorStream, ResultSuccess) +from ..util.state import State +from ..util.timestamp import now_isodt_str rng = random.SystemRandom() end_event = Event() @@ -28,9 +29,9 @@ def timed_recv_from_server(session, dest, byte_range): - ''' Request the **byte_range** from the URL at **dest**. If successful, + """Request the **byte_range** from the URL at **dest**. If successful, return True and the time it took to download. Otherwise return False and an - exception. ''' + exception. """ headers = {'Range': byte_range, 'Accept-Encoding': 'identity'} start_time = time.time() # TODO: @@ -48,13 +49,13 @@ def timed_recv_from_server(session, dest, byte_range): def get_random_range_string(content_length, size): - ''' + """ Return a random range of bytes of length **size**. **content_length** is the size of the file we will be requesting a range of bytes from. For example, for content_length of 100 and size 10, this function will return one of the following: '0-9', '1-10', '2-11', [...] '89-98', '90-99' - ''' + """ assert size <= content_length # start can be anywhere in the content_length as long as it is **size** # bytes away from the end or more. Because range is [start, end) (doesn't @@ -72,7 +73,7 @@ def get_random_range_string(content_length, size): def measure_rtt_to_server(session, conf, dest, content_length): - ''' Make multiple end-to-end RTT measurements by making small HTTP requests + """Make multiple end-to-end RTT measurements by making small HTTP requests over a circuit + stream that should already exist, persist, and not need rebuilding. If something goes wrong and not all of the RTT measurements can be made, return None. Otherwise return a list of the RTTs (in seconds). @@ -80,7 +81,7 @@ def measure_rtt_to_server(session, conf, dest, content_length): :returns tuple: results or None if the if the measurement fail. None or exception if the measurement fail. - ''' + """ rtts = [] size = conf.getint('scanner', 'min_download_size') for _ in range(0, conf.getint('scanner', 'num_rtts')): @@ -141,11 +142,11 @@ def measure_bandwidth_to_server(session, conf, dest, content_length): def _pick_ideal_second_hop(relay, dest, rl, cont, is_exit): - ''' + """ Sbws builds two hop circuits. Given the **relay** to measure with destination **dest**, pick a second relay that is or is not an exit according to **is_exit**. - ''' + """ candidates = rl.exits_not_bad_allowing_port(dest.port) if is_exit \ else rl.non_exits if not len(candidates): @@ -313,17 +314,17 @@ def _next_expected_amount(expected_amount, result_time, download_times, def result_putter(result_dump): - ''' Create a function that takes a single argument -- the measurement - result -- and return that function so it can be used by someone else ''' + """Create a function that takes a single argument -- the measurement + result -- and return that function so it can be used by someone else """ def closure(measurement_result): return result_dump.queue.put(measurement_result) return closure def result_putter_error(target): - ''' Create a function that takes a single argument -- an error from a + """Create a function that takes a single argument -- an error from a measurement -- and return that function so it can be used by someone else - ''' + """ def closure(err): log.error('Unhandled exception caught while measuring %s: %s %s', target.nickname, type(err), err) diff --git a/sbws/core/stats.py b/sbws/core/stats.py index f865b92a..2a5ef1f7 100644 --- a/sbws/core/stats.py +++ b/sbws/core/stats.py @@ -1,16 +1,13 @@ -from sbws.globals import fail_hard -from sbws.lib.resultdump import Result -from sbws.lib.resultdump import ResultError -from sbws.lib.resultdump import ResultErrorCircuit -from sbws.lib.resultdump import ResultErrorStream -from sbws.lib.resultdump import ResultSuccess -from sbws.lib.resultdump import load_recent_results_in_datadir -from argparse import ArgumentDefaultsHelpFormatter +import logging import os -from datetime import datetime -from datetime import timedelta +from argparse import ArgumentDefaultsHelpFormatter +from datetime import datetime, timedelta from statistics import mean -import logging + +from sbws.globals import fail_hard +from sbws.lib.resultdump import (Result, ResultError, ResultErrorCircuit, + ResultErrorStream, ResultSuccess, + load_recent_results_in_datadir) log = logging.getLogger(__name__) @@ -32,7 +29,8 @@ def _print_stats_error_types(data): continue number = counts[count_type] print('{}/{} ({:.2f}%) results were {}'.format( - number, counts['total'], 100*number/counts['total'], count_type)) + number, counts['total'], 100 * number / counts['total'], + count_type)) def _result_type_per_relay(data, result_type): @@ -43,10 +41,10 @@ def _result_type_per_relay(data, result_type): def _get_box_plot_values(iterable): - ''' Reutrn the min, q1, med, q1, and max of the input list or iterable. + """Reutrn the min, q1, med, q1, and max of the input list or iterable. This function is NOT perfect, and I think that's fine for basic statistical needs. Instead of median, it will return low or high median. Same for q1 - and q3. ''' + and q3. """ if not isinstance(iterable, list): iterable = list(iterable) iterable.sort() @@ -55,7 +53,7 @@ def _get_box_plot_values(iterable): q1_idx = round(length / 4) q3_idx = median_idx + q1_idx return [iterable[0], iterable[q1_idx], iterable[median_idx], - iterable[q3_idx], iterable[length-1]] + iterable[q3_idx], iterable[length - 1]] def _print_results_type_box_plot(data, result_type): @@ -78,14 +76,14 @@ def _print_averages(data): def _results_into_bandwidths(results, limit=5): - ''' + """ For all the given resutls, extract their download statistics and normalize them into bytes/second bandwidths. :param list results: list of :class:`sbws.list.resultdump.ResultSuccess` :param int limit: The maximum number of bandwidths to return :returns: list of up to `limit` bandwidths, with the largest first - ''' + """ downloads = [] for result in results: assert isinstance(result, ResultSuccess) @@ -95,14 +93,14 @@ def _results_into_bandwidths(results, limit=5): def print_stats(args, data): - ''' + """ Called from main to print various statistics about the organized **data** to stdout. :param argparse.Namespace args: command line arguments :param dict data: keyed by relay fingerprint, and with values of :class:`sbws.lib.resultdump.Result` subclasses - ''' + """ results = [] for fp in data: results.extend(data[fp]) @@ -127,7 +125,7 @@ def print_stats(args, data): print(len(success_results), 'success results and', len(error_results), 'error results') print('The fastest download was {:.2f} KiB/s'.format( - fastest_transfer/1024)) + fastest_transfer / 1024)) print('Results come from', first, 'to', last, 'over a period of', duration) if getattr(args, 'error_types', False) is True: @@ -135,12 +133,12 @@ def print_stats(args, data): def gen_parser(sub): - ''' + """ Helper function for the broader argument parser generating code that adds in all the possible command line arguments for the stats command. :param argparse._SubParsersAction sub: what to add a sub-parser to - ''' + """ d = 'Write some statistics about the data collected so far to stdout' p = sub.add_parser('stats', formatter_class=ArgumentDefaultsHelpFormatter, description=d) @@ -149,12 +147,12 @@ def gen_parser(sub): def main(args, conf): - ''' + """ Main entry point into the stats command. :param argparse.Namespace args: command line arguments :param configparser.ConfigParser conf: parsed config files - ''' + """ datadir = conf.getpath('paths', 'datadir') if not os.path.isdir(datadir): diff --git a/sbws/globals.py b/sbws/globals.py index 2277850d..aace1ca5 100644 --- a/sbws/globals.py +++ b/sbws/globals.py @@ -1,5 +1,5 @@ -import os import logging +import os log = logging.getLogger(__name__) @@ -55,13 +55,13 @@ def fail_hard(*a, **kw): - ''' Log something ... and then exit as fast as possible ''' + """Log something ... and then exit as fast as possible """ log.critical(*a, **kw) exit(1) def touch_file(fname, times=None): - ''' + """ If **fname** exists, update its last access and modified times to now. If **fname** does not exist, create it. If **times** are specified, pass them to os.utime for use. @@ -69,7 +69,7 @@ def touch_file(fname, times=None): :param str fname: Name of file to update or create :param tuple times: 2-tuple of floats for access time and modified time respectively - ''' + """ log.debug('Touching %s', fname) with open(fname, 'a') as fd: os.utime(fd.fileno(), times=times) diff --git a/sbws/lib/circuitbuilder.py b/sbws/lib/circuitbuilder.py index 948e6800..edb87098 100644 --- a/sbws/lib/circuitbuilder.py +++ b/sbws/lib/circuitbuilder.py @@ -1,13 +1,16 @@ -from stem import CircuitExtensionFailed, InvalidRequest, ProtocolError, Timeout -from stem import InvalidArguments, ControllerError +import logging import random + +from stem import (CircuitExtensionFailed, ControllerError, InvalidArguments, + InvalidRequest, ProtocolError, Timeout) + from .relaylist import Relay -import logging log = logging.getLogger(__name__) class PathLengthException(Exception): + def __init__(self, message=None, errors=None): if message is not None: super().__init__(message) @@ -24,7 +27,8 @@ def valid_circuit_length(path): class CircuitBuilder: - ''' The CircuitBuilder interface. + + """The CircuitBuilder interface. Subclasses must implement their own build_circuit() function. Subclasses may keep additional state if they'd find it helpful. @@ -36,7 +40,8 @@ class CircuitBuilder: It might be good practice to close circuits as you find you no longer need them, but CircuitBuilder will keep track of existing circuits and close them when it is deleted. - ''' + """ + def __init__(self, args, conf, controller, relay_list, close_circuits_on_exit=True): self.controller = controller @@ -51,8 +56,8 @@ def relays(self): return self.relay_list.relays def build_circuit(self, *a, **kw): - ''' Implementations of this method should build the circuit and return - its (str) ID. If it cannot be built, it should return None. ''' + """Implementations of this method should build the circuit and return + its (str) ID. If it cannot be built, it should return None. """ raise NotImplementedError() def close_circuit(self, circ_id): @@ -110,15 +115,17 @@ def __del__(self): class GapsCircuitBuilder(CircuitBuilder): - ''' The build_circuit member function takes a list. Falsey values in the + + """The build_circuit member function takes a list. Falsey values in the list will be replaced with relays chosen uniformally at random; Truthy - values will be assumed to be relays. ''' + values will be assumed to be relays. """ + def __init__(self, *a, **kw): super().__init__(*a, **kw) def _normalize_path(self, path): - ''' Change fingerprints/nicks to relay descriptor and change Falsey - values to None. Return the new path, or None if error ''' + """Change fingerprints/nicks to relay descriptor and change Falsey + values to None. Return the new path, or None if error """ new_path = [] for fp in path: if not fp: @@ -132,9 +139,9 @@ def _normalize_path(self, path): return new_path def _random_sample_relays(self, number, blacklist): - ''' Get random relays from self.relays that are not in the + """Get random relays from self.relays that are not in the blacklist. Return None if it cannot be done because too many are - blacklisted. Otherwise return a list of relays. ''' + blacklisted. Otherwise return a list of relays. """ all_fps = [r.fingerprint for r in self.relays] black_fps = [r.fingerprint for r in blacklist] if len(black_fps) + number > len(all_fps): @@ -149,11 +156,11 @@ def _random_sample_relays(self, number, blacklist): return [Relay(fp, self.controller) for fp in chosen_fps] def build_circuit(self, path): - ''' is a list of relays and Falsey values. Relays can be + """ is a list of relays and Falsey values. Relays can be specified by fingerprint or nickname, and fingerprint is highly recommended. Falsey values (like None) will be replaced with relays chosen uniformally at random. A relay will not be in a circuit twice. - ''' + """ if not valid_circuit_length(path): raise PathLengthException() path = self._normalize_path(path) diff --git a/sbws/lib/destination.py b/sbws/lib/destination.py index e69fdbbc..c2543caf 100644 --- a/sbws/lib/destination.py +++ b/sbws/lib/destination.py @@ -1,13 +1,15 @@ import logging +import os import random import time -import os from threading import RLock -import requests from urllib.parse import urlparse + +import requests from stem.control import EventType -import sbws.util.stem as stem_utils + import sbws.util.requests as requests_utils +import sbws.util.stem as stem_utils log = logging.getLogger(__name__) @@ -29,7 +31,7 @@ def _parse_verify_option(conf_section): def connect_to_destination_over_circuit(dest, circ_id, session, cont, max_dl): - ''' + """ Connect to **dest* over the given **circ_id** using the given Requests **session**. Make sure the destination seems usable. Return True and a dictionary of helpful information if we connected and the destination is @@ -64,7 +66,7 @@ def connect_to_destination_over_circuit(dest, circ_id, session, cont, max_dl): :param cont Controller: them Stem library controller controlling Tor :returns: True and a dictionary if everything is in order and measurements should commence. False and an error string otherwise. - ''' + """ assert isinstance(dest, Destination) error_prefix = 'When sending HTTP HEAD to {}, '.format(dest.url) with stem_utils.stream_building_lock: @@ -85,7 +87,7 @@ def connect_to_destination_over_circuit(dest, circ_id, session, cont, max_dl): '{} not {}'.format(requests.codes.ok, head.status_code) if 'content-length' not in head.headers: return False, error_prefix + 'we except the header Content-Length '\ - 'to exist in the response' + 'to exist in the response' content_length = int(head.headers['content-length']) if max_dl > content_length: return False, error_prefix + 'our maximum configured download size '\ @@ -95,6 +97,7 @@ def connect_to_destination_over_circuit(dest, circ_id, session, cont, max_dl): class Destination: + def __init__(self, url, max_dl, verify): self._max_dl = max_dl u = urlparse(url) @@ -102,9 +105,9 @@ def __init__(self, url, max_dl, verify): self._verify = verify def is_usable(self, circ_id, session, cont): - ''' Use **connect_to_destination_over_circuit** to determine if this + """Use **connect_to_destination_over_circuit** to determine if this destination is usable and return what it returns. Just a small wrapper. - ''' + """ if not isinstance(self.verify, bool): if not os.path.isfile(self.verify): return False, '{} is believed to be a CA bundle file on disk '\ @@ -147,6 +150,7 @@ def from_config(conf_section, max_dl): class DestinationList: + def __init__(self, conf, dests, circuit_builder, relay_list, controller): assert len(dests) > 0 for dest in dests: @@ -240,9 +244,9 @@ def from_config(conf, circuit_builder, relay_list, controller): controller), '' def next(self): - ''' + """ Returns the next destination that should be used in a measurement - ''' + """ with self._usability_lock: while True: if self._should_perform_usability_test(): diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py index 38635ba2..67e89eda 100644 --- a/sbws/lib/relaylist.py +++ b/sbws/lib/relaylist.py @@ -1,24 +1,26 @@ -from stem.descriptor.router_status_entry import RouterStatusEntryV3 -from stem.descriptor.server_descriptor import ServerDescriptor -from stem import Flag, DescriptorUnavailable, ControllerError +import logging import random import time -import logging from threading import Lock +from stem import ControllerError, DescriptorUnavailable, Flag +from stem.descriptor.router_status_entry import RouterStatusEntryV3 +from stem.descriptor.server_descriptor import ServerDescriptor + log = logging.getLogger(__name__) class Relay: + def __init__(self, fp, cont, ns=None, desc=None): - ''' + """ Given a relay fingerprint, fetch all the information about a relay that sbws currently needs and store it in this class. Acts as an abstraction to hide the confusion that is Tor consensus/descriptor stuff. :param str fp: fingerprint of the relay. :param cont: active and valid stem Tor controller connection - ''' + """ assert isinstance(fp, str) assert len(fp) == 40 if ns is not None: @@ -127,10 +129,11 @@ def is_exit_not_bad_allowing_port(self, port): class RelayList: - ''' Keeps a list of all relays in the current Tor network and updates it + + """Keeps a list of all relays in the current Tor network and updates it transparently in the background. Provides useful interfaces for getting only relays of a certain type. - ''' + """ REFRESH_INTERVAL = 300 # seconds def __init__(self, args, conf, controller): diff --git a/sbws/lib/relayprioritizer.py b/sbws/lib/relayprioritizer.py index 8d3b0c52..ac3ef7eb 100644 --- a/sbws/lib/relayprioritizer.py +++ b/sbws/lib/relayprioritizer.py @@ -1,20 +1,21 @@ -from decimal import Decimal -from ..lib.resultdump import ResultDump -from ..lib.resultdump import Result -from ..lib.resultdump import ResultError -from ..lib.relaylist import RelayList import copy -import time import logging +import time +from decimal import Decimal + +from ..lib.relaylist import RelayList +from ..lib.resultdump import Result, ResultDump, ResultError log = logging.getLogger(__name__) class RelayPrioritizer: + def __init__(self, args, conf, relay_list, result_dump): assert isinstance(relay_list, RelayList) assert isinstance(result_dump, ResultDump) - self.fresh_seconds = conf.getint('general', 'data_period')*24*60*60 + self.fresh_seconds = conf.getint('general', 'data_period') \ + * 24 * 60 * 60 self.relay_list = relay_list self.result_dump = result_dump self.measure_authorities = conf.getboolean( @@ -24,7 +25,7 @@ def __init__(self, args, conf, relay_list, result_dump): 'relayprioritizer', 'fraction_relays') def best_priority(self): - ''' Return a generator containing the best priority relays. + """Return a generator containing the best priority relays. NOTE: A lower value for priority means better priority. Remember your data structures class in university and consider this something like a @@ -44,7 +45,7 @@ def best_priority(self): with equal weight as successful results, then it would take a while to get around to giving the relay another chance at a getting a successful measurement. - ''' + """ fn_tstart = Decimal(time.time()) relays = set(copy.deepcopy(self.relay_list.relays)) if not self.measure_authorities: diff --git a/sbws/lib/resultdump.py b/sbws/lib/resultdump.py index 52a0b7cc..1c9ad061 100644 --- a/sbws/lib/resultdump.py +++ b/sbws/lib/resultdump.py @@ -1,29 +1,26 @@ -import os import json -import time import logging -from glob import glob -from threading import Thread -from threading import Event -from threading import RLock -from queue import Queue -from queue import Empty -from datetime import datetime -from datetime import timedelta +import os +import time +from datetime import datetime, timedelta from enum import Enum +from glob import glob +from queue import Empty, Queue +from threading import Event, RLock, Thread + from sbws.globals import RESULT_VERSION, fail_hard -from sbws.util.filelock import DirectoryLock from sbws.lib.relaylist import Relay +from sbws.util.filelock import DirectoryLock log = logging.getLogger(__name__) def merge_result_dicts(d1, d2): - ''' + """ Given two dictionaries that contain Result data, merge them. Result dictionaries have keys of relay fingerprints and values of lists of results for those relays. - ''' + """ for key in d2: if key not in d1: d1[key] = [] @@ -32,10 +29,10 @@ def merge_result_dicts(d1, d2): def load_result_file(fname, success_only=False): - ''' Reads in all lines from the given file, and parses them into Result + """Reads in all lines from the given file, and parses them into Result structures (or subclasses of Result). Optionally only keeps ResultSuccess. Returns all kept Results as a result dictionary. This function does not - care about the age of the results ''' + care about the age of the results """ assert os.path.isfile(fname) d = {} num_total = 0 @@ -67,11 +64,11 @@ def load_result_file(fname, success_only=False): def trim_results(fresh_days, result_dict): - ''' Given a result dictionary, remove all Results that are no longer valid - and return the new dictionary ''' + """Given a result dictionary, remove all Results that are no longer valid + and return the new dictionary """ assert isinstance(fresh_days, int) assert isinstance(result_dict, dict) - data_period = fresh_days * 24*60*60 + data_period = fresh_days * 24 * 60 * 60 oldest_allowed = time.time() - data_period out_results = {} for fp in result_dict: @@ -128,9 +125,9 @@ def trim_results_ip_changed(result_dict, on_changed_ipv4=False, def load_recent_results_in_datadir(fresh_days, datadir, success_only=False, on_changed_ipv4=False, on_changed_ipv6=False): - ''' Given a data directory, read all results files in it that could have + """Given a data directory, read all results files in it that could have results in them that are still valid. Trim them, and return the valid - Results as a list ''' + Results as a list """ assert isinstance(fresh_days, int) assert os.path.isdir(datadir) # Inform the results are being loaded, since it takes some seconds. @@ -169,7 +166,7 @@ def load_recent_results_in_datadir(fresh_days, datadir, success_only=False, def write_result_to_datadir(result, datadir): - ''' Can be called from any thread ''' + """Can be called from any thread """ assert isinstance(result, Result) assert os.path.isdir(datadir) dt = datetime.utcfromtimestamp(result.time) @@ -195,12 +192,15 @@ class _ResultType(_StrEnum): class Result: - ''' A simple struct to pack a measurement result into so that other code - can be confident it is handling a well-formed result. ''' + + """A simple struct to pack a measurement result into so that other code + can be confident it is handling a well-formed result. """ class Relay: - ''' Implements just enough of a stem RouterStatusEntryV3 for this - Result class to be happy ''' + + """Implements just enough of a stem RouterStatusEntryV3 for this + Result class to be happy """ + def __init__(self, fingerprint, nickname, address, master_key_ed25519, average_bandwidth=None, burst_bandwidth=None, observed_bandwidth=None, consensus_bandwidth=None, @@ -305,11 +305,11 @@ def to_dict(self): @staticmethod def from_dict(d): - ''' Given a dict, returns the Result* subtype that is represented by + """Given a dict, returns the Result* subtype that is represented by the dict. If we don't know how to parse the dict into a Result and it's likely because the programmer forgot to implement something, raises NotImplementedError. If we can't parse the dict for some other reason, - return None. ''' + return None. """ assert 'version' in d if d['version'] != RESULT_VERSION: return None @@ -333,6 +333,7 @@ def __str__(self): class ResultError(Result): + def __init__(self, *a, msg=None, **kw): super().__init__(*a, **kw) self._msg = msg @@ -343,7 +344,7 @@ def type(self): @property def freshness_reduction_factor(self): - ''' + """ When the RelayPrioritizer encounters this Result, how much should it adjust its freshness? (See RelayPrioritizer.best_priority() for more information about "freshness") @@ -355,7 +356,7 @@ def freshness_reduction_factor(self): The value 0.5 was chosen somewhat arbitrarily, but a few weeks of live network testing verifies that sbws is still able to perform useful measurements in a reasonable amount of time. - ''' + """ return 0.5 @property @@ -381,6 +382,7 @@ def to_dict(self): class ResultErrorCircuit(ResultError): + def __init__(self, *a, **kw): super().__init__(*a, **kw) @@ -390,7 +392,7 @@ def type(self): @property def freshness_reduction_factor(self): - ''' + """ There are a few instances when it isn't the relay's fault that the circuit failed to get built. Maybe someday we'll try detecting whose fault it most likely was and subclassing ResultErrorCircuit. But for @@ -400,7 +402,7 @@ def freshness_reduction_factor(self): A (hopefully very very rare) example of when a circuit would fail to get built is when the sbws client machine suddenly loses Internet access. - ''' + """ return 0.6 @staticmethod @@ -419,6 +421,7 @@ def to_dict(self): class ResultErrorStream(ResultError): + def __init__(self, *a, **kw): super().__init__(*a, **kw) @@ -442,6 +445,7 @@ def to_dict(self): class ResultErrorAuth(ResultError): + def __init__(self, *a, **kw): super().__init__(*a, **kw) @@ -451,7 +455,7 @@ def type(self): @property def freshness_reduction_factor(self): - ''' + """ Override the default ResultError.freshness_reduction_factor because a ResultErrorAuth is most likely not the measured relay's fault, so we shouldn't hurt its priority as much. A higher reduction factor means a @@ -459,7 +463,7 @@ def freshness_reduction_factor(self): priority better. The value 0.9 was chosen somewhat arbitrarily. - ''' + """ return 0.9 @staticmethod @@ -478,6 +482,7 @@ def to_dict(self): class ResultSuccess(Result): + def __init__(self, rtts, downloads, *a, **kw): super().__init__(*a, **kw) self._rtts = rtts @@ -525,8 +530,10 @@ def to_dict(self): class ResultDump: - ''' Runs the enter() method in a new thread and collects new Results on its - queue. Writes them to daily result files in the data directory ''' + + """Runs the enter() method in a new thread and collects new Results on its + queue. Writes them to daily result files in the data directory """ + def __init__(self, args, conf, end_event): assert os.path.isdir(conf.getpath('paths', 'datadir')) assert isinstance(end_event, Event) @@ -544,7 +551,7 @@ def __init__(self, args, conf, end_event): fail_hard(e) def store_result(self, result): - ''' Call from ResultDump thread ''' + """Call from ResultDump thread """ assert isinstance(result, Result) with self.data_lock: fp = result.fingerprint @@ -558,8 +565,8 @@ def store_result(self, result): # file. def handle_result(self, result): - ''' Call from ResultDump thread. If we are shutting down, ignores - ResultError* types ''' + """Call from ResultDump thread. If we are shutting down, ignores + ResultError* types """ assert isinstance(result, Result) fp = result.fingerprint nick = result.nickname @@ -572,17 +579,17 @@ def handle_result(self, result): if result.type == "success": msg = "Success measuring {} ({}) via circuit {} and " \ "destination {}".format( - result.fingerprint, result.nickname, result.circ, - result.dest_url) + result.fingerprint, result.nickname, result.circ, + result.dest_url) else: msg = "Error measuring {} ({}) via circuit {} and " \ "destination {}: {}".format( - result.fingerprint, result.nickname, result.circ, - result.dest_url, result.msg) + result.fingerprint, result.nickname, result.circ, + result.dest_url, result.msg) log.info(msg) def enter(self): - ''' Main loop for the ResultDump thread ''' + """Main loop for the ResultDump thread """ with self.data_lock: self.data = load_recent_results_in_datadir( self.fresh_days, self.datadir) diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py index 8cb9226f..88944786 100644 --- a/sbws/lib/v3bwfile.py +++ b/sbws/lib/v3bwfile.py @@ -7,19 +7,19 @@ import math import os from itertools import combinations -from statistics import median, mean +from statistics import mean, median + from stem.descriptor import parse_file from sbws import __version__ -from sbws.globals import (SPEC_VERSION, BW_LINE_SIZE, SBWS_SCALE_CONSTANT, - TORFLOW_SCALING, SBWS_SCALING, TORFLOW_BW_MARGIN, - TORFLOW_OBS_LAST, TORFLOW_OBS_MEAN, - PROP276_ROUND_DIG, MIN_REPORT, MAX_BW_DIFF_PERC) +from sbws.globals import (BW_LINE_SIZE, MAX_BW_DIFF_PERC, MIN_REPORT, + PROP276_ROUND_DIG, SBWS_SCALE_CONSTANT, SBWS_SCALING, + SPEC_VERSION, TORFLOW_BW_MARGIN, TORFLOW_OBS_LAST, + TORFLOW_OBS_MEAN, TORFLOW_SCALING) from sbws.lib.resultdump import ResultSuccess, _ResultType from sbws.util.filelock import DirectoryLock -from sbws.util.timestamp import (now_isodt_str, unixts_to_isodt_str, - now_unixts) from sbws.util.state import State +from sbws.util.timestamp import now_isodt_str, now_unixts, unixts_to_isodt_str log = logging.getLogger(__name__) @@ -35,7 +35,7 @@ KEYVALUES_INT = STATS_KEYVALUES # List of all unordered KeyValues currently being used to generate the file UNORDERED_KEYVALUES = EXTRA_ARG_KEYVALUES + STATS_KEYVALUES + \ - ['latest_bandwidth'] + ['latest_bandwidth'] # List of all the KeyValues currently being used to generate the file ALL_KEYVALUES = ['version'] + UNORDERED_KEYVALUES TERMINATOR = '=====' @@ -48,8 +48,8 @@ # not inclding in the files the extra bws for now BW_KEYVALUES_BASIC = ['node_id', 'bw'] BW_KEYVALUES_FILE = BW_KEYVALUES_BASIC + \ - ['master_key_ed25519', 'nick', 'rtt', 'time', - 'success', 'error_stream', 'error_circ', 'error_misc'] + ['master_key_ed25519', 'nick', 'rtt', 'time', + 'success', 'error_stream', 'error_circ', 'error_misc'] BW_KEYVALUES_EXTRA_BWS = ['bw_median', 'bw_mean', 'desc_bw_avg', 'desc_bw_bur', 'desc_bw_obs_last', 'desc_bw_obs_mean', 'consensus_bandwidth', @@ -102,6 +102,7 @@ def result_type_to_key(type_str): class V3BWHeader(object): + """ Create a bandwidth measurements (V3bw) header following bandwidth measurements document spec version 1.X.X. @@ -118,6 +119,7 @@ class V3BWHeader(object): - generator_started: str, ISO 8601 timestamp in UTC time zone when the generator started """ + def __init__(self, timestamp, **kwargs): assert isinstance(timestamp, str) for v in kwargs.values(): @@ -196,10 +198,10 @@ def from_lines_v100(cls, lines): @staticmethod def generator_started_from_file(state_fpath): - ''' + """ ISO formatted timestamp for the time when the scanner process most recently started. - ''' + """ state = State(state_fpath) if 'scanner_started' in state: return state['scanner_started'] @@ -266,6 +268,7 @@ def add_stats(self, **kwargs): class V3BWLine(object): + """ Create a Bandwidth List line following the spec version 1.X.X. @@ -282,6 +285,7 @@ class V3BWLine(object): - error_circ, int - error_misc, int """ + def __init__(self, node_id, bw, **kwargs): assert isinstance(node_id, str) assert isinstance(bw, int) @@ -392,8 +396,7 @@ def results_recent_than(results, secs_recent=None): if secs_recent is None: return results results_recent = list(filter( - lambda x: (now_unixts() - x.time) < secs_recent, - results)) + lambda x: (now_unixts() - x.time) < secs_recent, results)) # if not results_recent: # log.debug("Results are NOT more recent than %ss: %s", # secs_recent, @@ -499,7 +502,7 @@ def bw_keyvalue_v1str_ls(self): def bw_strv1(self): """Return Bandwidth Line string following spec v1.X.X.""" bw_line_str = BW_KEYVALUE_SEP_V1.join( - self.bw_keyvalue_v1str_ls) + LINE_SEP + self.bw_keyvalue_v1str_ls) + LINE_SEP if len(bw_line_str) > BW_LINE_SIZE: # if this is the case, probably there are too many KeyValues, # or the limit needs to be changed in Tor @@ -509,12 +512,14 @@ def bw_strv1(self): class V3BWFile(object): + """ Create a Bandwidth List file following spec version 1.X.X :param V3BWHeader v3bwheader: header :param list v3bwlines: V3BWLines """ + def __init__(self, v3bwheader, v3bwlines): self.header = v3bwheader self.bw_lines = v3bwlines diff --git a/sbws/sbws.py b/sbws/sbws.py index 1ac13dd1..e5ab3245 100644 --- a/sbws/sbws.py +++ b/sbws/sbws.py @@ -1,20 +1,18 @@ +import logging import os +import platform + +from requests.__version__ import __version__ as requests_version +from stem import __version__ as stem_version import sbws.core.cleanup -import sbws.core.scanner import sbws.core.generate +import sbws.core.scanner import sbws.core.stats -from sbws.util.config import get_config -from sbws.util.config import validate_config -from sbws.util.config import configure_logging -from sbws.util.parser import create_parser from sbws import __version__ as version -from stem import __version__ as stem_version -from requests.__version__ import __version__ as requests_version -import platform -import logging - +from sbws.util.config import configure_logging, get_config, validate_config from sbws.util.fs import sbws_required_disk_space +from sbws.util.parser import create_parser log = logging.getLogger(__name__) diff --git a/sbws/util/config.py b/sbws/util/config.py index 22f7cbdc..16b317f2 100644 --- a/sbws/util/config.py +++ b/sbws/util/config.py @@ -1,16 +1,17 @@ """Util functions to manage sbws configuration files.""" -from configparser import (ConfigParser, ExtendedInterpolation) -from configparser import InterpolationMissingOptionError -import os import logging import logging.config -from urllib.parse import urlparse +import os +from configparser import (ConfigParser, ExtendedInterpolation, + InterpolationMissingOptionError) from string import Template from tempfile import NamedTemporaryFile +from urllib.parse import urlparse + from sbws.globals import (DEFAULT_CONFIG_PATH, DEFAULT_LOG_CONFIG_PATH, - USER_CONFIG_PATH, SUPERVISED_RUN_DPATH, - SUPERVISED_USER_CONFIG_PATH) + SUPERVISED_RUN_DPATH, SUPERVISED_USER_CONFIG_PATH, + USER_CONFIG_PATH) _ALPHANUM = 'abcdefghijklmnopqrstuvwxyz' _ALPHANUM += _ALPHANUM.upper() @@ -100,7 +101,7 @@ def get_config(args): def _can_log_to_file(conf): - ''' + """ Checks all the known reasons for why we might not be able to log to a file, and returns whether or not we think we will be able to do so. This is useful because if we can't log to a file, we might want to force logging to @@ -108,7 +109,7 @@ def _can_log_to_file(conf): If we can't log to file, return False and the reason. Otherwise return True and an empty string. - ''' + """ # We won't be able to get paths.log_dname from the config when we are first # initializing sbws because it depends on paths.sbws_home (by default). # If there is an issue getting this option, tell the caller that we can't @@ -176,9 +177,9 @@ def configure_logging(args, conf): def validate_config(conf): - ''' Checks the given conf for bad values or bad combinations of values. If + """Checks the given conf for bad values or bad combinations of values. If there's something wrong, returns False and a list of error messages. - Otherwise, return True and an empty list ''' + Otherwise, return True and an empty list """ errors = [] errors.extend(_validate_general(conf)) errors.extend(_validate_cleanup(conf)) diff --git a/sbws/util/filelock.py b/sbws/util/filelock.py index 882c4e6e..4f6ab58a 100644 --- a/sbws/util/filelock.py +++ b/sbws/util/filelock.py @@ -1,12 +1,14 @@ -import os import fcntl import logging +import os + from sbws.globals import fail_hard log = logging.getLogger(__name__) class _FLock: + def __init__(self, lock_fname): self._lock_fname = lock_fname self._fd = None @@ -29,7 +31,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): class DirectoryLock(_FLock): - ''' + + """ Holds a lock on a file in **dname** so that other sbws processes/threads won't try to read/write while we are reading/writing in this directory. @@ -40,7 +43,8 @@ class DirectoryLock(_FLock): Note: The directory must already exist. :param str dname: Name of directory for which we want to obtain a lock - ''' + """ + def __init__(self, dname): assert os.path.isdir(dname) lock_fname = os.path.join(dname, '.lockfile') @@ -48,7 +52,8 @@ def __init__(self, dname): class FileLock(_FLock): - ''' + + """ Holds a lock on **fname** so that other sbws processes/threads won't try to read/write while we are reading/writing this file. @@ -57,7 +62,8 @@ class FileLock(_FLock): >>> # no longer have the lock :param str fname: Name of the file for which we want to obtain a lock - ''' + """ + def __init__(self, fname): lock_fname = fname + '.lockfile' super().__init__(lock_fname) diff --git a/sbws/util/parser.py b/sbws/util/parser.py index 7ff39d2a..701c2831 100644 --- a/sbws/util/parser.py +++ b/sbws/util/parser.py @@ -1,12 +1,12 @@ +import os +from argparse import ArgumentParser, RawTextHelpFormatter + import sbws.core.cleanup -import sbws.core.scanner import sbws.core.generate +import sbws.core.scanner import sbws.core.stats from sbws import __version__ -from argparse import ArgumentParser, RawTextHelpFormatter -import os - def _default_dot_sbws_dname(): home = os.path.expanduser('~') diff --git a/sbws/util/requests.py b/sbws/util/requests.py index 449ced77..68a26e34 100644 --- a/sbws/util/requests.py +++ b/sbws/util/requests.py @@ -1,4 +1,5 @@ import requests + import sbws.util.stem as stem_utils diff --git a/sbws/util/state.py b/sbws/util/state.py index 58da699b..0794e360 100644 --- a/sbws/util/state.py +++ b/sbws/util/state.py @@ -1,10 +1,12 @@ -from sbws.util.filelock import FileLock -import os import json +import os + +from sbws.util.filelock import FileLock class State: - ''' + + """ State allows one to atomically access and update a simple state file on disk across threads and across processes. @@ -40,7 +42,7 @@ class State: >>> # We can do many of the same things with a State object as with a dict >>> for key in state: print(key) >>> # Prints 'linux', 'age', and 'name' - ''' + """ _ALLOWED_TYPES = (int, float, str, bool, type(None)) def __init__(self, fname): diff --git a/sbws/util/stem.py b/sbws/util/stem.py index d8e5cf80..6f9b0b95 100644 --- a/sbws/util/stem.py +++ b/sbws/util/stem.py @@ -1,24 +1,25 @@ -from stem.control import (Controller, Listener) -from stem import (SocketError, InvalidRequest, UnsatisfiableRequest, - OperationFailed, ControllerError, InvalidArguments, - ProtocolError) -from stem.connection import IncorrectSocketType -import stem.process -from configparser import ConfigParser -from threading import RLock import copy import logging import os -from sbws.globals import fail_hard -from sbws.globals import TORRC_STARTING_POINT +from configparser import ConfigParser +from threading import RLock + +import stem.process +from stem import (ControllerError, InvalidArguments, InvalidRequest, + OperationFailed, ProtocolError, SocketError, + UnsatisfiableRequest) +from stem.connection import IncorrectSocketType +from stem.control import Controller, Listener + +from sbws.globals import TORRC_STARTING_POINT, fail_hard log = logging.getLogger(__name__) stream_building_lock = RLock() def attach_stream_to_circuit_listener(controller, circ_id): - ''' Returns a function that should be given to add_event_listener(). It - looks for newly created streams and attaches them to the given circ_id ''' + """Returns a function that should be given to add_event_listener(). It + looks for newly created streams and attaches them to the given circ_id """ def closure_stream_event_listener(st): if st.status == 'NEW' and st.purpose == 'USER': @@ -210,8 +211,8 @@ def launch_tor(conf): def get_socks_info(controller): - ''' Returns the first SocksPort Tor is configured to listen on, in the form - of an (address, port) tuple ''' + """Returns the first SocksPort Tor is configured to listen on, in the form + of an (address, port) tuple """ try: socks_ports = controller.get_listeners(Listener.SOCKS) return socks_ports[0] @@ -221,12 +222,12 @@ def get_socks_info(controller): def only_relays_with_bandwidth(controller, relays, min_bw=None, max_bw=None): - ''' + """ Given a list of relays, only return those that optionally have above **min_bw** and optionally have below **max_bw**, inclusively. If neither min_bw nor max_bw are given, essentially just returns the input list of relays. - ''' + """ assert min_bw is None or min_bw >= 0 assert max_bw is None or max_bw >= 0 ret = [] diff --git a/sbws/util/userquery.py b/sbws/util/userquery.py index 1bdeb22e..eb8bf78f 100644 --- a/sbws/util/userquery.py +++ b/sbws/util/userquery.py @@ -1,6 +1,6 @@ # Based on https://stackoverflow.com/a/3041990 def query_yes_no(question, default='yes'): - ''' + """ Ask a yes/no question via input() and return the user's answer. :param str question: Prompt given to the user. @@ -9,7 +9,7 @@ def query_yes_no(question, default='yes'): ``None`` (meaning an answer is required from the user). :returns: ``True`` if we ended up with a 'yes' answer, otherwise ``False``. - ''' + """ valid = {'yes': True, 'y': True, 'ye': True, 'no': False, 'n': False} if default is None: prompt = ' [y/n] ' diff --git a/scripts/tools/sbws-http-server.py b/scripts/tools/sbws-http-server.py index f21499fd..161cdb3e 100755 --- a/scripts/tools/sbws-http-server.py +++ b/scripts/tools/sbws-http-server.py @@ -21,7 +21,7 @@ import http.server # import time -FILE_SIZE = 1*1024*1024*1024 # 1 GiB +FILE_SIZE = 1 * 1024 * 1024 * 1024 # 1 GiB def _get_resp_size_from_range(range_str): diff --git a/setup.py b/setup.py index d27ae716..a4a5bb4d 100755 --- a/setup.py +++ b/setup.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 # Always prefer setuptools over distutils -from setuptools import setup, find_packages +import os # To use a consistent encoding from codecs import open -import os +from setuptools import find_packages, setup here = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/conftest.py b/tests/conftest.py index 1d5da5ba..d2e150ca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ """Common pytest configuration for unit and integration tests.""" import pytest + from sbws.util.parser import create_parser @@ -12,6 +13,7 @@ def parser(): def datadir(request): """get, read, open test files from the tests "data" directory.""" class D: + def __init__(self, basepath): self.basepath = basepath diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b1312d9c..36e0bcdd 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,8 +1,9 @@ """pytest configuration for integration tests.""" import argparse -import pytest import os +import pytest + from sbws.lib.circuitbuilder import GapsCircuitBuilder as CB from sbws.lib.destination import DestinationList from sbws.lib.relaylist import RelayList diff --git a/tests/integration/core/test_scanner.py b/tests/integration/core/test_scanner.py index 4a218f71..e86a0979 100644 --- a/tests/integration/core/test_scanner.py +++ b/tests/integration/core/test_scanner.py @@ -1,17 +1,18 @@ +import logging + import pytest from sbws.core.scanner import measure_relay from sbws.lib.resultdump import ResultSuccess -import logging def assert_within(value, target, radius): - ''' + """ Assert that **value** is within **radius** of **target** If target is 10 and radius is 2, value can be anywhere between 8 and 12 inclusive - ''' + """ assert target - radius < value, 'Value is too small. {} is not within '\ '{} of {}'.format(value, radius, target) assert target + radius > value, 'Value is too big. {} is not within '\ diff --git a/tests/integration/lib/test_relayprioritizer.py b/tests/integration/lib/test_relayprioritizer.py index d5464b8e..49ca677a 100644 --- a/tests/integration/lib/test_relayprioritizer.py +++ b/tests/integration/lib/test_relayprioritizer.py @@ -1,9 +1,9 @@ -from sbws.lib.resultdump import ResultDump -from sbws.lib.resultdump import ResultSuccess, ResultErrorCircuit -from sbws.lib.relayprioritizer import RelayPrioritizer from threading import Event from unittest.mock import patch +from sbws.lib.relayprioritizer import RelayPrioritizer +from sbws.lib.resultdump import ResultDump, ResultErrorCircuit, ResultSuccess + def static_time(value): while True: @@ -60,7 +60,7 @@ def test_relayprioritizer_general(time_mock, sbwshome_empty, args, # results for will have the highest priority, but don't test the order # of them. Skip to the end of the list and check those guys since they # should have a defined order. - for i in range(1, 5+1): + for i in range(1, 5 + 1): nick = 'relay{}'.format(i) pos = i * -1 relay = best_list[pos] diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 4dd442b7..5a6586ea 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,15 +1,15 @@ """pytest configuration for unit tests.""" import argparse -import pytest -from datetime import datetime import os +from datetime import datetime + +import pytest from sbws.globals import RESULT_VERSION -from sbws.lib.resultdump import (ResultErrorStream, ResultSuccess, Result) -from sbws.lib.resultdump import write_result_to_datadir +from sbws.lib.resultdump import (Result, ResultErrorStream, ResultSuccess, + write_result_to_datadir) from sbws.util.config import _get_default_config - TIME1 = 1529232277.9028733 TIME2 = datetime.utcnow().timestamp() FP1 = 'A' * 40 @@ -226,9 +226,9 @@ def resultdict_ip_changed_trimmed(): @pytest.fixture(scope='function') def sbwshome_error_result(sbwshome_only_datadir, conf): - ''' + """ Creates an ~/.sbws with a single fresh ResultError in it - ''' + """ dd = conf.getpath('paths', 'datadir') write_result_to_datadir(RESULT_ERROR_STREAM, dd) return sbwshome_only_datadir @@ -236,9 +236,9 @@ def sbwshome_error_result(sbwshome_only_datadir, conf): @pytest.fixture(scope='function') def sbwshome_success_result(sbwshome_only_datadir, conf): - ''' + """ Creates an ~/.sbws with a single fresh ResultSuccess in it - ''' + """ dd = conf.getpath('paths', 'datadir') write_result_to_datadir(RESULT_SUCCESS1, dd) return sbwshome_only_datadir @@ -246,9 +246,9 @@ def sbwshome_success_result(sbwshome_only_datadir, conf): @pytest.fixture(scope='function') def sbwshome_success_result_one_relay(sbwshome_only_datadir, conf): - ''' + """ Creates an ~/.sbws with a a couple of fresh ResultSuccess for one relay - ''' + """ dd = conf.getpath('paths', 'datadir') write_result_to_datadir(RESULT_SUCCESS1, dd) write_result_to_datadir(RESULT_SUCCESS1, dd) @@ -257,10 +257,10 @@ def sbwshome_success_result_one_relay(sbwshome_only_datadir, conf): @pytest.fixture(scope='function') def sbwshome_success_result_two_relays(sbwshome_only_datadir, conf): - ''' + """ Creates an ~/.sbws with a a couple of fresh ResultSuccess for a couple or relays - ''' + """ dd = conf.getpath('paths', 'datadir') write_result_to_datadir(RESULT_SUCCESS1, dd) write_result_to_datadir(RESULT_SUCCESS1, dd) diff --git a/tests/unit/core/test_generate.py b/tests/unit/core/test_generate.py index aafe2713..97130b0a 100644 --- a/tests/unit/core/test_generate.py +++ b/tests/unit/core/test_generate.py @@ -1,8 +1,8 @@ """Unit tests for sbws.core.generate module.""" import argparse -from sbws.globals import TORFLOW_ROUND_DIG, PROP276_ROUND_DIG from sbws.core.generate import gen_parser +from sbws.globals import PROP276_ROUND_DIG, TORFLOW_ROUND_DIG def test_gen_parser_arg_round_digs(): diff --git a/tests/unit/core/test_stats.py b/tests/unit/core/test_stats.py index fb291176..986446ac 100644 --- a/tests/unit/core/test_stats.py +++ b/tests/unit/core/test_stats.py @@ -1,16 +1,16 @@ +import logging import os.path +from unittest.mock import patch import sbws.core.stats from tests.unit.globals import monotonic_time -from unittest.mock import patch -import logging def test_stats_initted(sbwshome_empty, args, conf, caplog): - ''' + """ An initialized but rather empty .sbws directory should fail about missing ~/.sbws/datadir - ''' + """ try: sbws.core.stats.main(args, conf) except SystemExit as e: @@ -23,10 +23,10 @@ def test_stats_initted(sbwshome_empty, args, conf, caplog): def test_stats_stale_result(args, conf, caplog, sbwshome_success_result): - ''' + """ An initialized .sbws directory with no fresh results should say so and exit cleanly - ''' + """ caplog.set_level(logging.DEBUG) sbws.core.stats.main(args, conf) assert 'No fresh results' == caplog.records[-1].getMessage() @@ -35,10 +35,10 @@ def test_stats_stale_result(args, conf, caplog, @patch('time.time') def test_stats_fresh_result(time_mock, sbwshome_error_result, args, conf, capsys, caplog): - ''' + """ An initialized .sbws directory with a fresh error result should have some boring stats and exit cleanly - ''' + """ args.error_types = False start = 1529232278 time_mock.side_effect = monotonic_time(start=start) @@ -67,10 +67,10 @@ def test_stats_fresh_result(time_mock, sbwshome_error_result, args, conf, @patch('time.time') def test_stats_fresh_results(time_mock, sbwshome_success_result_two_relays, args, conf, capsys, caplog): - ''' + """ An initialized .sbws directory with a fresh error and fresh success should have some exciting stats and exit cleanly - ''' + """ caplog.set_level(logging.DEBUG) start = 1529232278 time_mock.side_effect = monotonic_time(start=start) diff --git a/tests/unit/lib/test_results.py b/tests/unit/lib/test_results.py index 3c2a52e2..ca9a0833 100644 --- a/tests/unit/lib/test_results.py +++ b/tests/unit/lib/test_results.py @@ -1,20 +1,17 @@ from unittest.mock import patch + from sbws.globals import RESULT_VERSION -from sbws.lib.resultdump import Result -from sbws.lib.resultdump import ResultSuccess -from sbws.lib.resultdump import ResultError -from sbws.lib.resultdump import ResultErrorAuth -from sbws.lib.resultdump import ResultErrorCircuit -from sbws.lib.resultdump import ResultErrorStream -from sbws.lib.resultdump import _ResultType +from sbws.lib.resultdump import (Result, ResultError, ResultErrorAuth, + ResultErrorCircuit, ResultErrorStream, + ResultSuccess, _ResultType) from tests.unit.globals import monotonic_time def test_Result(result): - ''' + """ A standard Result should not be convertible to a string because Result.type is not implemented. - ''' + """ try: str(result) print(str(result)) @@ -25,20 +22,20 @@ def test_Result(result): def test_Result_from_dict_bad_version(): - ''' + """ The first thing that is checked is the version field, and a wrong one should return None - ''' + """ d = {'version': RESULT_VERSION + 1} r = Result.from_dict(d) assert r is None def test_Result_from_dict_bad_type(): - ''' + """ If the result type string doesn't match any of the known types, then it should throw NotImplementedError - ''' + """ d = {'version': RESULT_VERSION, 'type': 'NotARealType'} try: Result.from_dict(d) diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py index 2d38f482..ebe208e5 100644 --- a/tests/unit/lib/test_v3bwfile.py +++ b/tests/unit/lib/test_v3bwfile.py @@ -5,12 +5,12 @@ import os.path from sbws import __version__ as version -from sbws.globals import (SPEC_VERSION, SBWS_SCALING, TORFLOW_SCALING, - MIN_REPORT, TORFLOW_ROUND_DIG, PROP276_ROUND_DIG) -from sbws.lib.resultdump import Result, load_result_file, ResultSuccess -from sbws.lib.v3bwfile import (V3BWHeader, V3BWLine, TERMINATOR, LINE_SEP, - KEYVALUE_SEP_V1, num_results_of_type, - V3BWFile, round_sig_dig) +from sbws.globals import (MIN_REPORT, PROP276_ROUND_DIG, SBWS_SCALING, + SPEC_VERSION, TORFLOW_ROUND_DIG, TORFLOW_SCALING) +from sbws.lib.resultdump import Result, ResultSuccess, load_result_file +from sbws.lib.v3bwfile import (KEYVALUE_SEP_V1, LINE_SEP, TERMINATOR, V3BWFile, + V3BWHeader, V3BWLine, num_results_of_type, + round_sig_dig) from sbws.util.timestamp import now_fname, now_isodt_str, now_unixts timestamp = 1523974147 diff --git a/tests/unit/util/test_config.py b/tests/unit/util/test_config.py index 95dde6fb..c1c9be43 100644 --- a/tests/unit/util/test_config.py +++ b/tests/unit/util/test_config.py @@ -1,8 +1,10 @@ -import sbws.util.config as con from configparser import ConfigParser +import sbws.util.config as con + class PseudoSection: + def __init__(self, key, value, mini=None, maxi=None): self.key = key self.value = value diff --git a/tests/unit/util/test_state.py b/tests/unit/util/test_state.py index dd23d1af..dbf2e3fa 100644 --- a/tests/unit/util/test_state.py +++ b/tests/unit/util/test_state.py @@ -1,5 +1,7 @@ -from sbws.util.state import State import os + +from sbws.util.state import State + # from tempfile import NamedTemporaryFile as NTF diff --git a/tests/unit/util/test_timestamp.py b/tests/unit/util/test_timestamp.py index 8ebc2f6e..58079d8a 100644 --- a/tests/unit/util/test_timestamp.py +++ b/tests/unit/util/test_timestamp.py @@ -5,7 +5,6 @@ from sbws.util.timestamp import (dt_obj_to_isodt_str, unixts_to_dt_obj, unixts_to_isodt_str, unixts_to_str) - isodt_str = '2018-05-23T12:55:04' dt_obj = datetime.strptime(isodt_str, '%Y-%m-%dT%H:%M:%S') unixts = int(dt_obj.replace(tzinfo=timezone.utc).timestamp()) diff --git a/tests/unit/util/test_userquery.py b/tests/unit/util/test_userquery.py index d1e40124..2de32b5d 100644 --- a/tests/unit/util/test_userquery.py +++ b/tests/unit/util/test_userquery.py @@ -1,4 +1,5 @@ from unittest.mock import patch + from sbws.util.userquery import query_yes_no