Ticket30406 refactor header constants

tests/conftest.py

@@ -1,8 +1,17 @@
 """Common pytest configuration for unit and integration tests."""
+import argparse
+import os.path
 import pytest
+from unittest import mock
+
+from sbws.lib import resultdump
+from sbws.util import config
 from sbws.util.parser import create_parser
 
 
+FAKE_TIME = 1557556118
+
+
 @pytest.fixture(scope='session')
 def parser():
     return create_parser()
@@ -29,3 +38,59 @@ def readlines(self, name):
             with self.open(name, "r") as f:
                 return f.readlines()
     return D(request.fspath.dirpath("data"))
+
+
+@pytest.fixture()
+def tests_data_dir(scope="global"):
+    """Root data directory, common to all the tests."""
+    here = os.path.abspath(os.path.dirname(__file__))
+    return os.path.join(here, "data")
+
+
+@pytest.fixture()
+def sbws_dir(tests_data_dir):
+    return os.path.join(tests_data_dir, ".sbws")
+
+
+# The following fixtures are similar to the ones defined in unit and
+# integration sub-directories, but they are for the tests data to be read,
+# not to write.
+@pytest.fixture()
+def sbws_args(tests_data_dir):
+    config_fpath = os.path.join(tests_data_dir, ".sbws.ini")
+    sbws_args = argparse.Namespace(**{'config': config_fpath})
+    return sbws_args
+
+
+@pytest.fixture()
+def sbws_conf(sbws_args, sbws_dir):
+    """Default configuration with sbws home in the tmp test dir."""
+    conf = config.get_config(sbws_args)
+    conf['paths']['sbws_home'] = sbws_dir
+    conf['paths']['v3bw_fname'] = "${v3bw_dname}/latest.v3bw"
+    return conf
+
+
+@pytest.fixture()
+@mock.patch('time.time')
+def measurements(mock_time, sbws_conf):
+    # Because load_recent_results_in_datadir will use time.time()
+    # to decide which results are recent.
+    mock_time.return_value = FAKE_TIME
+    print("measurements", sbws_conf.getpath('paths', 'datadir'))
+    measurements = resultdump \
+        .load_recent_results_in_datadir(
+            sbws_conf.getint('general', 'data_period'),
+            sbws_conf.getpath('paths', 'datadir'))
+    return measurements
+
+
+@pytest.fixture
+def bandwidth_file_headers():
+    d = {
+        'earliest_bandwidth': "2019-05-11T06:26:41",
+        'file_created': "2019-05-11T06:32:32",
+        'generator_started': "2019-05-11T06:26:28",
+        'latest_bandwidth': "2019-05-11T06:28:38",
+    }
+    return d

tests/data/.sbws.ini

@@ -0,0 +1,30 @@
+[general]
+# Days into the past that measurements are considered valid
+data_period = 1
+
+[paths]
+sbws_home = .sbws
+
+[scanner]
+country = ZZ
+
+[destinations]
+local = on
+
+[destinations.local]
+; url = https://localhost:28888/sbws.bin
+url = http://127.0.0.1:28888/sbws.bin
+verify = False
+country = ZZ
+
+[tor]
+extra_lines =
+    DirAuthority auth1 orport=2002 no-v2 v3ident=D7DBC517EFD2BA1A5012CF1BD0BB38F17C8160BD 127.10.0.1:2003 AA45C13025C037F056E734169891878ED0880231
+    DirAuthority auth2 orport=2002 no-v2 v3ident=4EE103A081F400E6622F5461D51782B876BB5C24 127.10.0.2:2003 E7B3C9A0040D628DAC88B0251AE6334D28E8F531
+    DirAuthority auth3 orport=2002 no-v2 v3ident=8B85069C7FC0593801E6491A34100264FCE28980 127.10.0.3:2003 35E3B8BB71C81355649AEC5862ECB7ED7EFDBC5C
+    TestingTorNetwork 1
+    NumCPUs 1
+
+[logging]
+level = debug
+to_stdout_level = ${level}

tests/data/.sbws/datadir/2019-05-11.txt

@@ -0,0 +1,15 @@
+{"master_key_ed25519": "gnREYdkUN0uYI3zF8oB+Wwm4MhM6ETPX1hEyt2m2a+Y", "fingerprint": "C7C5094677013F5BC124183C71A482D0156CDCFE", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556001.1935074, "circ": ["C7C5094677013F5BC124183C71A482D0156CDCFE", "270A861ABED22EC2B625198BCCD7B2B9DBFFC93C"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 5 [relay6 (C7C50946) -> exit1 (270A861A)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "relay6", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.9"}
+{"master_key_ed25519": "zOZX+yyD7EN1W/Y2wgjuvObpFOOWK+LZIWlKW0AOcIE", "fingerprint": "2ABFBACE61167A1019A56CB35B2E3362B97D8570", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556011.233926, "circ": ["2ABFBACE61167A1019A56CB35B2E3362B97D8570", "270A861ABED22EC2B625198BCCD7B2B9DBFFC93C"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 6 [relay1 (2ABFBACE) -> exit1 (270A861A)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "relay1", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.4"}
+{"master_key_ed25519": "NQspfAK/xkywFHV5LsQ5lLi2BmTKx2Imu4jacJ0d9os", "fingerprint": "4D664E247E530CA5CD5176B8C1A6DABC9531F0B0", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556021.2721329, "circ": ["4D664E247E530CA5CD5176B8C1A6DABC9531F0B0", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 7 [relay4 (4D664E24) -> exit3 (FC264325)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "relay4", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.7"}
+{"master_key_ed25519": "2fXiF4T993i+vVwZZEQ4fYee+N8OThzCGeacvnVaNbo", "relay_observed_bandwidth": 0, "fingerprint": "117A456C911114076BEB4E757AC48B16CC0CCC5F", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "consensus_bandwidth": 0, "relay_in_recent_consensus_count": 1, "time": 1557556022.519254, "circ": ["117A456C911114076BEB4E757AC48B16CC0CCC5F", "270A861ABED22EC2B625198BCCD7B2B9DBFFC93C"], "rtts": [], "relay_burst_bandwidth": 1073741824, "nickname": "relay1mbyteMAB", "consensus_bandwidth_is_unmeasured": false, "type": "success", "dest_url": "http://127.0.0.1:28888/sbws.bin", "relay_average_bandwidth": 1048576, "address": "127.10.0.14", "downloads": [{"duration": 5.801189422607422, "amount": 33454356}, {"duration": 5.437012195587158, "amount": 33454356}, {"duration": 5.97099232673645, "amount": 33454356}, {"duration": 5.786560535430908, "amount": 33454356}, {"duration": 5.766895532608032, "amount": 33454356}]}
+{"master_key_ed25519": "cj7V+PYPJvSANsvBOjZSiCvXuGHXFrpSnPqC46I6DgU", "fingerprint": "E894C65997F8EC96558B554176EEEA39C6A43EF6", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556032.5962188, "circ": ["E894C65997F8EC96558B554176EEEA39C6A43EF6", "C0606B414423F9A2BBA2679B440056E3B07FEC85"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 11 [relay7 (E894C659) -> exit2 (C0606B41)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "relay7", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.10"}
+{"master_key_ed25519": "Wiv9uOemlcFzRY1gZfBOpbQ+aJn6NG0Z/IArZFCdSdk", "fingerprint": "FC264325EA99D597FF94DA88379DABB64304DD9D", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556042.6297724, "circ": ["32B7178F7201F76411A99D3552F340D3597D5629", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 12 [relay5 (32B7178F) -> exit3 (FC264325)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "exit3", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.13"}
+{"master_key_ed25519": "eR0HnYlzpOEGwxuFjZkGJZ6pu2eV1i6fd9lhF4UOMno", "fingerprint": "934E06F38A391CB71DF83ECDE05DFF5CDE3AC49D", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556052.6754208, "circ": ["934E06F38A391CB71DF83ECDE05DFF5CDE3AC49D", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 14 [relay1mbyteRBR (934E06F3) -> exit3 (FC264325)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "relay1mbyteRBR", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.15"}
+{"master_key_ed25519": "Pymu1Z1eZRWgkE42xzDCYFLVKNtY743GKZzt6Im0OUw", "relay_observed_bandwidth": 0, "fingerprint": "8E687E91DCAB967F6E4EE8E46E66F6AD05C7C625", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "consensus_bandwidth": 0, "relay_in_recent_consensus_count": 1, "time": 1557556054.3331225, "circ": ["8E687E91DCAB967F6E4EE8E46E66F6AD05C7C625", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "rtts": [], "relay_burst_bandwidth": 1073741824, "nickname": "relay2", "consensus_bandwidth_is_unmeasured": false, "type": "success", "dest_url": "http://127.0.0.1:28888/sbws.bin", "relay_average_bandwidth": 1073741824, "address": "127.10.0.5", "downloads": [{"duration": 6.025712728500366, "amount": 33632847}, {"duration": 5.685051441192627, "amount": 33632847}, {"duration": 5.941658973693848, "amount": 33632847}, {"duration": 5.831774950027466, "amount": 33632847}, {"duration": 5.711573600769043, "amount": 33632847}]}
+{"master_key_ed25519": "uPz8ZZNm2Gcra7BauJP5PH+7uANRraYpCj7NFtp1KdM", "fingerprint": "E7B3C9A0040D628DAC88B0251AE6334D28E8F531", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556064.441996, "circ": ["E7B3C9A0040D628DAC88B0251AE6334D28E8F531", "C0606B414423F9A2BBA2679B440056E3B07FEC85"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 17 [auth2 (E7B3C9A0) -> exit2 (C0606B41)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "auth2", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.2"}
+{"master_key_ed25519": "r29RWlFMIdU5GUvKsWGdhQIKjIpUzqgw0yNx/7IpPFM", "fingerprint": "35E3B8BB71C81355649AEC5862ECB7ED7EFDBC5C", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556074.4799252, "circ": ["35E3B8BB71C81355649AEC5862ECB7ED7EFDBC5C", "270A861ABED22EC2B625198BCCD7B2B9DBFFC93C"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 18 [auth3 (35E3B8BB) -> exit1 (270A861A)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "auth3", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.3"}
+{"master_key_ed25519": "C506YEdasDDQqidu4G2VLMFOwaqMX28BYkuxr1+wI9o", "fingerprint": "270A861ABED22EC2B625198BCCD7B2B9DBFFC93C", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556084.5123796, "circ": ["C7C5094677013F5BC124183C71A482D0156CDCFE", "270A861ABED22EC2B625198BCCD7B2B9DBFFC93C"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 20 [relay6 (C7C50946) -> exit1 (270A861A)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "exit1", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.11"}
+{"master_key_ed25519": "wLglSEw9/DHfpNrlrqjVRSnGLVWfnm0vYxkryH4aT6Q", "relay_observed_bandwidth": 0, "fingerprint": "AA45C13025C037F056E734169891878ED0880231", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "consensus_bandwidth": 0, "relay_in_recent_consensus_count": 1, "time": 1557556088.1481652, "circ": ["AA45C13025C037F056E734169891878ED0880231", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "rtts": [], "relay_burst_bandwidth": 1073741824, "nickname": "auth1", "consensus_bandwidth_is_unmeasured": false, "type": "success", "dest_url": "http://127.0.0.1:28888/sbws.bin", "relay_average_bandwidth": 1073741824, "address": "127.10.0.1", "downloads": [{"duration": 6.3723015785217285, "amount": 35748124}, {"duration": 6.351818323135376, "amount": 35748124}, {"duration": 5.8544604778289795, "amount": 35748124}, {"duration": 6.5015482902526855, "amount": 35748124}, {"duration": 6.196664094924927, "amount": 35748124}]}
+{"master_key_ed25519": "/PrTbpen3BrKNgiNRhAa93JQtnrT3LJX3Ka1+jvbWj4", "fingerprint": "693F73187624BE760AAD2A12C5ED89DB1DE044F5", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556098.237717, "circ": ["693F73187624BE760AAD2A12C5ED89DB1DE044F5", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 23 [relay3 (693F7318) -> exit3 (FC264325)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "relay3", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.6"}
+{"master_key_ed25519": "iriD8sIKS25WGc6mLesQ2okT1Tcn81AuqnEbItJeuvY", "fingerprint": "C0606B414423F9A2BBA2679B440056E3B07FEC85", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "relay_in_recent_consensus_count": 1, "time": 1557556108.2805848, "circ": ["E7B3C9A0040D628DAC88B0251AE6334D28E8F531", "C0606B414423F9A2BBA2679B440056E3B07FEC85"], "msg": "Could not connect to http://127.0.0.1:28888/sbws.bin over circ 24 [auth2 (E7B3C9A0) -> exit2 (C0606B41)]: SOCKSHTTPConnectionPool(host='127.0.0.1', port=28888): Read timed out. (read timeout=10.0)", "nickname": "exit2", "type": "error-stream", "dest_url": "http://127.0.0.1:28888/sbws.bin", "address": "127.10.0.12"}
+{"master_key_ed25519": "5O+uCpFoBzsey33+Zzlgyy18/McmV1mpPJZ+DrZMKRc", "relay_observed_bandwidth": 0, "fingerprint": "32B7178F7201F76411A99D3552F340D3597D5629", "scanner": "IDidntEditTheSBWSConfig", "relay_recent_measurement_attempt_count": 1, "relay_recent_priority_list_count": 1, "version": 4, "consensus_bandwidth": 0, "relay_in_recent_consensus_count": 1, "time": 1557556118.4493086, "circ": ["32B7178F7201F76411A99D3552F340D3597D5629", "FC264325EA99D597FF94DA88379DABB64304DD9D"], "rtts": [], "relay_burst_bandwidth": 1073741824, "nickname": "relay5", "consensus_bandwidth_is_unmeasured": false, "type": "success", "dest_url": "http://127.0.0.1:28888/sbws.bin", "relay_average_bandwidth": 1073741824, "address": "127.10.0.8", "downloads": [{"duration": 5.454082012176514, "amount": 31498448}, {"duration": 5.785161018371582, "amount": 31498448}, {"duration": 5.272622108459473, "amount": 31498448}, {"duration": 5.410377025604248, "amount": 31498448}, {"duration": 5.6312878131866455, "amount": 31498448}]}

tests/data/.sbws/state.dat

@@ -1,9 +1,9 @@
 {
-    "uuid": "806218a0-3ce5-4778-b839-d6faf6798405",
-    "scanner_started": "2019-03-25T13:03:06",
-    "recent_consensus_count": 1,
-    "recent_priority_list_count": 1,
     "recent_measurement_attempt_count": 15,
+    "recent_priority_relay_count": 15,
+    "uuid": "714d1bf4-b08b-4d68-a56e-b917cda848f8",
+    "recent_priority_list_count": 1,
+    "recent_consensus_count": 1,
     "min_perc_reached": null,
-    "recent_priority_relay_count": 15
+    "scanner_started": "2019-05-11T06:26:28"
 }