feat: add ci tests

.github/workflows/tests.yml

@@ -0,0 +1,17 @@
+name: tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - run: uv sync --group dev --frozen
+      - run: uv run --no-sync pytest -q

AGENTS.md

@@ -1,5 +1,14 @@
 # workflow
 
+## commands
+
+run files in venv
+
+```
+.venv/bin/python main.py
+```
+
+
 ## git worktrees
 
 when asked to work on git worktrees

chain/effective_fee.py

@@ -1,4 +1,3 @@
-import argparse
 import itertools
 import json
 import os
@@ -158,21 +157,3 @@ def run(source, writer, chunk_blocks=None, flush_rows=None, verbose=True):
     if verbose:
         print(f"    wrote {writer.written} package rows", flush=True)
     return writer.written
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--chunk-blocks", type=int,
-                        default=config.UNIONFIND_CHUNK_BLOCKS)
-    parser.add_argument("--sqlite", help="run against a local database instead")
-    args = parser.parse_args()
-
-    if args.sqlite:
-        source, writer = SqliteSource(args.sqlite), ListWriter()
-    else:
-        source, writer = BigQuerySource(), BigQueryWriter()
-    run(source, writer, chunk_blocks=args.chunk_blocks)
-
-
-if __name__ == "__main__":
-    main()

chain/export_onchain_monthly_fee.py

@@ -1,4 +1,3 @@
-import argparse
 import json
 import os
 import sys
@@ -87,21 +86,13 @@ def report(months):
     print(f"{'total':<9}{'':>9}{'':>12}{total:>14.4f}")
 
 
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--out", default=os.path.join(config.DATA_DIR, FILENAME))
-    args = parser.parse_args()
-
+def export(out_path):
     fresh = build(monthly_rows())
     if not fresh:
         print("the onchain_monthly_fee table is empty; nothing to publish")
         return 1
 
-    months = merge(read(args.out), fresh)
+    months = merge(read(out_path), fresh)
     report(months)
-    write(payload(months), args.out)
+    write(payload(months), out_path)
     return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

chain/export_results.py

@@ -1,4 +1,3 @@
-import argparse
 import datetime
 import decimal
 import json
@@ -10,7 +9,6 @@
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "utils"))
 
 import bqio
-import config
 
 TABLES = {
     "monthly_summary": "SELECT * FROM `${dst}.monthly_summary` ORDER BY block_month",
@@ -219,8 +217,8 @@ def write_summary(out_dir, monthly, sensitivity_grid, pools):
         "- A number that moves by an order of magnitude across the threshold "
         "grid is a statement about the cut-offs, not about the chain.",
         "- Per-pool rows depend on coinbase tag attribution. Run "
-        "`sanity_check.py` and compare against a public hashrate chart before "
-        "quoting any of them.",
+        "`main.py sanity-check` and compare against a public hashrate chart "
+        "before quoting any of them.",
     ]
 
     path = os.path.join(out_dir, "summary.md")
@@ -272,20 +270,3 @@ def on_disk(name):
 
 def export_month(out_dir, replace=False):
     return merge_into(out_dir, fetch(), replace=replace)
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__,
-                                     formatter_class=argparse.RawDescriptionHelpFormatter)
-    parser.add_argument("--out", default=config.OUT_DIR)
-    parser.add_argument("--replace", action="store_true",
-                        help="ignore what is already in --out and write only "
-                             "the months the working dataset holds")
-    args = parser.parse_args()
-
-    print(f"writing to {args.out}/")
-    export_month(args.out, replace=args.replace)
-
-
-if __name__ == "__main__":
-    main()

chain/run_pipeline.py

@@ -1,4 +1,3 @@
-import argparse
 import os
 import sys
 
@@ -65,19 +64,19 @@ def sql_name(step):
     return os.path.join(SQL_DIR, f"{step}.sql")
 
 
-def select_steps(args):
+def select_steps(only=None, from_step=None):
     names = [s[0] for s in STEPS]
-    if args.only:
-        wanted = set(args.only.split(","))
+    if only:
+        wanted = set(only.split(","))
         unknown = wanted - set(names)
         if unknown:
             sys.exit(f"unknown step(s): {', '.join(sorted(unknown))}")
         return [s for s in STEPS if s[0] in wanted]
     start = 0
-    if getattr(args, "from_step", None):
-        if args.from_step not in names:
-            sys.exit(f"unknown step: {args.from_step}")
-        start = names.index(args.from_step)
+    if from_step:
+        if from_step not in names:
+            sys.exit(f"unknown step: {from_step}")
+        start = names.index(from_step)
     return STEPS[start:]
 
 
@@ -139,30 +138,16 @@ def headline():
     print("  read low_fee_sensitivity before quoting any of this")
 
 
-def main():
-    parser = argparse.ArgumentParser(description=__doc__,
-                                     formatter_class=argparse.RawDescriptionHelpFormatter)
-    parser.add_argument("--dry-run", action="store_true",
-                        help="report bytes per step and stop")
-    parser.add_argument("--month", metavar="YYYY-MM",
-                        help="run one month end to end, then merge it into out/")
-    parser.add_argument("--from", dest="from_step", metavar="STEP",
-                        help="start at this step")
-    parser.add_argument("--only", metavar="STEP[,STEP]",
-                        help="run only these steps")
-    parser.add_argument("--yes", action="store_true",
-                        help="do not ask before spending")
-    parser.add_argument("--skip-checks", action="store_true")
-    args = parser.parse_args()
-
-    if args.month:
-        config.set_month(args.month)
+def run(month=None, only=None, from_step=None, dry=False, yes=False,
+        skip_checks=False):
+    if month:
+        config.set_month(month)
         print(f"month run: {config.START_DATE} .. {config.END_DATE}")
 
-    steps = select_steps(args)
+    steps = select_steps(only, from_step)
     bqio.ensure_dataset()
 
-    if args.dry_run:
+    if dry:
         dry_run(steps)
         return
 
@@ -171,7 +156,7 @@ def main():
         estimate = bqio.dry_run(bqio.render(sql_name("01_tx_base")))
         print(f"step 01 will scan {bqio.human_bytes(estimate)} "
               f"(about ${bqio.usd(estimate):.2f})")
-        if not args.yes and not bqio.confirm("run it?"):
+        if not yes and not bqio.confirm("run it?"):
             sys.exit("stopped; nothing was run")
 
     for name, kind, what in steps:
@@ -182,16 +167,12 @@ def main():
             effective_fee.run(effective_fee.BigQuerySource(),
                               effective_fee.BigQueryWriter())
 
-    if not args.skip_checks:
+    if not skip_checks:
         run_checks()
     headline()
 
-    if args.month:
-        print(f"\nmerging {args.month} into {config.OUT_DIR}/")
+    if month:
+        print(f"\nmerging {month} into {config.OUT_DIR}/")
         export_results.export_month(config.OUT_DIR)
         print(f"\ndone. delete the BigQuery working dataset when ready:"
-              f"\n  python ../utils/delete_dataset.py")
-
-
-if __name__ == "__main__":
-    main()
+              f"\n  .venv/bin/python main.py delete-dataset")

chain/sanity_check.py

@@ -1,4 +1,3 @@
-import argparse
 import os
 import sys
 
@@ -72,20 +71,3 @@ def attribution_quality():
         print("(a big count here is a missing tag in pools.py)")
         for r in rows:
             print(f"  {r['blocks']:>5d}  {r['coinbase_text']}")
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__,
-                                     formatter_class=argparse.RawDescriptionHelpFormatter)
-    parser.add_argument("--top", type=int, default=12,
-                        help="pools listed per month")
-    parser.add_argument("--months", type=int, default=12)
-    parser.add_argument("--all", action="store_true", help="every month")
-    args = parser.parse_args()
-
-    attribution_quality()
-    monthly_shares(None if args.all else args.months, args.top)
-
-
-if __name__ == "__main__":
-    main()

chain/validate_against_mempool.py

@@ -1,4 +1,3 @@
-import argparse
 import json
 import os
 import random
@@ -60,23 +59,15 @@ def sample_blocks(sample, sensitivity_column):
     return rows
 
 
-def main():
-    parser = argparse.ArgumentParser(description=__doc__,
-                                     formatter_class=argparse.RawDescriptionHelpFormatter)
-    parser.add_argument("--sample", type=int, default=50)
-    parser.add_argument("--sleep", type=float, default=1.5,
-                        help="seconds between API calls")
-    parser.add_argument("--sensitivity", choices=["30", "50", "70"], default="50")
-    args = parser.parse_args()
-
-    column = f"low_fee_{args.sensitivity}"
-    blocks = sample_blocks(args.sample, column)
+def validate(sample, sleep, sensitivity):
+    column = f"low_fee_{sensitivity}"
+    blocks = sample_blocks(sample, column)
     if not blocks:
         print(f"no blocks with {column} above height {MIN_AUDITED_HEIGHT}")
         return
 
     print(f"checking {len(blocks)} blocks against mempool.space "
-          f"(sensitivity 0.{args.sensitivity})\n")
+          f"(sensitivity 0.{sensitivity})\n")
 
     audited = 0
     total_low_fee = 0
@@ -85,7 +76,7 @@ def main():
     blocks_with_any_overlap = 0
 
     for block in blocks:
-        data = fetch_audit(block["block_hash"], args.sleep)
+        data = fetch_audit(block["block_hash"], sleep)
         if not data:
             print(f"  {block['block_number']}  no audit data")
             continue
@@ -123,7 +114,3 @@ def main():
     print("Transactions in acceleratedTxs were bought out of band through a "
           "public service: they confirm the mechanism, and they are the part "
           "of the count that is not invisible.")
-
-
-if __name__ == "__main__":
-    main()