wpilog automatic struct decoding

examples/datalog/datalog_struct.py

@@ -0,0 +1,10 @@
+import dataclasses
+
+from wpiutil import wpistruct
+
+
+@wpistruct.make_wpistruct(name="ExampleRecord")
+@dataclasses.dataclass
+class ExampleRecord:
+    i: wpistruct.int32 = 0
+    j: wpistruct.int32 = 0

examples/datalog/printlog.py

@@ -5,88 +5,22 @@
 # the WPILib BSD license file in the root directory of this project.
 
 import argparse
-import datetime
 
 from wpilog import DataLogReader
 
+from datalog_struct import ExampleRecord
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
+    parser.add_argument("--predefined", action="store_true")
     parser.add_argument("infile")
     args = parser.parse_args()
 
     reader = DataLogReader(args.infile)
+    struct_types = (ExampleRecord,) if args.predefined else ()
 
-    entries = {}
-    for record in reader:
-        timestamp = record.get_timestamp() / 1000000
-        if record.is_start():
-            try:
-                data = record.get_start_data()
-                print(f"{data} [{timestamp}]")
-                if data.entry in entries:
-                    print("...DUPLICATE entry ID, overriding")
-                entries[data.entry] = data
-            except TypeError as e:
-                print("Start(INVALID)")
-        elif record.is_finish():
-            try:
-                entry = record.get_finish_entry()
-                print(f"Finish({entry}) [{timestamp}]")
-                if entry not in entries:
-                    print("...ID not found")
-                else:
-                    del entries[entry]
-            except TypeError as e:
-                print("Finish(INVALID)")
-        elif record.is_set_metadata():
-            try:
-                data = record.get_set_metadata_data()
-                print(f"{data} [{timestamp}]")
-                if data.entry not in entries:
-                    print("...ID not found")
-            except TypeError as e:
-                print("SetMetadata(INVALID)")
-        elif record.is_control():
-            print("Unrecognized control record")
-        else:
-            print(f"Data({record.get_entry()}, size={record.get_size()}) ", end="")
-            entry = entries.get(record.get_entry(), None)
-            if entry is None:
-                print("<ID not found>")
-                continue
-            print(f"<name='{entry.name}', type='{entry.type}'> [{timestamp}]")
-
-            try:
-                # handle systemTime specially
-                if entry.name == "systemTime" and entry.type == "int64":
-                    dt = datetime.fromtimestamp(record.get_integer() / 1000000)
-                    print(f"  {dt:%Y-%m-%d %H:%M:%S.%f}")
-                    continue
-
-                if entry.type == "double":
-                    print(f"  {record.get_double()}")
-                elif entry.type == "int64":
-                    print(f"  {record.get_integer()}")
-                elif entry.type == "string" or entry.type == "json":
-                    print(f"  '{record.get_string()}'")
-                elif entry.type == "boolean":
-                    print(f"  {record.get_boolean()}")
-                elif entry.type == "boolean[]":
-                    arr = record.get_boolean_array()
-                    print(f"  {arr}")
-                elif entry.type == "double[]":
-                    arr = record.get_double_array()
-                    print(f"  {arr}")
-                elif entry.type == "float[]":
-                    arr = record.get_float_array()
-                    print(f"  {arr}")
-                elif entry.type == "int64[]":
-                    arr = record.get_integer_array()
-                    print(f"  {arr}")
-                elif entry.type == "string[]":
-                    arr = record.get_string_array()
-                    print(f"  {arr}")
-                elif entry.type == "raw":
-                    print(f"  {record.get_raw()}")
-            except TypeError as e:
-                print("  invalid", e)
+    for record, entry, value in reader.iter_auto(*struct_types):
+        timestamp = record.get_timestamp() / 1_000_000
+        name = entry.name if entry is not None else f"entry:{record.get_entry()}"
+        advertised_type = entry.type if entry is not None else "unknown"
+        print(f"{name} [{advertised_type}] [{timestamp}] {value!r}")

examples/datalog/writelog.py

@@ -2,27 +2,73 @@
 
 import argparse
 import pathlib
+import time
+
+import wpilog
+
+from datalog_struct import ExampleRecord
+
+
+def _wait_for_complete_log(candidate_paths):
+    deadline = time.monotonic() + 5
+    while True:
+        for path in candidate_paths():
+            try:
+                values = wpilog.DataLogReader(str(path)).iter_auto(ExampleRecord)
+                record_count = sum(
+                    entry is not None and entry.name == "/record"
+                    for _, entry, _ in values
+                )
+            except (OSError, ValueError):
+                continue
+            if record_count == 2:
+                return
+
+        if time.monotonic() >= deadline:
+            raise TimeoutError("timed out waiting for the data log to flush")
+        time.sleep(0.01)
 
-from wpilog import DataLog, BooleanLogEntry, StringArrayLogEntry, RawLogEntry
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("out", type=pathlib.Path)
     args = parser.parse_args()
 
     if args.out.is_dir():
-        datalog = DataLog(str(args.out))
+        existing_paths = set(args.out.iterdir())
+        datalog = wpilog.DataLogBackgroundWriter(str(args.out))
+
+        def candidate_paths():
+            return set(args.out.iterdir()) - existing_paths
+
     else:
-        datalog = DataLog(str(args.out.parent), args.out.name)
+        datalog = wpilog.DataLogBackgroundWriter(str(args.out.parent), args.out.name)
+
+        def candidate_paths():
+            return (args.out,)
+
+    completed = False
+    try:
+        bools = wpilog.BooleanLogEntry(datalog, "/bools")
+        bools.append(True)
+        bools.append(False)
 
-    bools = BooleanLogEntry(datalog, "/bools")
-    bools.append(True)
-    bools.append(False)
+        strings = wpilog.StringArrayLogEntry(datalog, "/strings")
+        strings.append(["a", "b", "c"])
+        strings.append(["d", "e", "f"])
 
-    strings = StringArrayLogEntry(datalog, "/strings")
-    strings.append(["a", "b", "c"])
-    strings.append(["d", "e", "f"])
+        raw = wpilog.RawLogEntry(datalog, "/raws")
+        raw.append(b"\x01\x02\x03")
+        raw.append(b"\x04\x05\x06")
 
-    raw = RawLogEntry(datalog, "/raws")
-    raw.append(b"\x01\x02\x03")
-    raw.append(b"\x04\x05\x06")
+        record = wpilog.StructLogEntry(datalog, "/record", ExampleRecord)
+        record.append(ExampleRecord(1, 2))
+        record.append(ExampleRecord(3, 4))
+        completed = True
+    finally:
+        try:
+            if completed:
+                datalog.flush()
+                _wait_for_complete_log(candidate_paths)
+        finally:
+            datalog.stop()

subprojects/robotpy-wpilog/semiwrap/DataLogReader.yml

@@ -52,7 +52,8 @@ classes:
             }
             return ptr;
           }
-        return_value_policy: reference_internal
+        keepalive:
+        - [0, 1]
       GetFinishEntry:
         no_release_gil: true
         param_override:
@@ -83,7 +84,8 @@ classes:
             }
             return ptr;
           }
-        return_value_policy: reference_internal
+        keepalive:
+        - [0, 1]
       GetBoolean:
         no_release_gil: true
         param_override:
@@ -329,4 +331,11 @@ inline_code: |
       py::keep_alive<1, 2>())
     .def("__iter__", [](wpi::log::DataLogReader * that) {
       return py::make_iterator(that->begin(), that->end());
+    }, py::keep_alive<0,1>())
+    .def("_iter_stable", [](wpi::log::DataLogReader *that) {
+      return py::make_iterator<py::return_value_policy::copy>(
+          that->begin(), that->end(), py::keep_alive<0, 1>());
+    }, py::keep_alive<0, 1>())
+    .def("iter_auto", [](wpi::log::DataLogReader *self, py::args structTypes) {
+      return py::module_::import("wpilog._datalog").attr("_iter_auto")(self, structTypes);
     }, py::keep_alive<0,1>());

subprojects/robotpy-wpilog/tests/test_datalog_examples.py

@@ -0,0 +1,61 @@
+import runpy
+import subprocess
+import sys
+import types
+from pathlib import Path
+
+import pytest
+import wpilog
+
+EXAMPLE_DIR = Path(__file__).parents[3] / "examples" / "datalog"
+
+
+def test_writelog_output_is_immediately_readable(tmp_path):
+    output = tmp_path / "example.wpilog"
+
+    subprocess.run(
+        [sys.executable, str(EXAMPLE_DIR / "writelog.py"), str(output)],
+        check=True,
+        timeout=10,
+    )
+
+    values = list(wpilog.DataLogReader(str(output)).iter_auto())
+    assert (
+        sum(entry is not None and entry.name == "/record" for _, entry, _ in values)
+        == 2
+    )
+
+
+def test_writelog_stops_background_writer_when_append_fails(monkeypatch, tmp_path):
+    writers = []
+
+    class FakeWriter:
+        def __init__(self, *args):
+            self.stopped = False
+            writers.append(self)
+
+        def stop(self):
+            self.stopped = True
+
+    class FailingEntry:
+        def __init__(self, *args):
+            pass
+
+        def append(self, value):
+            raise RuntimeError("intentional append failure")
+
+    fake_wpilog = types.ModuleType("wpilog")
+    fake_wpilog.DataLogBackgroundWriter = FakeWriter
+    fake_wpilog.BooleanLogEntry = FailingEntry
+    fake_wpilog.StringArrayLogEntry = FailingEntry
+    fake_wpilog.RawLogEntry = FailingEntry
+    fake_wpilog.StructLogEntry = FailingEntry
+    monkeypatch.setitem(sys.modules, "wpilog", fake_wpilog)
+    monkeypatch.syspath_prepend(str(EXAMPLE_DIR))
+    monkeypatch.setattr(sys, "argv", ["writelog.py", str(tmp_path / "example.wpilog")])
+
+    with pytest.raises(RuntimeError, match="intentional append failure"):
+        runpy.run_path(str(EXAMPLE_DIR / "writelog.py"), run_name="__main__")
+
+    assert len(writers) == 1
+    assert writers[0].stopped