add datasets page to dashboard

servicex_app/servicex_app/dataset_manager.py

@@ -123,6 +123,7 @@ def from_file_list(
                 DatasetFile(paths=file, adler32="xxx", file_events=0, file_size=0)
                 for file in file_list
             ]
+            dataset.n_files = len(file_list)
 
             logger.info(
                 f"Upserted dataset for file list. Dataset Id is {dataset.id}",

servicex_app/servicex_app/resources/internal/transformer_file_complete.py

@@ -46,6 +46,8 @@
 
 from servicex_app import TransformerManager
 from servicex_app.models import (
+    Dataset,
+    DatasetFile,
     TransformRequest,
     TransformStatus,
     TransformationResult,
@@ -118,6 +120,10 @@ def put(self, request_id):
                     extra=log_extra,
                 )
 
+            # Backfill per-file and per-dataset stats from the transformer's report
+            # if the DID finder never populated them (e.g., user file-list datasets).
+            self.backfill_dataset_stats(session, info)
+
             # Lookup the transformation request and increment either the successful
             # or failed file count
             transform_req = self.record_file_complete(
@@ -250,6 +256,49 @@ def save_transform_result(request_id: str, info: dict[str, str], session: Sessio
             session.add(result)
         return orig_counts
 
+    @staticmethod
+    @file_complete_ops_retry
+    def backfill_dataset_stats(session: Session, info: dict):
+        if info.get("status") != "success":
+            return
+        events = info.get("total-events") or 0
+        size = info.get("total-bytes") or 0
+        if not events and not size:
+            return
+
+        with session.begin():
+            file_row = (
+                session.query(DatasetFile)
+                .filter_by(id=info["file-id"])
+                .with_for_update()
+                .one_or_none()
+            )
+            if file_row is None:
+                return
+
+            events_delta = 0
+            size_delta = 0
+            if not file_row.file_events and events:
+                file_row.file_events = events
+                events_delta = events
+            if not file_row.file_size and size:
+                file_row.file_size = size
+                size_delta = size
+
+            if events_delta == 0 and size_delta == 0:
+                return
+
+            dataset = (
+                session.query(Dataset)
+                .filter_by(id=file_row.dataset_id)
+                .with_for_update()
+                .one_or_none()
+            )
+            if dataset is None:
+                return
+            dataset.events = (dataset.events or 0) + events_delta
+            dataset.size = (dataset.size or 0) + size_delta
+
     @staticmethod
     @file_complete_ops_retry
     def transform_complete(

servicex_app/servicex_app/routes.py

@@ -94,6 +94,8 @@ def add_routes(
     from servicex_app.web.transformation_request import transformation_request
     from servicex_app.web.transformation_results import transformation_results
     from servicex_app.web.multiple_codegen_list import multiple_codegen_list
+    from servicex_app.web.datasets import datasets as datasets_page
+    from servicex_app.web.dataset import dataset as dataset_page
 
     # Must be its own module to allow patching
     from servicex_app.web.create_profile import create_profile
@@ -140,6 +142,8 @@ def add_routes(
     app.add_url_rule(
         "/multiple-codegen-list", "multiple_codegen_list", multiple_codegen_list
     )
+    app.add_url_rule("/datasets", "datasets", datasets_page)
+    app.add_url_rule("/datasets/<int:id_>", "dataset", dataset_page)
 
     # User management and Authentication Endpoints
     api.add_resource(TokenRefresh, "/token/refresh")

servicex_app/servicex_app/templates/base.html

@@ -44,6 +44,11 @@
             <li class="nav-item">
               <a class="nav-link" href="{{ url_for('home') }}">Dashboard</a>
             </li>
+            {% if not config["ENABLE_AUTH"] or session['is_authenticated'] %}
+            <li class="nav-item">
+              <a class="nav-link" href="{{ url_for('datasets') }}">Datasets</a>
+            </li>
+            {% endif %}
             <li class="nav-item">
               <a class="nav-link" href="{{ config['DOCS_BASE_URL'] }}" target="_blank">Docs</a>
             </li>

servicex_app/servicex_app/templates/dataset.html

@@ -0,0 +1,135 @@
+{% extends "base.html" %}
+{% block content %}
+<div class="content-section col-12">
+  <div class="row justify-content-between align-items-center">
+    <h4 class="mb-3 col-auto">Dataset</h4>
+    <div class="col-auto">
+      {% if not ds.stale %}
+      <button id="delete-{{ ds.id }}"
+        onclick="deleteDataset({{ ds.id }})"
+        type="button" class="btn btn-danger btn-sm">
+        Flush
+      </button>
+      {% elif ds.lookup_status and ds.lookup_status.value in ('bad_name', 'does_not_exist', 'internal_failure') %}
+      <span class="badge badge-danger">{{ ds.lookup_status.value }}</span>
+      {% else %}
+      <span class="badge badge-secondary">Flushed</span>
+      {% endif %}
+    </div>
+  </div>
+
+  <dl class="row">
+    <dt class="col-sm-3">ID</dt>
+    <dd class="col-sm-9">{{ ds.id }}</dd>
+
+    <dt class="col-sm-3">Name</dt>
+    <dd class="col-sm-9" style="word-break: break-all">
+      {% if ds.did_finder == "user" %}
+        (file list)
+        <div class="text-muted small">{{ ds.name }}</div>
+      {% else %}
+        {{ ds.name }}
+      {% endif %}
+    </dd>
+
+    <dt class="col-sm-3">DID Finder</dt>
+    <dd class="col-sm-9">{{ ds.did_finder }}</dd>
+
+    <dt class="col-sm-3">Lookup Status</dt>
+    <dd class="col-sm-9">{{ ds.lookup_status.value if ds.lookup_status else '-' }}</dd>
+
+    <dt class="col-sm-3">Files</dt>
+    <dd class="col-sm-9">{{ humanize.intcomma(ds.n_files or 0) }}</dd>
+
+    <dt class="col-sm-3">Total Events</dt>
+    <dd class="col-sm-9">{{ humanize.intcomma(ds.events or 0) }}</dd>
+
+    <dt class="col-sm-3">Total Size</dt>
+    <dd class="col-sm-9">{{ humanize.naturalsize(ds.size or 0) }}</dd>
+
+    <dt class="col-sm-3">Last Used</dt>
+    <dd class="col-sm-9">
+      {{ moment(ds.last_used).format("YYYY-MM-DD HH:mm:ss") if ds.last_used else "-" }}
+      <span class="tz"></span>
+    </dd>
+
+    <dt class="col-sm-3">Last Updated</dt>
+    <dd class="col-sm-9">
+      {{ moment(ds.last_updated).format("YYYY-MM-DD HH:mm:ss") if ds.last_updated else "-" }}
+      <span class="tz"></span>
+    </dd>
+
+    <dt class="col-sm-3">Transform Requests</dt>
+    <dd class="col-sm-9">
+      {% if ds.transform_requests %}
+        <ul class="list-unstyled mb-0">
+          {% for req in ds.transform_requests %}
+          <li>
+            <a href="{{ url_for('transformation_request', id_=req.id) }}">
+              {{ req.title or "Untitled" }}
+            </a>
+            <span class="text-muted small">({{ req.request_id }})</span>
+          </li>
+          {% endfor %}
+        </ul>
+      {% else %}
+        -
+      {% endif %}
+    </dd>
+  </dl>
+
+  <h5 class="mt-4">Files ({{ humanize.intcomma(ds.files|length) }})</h5>
+  {% if ds.files %}
+  <div class="table-responsive">
+    <table class="table table-sm table-bordered table-striped">
+      <thead class="thead-dark">
+        <tr>
+          <th scope="col">ID</th>
+          <th scope="col">Path(s)</th>
+          <th scope="col">Events</th>
+          <th scope="col">Size</th>
+          <th scope="col">adler32</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for f in ds.files %}
+        <tr>
+          <th scope="row">{{ f.id }}</th>
+          <td style="word-break: break-all">{{ f.paths }}</td>
+          <td>{{ humanize.intcomma(f.file_events or 0) }}</td>
+          <td>{{ humanize.naturalsize(f.file_size or 0) }}</td>
+          <td><code>{{ f.adler32 or '-' }}</code></td>
+        </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </div>
+  {% else %}
+    <p class="text-muted">No files.</p>
+  {% endif %}
+</div>
+{% endblock %}
+
+{% block scripts %}
+<script>
+  function deleteDataset(datasetId) {
+    if (!confirm(`Flush dataset ${datasetId}?\n\nThis marks it stale so a future request will refetch the file list.`)) {
+      return;
+    }
+    fetch(`/servicex/datasets/${datasetId}`, { method: 'DELETE' })
+      .then(async (res) => {
+        if (res.ok) {
+          location.reload();
+        } else {
+          const body = await res.json().catch(() => ({}));
+          alert(body.message || `Failed to flush dataset ${datasetId}`);
+        }
+      })
+      .catch((err) => alert(`Failed to flush dataset ${datasetId}: ${err}`));
+  }
+
+  $(document).ready(function () {
+    $(".tz").text(moment.tz(moment.tz.guess()).format("z"));
+  });
+</script>
+{% endblock %}

servicex_app/servicex_app/templates/dataset_table.html

@@ -0,0 +1,87 @@
+{% from 'bootstrap5/pagination.html' import render_pagination %}
+
+{% macro datasets_table(pagination, humanize) %}
+<div class="table-responsive">
+  <table class="table table-sm table-bordered table-striped">
+    <caption>All times in timezone: <span class="tz"></span>.</caption>
+    <thead class="thead-dark">
+      <tr>
+        <th scope="col">ID</th>
+        <th scope="col">Name</th>
+        <th scope="col">DID Finder</th>
+        <th scope="col">Lookup Status</th>
+        <th scope="col">Files</th>
+        <th scope="col">Events</th>
+        <th scope="col">Size</th>
+        <th scope="col">Last Used</th>
+        <th scope="col">Last Updated</th>
+        <th scope="col">Actions</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for ds in pagination.items %}
+      <tr id="dataset-row-{{ ds.id }}">
+        <th scope="row">{{ ds.id }}</th>
+        <td style="max-width: 400px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;"
+            title="{{ ds.name }}">
+          <a href="{{ url_for('dataset', id_=ds.id) }}">
+            {% if ds.did_finder == "user" %}(file list){% else %}{{ ds.name }}{% endif %}
+          </a>
+        </td>
+        <td>{{ ds.did_finder }}</td>
+        <td>{{ ds.lookup_status.value if ds.lookup_status else '-' }}</td>
+        <td>{{ humanize.intcomma(ds.n_files or 0) }}</td>
+        <td>{{ humanize.intcomma(ds.events or 0) }}</td>
+        <td>{{ humanize.naturalsize(ds.size or 0) }}</td>
+        <td>{{ moment(ds.last_used).format("YYYY-MM-DD HH:mm") if ds.last_used else "-" }}</td>
+        <td>{{ moment(ds.last_updated).format("YYYY-MM-DD HH:mm") if ds.last_updated else "-" }}</td>
+        <td>
+          {% if not ds.stale %}
+          <button id="delete-{{ ds.id }}"
+            onclick="deleteDataset({{ ds.id }}, '{{ ds.name|e }}')"
+            type="button" class="btn btn-danger btn-sm">
+            Flush
+          </button>
+          {% elif ds.lookup_status and ds.lookup_status.value in ('bad_name', 'does_not_exist', 'internal_failure') %}
+          <span class="badge badge-danger">{{ ds.lookup_status.value }}</span>
+          {% else %}
+          <span class="badge badge-secondary">Flushed</span>
+          {% endif %}
+        </td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% if pagination.items %}
+    {{ render_pagination(pagination, align='center') }}
+  {% else %}
+    <div class="text-center text-muted p-4">
+      No datasets found.
+    </div>
+  {% endif %}
+</div>
+{% endmacro %}
+
+{% macro datasets_table_scripts() %}
+<script>
+  function deleteDataset(datasetId, datasetName) {
+    if (!confirm(`Flush dataset "${datasetName}" (id ${datasetId})?\n\nThis marks it stale so a future request will refetch the file list.`)) {
+      return;
+    }
+    fetch(`/servicex/datasets/${datasetId}`, { method: 'DELETE' })
+      .then(async (res) => {
+        if (res.ok) {
+          location.reload();
+        } else {
+          const body = await res.json().catch(() => ({}));
+          alert(body.message || `Failed to flush dataset ${datasetId}`);
+        }
+      })
+      .catch((err) => alert(`Failed to flush dataset ${datasetId}: ${err}`));
+  }
+
+  $(document).ready(function () {
+    $(".tz").text(moment.tz(moment.tz.guess()).format("z"));
+  });
+</script>
+{% endmacro %}

servicex_app/servicex_app/templates/datasets.html

@@ -0,0 +1,37 @@
+{% extends "base.html" %}
+
+{% from 'dataset_table.html' import datasets_table, datasets_table_scripts with context %}
+{% from 'sort_dropdown.html' import sort_dropdown %}
+
+{% block content %}
+
+  <div class="col-12">
+    <div class="content-section">
+      <div class="row justify-content-between align-items-center">
+        <h4 class="mb-3 col-auto">Datasets</h4>
+        <div class="col-auto d-flex align-items-center">
+          <form method="get" class="form-inline mr-3">
+            <input type="hidden" name="sort" value="{{ active_sort }}">
+            <input type="hidden" name="order" value="{{ active_order }}">
+            <div class="form-check">
+              <input class="form-check-input" type="checkbox" name="show_deleted" value="true"
+                     id="show_deleted" {% if show_deleted %}checked{% endif %}
+                     onchange="this.form.submit()">
+              <label class="form-check-label" for="show_deleted">
+                Show flushed
+              </label>
+            </div>
+          </form>
+          {{ sort_dropdown(dropdown_options, active_sort, active_order) }}
+        </div>
+      </div>
+      {{ datasets_table(pagination, humanize) }}
+    </div>
+  </div>
+
+{% endblock %}
+
+{% block scripts %}
+  {{ super() }}
+  {{ datasets_table_scripts() }}
+{% endblock %}

servicex_app/servicex_app/templates/sort_dropdown.html

@@ -2,11 +2,11 @@
   <div class="dropdown col-auto">
     <button class="btn btn-sm btn-dark dropdown-toggle" type="button" id="dropdownMenuButton"
             data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
-      Sort: <b>{{ active_sort.title() }} ({{ active_order }})</b>
+      Sort: <b>{{ active_sort.replace('_', ' ').title() }} ({{ active_order }})</b>
     </button>
     <div class="dropdown-menu" aria-labelledby="dropdownMenuButton">
         {% for sort, order in options %}
-          <a href="?sort={{ sort }}&order={{ order }}" class="dropdown-item">{{ sort.title() }} ({{ order }})</a>
+          <a href="?sort={{ sort }}&order={{ order }}" class="dropdown-item">{{ sort.replace('_', ' ').title() }} ({{ order }})</a>
         {% endfor %}
     </div>
   </div>

servicex_app/servicex_app/web/dataset.py

@@ -0,0 +1,12 @@
+from flask import render_template, abort
+
+from servicex_app.decorators import oauth_required
+from servicex_app.models import Dataset
+
+
+@oauth_required
+def dataset(id_: int):
+    ds = Dataset.find_by_id(id_)
+    if not ds:
+        abort(404)
+    return render_template("dataset.html", ds=ds)