From 0f075124890b48adf4060e6c2fa40a5fef4aa047 Mon Sep 17 00:00:00 2001 From: p-kimberley Date: Wed, 2 Aug 2023 22:00:35 +1000 Subject: [PATCH] Back-port #3248 to 7.1 --- .../shared/DownloadSearchResultsRequest.java | 8 ++ .../dashboard/shared/TableResultRequest.java | 36 ++++++++- .../client/table/DownloadPresenter.java | 6 ++ .../client/table/DownloadViewImpl.java | 19 +++++ .../client/table/TablePresenter.java | 28 ++++--- .../client/table/DownloadViewImpl.ui.xml | 6 ++ .../dashboard/impl/DashboardServiceImpl.java | 76 ++++++------------- .../impl/download/DelimitedTarget.java | 10 +++ .../dashboard/impl/download/ExcelTarget.java | 21 +++-- .../impl/download/SearchResultWriter.java | 47 +++++++++--- .../impl/logging/SearchEventLogImpl.java | 5 ++ .../20230213_150003_377__3248.md | 19 +++++ 12 files changed, 201 insertions(+), 80 deletions(-) create mode 100644 unreleased_changes/20230213_150003_377__3248.md diff --git a/stroom-core-shared/src/main/java/stroom/dashboard/shared/DownloadSearchResultsRequest.java b/stroom-core-shared/src/main/java/stroom/dashboard/shared/DownloadSearchResultsRequest.java index 3740a71224e..73f6e816dc9 100644 --- a/stroom-core-shared/src/main/java/stroom/dashboard/shared/DownloadSearchResultsRequest.java +++ b/stroom-core-shared/src/main/java/stroom/dashboard/shared/DownloadSearchResultsRequest.java @@ -31,6 +31,8 @@ public class DownloadSearchResultsRequest { @JsonProperty private final DownloadSearchResultFileType fileType; @JsonProperty + private final boolean downloadAllTables; + @JsonProperty private final boolean sample; @JsonProperty private final int percent; @@ -39,11 +41,13 @@ public class DownloadSearchResultsRequest { public DownloadSearchResultsRequest(@JsonProperty("searchRequest") final DashboardSearchRequest searchRequest, @JsonProperty("componentId") final String componentId, @JsonProperty("fileType") final DownloadSearchResultFileType fileType, + @JsonProperty("downloadAllTables") final boolean downloadAllTables, @JsonProperty("sample") final boolean sample, @JsonProperty("percent") final int percent) { this.searchRequest = searchRequest; this.componentId = componentId; this.fileType = fileType; + this.downloadAllTables = downloadAllTables; this.sample = sample; this.percent = percent; } @@ -60,6 +64,10 @@ public DownloadSearchResultFileType getFileType() { return fileType; } + public boolean isDownloadAllTables() { + return downloadAllTables; + } + public boolean isSample() { return sample; } diff --git a/stroom-core-shared/src/main/java/stroom/dashboard/shared/TableResultRequest.java b/stroom-core-shared/src/main/java/stroom/dashboard/shared/TableResultRequest.java index da91840eab8..19e06808f00 100644 --- a/stroom-core-shared/src/main/java/stroom/dashboard/shared/TableResultRequest.java +++ b/stroom-core-shared/src/main/java/stroom/dashboard/shared/TableResultRequest.java @@ -33,12 +33,15 @@ @JsonPropertyOrder({ "componentId", "fetch", + "tableName", "tableSettings", "requestedRange", "openGroups"}) @JsonInclude(Include.NON_NULL) public class TableResultRequest extends ComponentResultRequest { + @JsonProperty + private final String tableName; @JsonProperty private final TableSettings tableSettings; @JsonProperty @@ -49,10 +52,12 @@ public class TableResultRequest extends ComponentResultRequest { @JsonCreator public TableResultRequest(@JsonProperty("componentId") final String componentId, @JsonProperty("fetch") final Fetch fetch, + @JsonProperty("tableName") final String tableName, @JsonProperty("tableSettings") final TableSettings tableSettings, @JsonProperty("requestedRange") final OffsetRange requestedRange, @JsonProperty("openGroups") final Set openGroups) { super(componentId, fetch); + this.tableName = tableName; this.tableSettings = tableSettings; this.requestedRange = requestedRange; this.openGroups = openGroups; @@ -62,6 +67,10 @@ public static Builder builder() { return new Builder(); } + public String getTableName() { + return tableName; + } + public TableSettings getTableSettings() { return tableSettings; } @@ -87,20 +96,26 @@ public boolean equals(final Object o) { return false; } final TableResultRequest that = (TableResultRequest) o; - return Objects.equals(tableSettings, that.tableSettings) && + return Objects.equals(tableName, that.tableName) && + Objects.equals(tableSettings, that.tableSettings) && Objects.equals(requestedRange, that.requestedRange) && Objects.equals(openGroups, that.openGroups); } @Override public int hashCode() { - return Objects.hash(tableSettings, requestedRange, openGroups); + return Objects.hash( + tableName, + tableSettings, + requestedRange, + openGroups); } @Override public String toString() { return "TableResultRequest{" + - "tableSettings=" + tableSettings + + "tableName='" + tableName + '\'' + + ",tableSettings=" + tableSettings + ", requestedRange=" + requestedRange + ", openGroups=" + openGroups + '}'; @@ -114,6 +129,7 @@ public static final class Builder { private String componentId; private Fetch fetch; + private String tableName; private TableSettings tableSettings; private OffsetRange requestedRange = new OffsetRange(0, 100); private Set openGroups; @@ -124,6 +140,7 @@ private Builder() { private Builder(final TableResultRequest tableResultRequest) { this.componentId = tableResultRequest.getComponentId(); this.fetch = tableResultRequest.getFetch(); + this.tableName = tableResultRequest.tableName; this.tableSettings = tableResultRequest.tableSettings; this.requestedRange = tableResultRequest.requestedRange; this.openGroups = tableResultRequest.openGroups; @@ -139,6 +156,11 @@ public Builder fetch(final Fetch fetch) { return this; } + public Builder tableName(final String tableName) { + this.tableName = tableName; + return this; + } + public Builder tableSettings(final TableSettings tableSettings) { this.tableSettings = tableSettings; return this; @@ -169,7 +191,13 @@ public Builder openGroup(final String group, final boolean open) { } public TableResultRequest build() { - return new TableResultRequest(componentId, fetch, tableSettings, requestedRange, openGroups); + return new TableResultRequest( + componentId, + fetch, + tableName, + tableSettings, + requestedRange, + openGroups); } } } diff --git a/stroom-dashboard/stroom-dashboard-client/src/main/java/stroom/dashboard/client/table/DownloadPresenter.java b/stroom-dashboard/stroom-dashboard-client/src/main/java/stroom/dashboard/client/table/DownloadPresenter.java index b35cc721da7..cf172816474 100644 --- a/stroom-dashboard/stroom-dashboard-client/src/main/java/stroom/dashboard/client/table/DownloadPresenter.java +++ b/stroom-dashboard/stroom-dashboard-client/src/main/java/stroom/dashboard/client/table/DownloadPresenter.java @@ -34,6 +34,10 @@ public DownloadSearchResultFileType getFileType() { return getView().getFileType(); } + public boolean downloadAllTables() { + return getView().downloadAllTables(); + } + public boolean isSample() { return getView().isSample(); } @@ -46,6 +50,8 @@ public interface DownloadView extends View { DownloadSearchResultFileType getFileType(); + boolean downloadAllTables(); + boolean isSample(); int getPercent(); diff --git a/stroom-dashboard/stroom-dashboard-client/src/main/java/stroom/dashboard/client/table/DownloadViewImpl.java b/stroom-dashboard/stroom-dashboard-client/src/main/java/stroom/dashboard/client/table/DownloadViewImpl.java index 4820001e0b5..f338c9f35ef 100644 --- a/stroom-dashboard/stroom-dashboard-client/src/main/java/stroom/dashboard/client/table/DownloadViewImpl.java +++ b/stroom-dashboard/stroom-dashboard-client/src/main/java/stroom/dashboard/client/table/DownloadViewImpl.java @@ -37,6 +37,8 @@ public class DownloadViewImpl extends ViewImpl implements DownloadView { @UiField ItemListBox fileType; @UiField + TickBox downloadAllTables; + @UiField TickBox sample; @UiField ValueSpinner percent; @@ -55,6 +57,18 @@ public DownloadViewImpl(final Binder binder) { fileType.addItem(DownloadSearchResultFileType.TSV); fileType.setSelectedItem(DownloadSearchResultFileType.EXCEL); + + downloadAllTables.setEnabled(isExcelFileTypeSelected()); + fileType.addSelectionHandler(event -> { + downloadAllTables.setEnabled(isExcelFileTypeSelected()); + if (!isExcelFileTypeSelected()) { + downloadAllTables.setBooleanValue(false); + } + }); + } + + private boolean isExcelFileTypeSelected() { + return DownloadSearchResultFileType.EXCEL.equals(fileType.getSelectedItem()); } @Override @@ -67,6 +81,11 @@ public DownloadSearchResultFileType getFileType() { return fileType.getSelectedItem(); } + @Override + public boolean downloadAllTables() { + return downloadAllTables.getBooleanValue(); + } + @Override public boolean isSample() { return sample.getBooleanValue(); diff --git a/stroom-dashboard/stroom-dashboard-client/src/main/java/stroom/dashboard/client/table/TablePresenter.java b/stroom-dashboard/stroom-dashboard-client/src/main/java/stroom/dashboard/client/table/TablePresenter.java index 8ce89fcaff5..9edabb76d52 100644 --- a/stroom-dashboard/stroom-dashboard-client/src/main/java/stroom/dashboard/client/table/TablePresenter.java +++ b/stroom-dashboard/stroom-dashboard-client/src/main/java/stroom/dashboard/client/table/TablePresenter.java @@ -437,16 +437,19 @@ private void download() { @Override public void onHideRequest(final boolean autoClose, final boolean ok) { if (ok) { - final TableResultRequest tableResultRequest = TableResultRequest - .builder() - .componentId(getComponentConfig().getId()) - .requestedRange(new OffsetRange(0, Integer.MAX_VALUE)) - .tableSettings(TablePresenter.this.tableResultRequest.getTableSettings()) - .fetch(Fetch.ALL) - .build(); - final List requests = new ArrayList<>(); - requests.add(tableResultRequest); + currentSearch.getComponentSettingsMap().entrySet() + .stream() + .filter(settings -> settings.getValue() instanceof TableComponentSettings) + .forEach(tableSettings -> requests.add(TableResultRequest + .builder() + .componentId(tableSettings.getKey()) + .requestedRange(new OffsetRange(0, Integer.MAX_VALUE)) + .tableName(getTableName(tableSettings.getKey())) + .tableSettings(((TableComponentSettings) tableSettings.getValue()).copy() + .buildTableSettings()) + .fetch(Fetch.ALL) + .build())); final Search search = Search .builder() @@ -472,6 +475,7 @@ public void onHideRequest(final boolean autoClose, final boolean ok) { searchRequest, getComponentConfig().getId(), downloadPresenter.getFileType(), + downloadPresenter.downloadAllTables(), downloadPresenter.isSample(), downloadPresenter.getPercent()); final Rest rest = restFactory.create(); @@ -497,6 +501,12 @@ public void onHide(final boolean autoClose, final boolean ok) { } } + private String getTableName(final String componentId) { + return Optional.ofNullable(getComponents().get(componentId)) + .map(component -> component.getComponentConfig().getName()) + .orElse(null); + } + private DateTimeSettings getDateTimeSettings() { final UserPreferences userPreferences = userPreferencesManager.getCurrentPreferences(); return DateTimeSettings diff --git a/stroom-dashboard/stroom-dashboard-client/src/main/resources/stroom/dashboard/client/table/DownloadViewImpl.ui.xml b/stroom-dashboard/stroom-dashboard-client/src/main/resources/stroom/dashboard/client/table/DownloadViewImpl.ui.xml index d3e3e1a72c4..82685e91539 100644 --- a/stroom-dashboard/stroom-dashboard-client/src/main/resources/stroom/dashboard/client/table/DownloadViewImpl.ui.xml +++ b/stroom-dashboard/stroom-dashboard-client/src/main/resources/stroom/dashboard/client/table/DownloadViewImpl.ui.xml @@ -10,6 +10,12 @@ + + Download All Tables: + + + + Sample Only: diff --git a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/DashboardServiceImpl.java b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/DashboardServiceImpl.java index 45d7c65ef3f..122fa9dcafc 100644 --- a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/DashboardServiceImpl.java +++ b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/DashboardServiceImpl.java @@ -30,7 +30,6 @@ import stroom.dashboard.shared.DashboardSearchRequest; import stroom.dashboard.shared.DashboardSearchResponse; import stroom.dashboard.shared.DestroySearchRequest; -import stroom.dashboard.shared.DownloadSearchResultFileType; import stroom.dashboard.shared.DownloadSearchResultsRequest; import stroom.dashboard.shared.FunctionSignature; import stroom.dashboard.shared.Search; @@ -43,14 +42,11 @@ import stroom.docref.DocRefInfo; import stroom.docstore.api.DocumentResourceHelper; import stroom.event.logging.rs.api.AutoLogged; -import stroom.query.api.v2.DateTimeSettings; import stroom.query.api.v2.Field; import stroom.query.api.v2.Param; import stroom.query.api.v2.Query; import stroom.query.api.v2.QueryKey; -import stroom.query.api.v2.Result; import stroom.query.api.v2.ResultRequest.Fetch; -import stroom.query.api.v2.Row; import stroom.query.api.v2.SearchRequest; import stroom.query.api.v2.SearchResponse; import stroom.query.api.v2.TableResult; @@ -242,7 +238,6 @@ public ResourceGeneration downloadSearchResults(final DownloadSearchResultsReque final DashboardSearchRequest searchRequest = request.getSearchRequest(); final QueryKey queryKey = searchRequest.getQueryKey(); - final Search search = searchRequest.getSearch(); Integer rowCount = null; try { @@ -260,49 +255,28 @@ public ResourceGeneration downloadSearchResults(final DownloadSearchResultsReque throw new EntityServiceException("No results can be found"); } - Result result = null; - for (final Result res : searchResponse.getResults()) { - if (res.getComponentId().equals(request.getComponentId())) { - result = res; - break; - } - } + final String componentId = request.getComponentId(); + final List tableResults = searchResponse.getResults() + .stream() + .filter(result -> result instanceof TableResult) + .filter(result -> request.isDownloadAllTables() || result.getComponentId().equals(componentId)) + .map(result -> (TableResult) result) + .toList(); + rowCount = tableResults + .stream() + .map(TableResult::getTotalResults) + .reduce(0, Integer::sum); - if (result == null) { + if (tableResults.isEmpty()) { throw new EntityServiceException("No result for component can be found"); } - if (!(result instanceof TableResult)) { - throw new EntityServiceException("Result is not a table"); - } - - final TableResult tableResult = (TableResult) result; - // Import file. - String fileName = getResultsFilename(request); - + final String fileName = getResultsFilename(request); resourceKey = resourceStore.createTempFile(fileName); final Path file = resourceStore.getTempFile(resourceKey); - final Optional optional = searchRequest.getComponentResultRequests() - .stream() - .filter(r -> r.getComponentId().equals(request.getComponentId())) - .findFirst(); - if (optional.isEmpty()) { - throw new EntityServiceException("No component result request found"); - } - - if (!(optional.get() instanceof TableResultRequest)) { - throw new EntityServiceException("Component result request is not a table"); - } - - final TableResultRequest tableResultRequest = (TableResultRequest) optional.get(); - final List fields = tableResultRequest.getTableSettings().getFields(); - final List rows = tableResult.getRows(); - rowCount = tableResult.getTotalResults(); - - download(fields, rows, file, request.getFileType(), request.isSample(), request.getPercent(), - searchRequest.getDateTimeSettings()); + download(request, searchRequest, tableResults, file); searchEventLog.downloadResults(request, rowCount); } catch (final RuntimeException e) { @@ -341,18 +315,15 @@ private String getFileName(final String baseName, return fileName; } - private void download(final List fields, - final List rows, - final Path file, - final DownloadSearchResultFileType fileType, - final boolean sample, - final int percent, - final DateTimeSettings dateTimeSettings) { + private void download(final DownloadSearchResultsRequest request, + final DashboardSearchRequest searchRequest, + final List tableResults, + final Path file) { try (final OutputStream outputStream = new BufferedOutputStream(Files.newOutputStream(file))) { SearchResultWriter.Target target = null; // Write delimited file. - switch (fileType) { + switch (request.getFileType()) { case CSV: target = new DelimitedTarget(outputStream, ","); break; @@ -360,12 +331,15 @@ private void download(final List fields, target = new DelimitedTarget(outputStream, "\t"); break; case EXCEL: - target = new ExcelTarget(outputStream, dateTimeSettings); + target = new ExcelTarget(outputStream, searchRequest.getDateTimeSettings()); break; } - final SampleGenerator sampleGenerator = new SampleGenerator(sample, percent); - final SearchResultWriter searchResultWriter = new SearchResultWriter(fields, rows, sampleGenerator); + final SampleGenerator sampleGenerator = new SampleGenerator(request.isSample(), request.getPercent()); + final SearchResultWriter searchResultWriter = new SearchResultWriter( + searchRequest, + tableResults, + sampleGenerator); searchResultWriter.write(target); } catch (final IOException e) { diff --git a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/download/DelimitedTarget.java b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/download/DelimitedTarget.java index 6de633181d3..c7740969e57 100644 --- a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/download/DelimitedTarget.java +++ b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/download/DelimitedTarget.java @@ -48,6 +48,16 @@ public void end() throws IOException { delimitedWriter.close(); } + @Override + public void startTable(final String tableName) { + // Do nothing + } + + @Override + public void endTable() { + // Do nothing + } + @Override public void startLine() { // Do nothing diff --git a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/download/ExcelTarget.java b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/download/ExcelTarget.java index 83872a48915..b247498d0ca 100644 --- a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/download/ExcelTarget.java +++ b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/download/ExcelTarget.java @@ -74,7 +74,6 @@ public void start() { // Create a workbook with 100 rows in memory. Exceeding rows will be // flushed to disk. workbook = new SXSSFWorkbook(100); - sheet = workbook.createSheet(); // Create a style for headings. final Font headingFont = workbook.createFont(); @@ -88,11 +87,6 @@ public void start() { @Override public void end() throws IOException { - // Auto-size tracked columns - for (var columnIndex : sheet.getTrackedColumnsForAutoSizing()) { - sheet.autoSizeColumn(columnIndex); - } - // Write the workbook to the output stream. workbook.write(outputStream); outputStream.close(); @@ -104,6 +98,21 @@ public void end() throws IOException { workbook.dispose(); } + @Override + public void startTable(final String tableName) { + sheet = workbook.createSheet(tableName); + rowNum = 0; + colNum = 0; + } + + @Override + public void endTable() { + // Auto-size tracked columns + for (var columnIndex : sheet.getTrackedColumnsForAutoSizing()) { + sheet.autoSizeColumn(columnIndex); + } + } + @Override public void startLine() { row = sheet.createRow(rowNum++); diff --git a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/download/SearchResultWriter.java b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/download/SearchResultWriter.java index ee48373fa54..0806b28b315 100644 --- a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/download/SearchResultWriter.java +++ b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/download/SearchResultWriter.java @@ -17,23 +17,28 @@ package stroom.dashboard.impl.download; import stroom.dashboard.impl.SampleGenerator; +import stroom.dashboard.shared.DashboardSearchRequest; +import stroom.dashboard.shared.TableResultRequest; import stroom.query.api.v2.Field; import stroom.query.api.v2.Row; +import stroom.query.api.v2.TableResult; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class SearchResultWriter { - private final List fields; - private final List rows; + private final DashboardSearchRequest searchRequest; + private final List tableResults; private final SampleGenerator sampleGenerator; - public SearchResultWriter(final List fields, - final List rows, + public SearchResultWriter(final DashboardSearchRequest searchRequest, + final List tableResults, final SampleGenerator sampleGenerator) { - this.fields = fields; - this.rows = rows; + this.searchRequest = searchRequest; + this.tableResults = tableResults; this.sampleGenerator = sampleGenerator; } @@ -41,11 +46,29 @@ public void write(final Target target) throws IOException { // Start writing. target.start(); - // Write heading. - writeHeadings(fields, target); + // Map each component ID to its corresponding table request + final Map tableRequestMap = new HashMap<>(); + searchRequest.getComponentResultRequests().forEach(request -> { + if (request instanceof final TableResultRequest tableResultRequest) { + tableRequestMap.put(tableResultRequest.getComponentId(), tableResultRequest); + } + }); + + for (final TableResult tableResult : tableResults) { + final TableResultRequest tableResultRequest = tableRequestMap.get(tableResult.getComponentId()); + final List fields = tableResultRequest.getTableSettings().getFields(); + final List rows = tableResult.getRows(); + + target.startTable(tableResultRequest.getTableName()); + + // Write heading. + writeHeadings(fields, target); - // Write content. - writeContent(rows, fields, sampleGenerator, target); + // Write content. + writeContent(rows, fields, sampleGenerator, target); + + target.endTable(); + } // End writing. target.end(); @@ -90,6 +113,10 @@ public interface Target { void end() throws IOException; + void startTable(final String tableName); + + void endTable(); + void startLine() throws IOException; void endLine() throws IOException; diff --git a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/logging/SearchEventLogImpl.java b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/logging/SearchEventLogImpl.java index fb77e5c2669..5da821ecf37 100644 --- a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/logging/SearchEventLogImpl.java +++ b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/logging/SearchEventLogImpl.java @@ -178,6 +178,11 @@ public void downloadResults(final DownloadSearchResultsRequest request, .withName("percent") .withValue(String.valueOf(request.getPercent())) .build()) + .addData(Data.builder() + .withName("allTables") + .withValue(String.valueOf( + request.isDownloadAllTables())) + .build()) .build()) .build()) .withOutcome(EventLoggingUtil.createOutcome(e)) diff --git a/unreleased_changes/20230213_150003_377__3248.md b/unreleased_changes/20230213_150003_377__3248.md new file mode 100644 index 00000000000..6a6f22787a1 --- /dev/null +++ b/unreleased_changes/20230213_150003_377__3248.md @@ -0,0 +1,19 @@ +* Issue **#3248** : Add ability to download multiple dashboard tables as Excel worksheets in a single file. + + +```sh +# ONLY the top line will be included as a change entry in the CHANGELOG. +# The entry should be in GitHub flavour markdown and should be written on a SINGLE +# line with no hard breaks. You can have multiple change files for a single GitHub issue. +# The entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than +# 'Fixed nasty bug'. +# +# Examples of acceptable entries are: +# +# +# * Issue **123** : Fix bug with an associated GitHub issue in this repository +# +# * Issue **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository +# +# * Fix bug with no associated GitHub issue. +```