Support local copy across storage areas

compose/assets/etc/storm/webdav/config/application-policies.yml

@@ -5,6 +5,13 @@
 storm:
   authz:
     policies:
+    - sa: noauth
+      actions:
+      - all
+      effect: permit
+      description: Grant read/write access to all users
+      principals:
+      - type: anyone
     - sa: fga
       actions:
       - all

compose/assets/etc/storm/webdav/sa.d/noauth.properties

@@ -9,3 +9,4 @@ accessPoints=/noauth
 authenticatedReadEnabled=true
 anonymousReadEnabled=true
 voMapGrantsWritePermission=false
+fineGrainedAuthzEnabled=true

robot/common/setup_and_teardown.robot

@@ -10,9 +10,9 @@ Default Setup
 Default Teardown
     Unset VOMS credential
 
-Setup file  [Arguments]  ${file_name}  ${content}=Hello World!
+Setup file  [Arguments]  ${file_name}  ${content}=Hello World!  ${sa}=${sa.default}
     Default Setup
-    Create Test File   ${file_name}  ${content}
+    Create Test File   ${file_name}  ${content}  ${sa}
 
 Setup directory  [Arguments]  ${dir_name}
     Default Setup
@@ -23,12 +23,12 @@ Teardown file  [Arguments]  ${file_name}
     Remove Test File   ${file_name}
     Remove Test File   ${file_name}.dest
 
-Teardown file cross sa  [Arguments]  ${file_name}
+Teardown file cross sa  [Arguments]  ${file_name}  ${sa_source}=${sa.default}  ${sa_dest}=${sa.oauth}
     Default Teardown
-    Remove Test File   ${file_name}
-    Remove Test File   ${file_name}.dest  sa=${sa.oauth}
+    Remove Test File   ${file_name}  sa=${sa_source}
+    Remove Test File   ${file_name}.dest  sa=${sa_dest}
 
 Teardown directory  [Arguments]  ${dir_name}
     Default Teardown
     Remove Test Directory   ${dir_name}
-    Remove Test Directory   ${dir_name}.dest
+    Remove Test Directory   ${dir_name}.dest

robot/test/copy.robot

@@ -86,12 +86,21 @@ Copy with destination equal to source
     Should Contain  ${out}  403
     [Teardown]   Teardown file  copy_works
 
-Copy across storage areas fails
+Copy across storage areas with Source header works
+    [Tags]  voms
+    [Setup]  Setup file  copy_x_sa_works  sa=${sa.noauth}
+    ${dest}  DAVS URL  copy_x_sa_works.dest
+    ${source}  DAVS URL  copy_x_sa_works  sa=${sa.noauth}
+    ${rc}  ${out}  Curl Voms Pull COPY Success  ${dest}  ${source}
+    Davix Get Success   ${dest}  ${davix.opts.voms}
+    [Teardown]   Teardown file cross sa   copy_x_sa_works  sa_source=${sa.noauth}
+
+Copy across storage areas with Destination header fails
     [Tags]  voms
     [Setup]  Setup file  copy_x_sa_works
     ${dest}  DAVS URL  copy_x_sa_works.dest  sa=${sa.oauth}
     ${source}  DAVS URL  copy_x_sa_works
     ${rc}  ${out}  Curl Voms Push COPY  ${dest}  ${source}
     Should Contain  ${out}   400
-    Should Contain  ${out}   Local copy across storage areas is not supported
-    [Teardown]   Teardown file cross sa   copy_x_sa_works
+    Should Contain  ${out}   Local copy across storage areas with Destination header is not supported
+    [Teardown]   Teardown file cross sa   copy_x_sa_works

src/main/java/org/italiangrid/storm/webdav/error/Forbidden.java

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: 2014 Istituto Nazionale di Fisica Nucleare
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package org.italiangrid.storm.webdav.error;
+
+public class Forbidden extends StoRMWebDAVError {
+
+  /** */
+  private static final long serialVersionUID = 1L;
+
+  public Forbidden(String message) {
+    super(message);
+  }
+}

src/main/java/org/italiangrid/storm/webdav/fs/DefaultFSStrategy.java

@@ -15,6 +15,7 @@
 import org.apache.commons.io.IOUtils;
 import org.italiangrid.storm.webdav.checksum.Adler32ChecksumInputStream;
 import org.italiangrid.storm.webdav.error.SameFileError;
+import org.italiangrid.storm.webdav.error.StoRMWebDAVError;
 import org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributesHelper;
 import org.italiangrid.storm.webdav.utils.IOExceptionHelper;
 import org.slf4j.Logger;
@@ -97,12 +98,18 @@ public void cp(File source, File dest) {
         FileUtils.copyDirectory(source, dest);
 
       } else {
-
-        Files.copy(source.toPath(), dest.toPath());
+        Process process =
+            Runtime.getRuntime().exec(new String[] {"cp", "-a", source.getPath(), dest.getPath()});
+        int returnCode = process.waitFor();
+        if (returnCode != 0) {
+          throw new IOException("cp -a error");
+        }
       }
 
     } catch (IOException e) {
       throw IOExceptionHelper.getStoRMWebDAVError(e);
+    } catch (InterruptedException e) {
+      throw new StoRMWebDAVError(e.getMessage(), e);
     }
   }
 

src/main/java/org/italiangrid/storm/webdav/milton/StoRMMiltonBehaviour.java

@@ -16,10 +16,12 @@
 import io.milton.http.exceptions.NotAuthorizedException;
 import io.milton.http.http11.Http11ResponseHandler;
 import java.io.IOException;
+import java.time.Clock;
 import org.italiangrid.storm.webdav.error.DirectoryNotEmpty;
 import org.italiangrid.storm.webdav.error.DiskQuotaExceeded;
 import org.italiangrid.storm.webdav.error.ResourceNotFound;
 import org.italiangrid.storm.webdav.error.SameFileError;
+import org.italiangrid.storm.webdav.tpc.transfer.TransferStatus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.web.server.MethodNotAllowedException;
@@ -49,6 +51,17 @@ public void process(FilterChain chain, Request request, Response response) {
       if (response.getEntity() != null) {
         manager.sendResponseEntity(response);
       }
+      // If it was a TPC send a success PerfMarker, so davix knows that the COPY was successful
+      if (request instanceof StoRMMiltonRequest stoRMMiltonRequest
+          && stoRMMiltonRequest.sendSuccessPerfMarker()) {
+        TransferStatus.Builder statusBuilder = TransferStatus.builder(Clock.systemDefaultZone());
+        try {
+          response.getOutputStream().write(statusBuilder.done(0).asPerfMarker().getBytes());
+          response.getOutputStream().close();
+        } catch (IOException e) {
+          LOG.error("Error sending success PerfMarker: {}", e.getMessage(), e);
+        }
+      }
     } catch (DiskQuotaExceeded e) {
       // responseHandler does not support sending insufficient storage
       response.sendError(Status.SC_INSUFFICIENT_STORAGE, e.getMessage());

src/main/java/org/italiangrid/storm/webdav/milton/StoRMMiltonRequest.java

@@ -11,15 +11,20 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.eclipse.jetty.util.URIUtil;
+import org.italiangrid.storm.webdav.tpc.SwappedServletRequest;
 
 public class StoRMMiltonRequest extends ServletRequest {
 
   private static final String REGEX = "(http.*:\\d*)/webdav/(.*)$";
   private static final Pattern PATTERN = Pattern.compile(REGEX);
+  private boolean sendSuccessPerfMarker = false;
 
   public StoRMMiltonRequest(HttpServletRequest r, ServletContext servletContext) {
 
     super(r, servletContext);
+    if (r instanceof SwappedServletRequest) {
+      this.sendSuccessPerfMarker = true;
+    }
   }
 
   @Override
@@ -47,4 +52,8 @@ public Auth getAuthorization() {
     // Always return null as milton is confused by the OAuth2 Bearer scheme
     return null;
   }
+
+  public boolean sendSuccessPerfMarker() {
+    return sendSuccessPerfMarker;
+  }
 }

src/main/java/org/italiangrid/storm/webdav/tpc/SwappedServletRequest.java

@@ -0,0 +1,71 @@
+// SPDX-FileCopyrightText: 2014 Istituto Nazionale di Fisica Nucleare
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package org.italiangrid.storm.webdav.tpc;
+
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletRequestWrapper;
+import java.net.URI;
+import java.net.URISyntaxException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.HttpHeaders;
+
+// Wraps a request so that the Source header becomes the request URL and the request URL becomes the
+// Destination header. This is necessary because the WebDAV protocol defines COPY only with the
+// Destination header.
+public class SwappedServletRequest extends HttpServletRequestWrapper {
+  public static final Logger LOG = LoggerFactory.getLogger(SwappedServletRequest.class);
+
+  final HttpServletRequest req;
+
+  public SwappedServletRequest(HttpServletRequest req) {
+    super(req);
+    this.req = req;
+  }
+
+  // The getHeader method is used by milton to obtain information about the COPY request
+  // https://github.com/miltonio/milton2/blob/master/milton-server-ce/src/main/java/io/milton/servlet/ServletRequest.java#L122
+  @Override
+  public String getHeader(String name) {
+    if (TransferConstants.DESTINATION_HEADER.equals(name)) {
+      // The Destination header is the requested URL of the original request
+      return req.getRequestURL().toString();
+    } else if (HttpHeaders.HOST.equals(name)) {
+      // The Host header is the Host specified in the Source header of the
+      // original request if it is present
+      try {
+        URI sourceUri = new URI(req.getHeader(TransferConstants.SOURCE_HEADER));
+        String sourceHost = sourceUri.getHost();
+        if (sourceHost != null) {
+          return sourceHost + ':' + sourceUri.getPort();
+        }
+      } catch (URISyntaxException e) {
+        LOG.warn("Error parsing Source header: {}", e.getMessage(), e);
+      }
+    }
+    return req.getHeader(name);
+  }
+
+  // The getRequestURL method is used by milton to obtain information about the COPY request
+  // https://github.com/miltonio/milton2/blob/master/milton-server-ce/src/main/java/io/milton/http/UrlAdapterImpl.java#L32
+  @Override
+  public StringBuffer getRequestURL() {
+    String source = req.getHeader(TransferConstants.SOURCE_HEADER);
+    try {
+      // If the Source header of the original request includes the host, just
+      // use it as the request URL
+      String sourceHost = new URI(source).getHost();
+      if (sourceHost != null) {
+        return new StringBuffer(source);
+      }
+    } catch (URISyntaxException e) {
+      LOG.warn("Error parsing Source header: {}", e.getMessage(), e);
+    }
+    // Otherwise get the scheme, host and port from the original request and use the Source header
+    // as the path
+    return new StringBuffer(
+        req.getScheme() + "://" + req.getServerName() + ":" + req.getServerPort() + source);
+  }
+}

src/main/java/org/italiangrid/storm/webdav/tpc/TpcUtils.java

@@ -40,6 +40,22 @@ default boolean requestHasSourceHeader(HttpServletRequest request) {
     return Optional.ofNullable(request.getHeader(TransferConstants.SOURCE_HEADER)).isPresent();
   }
 
+  default boolean requestHasLocalSourceHeader(
+      HttpServletRequest request, LocalURLService localURLService) {
+    Optional<String> source =
+        Optional.ofNullable(request.getHeader(TransferConstants.SOURCE_HEADER));
+
+    return source.isPresent() && localURLService.isLocalURL(source.get());
+  }
+
+  default boolean requestHasRemoteSourceHeader(
+      HttpServletRequest request, LocalURLService localURLService) {
+    Optional<String> source =
+        Optional.ofNullable(request.getHeader(TransferConstants.SOURCE_HEADER));
+
+    return source.isPresent() && !localURLService.isLocalURL(source.get());
+  }
+
   default boolean requestHasDestinationHeader(HttpServletRequest request) {
     return Optional.ofNullable(request.getHeader(TransferConstants.DESTINATION_HEADER)).isPresent();
   }
@@ -111,11 +127,27 @@ default boolean requestHasTranferHeader(HttpServletRequest request) {
     return false;
   }
 
+  default boolean requestHasTranferHeaderOtherThanAuthorization(HttpServletRequest request) {
+    Enumeration<String> headerNames = request.getHeaderNames();
+    while (headerNames.hasMoreElements()) {
+      String headerName = headerNames.nextElement();
+      if (headerName.toLowerCase().startsWith(TransferConstants.TRANSFER_HEADER_LC)
+          && !headerName
+              .toLowerCase()
+              .equals(
+                  TransferConstants.TRANSFER_HEADER_LC
+                      + TransferConstants.AUTHORIZATION_HEADER.toLowerCase())) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   default boolean isTpc(HttpServletRequest request, LocalURLService localUrlService) {
     return "COPY".equals(request.getMethod())
-        && (requestHasSourceHeader(request)
+        && (requestHasRemoteSourceHeader(request, localUrlService)
             || requestHasRemoteDestinationHeader(request, localUrlService)
-            || requestHasTranferHeader(request));
+            || requestHasTranferHeaderOtherThanAuthorization(request));
   }
 
   default boolean isCopyOrMoveRequest(HttpServletRequest request) {

src/main/java/org/italiangrid/storm/webdav/tpc/TransferFilter.java

@@ -20,6 +20,7 @@
 import org.apache.hc.client5.http.ClientProtocolException;
 import org.apache.hc.client5.http.HttpResponseException;
 import org.italiangrid.storm.webdav.error.BadRequest;
+import org.italiangrid.storm.webdav.error.Forbidden;
 import org.italiangrid.storm.webdav.error.ResourceNotFound;
 import org.italiangrid.storm.webdav.scitag.SciTag;
 import org.italiangrid.storm.webdav.server.PathResolver;
@@ -37,6 +38,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.MDC;
+import org.springframework.http.HttpHeaders;
 
 public class TransferFilter extends TransferFilterSupport implements Filter {
 
@@ -60,10 +62,32 @@ public TransferFilter(
 
   private void localCopySanityChecks(HttpServletRequest req) throws URISyntaxException {
     if (!requestPathAndDestinationHeaderAreInSameStorageArea(req, resolver)) {
-      throw new BadRequest("Local copy across storage areas is not supported");
+      throw new BadRequest(
+          "Local copy across storage areas with Destination header is not supported");
     }
   }
 
+  private void checkAccessPermission(SwappedServletRequest wrappedRequest)
+      throws ClientProtocolException {
+    URI uri = URI.create(wrappedRequest.getRequestURL().toString());
+    String path = getScopedPathInfo(wrappedRequest);
+    GetTransferRequest xferRequest =
+        GetTransferRequestBuilder.create()
+            .uuid(RequestIdHolder.getRequestId())
+            .uri(uri)
+            .path(path)
+            .headers(getTransferHeaders(wrappedRequest))
+            .addHeader("Range", "bytes=0-0")
+            .build();
+    client.handleCheckAccessPermission(
+        xferRequest,
+        (r, s) -> {
+          if (s.getStatus() == TransferStatus.Status.ERROR) {
+            throw new Forbidden(s.asPerfMarker());
+          }
+        });
+  }
+
   @Override
   public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
       throws IOException, ServletException {
@@ -73,13 +97,32 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha
 
     if (isTpc(req, localURLService)) {
       handleTpc(req, res);
-    } else if (isCopy(req) && requestHasLocalDestinationHeader(req, localURLService)) {
+    } else if (isCopy(req)) {
       try {
-        localCopySanityChecks(req);
+        if (requestHasLocalDestinationHeader(req, localURLService)) {
+          localCopySanityChecks(req);
+        } else if (requestHasLocalSourceHeader(req, localURLService)) {
+          SwappedServletRequest wrappedRequest = new SwappedServletRequest(req);
+          if (LOG.isInfoEnabled()) {
+            LOG.info(
+                "Wrapped the COPY request to swap Source/Destination, URL: {} -> {}, Host: {} -> {}, Source header {} -> Destination header {}",
+                req.getRequestURL(),
+                wrappedRequest.getRequestURL(),
+                req.getHeader(HttpHeaders.HOST),
+                wrappedRequest.getHeader(HttpHeaders.HOST),
+                req.getHeader(TransferConstants.SOURCE_HEADER),
+                wrappedRequest.getHeader(TransferConstants.DESTINATION_HEADER));
+          }
+          res.setStatus(HttpServletResponse.SC_ACCEPTED);
+          checkAccessPermission(wrappedRequest);
+          request = wrappedRequest;
+        }
         // Let milton handle the local copy
         chain.doFilter(request, response);
-      } catch (URISyntaxException | BadRequest | ResourceNotFound e) {
-        res.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+      } catch (URISyntaxException | BadRequest | Forbidden | ResourceNotFound e) {
+        if (!(e instanceof Forbidden)) {
+          res.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+        }
         res.setContentType("text/plain");
         res.getWriter().print(e.getMessage());
         res.flushBuffer();