Some parameter test

.gitignore

@@ -129,3 +129,6 @@ dmypy.json
 .pyre/
 
 mlruns/
+
+# VSCode
+.vscode

MLproject

@@ -28,6 +28,12 @@ entry_points:
       generation: {type: string, default: ""}
       analysis1: {type: string, default: ""}
     command: "python evaluation1.py --generation {generation} --analysis1 {analysis1}"
+  analysis12_evaluation1:
+    parameters:
+      generation: {type: string, default: ""}
+      threshold: {type: float, default: 50.0}
+      max_distance: {type: float, default: 50.0}
+    command: "python analysis12_evaluation1.py --generation {generation} --threshold {threshold} --max_distance {max_distance}"
   main:
     parameters:
       num_samples: {type: int, default: 1}

README.md

@@ -4,16 +4,47 @@
 
 - Ubuntu Linux 18.04 (This workflow does not work in WSL2 environment due to the uniqueness of the file system)
 
-## How to run this workflow
+## Setup mlflow environment
 
 1. Install [Miniconda](https://docs.conda.io/en/latest/miniconda.html)
 1. run `pip install mlflow` with Miniconda's pip
+
+## Setup environment variables
+
+If you use S3 or S3 compatible siystem like [MINIO](https://min.io/),
+ you need to set up these environment variables
+
+```
+export AWS_DEFAULT_REGION=ap-northeast-1
+export AWS_ACCESS_KEY_ID=YOURMINIOACCESSKEY
+export AWS_SECRET_ACCESS_KEY=YOURMINIOSECRETKEY
+export MLFLOW_S3_ENDPOINT_URL=http://xxx.xxx.xxx.xxx:yyyy/
+```
+
+## Setup mlflow server
+
 1. run `tmux` to create multiple shells and save the processes
 1. run `mlflow ui -h 0.0.0.0` and create a new tmux pane
+
+## How to run this workflow
+
+1. run `tmux` to create multiple shells and save the processes
 1. move to the another tmux pane and run `mlflow run https://github.com/ecell/bioimage_workflows.git`
 
 ## How to run specific workflow which is written as entrypoint
 
 If you want to execute `analysis1` in entrypoint, command is following
 
 `mlflow run -e analysis1 https://github.com/ecell/bioimage_workflows.git -P num_samples=1 -P num_frames=5`
+
+## How to run with specific experiment name
+
+If you want to execute `myexperiment1` in experiment, command is following
+
+`mlflow run -e analysis1 https://github.com/ecell/bioimage_workflows.git -P num_samples=1 -P num_frames=5 --experiment-name "myexperiment1"`
+
+## How to run with specific git commit id
+
+If you want to execute git commit id `"40b29386"`, command is following
+
+`mlflow run -e analysis1 https://github.com/ecell/bioimage_workflows.git -P num_samples=1 -P num_frames=5 --experiment-name "myexperiment1" --version "40b29386"`

analysis1.py

@@ -4,6 +4,11 @@
 import mlflow
 from mlflow import log_metric, log_param, log_artifacts
 
+from mlflow_utils import _is_existing_run
+from mlflow.tracking.context.git_context import _get_git_commit
+
+import sys
+
 entrypoint = "analysis1"
 parser = argparse.ArgumentParser(description='analysis1 step')
 parser.add_argument('--generation', type=str, default="")
@@ -13,15 +18,20 @@
 parser.add_argument('--overlap', type=float, default=0.5)
 args = parser.parse_args()
 
-active_run = mlflow.start_run()
-mlflow.set_tag("mlflow.runName", entrypoint)
-
 generation = args.generation
 min_sigma = args.min_sigma
 max_sigma = args.max_sigma
 threshold = args.threshold
 overlap = args.overlap
 
+git_commit = _get_git_commit(".")
+if _is_existing_run("analysis1", {"generation": generation, "threshold": threshold, "min_sigma": min_sigma}, git_commit):
+    sys.exit(0) 
+
+active_run = mlflow.start_run()
+mlflow.set_tag("mlflow.runName", entrypoint)
+#mlflow.tracking.MlflowClient().set_tag(run_id, "mlflow.runName", "run_name")
+
 for key, value in vars(args).items():
     log_param(key, value)
 

analysis12_evaluation1.py

@@ -0,0 +1,38 @@
+import pathlib
+import argparse
+import itertools
+
+import mlflow
+# from mlflow.utils import mlflow_tags
+from mlflow import log_metric, log_param, log_artifacts
+from mlflow_utils import _get_or_run
+
+entrypoint = "analysis12_evaluation1"
+parser = argparse.ArgumentParser(description='main step')
+parser.add_argument('--generation', type=str, default="")
+parser.add_argument('--threshold', type=float, default=50.0)
+parser.add_argument('--min_sigma', type=int, default=1)
+parser.add_argument('--max_distance', type=float, default=50.0)
+args = parser.parse_args()
+
+generation = args.generation
+threshold = args.threshold
+min_sigma = args.min_sigma
+max_distance = args.max_distance
+client = mlflow.tracking.MlflowClient()
+generation_run = client.get_run(generation)
+
+with mlflow.start_run(nested=True) as active_run:
+    git_commit = active_run.data.tags.get("mlflow.source.git.commit")
+    mlflow.set_tag("mlflow.runName", entrypoint)
+    for key, value in vars(args).items():
+        log_param(key, value)
+
+    analysis1_run = _get_or_run("analysis1", {"generation": generation, "threshold": threshold, "min_sigma": min_sigma}, git_commit)
+    analysis2_run = _get_or_run("analysis2", {"generation": generation, "analysis1": analysis1_run.info.run_id,"max_distance":max_distance}, git_commit)
+    evaluation1_run = _get_or_run("evaluation1", {"generation": generation, "analysis1": analysis1_run.info.run_id}, git_commit)
+
+    for run_obj in (generation_run, analysis1_run, analysis2_run, evaluation1_run):
+        for key, value in run_obj.data.metrics.items():
+            log_metric(key, value)
+

analysis2.py

@@ -4,6 +4,11 @@
 import mlflow
 from mlflow import log_metric, log_param, log_artifacts
 
+from mlflow_utils import _is_existing_run
+from mlflow.tracking.context.git_context import _get_git_commit
+
+import sys
+
 entrypoint = "analysis2"
 parser = argparse.ArgumentParser(description='analysis2 step')
 parser.add_argument('--generation', type=str, default="")
@@ -12,14 +17,18 @@
 parser.add_argument('--max_distance', type=float, default=50.0)
 args = parser.parse_args()
 
-active_run = mlflow.start_run()
-mlflow.set_tag("mlflow.runName", entrypoint)
-
 generation = args.generation
 analysis1 = args.analysis1
 seed = args.seed
 max_distance = args.max_distance
 
+git_commit = _get_git_commit(".")
+if _is_existing_run("analysis2", {"generation": generation, "analysis1": analysis1, "max_distance": max_distance}, git_commit):
+    sys.exit(0) 
+
+active_run = mlflow.start_run()
+mlflow.set_tag("mlflow.runName", entrypoint)
+
 for key, value in vars(args).items():
     log_param(key, value)
 

evaluation1.py

@@ -4,6 +4,11 @@
 import mlflow
 from mlflow import log_metric, log_param, log_artifacts
 
+from mlflow_utils import _is_existing_run
+from mlflow.tracking.context.git_context import _get_git_commit
+
+import sys
+
 entrypoint = "evaluation1"
 parser = argparse.ArgumentParser(description='evaluation1 step')
 parser.add_argument('--generation', type=str, default="")
@@ -12,14 +17,18 @@
 parser.add_argument('--max_distance', type=float, default=50.0)
 args = parser.parse_args()
 
-active_run = mlflow.start_run()
-mlflow.set_tag("mlflow.runName", entrypoint)
-
 generation = args.generation
 analysis1 = args.analysis1
 # analysis2 = args.analysis2
 max_distance = args.max_distance
 
+git_commit = _get_git_commit(".")
+if _is_existing_run("evaluation1", {"generation": generation, "analysis1": analysis1}, git_commit):
+    sys.exit(0) 
+
+active_run = mlflow.start_run()
+mlflow.set_tag("mlflow.runName", entrypoint)
+
 for key, value in vars(args).items():
     log_param(key, value)
 

generation.py

@@ -4,6 +4,11 @@
 import mlflow
 from mlflow import log_metric, log_param, log_artifacts
 
+from mlflow_utils import _is_existing_run
+from mlflow.tracking.context.git_context import _get_git_commit
+
+import sys
+
 entrypoint = "generation"
 parser = argparse.ArgumentParser(description='generation step')
 parser.add_argument('--seed', type=int, default=123)
@@ -13,15 +18,20 @@
 parser.add_argument('--exposure_time', type=float, default=0.033)
 args = parser.parse_args()
 
-active_run = mlflow.start_run()
-mlflow.set_tag("mlflow.runName", entrypoint)
-
 seed = args.seed
 interval = args.interval
 num_samples = args.num_samples
 num_frames = args.num_frames
 exposure_time = args.exposure_time
 
+
+git_commit = _get_git_commit(".")
+if _is_existing_run("generation", {"num_samples": num_samples, "num_frames": num_frames}, git_commit):
+    sys.exit(0) 
+
+active_run = mlflow.start_run()
+mlflow.set_tag("mlflow.runName", entrypoint)
+
 Nm = [100, 100, 100]
 Dm = [0.222e-12, 0.032e-12, 0.008e-12]
 transmat = [

mlflow_utils.py

@@ -64,4 +64,10 @@ def _get_or_run(entrypoint, parameters, git_commit, use_cache=True):
     print("Launching new run for entrypoint=%s and parameters=%s" % (entrypoint, parameters))
     submitted_run = mlflow.run(".", entrypoint, parameters=parameters)
     return mlflow.tracking.MlflowClient().get_run(submitted_run.run_id)
-
+
+def _is_existing_run(entrypoint, parameters, git_commit, use_cache=True):
+    existing_run = _already_ran(entrypoint, parameters, git_commit)
+    if use_cache and existing_run:
+        print("Found existing run for entrypoint=%s and parameters=%s run_id=%s" % (entrypoint, parameters,existing_run.info.run_id))
+        return True
+    return False

run_max_distance.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+max_distances=(50.0 60.0 70.0 80.0 90.0)
+for max_distance in ${max_distances[@]}; do
+  mlflow run . -e analysis12_evaluation1 -P generation=e3d8393844b9453f9a390e19305d8f3c -P max_distance=${max_distance} --experiment-name  "shareobjectstorage2"
+done

run_thresholds.sh

@@ -1,4 +1,4 @@
 thresholds=(50.0 60.0 70.0 80.0 90.0)
 for t in ${thresholds[@]}; do
-  mlflow run -e analysis1 . -P threshold=$t -P generated_data="file:///home/azureuser/mlruns/0/16ad0e8f04bc4d5dbbc667838d26e919/artifacts"
+  mlflow run . -e analysis12_evaluation1 -P generation=e3d8393844b9453f9a390e19305d8f3c -P threshold=$t --experiment-name  "shareobjectstorage2"
 done