diff --git a/.gitignore b/.gitignore index 4564a6c..cdba437 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,6 @@ dmypy.json .pyre/ mlruns/ + +# VSCode +.vscode diff --git a/MLproject b/MLproject index 3588e7e..01f9d27 100644 --- a/MLproject +++ b/MLproject @@ -28,6 +28,12 @@ entry_points: generation: {type: string, default: ""} analysis1: {type: string, default: ""} command: "python evaluation1.py --generation {generation} --analysis1 {analysis1}" + analysis12_evaluation1: + parameters: + generation: {type: string, default: ""} + threshold: {type: float, default: 50.0} + max_distance: {type: float, default: 50.0} + command: "python analysis12_evaluation1.py --generation {generation} --threshold {threshold} --max_distance {max_distance}" main: parameters: num_samples: {type: int, default: 1} diff --git a/README.md b/README.md index f1ec362..b4fdf5e 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,31 @@ - Ubuntu Linux 18.04 (This workflow does not work in WSL2 environment due to the uniqueness of the file system) -## How to run this workflow +## Setup mlflow environment 1. Install [Miniconda](https://docs.conda.io/en/latest/miniconda.html) 1. run `pip install mlflow` with Miniconda's pip + +## Setup environment variables + +If you use S3 or S3 compatible siystem like [MINIO](https://min.io/), + you need to set up these environment variables + +``` +export AWS_DEFAULT_REGION=ap-northeast-1 +export AWS_ACCESS_KEY_ID=YOURMINIOACCESSKEY +export AWS_SECRET_ACCESS_KEY=YOURMINIOSECRETKEY +export MLFLOW_S3_ENDPOINT_URL=http://xxx.xxx.xxx.xxx:yyyy/ +``` + +## Setup mlflow server + 1. run `tmux` to create multiple shells and save the processes 1. run `mlflow ui -h 0.0.0.0` and create a new tmux pane + +## How to run this workflow + +1. run `tmux` to create multiple shells and save the processes 1. move to the another tmux pane and run `mlflow run https://github.com/ecell/bioimage_workflows.git` ## How to run specific workflow which is written as entrypoint @@ -17,3 +36,15 @@ If you want to execute `analysis1` in entrypoint, command is following `mlflow run -e analysis1 https://github.com/ecell/bioimage_workflows.git -P num_samples=1 -P num_frames=5` + +## How to run with specific experiment name + +If you want to execute `myexperiment1` in experiment, command is following + +`mlflow run -e analysis1 https://github.com/ecell/bioimage_workflows.git -P num_samples=1 -P num_frames=5 --experiment-name "myexperiment1"` + +## How to run with specific git commit id + +If you want to execute git commit id `"40b29386"`, command is following + +`mlflow run -e analysis1 https://github.com/ecell/bioimage_workflows.git -P num_samples=1 -P num_frames=5 --experiment-name "myexperiment1" --version "40b29386"` diff --git a/analysis1.py b/analysis1.py index e2dea63..8ac61a6 100644 --- a/analysis1.py +++ b/analysis1.py @@ -4,6 +4,11 @@ import mlflow from mlflow import log_metric, log_param, log_artifacts +from mlflow_utils import _is_existing_run +from mlflow.tracking.context.git_context import _get_git_commit + +import sys + entrypoint = "analysis1" parser = argparse.ArgumentParser(description='analysis1 step') parser.add_argument('--generation', type=str, default="") @@ -13,15 +18,20 @@ parser.add_argument('--overlap', type=float, default=0.5) args = parser.parse_args() -active_run = mlflow.start_run() -mlflow.set_tag("mlflow.runName", entrypoint) - generation = args.generation min_sigma = args.min_sigma max_sigma = args.max_sigma threshold = args.threshold overlap = args.overlap +git_commit = _get_git_commit(".") +if _is_existing_run("analysis1", {"generation": generation, "threshold": threshold, "min_sigma": min_sigma}, git_commit): + sys.exit(0) + +active_run = mlflow.start_run() +mlflow.set_tag("mlflow.runName", entrypoint) +#mlflow.tracking.MlflowClient().set_tag(run_id, "mlflow.runName", "run_name") + for key, value in vars(args).items(): log_param(key, value) diff --git a/analysis12_evaluation1.py b/analysis12_evaluation1.py new file mode 100644 index 0000000..89f7e9b --- /dev/null +++ b/analysis12_evaluation1.py @@ -0,0 +1,38 @@ +import pathlib +import argparse +import itertools + +import mlflow +# from mlflow.utils import mlflow_tags +from mlflow import log_metric, log_param, log_artifacts +from mlflow_utils import _get_or_run + +entrypoint = "analysis12_evaluation1" +parser = argparse.ArgumentParser(description='main step') +parser.add_argument('--generation', type=str, default="") +parser.add_argument('--threshold', type=float, default=50.0) +parser.add_argument('--min_sigma', type=int, default=1) +parser.add_argument('--max_distance', type=float, default=50.0) +args = parser.parse_args() + +generation = args.generation +threshold = args.threshold +min_sigma = args.min_sigma +max_distance = args.max_distance +client = mlflow.tracking.MlflowClient() +generation_run = client.get_run(generation) + +with mlflow.start_run(nested=True) as active_run: + git_commit = active_run.data.tags.get("mlflow.source.git.commit") + mlflow.set_tag("mlflow.runName", entrypoint) + for key, value in vars(args).items(): + log_param(key, value) + + analysis1_run = _get_or_run("analysis1", {"generation": generation, "threshold": threshold, "min_sigma": min_sigma}, git_commit) + analysis2_run = _get_or_run("analysis2", {"generation": generation, "analysis1": analysis1_run.info.run_id,"max_distance":max_distance}, git_commit) + evaluation1_run = _get_or_run("evaluation1", {"generation": generation, "analysis1": analysis1_run.info.run_id}, git_commit) + + for run_obj in (generation_run, analysis1_run, analysis2_run, evaluation1_run): + for key, value in run_obj.data.metrics.items(): + log_metric(key, value) + diff --git a/analysis2.py b/analysis2.py index 61814ec..5c5bc0d 100644 --- a/analysis2.py +++ b/analysis2.py @@ -4,6 +4,11 @@ import mlflow from mlflow import log_metric, log_param, log_artifacts +from mlflow_utils import _is_existing_run +from mlflow.tracking.context.git_context import _get_git_commit + +import sys + entrypoint = "analysis2" parser = argparse.ArgumentParser(description='analysis2 step') parser.add_argument('--generation', type=str, default="") @@ -12,14 +17,18 @@ parser.add_argument('--max_distance', type=float, default=50.0) args = parser.parse_args() -active_run = mlflow.start_run() -mlflow.set_tag("mlflow.runName", entrypoint) - generation = args.generation analysis1 = args.analysis1 seed = args.seed max_distance = args.max_distance +git_commit = _get_git_commit(".") +if _is_existing_run("analysis2", {"generation": generation, "analysis1": analysis1, "max_distance": max_distance}, git_commit): + sys.exit(0) + +active_run = mlflow.start_run() +mlflow.set_tag("mlflow.runName", entrypoint) + for key, value in vars(args).items(): log_param(key, value) diff --git a/evaluation1.py b/evaluation1.py index 749f41e..ca58645 100644 --- a/evaluation1.py +++ b/evaluation1.py @@ -4,6 +4,11 @@ import mlflow from mlflow import log_metric, log_param, log_artifacts +from mlflow_utils import _is_existing_run +from mlflow.tracking.context.git_context import _get_git_commit + +import sys + entrypoint = "evaluation1" parser = argparse.ArgumentParser(description='evaluation1 step') parser.add_argument('--generation', type=str, default="") @@ -12,14 +17,18 @@ parser.add_argument('--max_distance', type=float, default=50.0) args = parser.parse_args() -active_run = mlflow.start_run() -mlflow.set_tag("mlflow.runName", entrypoint) - generation = args.generation analysis1 = args.analysis1 # analysis2 = args.analysis2 max_distance = args.max_distance +git_commit = _get_git_commit(".") +if _is_existing_run("evaluation1", {"generation": generation, "analysis1": analysis1}, git_commit): + sys.exit(0) + +active_run = mlflow.start_run() +mlflow.set_tag("mlflow.runName", entrypoint) + for key, value in vars(args).items(): log_param(key, value) diff --git a/generation.py b/generation.py index d0fbb8a..ccba690 100755 --- a/generation.py +++ b/generation.py @@ -4,6 +4,11 @@ import mlflow from mlflow import log_metric, log_param, log_artifacts +from mlflow_utils import _is_existing_run +from mlflow.tracking.context.git_context import _get_git_commit + +import sys + entrypoint = "generation" parser = argparse.ArgumentParser(description='generation step') parser.add_argument('--seed', type=int, default=123) @@ -13,15 +18,20 @@ parser.add_argument('--exposure_time', type=float, default=0.033) args = parser.parse_args() -active_run = mlflow.start_run() -mlflow.set_tag("mlflow.runName", entrypoint) - seed = args.seed interval = args.interval num_samples = args.num_samples num_frames = args.num_frames exposure_time = args.exposure_time + +git_commit = _get_git_commit(".") +if _is_existing_run("generation", {"num_samples": num_samples, "num_frames": num_frames}, git_commit): + sys.exit(0) + +active_run = mlflow.start_run() +mlflow.set_tag("mlflow.runName", entrypoint) + Nm = [100, 100, 100] Dm = [0.222e-12, 0.032e-12, 0.008e-12] transmat = [ diff --git a/mlflow_utils.py b/mlflow_utils.py index 65056c4..f7c7a38 100644 --- a/mlflow_utils.py +++ b/mlflow_utils.py @@ -64,4 +64,10 @@ def _get_or_run(entrypoint, parameters, git_commit, use_cache=True): print("Launching new run for entrypoint=%s and parameters=%s" % (entrypoint, parameters)) submitted_run = mlflow.run(".", entrypoint, parameters=parameters) return mlflow.tracking.MlflowClient().get_run(submitted_run.run_id) - \ No newline at end of file + +def _is_existing_run(entrypoint, parameters, git_commit, use_cache=True): + existing_run = _already_ran(entrypoint, parameters, git_commit) + if use_cache and existing_run: + print("Found existing run for entrypoint=%s and parameters=%s run_id=%s" % (entrypoint, parameters,existing_run.info.run_id)) + return True + return False diff --git a/run_max_distance.sh b/run_max_distance.sh new file mode 100755 index 0000000..5789c3b --- /dev/null +++ b/run_max_distance.sh @@ -0,0 +1,5 @@ +#!/bin/bash +max_distances=(50.0 60.0 70.0 80.0 90.0) +for max_distance in ${max_distances[@]}; do + mlflow run . -e analysis12_evaluation1 -P generation=e3d8393844b9453f9a390e19305d8f3c -P max_distance=${max_distance} --experiment-name "shareobjectstorage2" +done diff --git a/run_thresholds.sh b/run_thresholds.sh old mode 100644 new mode 100755 index 5b769fe..9271679 --- a/run_thresholds.sh +++ b/run_thresholds.sh @@ -1,4 +1,4 @@ thresholds=(50.0 60.0 70.0 80.0 90.0) for t in ${thresholds[@]}; do - mlflow run -e analysis1 . -P threshold=$t -P generated_data="file:///home/azureuser/mlruns/0/16ad0e8f04bc4d5dbbc667838d26e919/artifacts" + mlflow run . -e analysis12_evaluation1 -P generation=e3d8393844b9453f9a390e19305d8f3c -P threshold=$t --experiment-name "shareobjectstorage2" done