Add a helper script to copy files from cluster

README.md

@@ -80,20 +80,8 @@ It is related to https://github.com/nbiederbeck/lst-agn-analysis/issues/26
 If you have run snakemake on the cluster, you can create the plots and tex files locally (using your own matplotlibrc for example).
 We separate the calculation of metrics and the plotting to make sure you can finetune plots later on without needing to
 run the expensive steps on the local machine. The tables for that are saved as either `fits.gz` or `h5`.
-
-For the data-selection plots, you need to download `build/dl1-datacheck-masked.h5`, e.g.:
-
-```
-mkdir -p build
-scp <host>:<path-to>/lst-data-selection/build/dl1-datachecks-masked.h5 build/
-```
-
-Afterwards:
+We create a helper script for that that uses `rsync`. Check how to use it:
 
 ```
-make -f local.mk
+./copy-from-cluster.py --help
 ```
-
-DEV-TODO: Do the same for the other plots (https://github.com/nbiederbeck/lst-agn-analysis/issues/29)
-If you do some `cp **/*.fits.gz` shenanigans, beware that the dl3 files are saved with
-the extension `fits.gz` as well.

copy-from-cluster.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+from argparse import ArgumentParser
+from subprocess import run
+
+parser = ArgumentParser(
+ description="Copy built data files from the cluster to locally create plots.",
+)
+parser.add_argument(
+ "--hostname",
+ help="The hostname as configured in you ~/.ssh/config, e.g. `cp01`.",
+ required=True,
+)
+parser.add_argument(
+ "--remote-path",
+ help="Path to your directory on the cluster, "
+ "e.g. `/fefs/aswg/workspace/<username>/lst-agn-analysis/build`. "
+ "Can be absolute (start with `/`) otherwise it is relative to the home. "
+ "Get the absoulte path of any directory with "
+ "`realpath <directory>` on the cluster.",
+ required=True,
+)
+parser.add_argument(
+ "--exclude",
+ help="Patterns to additionally exclude from copying. Comma separated string. "
+ "When you use globs (`*`), remember to quote it in single quotes, "
+ "e.g. `--exclude='*.pdf,*.png'`.",
+ default=None,
+ type=str,
+)
+parser.add_argument(
+ "--rsync-args",
+ help="Custom command line arguments for rsync. Check `man rsync` for info. "
+ "Try `--rsync-args='-nv'` for a verbose dry-run.",
+ default="",
+ type=str,
+)
+args = parser.parse_args()
+
+exclude_patterns = [
+ # run files
+ "dl1_*.h5",
+ "dl2_*.h5",
+ "dl3_*.fits.gz",
+ # DL4 Datasets
+ "phaobs_*.fits",
+ "dl4/*/datasets.fits.gz",
+ # log files
+ "*.log",
+ "logs/*",
+ # models
+ "models/model*",
+ # plots
+ "*.pdf",
+]
+
+
+def main():
+ rsync = "rsync -auh --info=progress2 "
+ rsync += args.rsync_args + " "
+ for pat in exclude_patterns:
+ rsync += f"--exclude='{pat}' "
+ if args.exclude is not None:
+ for pat in args.exclude.split(","):
+ rsync += f"--exclude='{pat}' "
+ cmd = f"{rsync} '{args.hostname}:{args.remote_path.rstrip('/')}' ."
+ print(cmd)
+ run(cmd, shell=True, capture_output=False, check=True)
+
+
+if __name__ == "__main__":
+ main()