diff --git a/src/twomartens/masterthesis/cli.py b/src/twomartens/masterthesis/cli.py index 1409a5d..cf3f7cd 100644 --- a/src/twomartens/masterthesis/cli.py +++ b/src/twomartens/masterthesis/cli.py @@ -91,6 +91,12 @@ def visualise(args: argparse.Namespace) -> None: _visualise_gt(args, file_names, instances, cats_to_classes, cats_to_names, output_path) +def visualise_metrics(args: argparse.Namespace) -> None: + output_path, evaluation_path = _visualise_metrics_get_config_values(conf.get_property) + output_path, metrics_file = _visualise_metrics_prepare_paths(args, output_path, evaluation_path) + _visualise_metrics(_visualise_precision_recall, output_path, metrics_file) + + def measure_mapping(args: argparse.Namespace) -> None: from twomartens.masterthesis.ssd_keras.eval_utils import coco_utils @@ -415,6 +421,27 @@ def _visualise_gt(args: argparse.Namespace, i += 1 +def _visualise_metrics(visualise_precision_recall: callable, + output_path: str, + metrics_file: str) -> None: + import pickle + + with open(metrics_file, "rb") as file: + metrics = pickle.load(file) + + precision_micro = metrics["cumulative_precision_micro"] + recall_micro = metrics["cumulative_recall_micro"] + visualise_precision_recall(precision_micro, recall_micro, + output_path, "micro") + + precision_macro = metrics["cumulative_precision_macro"] + recall_macro = metrics["cumulative_recall_macro"] + visualise_precision_recall(precision_macro, recall_macro, + output_path, "macro") + + # TODO add further metrics + + def _init_eager_mode() -> None: tf.enable_eager_execution() @@ -620,6 +647,14 @@ def _visualise_get_config_values(config_get: Callable[[str], Union[str, int, flo return output_path, coco_path, ground_truth_path +def _visualise_metrics_get_config_values(config_get: Callable[[str], Union[str, int, float, bool]] + ) -> Tuple[str, str]: + output_path = config_get("Paths.output") + evaluation_path = config_get("Paths.evaluation") + + return output_path, evaluation_path + + def _ssd_is_dropout(args: argparse.Namespace) -> bool: return False if args.network == "ssd" else True @@ -703,6 +738,19 @@ def _visualise_prepare_paths(args: argparse.Namespace, return output_path, annotation_file_train +def _visualise_metrics_prepare_paths(args: argparse.Namespace, + output_path: str, + evaluation_path: str) -> Tuple[str, str]: + import os + + metrics_file = f"{evaluation_path}/{args.network}/results-{args.iteration}.bin" + output_path = f"{output_path}/{args.network}/visualise/{args.iteration}" + + os.makedirs(output_path, exist_ok=True) + + return output_path, metrics_file + + def _ssd_train_load_gt(train_gt_path: str, val_gt_path: str ) -> Tuple[Sequence[Sequence[str]], Sequence[Sequence[Sequence[dict]]], @@ -950,6 +998,20 @@ def _ssd_evaluate_get_results(true_positives: Sequence[np.ndarray], return results +def _visualise_precision_recall(precision: np.ndarray, recall: np.ndarray, + output_path: str, file_suffix: str) -> None: + from matplotlib import pyplot + + figure = pyplot.figure() + + pyplot.ylabel("precision") + pyplot.xlabel("recall") + pyplot.plot(recall, precision) + + pyplot.savefig(f"{output_path}/precision-recall-{file_suffix}.png") + pyplot.close(figure) + + def _auto_encoder_train(args: argparse.Namespace) -> None: import os diff --git a/src/twomartens/masterthesis/main.py b/src/twomartens/masterthesis/main.py index 5593825..33f6ce8 100644 --- a/src/twomartens/masterthesis/main.py +++ b/src/twomartens/masterthesis/main.py @@ -43,7 +43,8 @@ def main() -> None: _build_test(sub_parsers[3]) _build_evaluate(sub_parsers[4]) _build_visualise(sub_parsers[5]) - _build_measure(sub_parsers[6]) + _build_visualise_metrics(sub_parsers[6]) + _build_measure(sub_parsers[7]) args = _get_user_input(parser) _execute_action(args, @@ -53,6 +54,7 @@ def main() -> None: cli.test, cli.evaluate, cli.visualise, + cli.visualise_metrics, cli.measure_mapping) @@ -71,6 +73,7 @@ def _build_sub_parsers(parser: argparse.ArgumentParser) -> List[argparse.Argumen test_parser = sub_parsers.add_parser("test", help="Test a network") evaluate_parser = sub_parsers.add_parser("evaluate", help="Evaluate a network") visualise_parser = sub_parsers.add_parser("visualise", help="Visualise the ground truth") + visualise_metrics_parser = sub_parsers.add_parser("visualise_metrics", help="Visualise the evaluation results") measure_parser = sub_parsers.add_parser("measure_mapping", help="Measure the number of instances per COCO category") return [ @@ -80,6 +83,7 @@ def _build_sub_parsers(parser: argparse.ArgumentParser) -> List[argparse.Argumen test_parser, evaluate_parser, visualise_parser, + visualise_metrics_parser, measure_parser ] @@ -99,6 +103,7 @@ def _execute_action(args: argparse.Namespace, on_test: callable, on_evaluate: callable, on_visualise: callable, + on_visualise_metrics: callable, on_measure: callable) -> None: if args.component == "config": on_config(args) @@ -112,6 +117,8 @@ def _execute_action(args: argparse.Namespace, on_evaluate(args) elif args.component == "visualise": on_visualise(args) + elif args.component == "visualise_metrics": + on_visualise_metrics(args) elif args.component == "measure_mapping": on_measure(args) @@ -214,6 +221,10 @@ def _build_visualise(parser: argparse.ArgumentParser) -> None: parser.add_argument("trajectory", type=int, help="trajectory to visualise") +def _build_visualise_metrics(parser: argparse.ArgumentParser) -> None: + parser.add_argument("iteration", type=int, help="the validation iteration to use") + + def _build_measure(parser: argparse.ArgumentParser) -> None: parser.add_argument("tarball_id", type=str, help="id of the used tarball. number for training tarball or 'test'")