Temp changes.

src/av2/evaluation/detection/eval.py

@@ -65,7 +65,7 @@
 
 logger = logging.getLogger(__name__)
 
-
+@profile
 def evaluate(
     dts: pd.DataFrame,
     gts: pd.DataFrame,
@@ -84,27 +84,44 @@ def evaluate(
         (C+1,K) Table of evaluation metrics where C is the number of classes. Plus a row for their means.
         K refers to the number of evaluation metrics.
     """
-    log_ids: List[str] = gts["log_id"].unique().tolist()
-    log_id_to_avm: Dict[str, ArgoverseStaticMap] = {}
-    log_id_to_timestamped_poses: Dict[str, TimestampedCitySE3EgoPoses] = {}
-    if cfg.eval_only_roi_instances and cfg.dataset_dir is not None:
-        for log_id in log_ids:
-            log_dir = cfg.dataset_dir / log_id
-            avm_dir = log_dir / "map"
-            avm = ArgoverseStaticMap.from_map_dir(avm_dir, build_raster=True)
-            log_id_to_avm[log_id] = avm
-            log_id_to_timestamped_poses[log_id] = read_city_SE3_ego(log_dir)
-
-    dts_mapping = {uuid: x for uuid, x in dts.groupby(["log_id", "timestamp_ns"])}
-    gts_mapping = {uuid: x for uuid, x in gts.groupby(["log_id", "timestamp_ns"])}
+    # dts_npy = dts.to_numpy()
+    # gts_npy = gts.to_numpy()
+    # log_ids: List[str] = gts["log_id"].unique().tolist()
+    # log_id_to_avm: Dict[str, ArgoverseStaticMap] = {}
+    # log_id_to_timestamped_poses: Dict[str, TimestampedCitySE3EgoPoses] = {}
+    # if cfg.eval_only_roi_instances and cfg.dataset_dir is not None:
+    #     for log_id in log_ids:
+    #         log_dir = cfg.dataset_dir / log_id
+    #         avm_dir = log_dir / "map"
+    #         avm = ArgoverseStaticMap.from_map_dir(avm_dir, build_raster=True)
+    #         log_id_to_avm[log_id] = avm
+    #         log_id_to_timestamped_poses[log_id] = read_city_SE3_ego(log_dir)
+    # dts = dts.sort_values(["log_id", "timestamp_ns"], ascending=True).reset_index(drop=True)
+    # breakpoint()
+    # uuids = gts.loc[:, ["log_id", "timestamp_ns"]].drop_duplicates().melt()
+    # breakpoint()
+
+    # dts = dts.set_index(["log_id", "timestamp_ns"]).sort_values("score", ascending=False)
+    # gts = gts.set_index(["log_id", "timestamp_ns"])
+    # breakpoint()
+    dts_mapping = {uuid: x for uuid, x in dts.groupby(["log_id", "timestamp_ns", "category"])}
+    gts_mapping = {uuid: x for uuid, x in gts.groupby(["log_id", "timestamp_ns", "category"])}
     args_list = [(dts_mapping[uuid], sweep_gts, cfg, None, None) for uuid, sweep_gts in gts_mapping.items()]
 
     # Accumulate and gather the processed detections and ground truth annotations.
     results = [accumulate(*x) for x in args_list]
     dts_list, gts_list = zip(*results)
 
-    dts = pd.concat(dts_list).reset_index(drop=True)
-    gts = pd.concat(gts_list).reset_index(drop=True)
+    cols = cfg.affinity_thresholds_m + tuple(x.value for x in TruePositiveErrorNames)
+    dts_npy = np.concatenate(dts_list)
+    gts_npy = np.concatenate(gts_list)
+
+    dts.loc[:len(dts_npy) - 1, cols + ("is_evaluated",)] = dts_npy
+    gts.loc[:len(gts_npy) - 1, cfg.affinity_thresholds_m + ("is_evaluated",)] = gts_npy
+    # breakpoint()
+
+    # dts = pd.concat(dts_list).reset_index(drop=True)
+    # gts = pd.concat(gts_list).reset_index(drop=True)
 
     # Compute summary metrics.
     metrics = summarize_metrics(dts, gts, cfg)

src/av2/evaluation/detection/utils.py

@@ -94,7 +94,7 @@ def tp_normalization_terms(self) -> Tuple[float, ...]:
             MAX_YAW_RAD_ERROR,
         )
 
-
+@profile
 def accumulate(
     dts: pd.DataFrame,
     gts: pd.DataFrame,
@@ -115,9 +115,14 @@ def accumulate(
     Returns:
         The detection and ground truth cuboids augmented with assigment and evaluation fields.
     """
+    breakpoint()
+    perm = (-dts["score"]).to_numpy().argsort()
+    dts = dts.iloc[perm]
+    # dts = dts[]
+    # breakpoint()
     dts = dts.sort_values("score", ascending=False).reset_index(drop=True)
-    dts_npy: NDArrayObject = dts[CUBOID_COLS].to_numpy()
-    gts_npy: NDArrayObject = gts[CUBOID_COLS].to_numpy()
+    dts_npy: NDArrayFloat = dts[CUBOID_COLS].to_numpy()
+    gts_npy: NDArrayFloat = gts[CUBOID_COLS].to_numpy()
 
     dts_categories: NDArrayObject = dts["category"].to_numpy()
     gts_categories: NDArrayObject = gts["category"].to_numpy()
@@ -126,6 +131,8 @@ def accumulate(
 
     dt_results: NDArrayFloat = np.zeros((len(dts), 8))
     gt_results: NDArrayFloat = np.zeros((len(gts), 5))
+
+
     for category in cfg.categories:
         category_dts_mask: NDArrayBool = dts_categories == category
         category_gts_mask: NDArrayBool = gts_categories == category
@@ -161,10 +168,9 @@ def accumulate(
         dt_results[category_dts_idx, :-1] = dts_assignments
         gt_results[category_gts_idx, :-1] = gts_assignments
 
-    cols = cfg.affinity_thresholds_m + tuple(x.value for x in TruePositiveErrorNames) 
-    dts.loc[:, cols + ("is_evaluated",)] = dt_results
-    gts.loc[:, cfg.affinity_thresholds_m + ("is_evaluated",)] = gt_results
-    return dts, gts
+    _, inv = np.unique(perm, return_index=True)
+    dt_results = dt_results[inv]
+    return dt_results, gt_results
 
 
 def assign(dts: NDArrayFloat, gts: NDArrayFloat, cfg: DetectionCfg) -> Tuple[NDArrayFloat, NDArrayFloat]:

tests/evaluation/detection/test_eval.py

@@ -165,16 +165,16 @@ def test_accumulate() -> None:
         dts, gts = accumulate(*job)
 
         # Check that there's a true positive under every threshold.
-        assert np.all(dts.loc[:, cfg.affinity_thresholds_m])
+        assert np.all(dts[:, :4])
 
         # Check that all error metrics are zero.
-        assert (dts.loc[:, tuple(x.value for x in TruePositiveErrorNames)] == 0).all(axis=None)
+        assert (dts[:, 4:7] == 0).all()
 
-        # Check that there are 2 regular vehicles.
-        assert gts["category"].value_counts()["REGULAR_VEHICLE"] == 2
+        # # Check that there are 2 regular vehicles.
+        # assert gts["category"].value_counts()["REGULAR_VEHICLE"] == 2
 
-        # Check that there are no other labels.
-        assert gts["category"].value_counts().sum() == 2
+        # # Check that there are no other labels.
+        # assert gts["category"].value_counts().sum() == 2
 
 
 def test_assign() -> None:
@@ -304,22 +304,22 @@ def test_compute_evaluated_gts_mask() -> None:
     np.testing.assert_array_equal(gts_mask, gts_mask_)  # type: ignore
 
 
-# def test_val_identity() -> None:
-#     root_dir = Path.home() / "data" / "datasets" / "av2" / "sensor" / "val"
-#     paths = sorted(root_dir.glob("*/annotations.feather"))
+def test_val_identity() -> None:
+    root_dir = Path.home() / "data" / "datasets" / "av2" / "sensor" / "val"
+    paths = sorted(root_dir.glob("*/annotations.feather"))
 
-#     annotations = []
-#     for p in paths:
-#         df = pd.read_feather(p)
-#         df["log_id"] = p.parent.stem
-#         annotations.append(df)
-#     annotations = pd.concat(annotations).reset_index(drop=True)
-#     dts = annotations.copy()
-#     dts["score"] = 1.0
-#     annotations["num_interior_pts"] = 1
+    annotations = []
+    for p in paths:
+        df = pd.read_feather(p)
+        df["log_id"] = p.parent.stem
+        annotations.append(df)
+    annotations = pd.concat(annotations).reset_index(drop=True)
+    dts = annotations.copy()
+    dts["score"] = 1.0
+    annotations["num_interior_pts"] = 1
 
-#     detection_cfg = DetectionCfg(eval_only_roi_instances=False, max_num_dts_per_category=1000)
-#     dts_, gts_, metrics_ = evaluate(dts, annotations, detection_cfg)
+    detection_cfg = DetectionCfg(eval_only_roi_instances=False, max_num_dts_per_category=1000)
+    dts_, gts_, metrics_ = evaluate(dts, annotations, detection_cfg)
 
 
 if __name__ == "__main__":