diff --git a/csrc/faster_eval_api/coco_eval/cocoeval.cpp b/csrc/faster_eval_api/coco_eval/cocoeval.cpp index fba7324..9d46a80 100644 --- a/csrc/faster_eval_api/coco_eval/cocoeval.cpp +++ b/csrc/faster_eval_api/coco_eval/cocoeval.cpp @@ -601,6 +601,141 @@ namespace coco_eval std::vector result = EvaluateImages(area_ranges, max_detections.back(), iou_thresholds, image_category_ious, image_category_ground_truth_instances, image_category_detection_instances); return Accumulate(params, result); } + + py::object deepcopy(const py::object &pyobj) + { + auto _pyobj = pyobj; + return _pyobj; + } + + void Dataset::append(int64_t img_id, int64_t cat_id, py::dict ann) + { + std::string key = std::to_string(img_id) + "_" + std::to_string(cat_id); + this->data[key].push_back(ann); + } + std::vector Dataset::get(int64_t img_id, int64_t cat_id) + { + std::string key = std::to_string(img_id) + "_" + std::to_string(cat_id); + return this->data[key]; + } + + InstanceAnnotation parseInstanceAnnotation(const py::dict &ann) + { + uint64_t id = 0; + double score = 0.; + double area = 0.; + bool is_crowd = false; + bool ignore = false; + bool lvis_mark = false; + + for (auto it : ann) + { + std::string key = it.first.cast(); + + if (key == "id") + { + id = it.second.cast(); + } + else if (key == "score") + { + score = it.second.cast(); + } + else if (key == "area") + { + area = it.second.cast(); + } + else if (key == "is_crowd") + { + is_crowd = it.second.cast(); + } + else if (key == "ignore") + { + ignore = it.second.cast(); + } + else if (key == "lvis_mark") + { + lvis_mark = it.second.cast(); + } + } + return InstanceAnnotation(id, score, area, is_crowd, ignore, lvis_mark); + } + + std::vector Dataset::get_cpp_annotations( + int64_t img_id, int64_t cat_id) + { + std::string key = std::to_string(img_id) + "_" + std::to_string(cat_id); + std::vector result; + std::vector anns = get(img_id, cat_id); + for (size_t i = 0; i < anns.size(); i++) + { + result.push_back(parseInstanceAnnotation(anns[i])); + } + return result; + } + + std::vector>> Dataset::get_cpp_instances( + std::vector img_ids, std::vector cat_ids, bool useCats) + { + std::vector>> result; + for (size_t i = 0; i < img_ids.size(); i++) + { + int64_t img_id = img_ids[i]; + result.push_back(std::vector>()); + if (!useCats) + { + result[i].push_back(std::vector()); + } + + for (size_t j = 0; j < cat_ids.size(); j++) + { + int64_t cat_id = cat_ids[j]; + + std::vector anns = this->get_cpp_annotations(img_id, cat_id); + + if (useCats) + { + result[i].push_back(anns); + } + else + { + result[i][0].insert(result[i][0].end(), anns.begin(), anns.end()); + } + } + } + return result; + } + + std::vector>> Dataset::get_instances( + std::vector img_ids, std::vector cat_ids, bool useCats) + { + std::vector>> result; + for (size_t i = 0; i < img_ids.size(); i++) + { + int64_t img_id = img_ids[i]; + result.push_back(std::vector>()); + if (!useCats) + { + result[i].push_back(std::vector()); + } + + for (size_t j = 0; j < cat_ids.size(); j++) + { + int64_t cat_id = cat_ids[j]; + + std::vector anns = this->get(img_id, cat_id); + + if (useCats) + { + result[i].push_back(anns); + } + else + { + result[i][0].insert(result[i][0].end(), anns.begin(), anns.end()); + } + } + } + return result; + } } // namespace COCOeval } // namespace coco_eval \ No newline at end of file diff --git a/csrc/faster_eval_api/coco_eval/cocoeval.h b/csrc/faster_eval_api/coco_eval/cocoeval.h index 039559f..2efa11b 100644 --- a/csrc/faster_eval_api/coco_eval/cocoeval.h +++ b/csrc/faster_eval_api/coco_eval/cocoeval.h @@ -57,6 +57,25 @@ namespace coco_eval std::vector detection_ignores; }; + class Dataset + { + public: + Dataset() {} + void append(int64_t img_id, int64_t cat_id, py::dict ann); + std::vector get(int64_t img_id, int64_t cat_id); + std::vector get_cpp_annotations( + int64_t img_id, int64_t cat_id); + + std::vector>> get_cpp_instances( + std::vector img_ids, std::vector cat_ids, bool useCats); + + std::vector>> get_instances( + std::vector img_ids, std::vector cat_ids, bool useCats); + + private: + std::unordered_map> data; + }; + template using ImageCategoryInstances = std::vector>>; @@ -100,5 +119,6 @@ namespace coco_eval const ImageCategoryInstances & image_category_detection_instances); + py::object deepcopy(const py::object &pyobj); } // namespace COCOeval } // namespace coco_eval \ No newline at end of file diff --git a/csrc/faster_eval_api/faster_eval_api.cpp b/csrc/faster_eval_api/faster_eval_api.cpp index f23947b..149789a 100644 --- a/csrc/faster_eval_api/faster_eval_api.cpp +++ b/csrc/faster_eval_api/faster_eval_api.cpp @@ -52,6 +52,15 @@ namespace coco_eval pybind11::class_(m, "InstanceAnnotation").def(pybind11::init()); pybind11::class_(m, "ImageEvaluation").def(pybind11::init<>()); + m.def("deepcopy", &COCOeval::deepcopy, "COCOeval::deepcopy"); + + pybind11::class_(m, "Dataset").def(pybind11::init<>()) + .def("append", &COCOeval::Dataset::append) + .def("get", &COCOeval::Dataset::get) + .def("get_instances", &COCOeval::Dataset::get_instances) + .def("get_cpp_annotations", &COCOeval::Dataset::get_cpp_annotations) + .def("get_cpp_instances", &COCOeval::Dataset::get_cpp_instances); + #ifdef VERSION_INFO m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); #else diff --git a/csrc/mask_api/mask_api.cpp b/csrc/mask_api/mask_api.cpp index 540224b..91b4fc2 100644 --- a/csrc/mask_api/mask_api.cpp +++ b/csrc/mask_api/mask_api.cpp @@ -73,7 +73,6 @@ namespace mask_api m.def("iou", &Mask::iou, "Mask::iou"); m.def("toBbox", &Mask::toBbox, "Mask::toBbox"); - // m.def("merge", &Mask::merge, "Mask::merge"); m.def("merge", py::overload_cast&, const uint64_t&>(&Mask::merge), "Mask::merge"); m.def("merge", py::overload_cast&>(&Mask::merge), "Mask::merge"); @@ -87,7 +86,6 @@ namespace mask_api m.def("frUncompressedRLE", &Mask::frUncompressedRLE, "Mask::frUncompressedRLE"); m.def("frPyObjects", &Mask::frPyObjects, "Mask::frPyObjects"); - m.def("deepcopy", &Mask::deepcopy, "Mask::deepcopy"); #ifdef VERSION_INFO m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); #else diff --git a/csrc/mask_api/src/mask.cpp b/csrc/mask_api/src/mask.cpp index ae33601..0ad9db5 100644 --- a/csrc/mask_api/src/mask.cpp +++ b/csrc/mask_api/src/mask.cpp @@ -378,13 +378,13 @@ namespace mask_api size_t n = R.size(); if (n > 0) { - + uint64_t s = R[0].h * R[0].w * n; for (uint64_t i = 0; i < R.size(); i++) { uint v = 0; std::vector> mask(R[i].h, std::vector(R[i].w)); - size_t x=0,y=0,c=0; + size_t x = 0, y = 0, c = 0; for (uint64_t j = 0; j < R[i].m; j++) { for (uint64_t k = 0; k < R[i].cnts[j]; k++) @@ -396,7 +396,7 @@ namespace mask_api { throw std::range_error("Invalid RLE mask representation"); } - y+=1; + y += 1; if (y >= R[i].h) { @@ -858,17 +858,13 @@ namespace mask_api printf("crowd_length=%zu, n=%zu\n", crowd_length, n); throw std::out_of_range("iscrowd must have the same length as gt"); } - if (m == 0 || n == 0){ + if (m == 0 || n == 0) + { return std::vector(0); } return py::array(iou.size(), iou.data()).reshape({m, n}); } - py::object deepcopy(const py::object &pyobj){ - auto _pyobj = pyobj; - return _pyobj; - } - std::variant> frPyObjects(const py::object &pyobj, const uint64_t &h, const uint64_t &w) { std::vector rles; @@ -930,7 +926,6 @@ namespace mask_api return _toString(rles); } - } // namespace Mask } \ No newline at end of file diff --git a/csrc/mask_api/src/mask.h b/csrc/mask_api/src/mask.h index 2fa43c9..e950ecb 100644 --- a/csrc/mask_api/src/mask.h +++ b/csrc/mask_api/src/mask.h @@ -14,8 +14,6 @@ namespace mask_api namespace Mask { - typedef std::list bbox; - struct RLE { RLE( @@ -50,7 +48,7 @@ namespace mask_api // pyx functions py::array_t decode(const std::vector &R); std::vector encode(const py::array_t &M); - + py::array_t toBbox(const std::vector &R); py::dict merge(const std::vector &rleObjs, const uint64_t &intersect); py::dict merge(const std::vector &rleObjs); @@ -66,7 +64,5 @@ namespace mask_api std::vector rleToUncompressedRLE(const std::vector &R); py::array_t rleToBbox(const std::vector R, const uint64_t &n); std::variant> frPyObjects(const py::object &pyobj, const uint64_t &h, const uint64_t &w); - - py::object deepcopy(const py::object &pyobj); } // namespace Mask } // namespace mask_api \ No newline at end of file diff --git a/faster_coco_eval/core/coco.py b/faster_coco_eval/core/coco.py index ffd2ffe..45da69d 100644 --- a/faster_coco_eval/core/coco.py +++ b/faster_coco_eval/core/coco.py @@ -52,6 +52,8 @@ import numpy as np +import faster_coco_eval.faster_eval_api_cpp as _C + from . import mask as maskUtils logger = logging.getLogger(__name__) @@ -88,7 +90,7 @@ def __init__(self, annotation_file=None): if type(annotation_file) is str: self.dataset = self.load_json(annotation_file) elif type(annotation_file) is dict: - self.dataset = maskUtils.deepcopy(annotation_file) + self.dataset = _C.deepcopy(annotation_file) else: self.dataset = None @@ -338,7 +340,7 @@ def loadRes(self, resFile, min_score=0): elif type(resFile) is np.ndarray: anns = self.loadNumpyAnnotations(resFile) else: - anns = maskUtils.deepcopy(resFile) + anns = _C.deepcopy(resFile) assert type(anns) is list, "results in not an array of objects" @@ -358,9 +360,7 @@ def loadRes(self, resFile, min_score=0): for id, ann in enumerate(anns): ann["id"] = id + 1 elif "bbox" in anns[0] and not anns[0]["bbox"] == []: - res.dataset["categories"] = maskUtils.deepcopy( - self.dataset["categories"] - ) + res.dataset["categories"] = _C.deepcopy(self.dataset["categories"]) for id, ann in enumerate(anns): bb = ann["bbox"] x1, x2, y1, y2 = [bb[0], bb[0] + bb[2], bb[1], bb[1] + bb[3]] @@ -370,9 +370,7 @@ def loadRes(self, resFile, min_score=0): ann["id"] = id + 1 ann["iscrowd"] = 0 elif "segmentation" in anns[0]: - res.dataset["categories"] = maskUtils.deepcopy( - self.dataset["categories"] - ) + res.dataset["categories"] = _C.deepcopy(self.dataset["categories"]) for id, ann in enumerate(anns): # now only support compressed RLE format as segmentation results ann["area"] = maskUtils.area(ann["segmentation"]) @@ -381,9 +379,7 @@ def loadRes(self, resFile, min_score=0): ann["id"] = id + 1 ann["iscrowd"] = 0 elif "keypoints" in anns[0]: - res.dataset["categories"] = maskUtils.deepcopy( - self.dataset["categories"] - ) + res.dataset["categories"] = _C.deepcopy(self.dataset["categories"]) for id, ann in enumerate(anns): s = ann["keypoints"] x = s[0::3] diff --git a/faster_coco_eval/core/cocoeval.py b/faster_coco_eval/core/cocoeval.py index f6a75b4..e798358 100644 --- a/faster_coco_eval/core/cocoeval.py +++ b/faster_coco_eval/core/cocoeval.py @@ -5,6 +5,8 @@ import numpy as np +import faster_coco_eval.faster_eval_api_cpp as _C + from . import mask as maskUtils from .coco import COCO @@ -87,8 +89,8 @@ def __init__( # per-image per-category evaluation results [KxAxI] elements self.evalImgs = defaultdict(list) self.eval: dict = {} # accumulated evaluation results - self._gts = defaultdict(list) # gt for evaluation - self._dts = defaultdict(list) # dt for evaluation + # self._gts = defaultdict(list) # gt for evaluation + # self._dts = defaultdict(list) # dt for evaluation self.params = Params( iouType=iouType, kpt_sigmas=kpt_oks_sigmas ) # parameters @@ -122,8 +124,9 @@ def __init__( ) self.print_function = print_function # output print function - self.load_data_time = [] - self.iou_data_time = [] + + self.dt_dataset = _C.Dataset() + self.gt_dataset = _C.Dataset() def _toMask(self, anns: list, coco: COCO): # modify ann['segmentation'] by reference @@ -159,8 +162,8 @@ def _prepare(self): gt["ignore"] = "iscrowd" in gt and gt["iscrowd"] if p.iouType == "keypoints": gt["ignore"] = (gt.get("num_keypoints") == 0) or gt["ignore"] - self._gts = defaultdict(list) # gt for evaluation - self._dts = defaultdict(list) # dt for evaluation + # self._gts = defaultdict(list) # gt for evaluation + # self._dts = defaultdict(list) # dt for evaluation img_pl = defaultdict( set ) # per image list of categories present in image @@ -189,7 +192,8 @@ def _prepare(self): self.freq_groups = self._prepare_freq_group() for gt in gts: - self._gts[gt["image_id"], gt["category_id"]].append(gt) + # self._gts[gt["image_id"], gt["category_id"]].append(gt) + self.gt_dataset.append(gt["image_id"], gt["category_id"], gt) for dt in dts: img_id, cat_id = dt["image_id"], dt["category_id"] @@ -204,7 +208,8 @@ def _prepare(self): dt["category_id"] in self.img_nel[dt["image_id"]] ) - self._dts[img_id, cat_id].append(dt) + # self._dts[img_id, cat_id].append(dt) + self.dt_dataset.append(img_id, cat_id, dt) def _prepare_freq_group(self): p = self.params @@ -217,14 +222,22 @@ def _prepare_freq_group(self): def computeIoU(self, imgId, catId): p = self.params - if p.useCats: - gt = self._gts[imgId, catId] - dt = self._dts[imgId, catId] - else: - gt = [ann for cId in p.catIds for ann in self._gts[imgId, cId]] - dt = [ann for cId in p.catIds for ann in self._dts[imgId, cId]] + + gt = self.gt_dataset.get_instances( + [imgId], [catId] if p.useCats else p.catIds, bool(p.useCats) + )[0][ + 0 + ] # 1 imgId 1 catId + + dt = self.dt_dataset.get_instances( + [imgId], [catId] if p.useCats else p.catIds, bool(p.useCats) + )[0][ + 0 + ] # 1 imgId 1 catId + if len(gt) == 0 and len(dt) == 0: return [] + inds = np.argsort([-d["score"] for d in dt], kind="mergesort") dt = [dt[i] for i in inds] if len(dt) > p.maxDets[-1]: @@ -247,8 +260,9 @@ def computeIoU(self, imgId, catId): def computeOks(self, imgId, catId): p = self.params # dimention here should be Nxm - gts = self._gts[imgId, catId] - dts = self._dts[imgId, catId] + gts = self.gt_dataset.get(imgId, catId) + dts = self.dt_dataset.get(imgId, catId) + inds = np.argsort([-d["score"] for d in dts], kind="mergesort") dts = [dts[i] for i in inds] if len(dts) > p.maxDets[-1]: diff --git a/faster_coco_eval/core/faster_eval_api.py b/faster_coco_eval/core/faster_eval_api.py index 26695aa..1f4d23a 100644 --- a/faster_coco_eval/core/faster_eval_api.py +++ b/faster_coco_eval/core/faster_eval_api.py @@ -8,7 +8,6 @@ import faster_coco_eval.faster_eval_api_cpp as _C -from . import mask as maskUtils from .cocoeval import COCOeval logger = logging.getLogger(__name__) @@ -19,25 +18,6 @@ class COCOeval_faster(COCOeval): functions evaluateImg() and accumulate() are implemented in C++ to speedup evaluation.""" - # <<<< Beginning of code differences with original COCO API - def convert_instances_to_cpp(self, instances, is_det=False): - # Convert annotations for a list of instances in an image to a format that's fast # noqa: E501 - # to access in C++ - instances_cpp = [] - for instance in instances: - instance_cpp = _C.InstanceAnnotation( - int(instance["id"]), - float( - instance["score"] if is_det else instance.get("score", 0.0) - ), - float(instance["area"]), - bool(instance.get("iscrowd", 0)), - bool(instance.get("ignore", 0)), - bool(instance.get("lvis_mark", False)) if is_det else False, - ) - instances_cpp.append(instance_cpp) - return instances_cpp - def evaluate(self): """Run per image evaluation on given images and store results in self.evalImgs_cpp, a datastructure that isn't readable from Python but @@ -78,38 +58,18 @@ def evaluate(self): } # bottleneck # Convert GT annotations, detections, and IOUs to a format that's fast to access in C++ # noqa: E501 - ground_truth_instances = [ - [ - self.convert_instances_to_cpp(self._gts[imgId, catId]) - for catId in p.catIds - ] - for imgId in p.imgIds - ] - - detected_instances = [ - [ - self.convert_instances_to_cpp( - self._dts[imgId, catId], is_det=True - ) - for catId in p.catIds - ] - for imgId in p.imgIds - ] + ground_truth_instances = self.gt_dataset.get_cpp_instances( + p.imgIds, p.catIds, bool(p.useCats) + ) + detected_instances = self.dt_dataset.get_cpp_instances( + p.imgIds, p.catIds, bool(p.useCats) + ) ious = [ [self.ious[imgId, catId] for catId in catIds] for imgId in p.imgIds ] - if not p.useCats: - # For each image, flatten per-category lists into a single list - ground_truth_instances = [ - [[o for c in i for o in c]] for i in ground_truth_instances - ] - detected_instances = [ - [[o for c in i for o in c]] for i in detected_instances - ] - - self._paramsEval = maskUtils.deepcopy(self.params) + self._paramsEval = _C.deepcopy(self.params) if self.separate_eval: # Call C++ implementation of self.evaluateImgs() diff --git a/faster_coco_eval/core/mask.py b/faster_coco_eval/core/mask.py index 4ddabd2..2854bfc 100644 --- a/faster_coco_eval/core/mask.py +++ b/faster_coco_eval/core/mask.py @@ -78,7 +78,6 @@ iou = _mask.iou merge = _mask.merge frPyObjects = _mask.frPyObjects -deepcopy = _mask.deepcopy def encode(bimask):