Re-enabled reading duplicate cells from CSV files.

The truth mapping code (currently) relies on possibly having
duplicate cells in the event record, for different particles
depositing energy in the same cell. The truth mapping code,
and some of the I/O unit tests, are now set up to disable the
cell de-duplication during CSV reading.

Reverted to the previous ordering scheme for traccc::cell-s,
and implemented a (different) custom ordering for the cell
collection. Since as it turns out, the project implicitly
relies on these two behaving a little differently.

core/include/traccc/edm/cell.hpp

@@ -73,10 +73,12 @@ inline bool operator<(const cell& lhs, const cell& rhs) {
 
     if (lhs.module_link != rhs.module_link) {
         return lhs.module_link < rhs.module_link;
+    } else if (lhs.channel0 != rhs.channel0) {
+        return (lhs.channel0 < rhs.channel0);
     } else if (lhs.channel1 != rhs.channel1) {
         return (lhs.channel1 < rhs.channel1);
     } else {
-        return (lhs.channel0 < rhs.channel0);
+        return lhs.activation < rhs.activation;
     }
 }
 

io/include/traccc/io/read_cells.hpp

@@ -1,6 +1,6 @@
 /** TRACCC library, part of the ACTS project (R&D line)
  *
- * (c) 2022-2023 CERN for the benefit of the ACTS project
+ * (c) 2022-2024 CERN for the benefit of the ACTS project
  *
  * Mozilla Public License Version 2.0
  */
@@ -37,13 +37,15 @@ namespace traccc::io {
 /// @param dconfig The detector's digitization configuration
 /// @param bardoce_map An object to perform barcode re-mapping with
 ///                    (For Acts->Detray identifier re-mapping, if necessary)
+/// @param deduplicate Whether to deduplicate the cells
 ///
 void read_cells(
     cell_reader_output &out, std::size_t event, std::string_view directory,
     data_format format = data_format::csv, const geometry *geom = nullptr,
     const digitization_config *dconfig = nullptr,
     const std::map<std::uint64_t, detray::geometry::barcode> *barcode_map =
-        nullptr);
+        nullptr,
+    bool deduplicate = true);
 
 /// Read cell data into memory
 ///
@@ -56,12 +58,14 @@ void read_cells(
 /// @param dconfig The detector's digitization configuration
 /// @param bardoce_map An object to perform barcode re-mapping with
 ///                    (For Acts->Detray identifier re-mapping, if necessary)
+/// @param deduplicate Whether to deduplicate the cells
 ///
 void read_cells(cell_reader_output &out, std::string_view filename,
                 data_format format = data_format::csv,
                 const geometry *geom = nullptr,
                 const digitization_config *dconfig = nullptr,
                 const std::map<std::uint64_t, detray::geometry::barcode>
-                    *barcode_map = nullptr);
+                    *barcode_map = nullptr,
+                bool deduplicate = true);
 
 }  // namespace traccc::io

io/src/csv/read_cells.cpp

@@ -11,15 +11,31 @@
 #include "traccc/io/csv/make_cell_reader.hpp"
 
 // System include(s).
+#include <algorithm>
 #include <cassert>
 #include <iostream>
 #include <map>
 #include <set>
 #include <stdexcept>
 #include <utility>
+#include <vector>
 
 namespace {
 
+/// Comparator used for sorting cells. This sorting is one of the assumptions
+/// made in the clusterization algorithm
+struct cell_order {
+    bool operator()(const traccc::cell& lhs, const traccc::cell& rhs) const {
+        if (lhs.module_link != rhs.module_link) {
+            return lhs.module_link < rhs.module_link;
+        } else if (lhs.channel1 != rhs.channel1) {
+            return (lhs.channel1 < rhs.channel1);
+        } else {
+            return (lhs.channel0 < rhs.channel0);
+        }
+    }
+};  // struct cell_order
+
 /// Helper function which finds module from csv::cell in the geometry and
 /// digitization config, and initializes the modules limits with the cell's
 /// properties
@@ -67,29 +83,23 @@ traccc::cell_module get_module(const std::uint64_t geometry_id,
     return result;
 }
 
-}  // namespace
-
-namespace traccc::io::csv {
-
-void read_cells(
-    cell_reader_output& out, std::string_view filename, const geometry* geom,
-    const digitization_config* dconfig,
-    const std::map<std::uint64_t, detray::geometry::barcode>* barcode_map) {
+std::map<std::uint64_t, std::set<traccc::cell, ::cell_order> >
+read_deduplicated_cells(std::string_view filename) {
 
     // Temporary storage for all the cells and modules.
-    std::map<std::uint64_t, std::set<traccc::cell> > cellsMap;
+    std::map<std::uint64_t, std::set<traccc::cell, ::cell_order> > result;
 
     // Construct the cell reader object.
-    auto reader = make_cell_reader(filename);
+    auto reader = traccc::io::csv::make_cell_reader(filename);
 
     // Read all cells from input file.
-    csv::cell iocell;
+    traccc::io::csv::cell iocell;
     unsigned int nduplicates = 0;
     while (reader.read(iocell)) {
 
         // Add the cell to the module. At this point the module link of the
         // cells is not set up correctly yet.
-        if (cellsMap[iocell.geometry_id]
+        if (result[iocell.geometry_id]
                 .insert({iocell.channel0, iocell.channel1, iocell.value,
                          iocell.timestamp, 0})
                 .second == false) {
@@ -101,30 +111,84 @@ void read_cells(
                   << " duplicate cells found in " << filename << std::endl;
     }
 
-    // Fill the output containers with the ordered cells and modules.
-    for (const auto& [original_geometry_id, cells] : cellsMap) {
-
-        // Modify the geometry ID of the module if a barcode map is provided.
-        std::uint64_t geometry_id = original_geometry_id;
-        if (barcode_map != nullptr) {
-            const auto it = barcode_map->find(geometry_id);
-            if (it != barcode_map->end()) {
-                geometry_id = it->second.value();
-            } else {
-                throw std::runtime_error(
-                    "Could not find barcode for geometry ID " +
-                    std::to_string(geometry_id));
+    // Return the container.
+    return result;
+}
+
+std::map<std::uint64_t, std::vector<traccc::cell> > read_all_cells(
+    std::string_view filename) {
+
+    // Temporary storage for all the cells and modules.
+    std::map<std::uint64_t, std::vector<traccc::cell> > result;
+
+    // Construct the cell reader object.
+    auto reader = traccc::io::csv::make_cell_reader(filename);
+
+    // Read all cells from input file.
+    traccc::io::csv::cell iocell;
+    while (reader.read(iocell)) {
+
+        // Add the cell to the module. At this point the module link of the
+        // cells is not set up correctly yet.
+        result[iocell.geometry_id].push_back({iocell.channel0, iocell.channel1,
+                                              iocell.value, iocell.timestamp,
+                                              0});
+    }
+
+    // Sort the cells. Deduplication or not, they do need to be sorted.
+    for (auto& [_, cells] : result) {
+        std::sort(cells.begin(), cells.end(), ::cell_order());
+    }
+
+    // Return the container.
+    return result;
+}
+
+}  // namespace
+
+namespace traccc::io::csv {
+
+void read_cells(
+    cell_reader_output& out, std::string_view filename, const geometry* geom,
+    const digitization_config* dconfig,
+    const std::map<std::uint64_t, detray::geometry::barcode>* barcode_map,
+    const bool deduplicate) {
+
+    // Helper lambda filling the output container.
+    const auto fill_output = [&out, geom, dconfig,
+                              barcode_map](const auto& cellsMap) {
+        // Fill the output containers with the ordered cells and modules.
+        for (const auto& [original_geometry_id, cells] : cellsMap) {
+            // Modify the geometry ID of the module if a barcode map is
+            // provided.
+            std::uint64_t geometry_id = original_geometry_id;
+            if (barcode_map != nullptr) {
+                const auto it = barcode_map->find(geometry_id);
+                if (it != barcode_map->end()) {
+                    geometry_id = it->second.value();
+                } else {
+                    throw std::runtime_error(
+                        "Could not find barcode for geometry ID " +
+                        std::to_string(geometry_id));
+                }
             }
-        }
 
-        // Add the module and its cells to the output.
-        out.modules.push_back(
-            get_module(geometry_id, geom, dconfig, original_geometry_id));
-        for (auto& cell : cells) {
-            out.cells.push_back(cell);
-            // Set the module link.
-            out.cells.back().module_link = out.modules.size() - 1;
+            // Add the module and its cells to the output.
+            out.modules.push_back(
+                get_module(geometry_id, geom, dconfig, original_geometry_id));
+            for (auto& cell : cells) {
+                out.cells.push_back(cell);
+                // Set the module link.
+                out.cells.back().module_link = out.modules.size() - 1;
+            }
         }
+    };
+
+    // Fill the output in the appropriate way.
+    if (deduplicate) {
+        fill_output(read_deduplicated_cells(filename));
+    } else {
+        fill_output(read_all_cells(filename));
     }
 }
 

io/src/csv/read_cells.hpp

@@ -1,6 +1,6 @@
 /** TRACCC library, part of the ACTS project (R&D line)
  *
- * (c) 2022-2023 CERN for the benefit of the ACTS project
+ * (c) 2022-2024 CERN for the benefit of the ACTS project
  *
  * Mozilla Public License Version 2.0
  */
@@ -27,11 +27,13 @@ namespace traccc::io::csv {
 /// @param geom The description of the detector geometry
 /// @param dconfig The detector's digitization configuration
 /// @param bardoce_map An object to perform barcode re-mapping with
+/// @param deduplicate Whether to deduplicate the cells
 ///
 void read_cells(cell_reader_output& out, std::string_view filename,
                 const geometry* geom = nullptr,
                 const digitization_config* dconfig = nullptr,
                 const std::map<std::uint64_t, detray::geometry::barcode>*
-                    barcode_map = nullptr);
+                    barcode_map = nullptr,
+                bool deduplicate = true);
 
 }  // namespace traccc::io::csv

io/src/mapper.cpp

@@ -207,7 +207,7 @@ generate_measurement_cell_map(std::size_t event,
     // Read the cells from the relevant event file
     traccc::io::cell_reader_output readOut(&resource);
     io::read_cells(readOut, event, cells_dir, traccc::data_format::csv,
-                   &surface_transforms, &digi_cfg);
+                   &surface_transforms, &digi_cfg, nullptr, false);
     cell_collection_types::host& cells_per_event = readOut.cells;
     cell_module_collection_types::host& modules_per_event = readOut.modules;
 

io/src/read_cells.cpp

@@ -1,6 +1,6 @@
 /** TRACCC library, part of the ACTS project (R&D line)
  *
- * (c) 2022-2023 CERN for the benefit of the ACTS project
+ * (c) 2022-2024 CERN for the benefit of the ACTS project
  *
  * Mozilla Public License Version 2.0
  */
@@ -18,14 +18,15 @@ void read_cells(
     cell_reader_output& out, std::size_t event, std::string_view directory,
     data_format format, const geometry* geom,
     const digitization_config* dconfig,
-    const std::map<std::uint64_t, detray::geometry::barcode>* barcode_map) {
+    const std::map<std::uint64_t, detray::geometry::barcode>* barcode_map,
+    bool deduplicate) {
 
     switch (format) {
         case data_format::csv: {
             read_cells(out,
                        data_directory() + directory.data() +
                            get_event_filename(event, "-cells.csv"),
-                       format, geom, dconfig, barcode_map);
+                       format, geom, dconfig, barcode_map, deduplicate);
             break;
         }
         case data_format::binary: {
@@ -45,11 +46,13 @@ void read_cells(
 void read_cells(
     cell_reader_output& out, std::string_view filename, data_format format,
     const geometry* geom, const digitization_config* dconfig,
-    const std::map<std::uint64_t, detray::geometry::barcode>* barcode_map) {
+    const std::map<std::uint64_t, detray::geometry::barcode>* barcode_map,
+    bool deduplicate) {
 
     switch (format) {
         case data_format::csv:
-            return csv::read_cells(out, filename, geom, dconfig, barcode_map);
+            return csv::read_cells(out, filename, geom, dconfig, barcode_map,
+                                   deduplicate);
 
         default:
             throw std::invalid_argument("Unsupported data format");

tests/io/test_csv.cpp

@@ -29,9 +29,9 @@ class io : public traccc::tests::data_test {};
 TEST_F(io, csv_read_single_module) {
 
     traccc::io::cell_reader_output single_module_cells;
-    traccc::io::read_cells(single_module_cells,
-                           get_datafile("single_module/cells.csv"),
-                           traccc::data_format::csv);
+    traccc::io::read_cells(
+        single_module_cells, get_datafile("single_module/cells.csv"),
+        traccc::data_format::csv, nullptr, nullptr, nullptr, false);
     auto& cells = single_module_cells.cells;
     auto& modules = single_module_cells.modules;
     ASSERT_EQ(cells.size(), 6u);
@@ -49,9 +49,9 @@ TEST_F(io, csv_read_single_module) {
 TEST_F(io, csv_read_two_modules) {
 
     traccc::io::cell_reader_output two_module_cells;
-    traccc::io::read_cells(two_module_cells,
-                           get_datafile("two_modules/cells.csv"),
-                           traccc::data_format::csv);
+    traccc::io::read_cells(
+        two_module_cells, get_datafile("two_modules/cells.csv"),
+        traccc::data_format::csv, nullptr, nullptr, nullptr, false);
     auto& cells = two_module_cells.cells;
     auto& modules = two_module_cells.modules;
     ASSERT_EQ(modules.size(), 2u);