Adds type inference and type conversion for leaf-columns to the nested JSON parser

cpp/include/cudf/io/json.hpp

@@ -83,6 +83,9 @@ class json_reader_options {
   // Whether to use the experimental reader
   bool _experimental = false;
 
+  // Whether to keep the quote characters of string values
+  bool _keep_quotes = false;
+
   /**
    * @brief Constructor from source info.
    *
@@ -203,6 +206,13 @@ class json_reader_options {
    */
   bool is_enabled_experimental() const { return _experimental; }
 
+  /**
+   * @brief Whether the experimental reader should keep quotes of string values.
+   *
+   * @returns true if the experimental reader should keep quotes, false otherwise
+   */
+  bool is_keeping_quotes() const { return _keep_quotes; }
+
   /**
    * @brief Set data types for columns to be read.
    *
@@ -258,6 +268,14 @@ class json_reader_options {
    * @param val Boolean value to enable/disable the experimental reader
    */
   void enable_experimental(bool val) { _experimental = val; }
+
+  /**
+   * @brief Set whether the experimental reader should keep quotes of string values.
+   *
+   * @param val Boolean value to indicate whether the experimental reader should keep quotes
+   * of string values
+   */
+  void keep_quotes(bool val) { _keep_quotes = val; }
 };
 
 /**
@@ -377,6 +395,19 @@ class json_reader_options_builder {
     return *this;
   }
 
+  /**
+   * @brief Set whether the experimental reader should keep quotes of string values.
+   *
+   * @param val Boolean value to indicate whether the experimental reader should keep quotes
+   * of string values
+   * @return this for chaining
+   */
+  json_reader_options_builder& keep_quotes(bool val)
+  {
+    options._keep_quotes = val;
+    return *this;
+  }
+
   /**
    * @brief move json_reader_options member once it's built.
    */

cpp/src/io/fst/agent_dfa.cuh

@@ -422,7 +422,7 @@ struct AgentDFA {
     OffsetT const num_total_symbols,
     StateIndexT& state,
     CallbackOpT& callback_op,
-    cub::Int2Type<BYPASS_LOAD> /**/)
+    cub::Int2Type<BYPASS_LOAD>)
   {
     using StateTransitionOpT = StateTransitionOp<CallbackOpT, TransitionTableT>;
 

cpp/src/io/json/nested_json.hpp

@@ -125,6 +125,7 @@ struct json_column {
   // Following "items" as the default child column's name of a list column
   // Using the struct's field names
   std::map<std::string, json_column> child_columns;
+  std::vector<std::string> column_order;
 
   // Counting the current number of items in this column
   row_offset_t current_offset = 0;
@@ -199,7 +200,21 @@ struct json_column {
                   uint32_t child_count)
   {
     // If, thus far, the column's type couldn't be inferred, we infer it to the given type
-    if (type == json_col_t::Unknown) { type = row_type; }
+    if (type == json_col_t::Unknown) {
+      type = row_type;
+    }
+    // If, at some point within a column, we encounter a nested type (list or struct),
+    // we change that columns type to that respective nested type and invalidate all previous rows
+    else if (type == json_col_t::StringColumn &&
+             (row_type == json_col_t::ListColumn || row_type == json_col_t::StructColumn)) {
+      // Change the column type
+      type = row_type;
+
+      // Invalidate all previous entries, as they were _not_ of the nested type to which we just
+      // converted
+      std::fill_n(validity.begin(), validity.size(), 0);
+      valid_count = 0U;
+    }
 
     // We shouldn't run into this, as we shouldn't be asked to append an "unknown" row type
     // CUDF_EXPECTS(type != json_col_t::Unknown, "Encountered invalid JSON token sequence");
@@ -216,13 +231,13 @@ struct json_column {
     // Struct   | List      => FAIL
     // Struct   | Struct    => valid
     // Struct   | String    => null
-    // String   | List      => null
-    // String   | Struct    => null
+    // String   | List      => valid (we switch col type to list, null'ing all previous rows)
+    // String   | Struct    => valid (we switch col type to list, null'ing all previous rows)
     // String   | String    => valid
     bool const is_valid = (type == row_type);
     if (static_cast<size_type>(validity.size()) < word_index(current_offset))
       validity.push_back({});
-    set_bit_unsafe(&validity.back(), intra_word_index(current_offset));
+    if (is_valid) { set_bit_unsafe(&validity.back(), intra_word_index(current_offset)); }
     valid_count += (is_valid) ? 1U : 0U;
     string_offsets.push_back(string_offset);
     string_lengths.push_back(string_end - string_offset);