Update PrefixIndexer.scala

To address Issue foursquare#53 (foursquare#53), I have wrapped prefix processing in a try/catch. I am not a scala programmer, so this may not be the best implementation. However, if we do not contain regex errors related to prefix processing, the processing fails, and corrupts the database/indices. Therefore, I feel like its worthwhile to trap the error here, and just skip over processing errors.
CorkHounds · Jul 12, 2018 · c1aab9e · c1aab9e
1 parent 5f133d7
commit c1aab9e
Showing 1 changed file with 22 additions and 18 deletions.
diff --git a/src/jvm/io/fsq/twofishes/indexer/output/PrefixIndexer.scala b/src/jvm/io/fsq/twofishes/indexer/output/PrefixIndexer.scala
@@ -112,32 +112,36 @@ class PrefixIndexer(
       if (index % 1000 == 0) {
         log.info("done with %d of %d prefixes".format(index, numPrefixes))
       }
-      val records = getRecordsByPrefix(prefix, PrefixIndexer.MaxNamesToConsider)
+      try {
+        val records = getRecordsByPrefix(prefix, PrefixIndexer.MaxNamesToConsider)
 
-      val (woeMatches, woeMismatches) = records.partition(r =>
-        bestWoeTypes.contains(r.woeTypeOrThrow))
+        val (woeMatches, woeMismatches) = records.partition(r =>
+          bestWoeTypes.contains(r.woeTypeOrThrow))
 
-      val (prefSortedRecords, unprefSortedRecords) =
-        sortRecordsByNames(woeMatches.toList)
+        val (prefSortedRecords, unprefSortedRecords) =
+          sortRecordsByNames(woeMatches.toList)
 
-      val fids = new HashSet[StoredFeatureId]
-      //roundRobinByCountryCode(prefSortedRecords).foreach(f => {
-      prefSortedRecords.foreach(f => {
-        if (fids.size < PrefixIndexer.MaxFidsToStorePerPrefix) {
-          fids.add(f.fidAsFeatureId)
-        }
-      })
-
-      if (fids.size < PrefixIndexer.MaxFidsWithPreferredNamesBeforeConsideringNonPreferred) {
-        //roundRobinByCountryCode(unprefSortedRecords).foreach(f => {
-        unprefSortedRecords.foreach(f => {
+        val fids = new HashSet[StoredFeatureId]
+        //roundRobinByCountryCode(prefSortedRecords).foreach(f => {
+        prefSortedRecords.foreach(f => {
           if (fids.size < PrefixIndexer.MaxFidsToStorePerPrefix) {
             fids.add(f.fidAsFeatureId)
           }
         })
-      }
 
-      prefixWriter.append(prefix, fidsToCanonicalFids(fids.toList))
+        if (fids.size < PrefixIndexer.MaxFidsWithPreferredNamesBeforeConsideringNonPreferred) {
+          //roundRobinByCountryCode(unprefSortedRecords).foreach(f => {
+          unprefSortedRecords.foreach(f => {
+            if (fids.size < PrefixIndexer.MaxFidsToStorePerPrefix) {
+              fids.add(f.fidAsFeatureId)
+            }
+          })
+        }
+
+        prefixWriter.append(prefix, fidsToCanonicalFids(fids.toList))
+      } catch {
+         case e: Exception => println("Skipping due to error processing prefixes")
+      }
     }
 
     prefixWriter.close()