diff --git a/src/jvm/io/fsq/twofishes/indexer/output/PrefixIndexer.scala b/src/jvm/io/fsq/twofishes/indexer/output/PrefixIndexer.scala index 6b4e76be..d5457cbb 100644 --- a/src/jvm/io/fsq/twofishes/indexer/output/PrefixIndexer.scala +++ b/src/jvm/io/fsq/twofishes/indexer/output/PrefixIndexer.scala @@ -112,32 +112,36 @@ class PrefixIndexer( if (index % 1000 == 0) { log.info("done with %d of %d prefixes".format(index, numPrefixes)) } - val records = getRecordsByPrefix(prefix, PrefixIndexer.MaxNamesToConsider) + try { + val records = getRecordsByPrefix(prefix, PrefixIndexer.MaxNamesToConsider) - val (woeMatches, woeMismatches) = records.partition(r => - bestWoeTypes.contains(r.woeTypeOrThrow)) + val (woeMatches, woeMismatches) = records.partition(r => + bestWoeTypes.contains(r.woeTypeOrThrow)) - val (prefSortedRecords, unprefSortedRecords) = - sortRecordsByNames(woeMatches.toList) + val (prefSortedRecords, unprefSortedRecords) = + sortRecordsByNames(woeMatches.toList) - val fids = new HashSet[StoredFeatureId] - //roundRobinByCountryCode(prefSortedRecords).foreach(f => { - prefSortedRecords.foreach(f => { - if (fids.size < PrefixIndexer.MaxFidsToStorePerPrefix) { - fids.add(f.fidAsFeatureId) - } - }) - - if (fids.size < PrefixIndexer.MaxFidsWithPreferredNamesBeforeConsideringNonPreferred) { - //roundRobinByCountryCode(unprefSortedRecords).foreach(f => { - unprefSortedRecords.foreach(f => { + val fids = new HashSet[StoredFeatureId] + //roundRobinByCountryCode(prefSortedRecords).foreach(f => { + prefSortedRecords.foreach(f => { if (fids.size < PrefixIndexer.MaxFidsToStorePerPrefix) { fids.add(f.fidAsFeatureId) } }) - } - prefixWriter.append(prefix, fidsToCanonicalFids(fids.toList)) + if (fids.size < PrefixIndexer.MaxFidsWithPreferredNamesBeforeConsideringNonPreferred) { + //roundRobinByCountryCode(unprefSortedRecords).foreach(f => { + unprefSortedRecords.foreach(f => { + if (fids.size < PrefixIndexer.MaxFidsToStorePerPrefix) { + fids.add(f.fidAsFeatureId) + } + }) + } + + prefixWriter.append(prefix, fidsToCanonicalFids(fids.toList)) + } catch { + case e: Exception => println("Skipping due to error processing prefixes") + } } prefixWriter.close()