Rename filesToRoiImageFrame to filesToRoiImageFeatures (intel-analyti…

…cs#2949) * Update the RoiLabel related files from Sequence-file related PR * var -> val * Bug fix for curBatchSize < batchSize. toRGB default to false * add ROISIZE * update documents * update documents * add UT * fix document * filesToRoiImageFrame -> filesToRoiImageFeatures, to public
dding3 · Oct 29, 2019 · 3fa2ea4 · 3fa2ea4
1 parent 1ab4e07
commit 3fa2ea4
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 4 deletions.
diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/dataset/DataSet.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/dataset/DataSet.scala
@@ -598,13 +598,13 @@ object DataSet {
       false
     }
     /**
-     * Extract hadoop sequence files from an HDFS path as ImageFrame
+     * Extract hadoop sequence files from an HDFS path as ImageFeatures
      * @param url sequence files folder path
      * @param sc spark context
      * @param partitionNum partition number, default: Engine.nodeNumber() * Engine.coreNumber()
      * @return
      */
-    private[bigdl] def filesToRoiImageFrame(url: String, sc: SparkContext,
+    def filesToRoiImageFeatures(url: String, sc: SparkContext,
       partitionNum: Option[Int] = None): DataSet[ImageFeature] = {
       val num = partitionNum.getOrElse(Engine.nodeNumber() * Engine.coreNumber())
       val rawData = sc.sequenceFile(url, classOf[BytesWritable], classOf[BytesWritable], num)

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/maskrcnn/Test.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/maskrcnn/Test.scala
@@ -62,7 +62,8 @@ object Test {
       val partitionNum = if (param.partitionNum > 0) param.partitionNum
       else Engine.nodeNumber() * Engine.coreNumber()
 
-      val rddData = DataSet.SeqFileFolder.filesToRoiImageFrame(param.folder, sc, Some(partitionNum))
+      val rddData = DataSet.SeqFileFolder.filesToRoiImageFeatures(param.folder,
+        sc, Some(partitionNum))
         .toDistributed().data(train = false)
 
       val transformer = MTImageFeatureToBatchWithResize(

diff --git a/spark/dl/src/test/scala/com/intel/analytics/bigdl/dataset/DataSetSpec.scala b/spark/dl/src/test/scala/com/intel/analytics/bigdl/dataset/DataSetSpec.scala
@@ -95,7 +95,7 @@ class DataSetSpec extends SparkContextLifeCycle with Matchers {
       "-b", "2", "-m", dataSetFolder + "cocomini.json"))
 
     // write done, now read and check
-    DataSet.SeqFileFolder.filesToRoiImageFrame(tmpFile.getPath, sc).toDistributed()
+    DataSet.SeqFileFolder.filesToRoiImageFeatures(tmpFile.getPath, sc).toDistributed()
       .data(false)
       .map(imf => {
         (imf(ImageFeature.uri).asInstanceOf[String], imf.getOriginalSize, imf.getLabel[RoiLabel],