From 095e11debd5c0dde27d716f1a55fe078223830dc Mon Sep 17 00:00:00 2001 From: Xin Qiu Date: Wed, 31 Jul 2019 13:44:00 +0800 Subject: [PATCH] Incremental Training for imagenet (#1391) --- python/dllib/src/bigdl/dllib/feature/common.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python/dllib/src/bigdl/dllib/feature/common.py b/python/dllib/src/bigdl/dllib/feature/common.py index c3ba2568ea3..4fc47c6f236 100644 --- a/python/dllib/src/bigdl/dllib/feature/common.py +++ b/python/dllib/src/bigdl/dllib/feature/common.py @@ -219,9 +219,13 @@ def image_frame(cls, image_frame, memory_type="DRAM", bigdl_type="float"): """ Create FeatureSet from ImageFrame. :param image_frame: ImageFrame - :param memory_type: string, DRAM or PMEM + :param memory_type: string, DRAM, PMEM or a Int number. If it's DRAM, will cache dataset into dynamic random-access memory If it's PMEM, will cache dataset into Intel Optane DC Persistent Memory + If it's a Int number n, will cache dataset into disk, and only hold 1/n + of the data into memory during the training. After going through the + 1/n, we will release the current cache, and load another 1/n into + memory. :param bigdl_type: numeric type :return: A feature set """ @@ -249,9 +253,13 @@ def rdd(cls, rdd, memory_type="DRAM", bigdl_type="float"): """ Create FeatureSet from RDD. :param rdd: A RDD - :param memory_type: string, DRAM or PMEM + :param memory_type: string, DRAM, PMEM or a Int number. If it's DRAM, will cache dataset into dynamic random-access memory If it's PMEM, will cache dataset into Intel Optane DC Persistent Memory + If it's a Int number n, will cache dataset into disk, and only hold 1/n + of the data into memory during the training. After going through the + 1/n, we will release the current cache, and load another 1/n into + memory. :param bigdl_type:numeric type :return: A feature set """