Experimental market migration memory reduction (#260)

* 1000x less memory held during market migration

* Review Response

---------

Co-authored-by: zenground0 <ZenGround0@users.noreply.github.com>

builtin/v13/migration/market.go

@@ -309,10 +309,22 @@ func (m *marketMigrator) migrateProviderSectorsAndStatesFromScratch(ctx context.
 		return cid.Undef, cid.Undef, xerrors.Errorf("failed to load proposals array: %w", err)
 	}
 
+	proposalsSize := proposalsArr.Length()
+
 	var oldState market12.DealState
 	var newState market13.DealState
-
+	var cnt int
 	err = oldStateArray.ForEach(&oldState, func(i int64) error {
+		// Refresh proposals array periodically to avoid holding onto all ~10GB of memory
+		// throughout whole migration.
+		// This has limited impact on caching speedups because the access pattern is sequential.
+		cnt += 1
+		if proposalsSize > 0 && cnt%(int(proposalsSize/1000)) == 0 {
+			proposalsArr, err = adt.AsArray(ctxStore, proposals, market12.ProposalsAmtBitwidth)
+			if err != nil {
+				return xerrors.Errorf("failed to load proposals array: %w", err)
+			}
+		}
 		deal := abi.DealID(i)
 
 		newState.SlashEpoch = oldState.SlashEpoch