feat: add support for Non-interactive PoRep

This commit adds support for ni-porep.

It's a combination of the indivdual PRs #453, #456 and #458. Thanks everyone
involved working on those.

The changes are for the Go side (actors and FVM) as well as the Rust side
(proofs API).

Changes on the CI are:

 - Makes it match the Rust version in the rust-toolchain.toml
 - Newer Go version (1.21) on all platforms
 - Larger instance for the testsas the Ni-PoRep synthesis phase takes
   more resources.

Closes #453, #456, #458.

.circleci/config.yml

@@ -6,7 +6,7 @@ orbs:
 executors:
   golang:
     docker:
-      - image: cimg/go:1.20
+      - image: cimg/go:1.21
     resource_class: small
     environment:
       # Build the kernel only for the single architecture. This should reduce
@@ -56,7 +56,7 @@ jobs:
         type: boolean
         default: true
     machine:
-      image: ubuntu-2204:2023.07.2
+      image: default
     resource_class: arm.large
     working_directory: ~/go/src/github.com/filecoin-project/filecoin-ffi
     steps:
@@ -78,7 +78,7 @@ jobs:
         type: boolean
         default: true
     executor: golang
-    resource_class: medium
+    resource_class: medium+
     working_directory: ~/go/src/github.com/filecoin-project/filecoin-ffi
     steps:
       - configure_environment_variables
@@ -232,7 +232,7 @@ commands:
             - run:
                 name: Install Go
                 command: |
-                  curl https://dl.google.com/go/go1.20.10.darwin-arm64.pkg -o /tmp/go.pkg && \
+                  curl https://go.dev/dl/go1.21.11.darwin-arm64.pkg -o /tmp/go.pkg && \
                   sudo installer -pkg /tmp/go.pkg -target /
                   go version
             - run:
@@ -352,13 +352,13 @@ commands:
           no_output_timeout: 90m
       - run:
           name: Run the Go tests
-          command: GODEBUG=cgocheck=2 RUST_LOG=info go test -p 1 -timeout 60m
+          command: GOEXPERIMENT=cgocheck2 RUST_LOG=info go test -p 1 -timeout 60m
           no_output_timeout: 60m
   compile_tests:
     steps:
       - run:
           name: Build project and tests, but don't actually run the tests (used to verify that build/link works with Darwin)
-          command: GODEBUG=cgocheck=2 RUST_LOG=info go test -run=^$
+          command: GOEXPERIMENT=cgocheck2 RUST_LOG=info go test -run=^$
   restore_parameter_cache:
     steps:
       - restore_cache:

cgo/const.go

@@ -16,21 +16,26 @@ const (
 )
 
 const (
-	RegisteredSealProofStackedDrg2KiBV1                        = C.REGISTERED_SEAL_PROOF_STACKED_DRG2_KI_B_V1
-	RegisteredSealProofStackedDrg8MiBV1                        = C.REGISTERED_SEAL_PROOF_STACKED_DRG8_MI_B_V1
-	RegisteredSealProofStackedDrg512MiBV1                      = C.REGISTERED_SEAL_PROOF_STACKED_DRG512_MI_B_V1
-	RegisteredSealProofStackedDrg32GiBV1                       = C.REGISTERED_SEAL_PROOF_STACKED_DRG32_GI_B_V1
-	RegisteredSealProofStackedDrg64GiBV1                       = C.REGISTERED_SEAL_PROOF_STACKED_DRG64_GI_B_V1
-	RegisteredSealProofStackedDrg2KiBV11                       = C.REGISTERED_SEAL_PROOF_STACKED_DRG2_KI_B_V1_1
-	RegisteredSealProofStackedDrg8MiBV11                       = C.REGISTERED_SEAL_PROOF_STACKED_DRG8_MI_B_V1_1
-	RegisteredSealProofStackedDrg512MiBV11                     = C.REGISTERED_SEAL_PROOF_STACKED_DRG512_MI_B_V1_1
-	RegisteredSealProofStackedDrg32GiBV11                      = C.REGISTERED_SEAL_PROOF_STACKED_DRG32_GI_B_V1_1
-	RegisteredSealProofStackedDrg64GiBV11                      = C.REGISTERED_SEAL_PROOF_STACKED_DRG64_GI_B_V1_1
-	RegisteredSealProofStackedDrg2KiBV11_Feat_SyntheticPoRep   = C.REGISTERED_SEAL_PROOF_STACKED_DRG2_KI_B_V1_1__FEAT__SYNTHETIC_PO_REP
-	RegisteredSealProofStackedDrg8MiBV11_Feat_SyntheticPoRep   = C.REGISTERED_SEAL_PROOF_STACKED_DRG8_MI_B_V1_1__FEAT__SYNTHETIC_PO_REP
-	RegisteredSealProofStackedDrg512MiBV11_Feat_SyntheticPoRep = C.REGISTERED_SEAL_PROOF_STACKED_DRG512_MI_B_V1_1__FEAT__SYNTHETIC_PO_REP
-	RegisteredSealProofStackedDrg32GiBV11_Feat_SyntheticPoRep  = C.REGISTERED_SEAL_PROOF_STACKED_DRG32_GI_B_V1_1__FEAT__SYNTHETIC_PO_REP
-	RegisteredSealProofStackedDrg64GiBV11_Feat_SyntheticPoRep  = C.REGISTERED_SEAL_PROOF_STACKED_DRG64_GI_B_V1_1__FEAT__SYNTHETIC_PO_REP
+	RegisteredSealProofStackedDrg2KiBV1                              = C.REGISTERED_SEAL_PROOF_STACKED_DRG2_KI_B_V1
+	RegisteredSealProofStackedDrg8MiBV1                              = C.REGISTERED_SEAL_PROOF_STACKED_DRG8_MI_B_V1
+	RegisteredSealProofStackedDrg512MiBV1                            = C.REGISTERED_SEAL_PROOF_STACKED_DRG512_MI_B_V1
+	RegisteredSealProofStackedDrg32GiBV1                             = C.REGISTERED_SEAL_PROOF_STACKED_DRG32_GI_B_V1
+	RegisteredSealProofStackedDrg64GiBV1                             = C.REGISTERED_SEAL_PROOF_STACKED_DRG64_GI_B_V1
+	RegisteredSealProofStackedDrg2KiBV11                             = C.REGISTERED_SEAL_PROOF_STACKED_DRG2_KI_B_V1_1
+	RegisteredSealProofStackedDrg8MiBV11                             = C.REGISTERED_SEAL_PROOF_STACKED_DRG8_MI_B_V1_1
+	RegisteredSealProofStackedDrg512MiBV11                           = C.REGISTERED_SEAL_PROOF_STACKED_DRG512_MI_B_V1_1
+	RegisteredSealProofStackedDrg32GiBV11                            = C.REGISTERED_SEAL_PROOF_STACKED_DRG32_GI_B_V1_1
+	RegisteredSealProofStackedDrg64GiBV11                            = C.REGISTERED_SEAL_PROOF_STACKED_DRG64_GI_B_V1_1
+	RegisteredSealProofStackedDrg2KiBV11_Feat_SyntheticPoRep         = C.REGISTERED_SEAL_PROOF_STACKED_DRG2_KI_B_V1_1__FEAT__SYNTHETIC_PO_REP
+	RegisteredSealProofStackedDrg8MiBV11_Feat_SyntheticPoRep         = C.REGISTERED_SEAL_PROOF_STACKED_DRG8_MI_B_V1_1__FEAT__SYNTHETIC_PO_REP
+	RegisteredSealProofStackedDrg512MiBV11_Feat_SyntheticPoRep       = C.REGISTERED_SEAL_PROOF_STACKED_DRG512_MI_B_V1_1__FEAT__SYNTHETIC_PO_REP
+	RegisteredSealProofStackedDrg32GiBV11_Feat_SyntheticPoRep        = C.REGISTERED_SEAL_PROOF_STACKED_DRG32_GI_B_V1_1__FEAT__SYNTHETIC_PO_REP
+	RegisteredSealProofStackedDrg64GiBV11_Feat_SyntheticPoRep        = C.REGISTERED_SEAL_PROOF_STACKED_DRG64_GI_B_V1_1__FEAT__SYNTHETIC_PO_REP
+	RegisteredSealProofStackedDrg2KiBV1_2_Feat_NonInteractivePoRep   = C.REGISTERED_SEAL_PROOF_STACKED_DRG2_KI_B_V1_2__FEAT__NON_INTERACTIVE_PO_REP
+	RegisteredSealProofStackedDrg8MiBV1_2_Feat_NonInteractivePoRep   = C.REGISTERED_SEAL_PROOF_STACKED_DRG8_MI_B_V1_2__FEAT__NON_INTERACTIVE_PO_REP
+	RegisteredSealProofStackedDrg512MiBV1_2_Feat_NonInteractivePoRep = C.REGISTERED_SEAL_PROOF_STACKED_DRG512_MI_B_V1_2__FEAT__NON_INTERACTIVE_PO_REP
+	RegisteredSealProofStackedDrg32GiBV1_2_Feat_NonInteractivePoRep  = C.REGISTERED_SEAL_PROOF_STACKED_DRG32_GI_B_V1_2__FEAT__NON_INTERACTIVE_PO_REP
+	RegisteredSealProofStackedDrg64GiBV1_2_Feat_NonInteractivePoRep  = C.REGISTERED_SEAL_PROOF_STACKED_DRG64_GI_B_V1_2__FEAT__NON_INTERACTIVE_PO_REP
 )
 
 const (

cgo/proofs.go

@@ -130,6 +130,15 @@ func SealCommitPhase2(sealCommitPhase1Output SliceRefUint8, sectorId uint64, pro
 	return resp.value.copy(), nil
 }
 
+func SealCommitPhase2CircuitProofs(sealCommitPhase1Output SliceRefUint8, sectorId uint64) ([]byte, error) {
+	resp := C.seal_commit_phase2_circuit_proofs(sealCommitPhase1Output, C.uint64_t(sectorId))
+	defer resp.destroy()
+	if err := CheckErr(resp); err != nil {
+		return nil, err
+	}
+	return resp.value.copy(), nil
+}
+
 func AggregateSealProofs(registeredProof RegisteredSealProof, registeredAggregation RegisteredAggregationProof, commRs SliceRefByteArray32, seeds SliceRefByteArray32, sealCommitResponses SliceRefSliceBoxedUint8) ([]byte, error) {
 	resp := C.aggregate_seal_proofs(registeredProof, registeredAggregation, commRs, seeds, sealCommitResponses)
 	defer resp.destroy()

go.mod

@@ -5,7 +5,7 @@ go 1.18
 require (
 	github.com/filecoin-project/go-address v1.1.0
 	github.com/filecoin-project/go-fil-commcid v0.1.0
-	github.com/filecoin-project/go-state-types v0.13.1
+	github.com/filecoin-project/go-state-types v0.14.0-rc1
 	github.com/ipfs/go-block-format v0.0.3
 	github.com/ipfs/go-cid v0.3.2
 	github.com/ipfs/go-ipfs-blockstore v1.2.0

go.sum

@@ -9,8 +9,8 @@ github.com/filecoin-project/go-crypto v0.0.1 h1:AcvpSGGCgjaY8y1az6AMfKQWreF/pWO2
 github.com/filecoin-project/go-crypto v0.0.1/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-fil-commcid v0.1.0 h1:3R4ds1A9r6cr8mvZBfMYxTS88OqLYEo6roi+GiIeOh8=
 github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
-github.com/filecoin-project/go-state-types v0.13.1 h1:4CivvlcHAIoAtFFVVlZtokynaMQu5XLXGoTKhQkfG1I=
-github.com/filecoin-project/go-state-types v0.13.1/go.mod h1:cHpOPup9H1g2T29dKHAjC2sc7/Ef5ypjuW9A3I+e9yY=
+github.com/filecoin-project/go-state-types v0.14.0-rc1 h1:kWBGX/uqZmYotYMNmw+R/fIuot/k0KMcEtB7PKFy1SQ=
+github.com/filecoin-project/go-state-types v0.14.0-rc1/go.mod h1:cHpOPup9H1g2T29dKHAjC2sc7/Ef5ypjuW9A3I+e9yY=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=

proofs.go

@@ -414,6 +414,13 @@ func SealCommitPhase2(
 	return cgo.SealCommitPhase2(cgo.AsSliceRefUint8(phase1Output), uint64(sectorNum), &proverID)
 }
 
+// SealCommitPhase2CircuitProofs runs a non-interactive proof and returns the circuit proof bytes
+// rather than the aggregated proof bytes. This is used to aggregate multiple non-interactive
+// proofs as the aggregated single proof outputs can't further be aggregated.
+func SealCommitPhase2CircuitProofs(phase1Output []byte, sectorNum abi.SectorNumber) ([]byte, error) {
+	return cgo.SealCommitPhase2CircuitProofs(cgo.AsSliceRefUint8(phase1Output), uint64(sectorNum))
+}
+
 // TODO AggregateSealProofs it only needs InteractiveRandomness out of the aggregateInfo.Infos
 func AggregateSealProofs(aggregateInfo proof.AggregateSealVerifyProofAndInfos, proofs [][]byte) (out []byte, err error) {
 	sp, err := toFilRegisteredSealProof(aggregateInfo.SealProof)
@@ -1030,6 +1037,17 @@ func toFilRegisteredSealProof(p abi.RegisteredSealProof) (cgo.RegisteredSealProo
 	case abi.RegisteredSealProof_StackedDrg64GiBV1_1_Feat_SyntheticPoRep:
 		return cgo.RegisteredSealProofStackedDrg64GiBV11_Feat_SyntheticPoRep, nil
 
+	case abi.RegisteredSealProof_StackedDrg2KiBV1_2_Feat_NiPoRep:
+		return cgo.RegisteredSealProofStackedDrg2KiBV1_2_Feat_NonInteractivePoRep, nil
+	case abi.RegisteredSealProof_StackedDrg8MiBV1_2_Feat_NiPoRep:
+		return cgo.RegisteredSealProofStackedDrg8MiBV1_2_Feat_NonInteractivePoRep, nil
+	case abi.RegisteredSealProof_StackedDrg512MiBV1_2_Feat_NiPoRep:
+		return cgo.RegisteredSealProofStackedDrg512MiBV1_2_Feat_NonInteractivePoRep, nil
+	case abi.RegisteredSealProof_StackedDrg32GiBV1_2_Feat_NiPoRep:
+		return cgo.RegisteredSealProofStackedDrg32GiBV1_2_Feat_NonInteractivePoRep, nil
+	case abi.RegisteredSealProof_StackedDrg64GiBV1_2_Feat_NiPoRep:
+		return cgo.RegisteredSealProofStackedDrg64GiBV1_2_Feat_NonInteractivePoRep, nil
+
 	default:
 		return 0, errors.Errorf("no mapping to C.FFIRegisteredSealProof value available for: %v", p)
 	}

rust/Cargo.toml

@@ -31,8 +31,8 @@ rayon = "1.2.1"
 anyhow = "1.0.23"
 serde_json = "1.0.46"
 rust-gpu-tools = { version = "0.7", optional = true, default-features = false }
-fvm4 = { package = "fvm", version = "~4.2.0", default-features = false, features = ["nv23-dev"] }
-fvm4_shared = { package = "fvm_shared", version = "~4.2.0" }
+fvm4 = { package = "fvm", version = "~4.3.0", default-features = false, features = ["nv23-dev"] }
+fvm4_shared = { package = "fvm_shared", version = "~4.3.0" }
 fvm3 = { package = "fvm", version = "~3.9.0", default-features = false }
 fvm3_shared = { package = "fvm_shared", version = "~3.6.0" }
 fvm2 = { package = "fvm", version = "~2.7", default-features = false }
@@ -45,7 +45,7 @@ lazy_static = "1.4.0"
 serde = "1.0.117"
 serde_tuple = "0.5"
 safer-ffi = { version = "0.0.7", features = ["proc_macros"] }
-filecoin-proofs-api = { version = "17.0", default-features = false }
+filecoin-proofs-api = { version = "18.0", default-features = false }
 yastl = "0.1.2"
 
 [dev-dependencies]