Trigger next release (#76)

* [maven-release-plugin] prepare release 3.0.7

* [maven-release-plugin] prepare for next development iteration

* Add new user role for authenticated users. (#71)

* Add new user role for authenticated users.
* Accept the isAuthenticated() keyword also for admins

* Add support for access groups to authenticate users and admins (#72)

* Add new user role for authenticated users.
* Accept the isAuthenticated() keyword also for admins
* Add support for access groups to authenticate users and admins.

Internal ticket ID: DPSTAT-726

* Add endpoint to check fnr for missing sid mapping (#73)

* Add endpoint to check fnr for missing sid mapping

* Add 'keycloak_token' field in README

* Add remoted debug run configuration

---------

Co-authored-by: dapla-bot[bot] <143391972+dapla-bot[bot]@users.noreply.github.com>
Co-authored-by: Michael Moen Allport <mmallport@gmail.com>

conf/application-local.yml

@@ -41,6 +41,13 @@ micronaut:
             keycloak-staging:
               url: 'https://keycloak.staging-bip-app.ssb.no/auth/realms/ssb/protocol/openid-connect/certs'
 
+  http:
+    services:
+      cloud-identity-service:
+        url: 'https://cloudidentity.googleapis.com'
+        path: '/v1'
+        read-timeout: 60s
+
   object-storage:
     gcp:
       sid:
@@ -63,6 +70,19 @@ gcp:
       - gcp-kms://projects/dev-sirius/locations/europe-north1/keyRings/pseudo-service-common-keyring/cryptoKeys/pseudo-service-common-kek-1
       - gcp-kms://projects/dev-sirius/locations/europe-north1/keyRings/crypto-keyring/cryptoKeys/crypto-key
 
+  http:
+    client:
+      filter:
+        services:
+          cloud-identity-service:
+            audience: "https://www.googleapis.com/auth/cloud-identity.groups.readonly"
+
+        credentials-path: private/gcp/sa-keys/dev-dapla-pseudo-service-test-sa-key.json
+
+  caches:
+    cloud-identity-service-cache:
+      expire-after-write: 30s
+
 sid:
   mapping.filename: "freg-snr/snr-kat-latest"
   index.filename: "sid/index"
@@ -97,10 +117,13 @@ pseudo.secrets:
     type: TINK_WDEK
 
 app-roles:
+  users:
+    - isAuthenticated()
   admins:
     - kons_schu@ssb.no
-    - kons-skaar@ssb.no
     - kons-lunde@ssb.no
     - mmw@ssb.no
     - mic@ssb.no
 
+  admins-group: pseudo-service-user-dev@ssb.no
+

conf/idea/runConfigurations/pseudo-service-remote-debug.run.xml

@@ -0,0 +1,16 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="pseudo-service-remote-debug" type="Remote">
+    <module name="dapla-dlp-pseudo-service" />
+    <option name="USE_SOCKET_TRANSPORT" value="true" />
+    <option name="SERVER_MODE" value="false" />
+    <option name="SHMEM_ADDRESS" />
+    <option name="HOST" value="localhost" />
+    <option name="PORT" value="5005" />
+    <option name="AUTO_RESTART" value="false" />
+    <RunnerSettings RunnerId="Debug">
+      <option name="DEBUG_PORT" value="5005" />
+      <option name="LOCAL" value="false" />
+    </RunnerSettings>
+    <method v="2" />
+  </configuration>
+</component>

doc/requests/README.md

@@ -12,6 +12,7 @@ the following structure:
 {
   "local": {
     "base_url": "localhost:10210",
+    "keycloak_token": "..."
   },
   "staging": {
     "base_url": "localhost:10210",

doc/requests/examples-sid.http

@@ -26,6 +26,19 @@ Content-Type: application/json
 Authorization: Bearer {{keycloak_token}}
 
 
+### Look up SID for a list of FNRs and return those that are not found
+
+POST {{base_url}}/sid/lookup/batch
+Content-Type: application/json
+Authorization: Bearer {{keycloak_token}}
+
+{
+  "fnrList": [
+    "20859374701",
+    "01234567890"
+  ]
+}
+
 ### Pseudonymize using SID mapping
 
 POST {{base_url}}/pseudonymize/file

pom.xml

@@ -3,7 +3,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>no.ssb.dapla.dlp.pseudo</groupId>
   <artifactId>dapla-dlp-pseudo-service</artifactId>
-  <version>3.0.7-SNAPSHOT</version>
+  <version>3.0.8-SNAPSHOT</version>
   <name>dapla-dlp-pseudo-service</name>
 
   <parent>

src/main/java/no/ssb/dlp/pseudo/service/accessgroups/CloudIdentityClient.java

@@ -0,0 +1,37 @@
+package no.ssb.dlp.pseudo.service.accessgroups;
+
+import io.micronaut.http.annotation.Get;
+import io.micronaut.http.client.annotation.Client;
+import io.micronaut.scheduling.TaskExecutors;
+import io.micronaut.scheduling.annotation.ExecuteOn;
+import no.ssb.dlp.pseudo.service.filters.AccessTokenFilterMatcher;
+import org.reactivestreams.Publisher;
+
+import javax.annotation.Nullable;
+
+@Client(id="cloud-identity-service")
+@AccessTokenFilterMatcher()
+public interface CloudIdentityClient {
+
+    /**
+     * Lookup a group by its email address.
+     * See: https://cloud.google.com/identity/docs/reference/rest/v1/groups/lookup
+     * @param groupKeyId the email address of the group
+     * @return a {@link Publisher} of {@link LookupResponse}
+     */
+    @Get( "/groups:lookup?groupKey.id={groupKeyId}")
+    @ExecuteOn(TaskExecutors.IO)
+    Publisher<LookupResponse> lookup(String groupKeyId);
+
+    /**
+     * List all members of a group.
+     * See: https://cloud.google.com/identity/docs/reference/rest/v1/groups.memberships/list
+     *
+     * @param groupId the id of the group
+     * @param pageToken for pagination
+     * @return a {@link Publisher} of {@link MembershipResponse}
+     */
+    @Get( "/groups/{groupId}/memberships?pageToken={pageToken}")
+    @ExecuteOn(TaskExecutors.IO)
+    Publisher<MembershipResponse> listMembers(String groupId, @Nullable String pageToken);
+}

src/main/java/no/ssb/dlp/pseudo/service/accessgroups/CloudIdentityService.java

@@ -0,0 +1,46 @@
+package no.ssb.dlp.pseudo.service.accessgroups;
+
+import io.micronaut.cache.annotation.Cacheable;
+import io.reactivex.Flowable;
+import lombok.RequiredArgsConstructor;
+
+import javax.inject.Singleton;
+import java.util.ArrayList;
+import java.util.List;
+
+@Singleton
+@RequiredArgsConstructor
+public class CloudIdentityService {
+    private final CloudIdentityClient cloudIdentityClient;
+
+    @Cacheable(value = "cloud-identity-service-cache", parameters = {"groupEmail"})
+    public List<Membership> listMembers(String groupEmail) {
+        return Flowable.fromPublisher(cloudIdentityClient.lookup(groupEmail))
+                .flatMap(lookupResponse -> fetchMemberships(lookupResponse.getGroupName(), null,
+                        new ArrayList<>()))
+                .blockingFirst();
+    }
+
+    /**
+     * Paginate through all memberships of a group.
+     *
+     * @param groupId the id of the group
+     * @param nextPageToken a token for pagination (will be null on first call)
+     * @param allMemberships a list that will be populated with all memberships
+     * @return the list of all memberships
+     */
+    private Flowable<List<Membership>> fetchMemberships(String groupId, String nextPageToken,
+                                                                   List<Membership> allMemberships) {
+        if (groupId == null || groupId.isEmpty()) {
+            return Flowable.just(allMemberships);
+        }
+        return Flowable.fromPublisher(cloudIdentityClient.listMembers(groupId, nextPageToken))
+                .flatMap(membershipResponse -> {
+                    allMemberships.addAll(membershipResponse.getMemberships());
+                    String nextToken = membershipResponse.getNextPageToken();
+                    return nextToken != null ?
+                            fetchMemberships(groupId, nextToken, allMemberships) :
+                            Flowable.just(allMemberships);
+                });
+    }
+}

src/main/java/no/ssb/dlp/pseudo/service/accessgroups/EntityKey.java

@@ -0,0 +1,18 @@
+package no.ssb.dlp.pseudo.service.accessgroups;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.extern.jackson.Jacksonized;
+
+/**
+ * A unique identifier for an entity in the Cloud Identity Groups API.
+ * An entity can represent either a group with an optional namespace or a user without a namespace. The combination of
+ * id and namespace must be unique; however, the same id can be used with different namespaces.
+ */
+@Data
+@Builder
+@Jacksonized
+public class EntityKey {
+    private final String id;
+    private final String namespace;
+}

src/main/java/no/ssb/dlp/pseudo/service/accessgroups/LookupResponse.java

@@ -0,0 +1,17 @@
+package no.ssb.dlp.pseudo.service.accessgroups;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.extern.jackson.Jacksonized;
+
+@Data
+@Builder
+@Jacksonized
+public class LookupResponse {
+    // The resource name of the looked-up Group.
+    private final String name;
+
+    public String getGroupName() {
+        return name != null ? name.substring(name.lastIndexOf('/') + 1) : null;
+    }
+}

src/main/java/no/ssb/dlp/pseudo/service/accessgroups/Membership.java

@@ -0,0 +1,17 @@
+package no.ssb.dlp.pseudo.service.accessgroups;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.extern.jackson.Jacksonized;
+
+/**
+ * A membership within the Cloud Identity Groups API.
+ * A Membership defines a relationship between a Group and an entity belonging to that Group, referred to as a "member".
+ */
+@Data
+@Builder
+@Jacksonized
+public class Membership {
+    private final String name;
+    private final EntityKey preferredMemberKey;
+}

src/main/java/no/ssb/dlp/pseudo/service/accessgroups/MembershipResponse.java

@@ -0,0 +1,15 @@
+package no.ssb.dlp.pseudo.service.accessgroups;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.extern.jackson.Jacksonized;
+
+import java.util.List;
+
+@Data
+@Builder
+@Jacksonized
+public class MembershipResponse {
+    private final List<Membership> memberships;
+    private final String nextPageToken;
+}

src/main/java/no/ssb/dlp/pseudo/service/filters/AccessTokenFilter.java

@@ -0,0 +1,93 @@
+package no.ssb.dlp.pseudo.service.filters;
+
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.GoogleCredentials;
+import io.micronaut.context.ApplicationContext;
+import io.micronaut.context.annotation.Value;
+import io.micronaut.http.HttpResponse;
+import io.micronaut.http.MutableHttpRequest;
+import io.micronaut.http.filter.ClientFilterChain;
+import io.micronaut.http.filter.HttpClientFilter;
+import io.micronaut.inject.qualifiers.Qualifiers;
+import lombok.Data;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.reactivestreams.Publisher;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import java.io.FileInputStream;
+import java.net.URI;
+import java.util.Optional;
+
+/**
+ * This filter will obtain an {@link AccessToken} and add it to the request. It can use credentials from either
+ * Google's default Application Default Credentials or a custom Service Account (as opposed to the
+ * {@see io.micronaut.gcp.http.client.GoogleAuthFilter} which only uses the Compute metadata server).
+ */
+@AccessTokenFilterMatcher
+@Singleton
+@Data
+@Slf4j
+public class AccessTokenFilter implements HttpClientFilter {
+
+    @Inject
+    private ApplicationContext applicationContext;
+    @Nullable
+    @Value("${gcp.http.client.filter.project-id}")
+    private String projectId;
+    private final GoogleCredentials credentials;
+
+    @SneakyThrows
+    public AccessTokenFilter(@Nullable @Value("${gcp.http.client.filter.credentials-path}") String credentialsPath) {
+        if (credentialsPath == null) {
+            log.info("Using Application Default Credentials");
+            this.credentials = GoogleCredentials.getApplicationDefault();
+        } else {
+            log.info("Using Credentials from Service Account file: {}", credentialsPath);
+            this.credentials = GoogleCredentials.fromStream(
+                    new FileInputStream(credentialsPath));
+        }
+    }
+
+    @SneakyThrows
+    public Publisher<? extends HttpResponse<?>> doFilter(MutableHttpRequest<?> request, ClientFilterChain chain) {
+        Optional<AccessTokenFilterConfig> config = getConfig(request);
+        if (config.isPresent()) {
+            request.bearerAuth(getAccessToken(config.get().getAudience()));
+            setProjectIdHeader(request);
+        } else {
+            request.bearerAuth(getAccessToken(getAudienceFromRequest(request)));
+            setProjectIdHeader(request);
+        }
+        return chain.proceed(request);
+    }
+
+    private void setProjectIdHeader(MutableHttpRequest<?> request) {
+        if (projectId != null) {
+            log.debug("Using projectId {} from config to override qoutaProjectId", projectId);
+            request.getHeaders().add("x-goog-user-project", projectId);
+        }
+    }
+
+    @SneakyThrows
+    private String getAccessToken(String audience) {
+        return credentials.createScoped(audience).refreshAccessToken().getTokenValue();
+    }
+
+    private Optional<AccessTokenFilterConfig> getConfig(MutableHttpRequest<?> request) {
+        final Optional<Object> serviceId = request.getAttribute("micronaut.http.serviceId");
+
+        if (applicationContext != null && serviceId.isPresent()) {
+            return applicationContext.findBean(AccessTokenFilterConfig.class, Qualifiers.byName(serviceId.get().toString()));
+        }
+        return Optional.empty();
+    }
+
+    private String getAudienceFromRequest(final MutableHttpRequest<?> request) {
+        URI fullURI = request.getUri();
+        return fullURI.getScheme() + "://" + fullURI.getHost();
+    }
+
+}

src/main/java/no/ssb/dlp/pseudo/service/filters/AccessTokenFilterConfig.java

@@ -0,0 +1,32 @@
+package no.ssb.dlp.pseudo.service.filters;
+
+import io.micronaut.context.annotation.EachProperty;
+import io.micronaut.context.annotation.Parameter;
+import lombok.Data;
+
+/**
+ * Creates a GoogleAuthServiceConfig for each Service configured under
+ * gcp.http.client.auth.services.*.audience. The audience can be configured per
+ * service and the correct config bean is selected in {@code AccessTokenFilter} via the service id
+ * inside the corresponding request.
+ *
+ * Requires the user to set the {@code gcp.http.client.auth.services.*.audience} property with the
+ * desired audience to create the corresponding config bean.
+ *
+ */
+@EachProperty(AccessTokenFilterConfig.PREFIX)
+@Data
+public class AccessTokenFilterConfig {
+    public static final String PREFIX = "gcp.http.client.filter.services";
+
+    private final String serviceId;
+
+    public AccessTokenFilterConfig(@Parameter String serviceId) {
+        this.serviceId = serviceId;
+    }
+
+    /**
+     * @param audience set the desired audience
+     */
+    private String audience;
+}

src/main/java/no/ssb/dlp/pseudo/service/filters/AccessTokenFilterMatcher.java

@@ -0,0 +1,7 @@
+package no.ssb.dlp.pseudo.service.filters;
+
+import io.micronaut.http.annotation.FilterMatcher;
+
+@FilterMatcher
+public @interface AccessTokenFilterMatcher {
+}