diff --git a/.gitignore b/.gitignore index 6704566..105a94b 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,5 @@ dist # TernJS port file .tern-port + +deploy/*/charts \ No newline at end of file diff --git a/.helmdocsignore b/.helmdocsignore new file mode 100644 index 0000000..dd512bf --- /dev/null +++ b/.helmdocsignore @@ -0,0 +1 @@ +deploy/*/charts \ No newline at end of file diff --git a/README.md.gotmpl b/README.md.gotmpl new file mode 100644 index 0000000..da7f270 --- /dev/null +++ b/README.md.gotmpl @@ -0,0 +1,95 @@ +# magda-function-esri-url-processor + +This is Magda ESRI URL processor (a serverless function) created from [this template repo](https://github.com/magda-io/magda-function-template) + +The url processor is used by dataset metadata creation tool to extract metadata from a ESRI API URL. + +Requirement can be found [here](https://github.com/magda-io/magda/issues/2810) + +### Function Spec + +The function source code can be found from [here](./src/index.ts). + +The function is defined as below: + +```typescript +export type UrlProcessorResult = { + dataset: Record; + distributions: Record[]; +}; + +export default async function myFunction( + input: string +): Promise; +``` + +It expects an url string as input and output an `UrlProcessorResult` type data. + +### Install Project Dependencies + +```bash +yarn install +``` + +### Build & Run Function in Minikube + +- Deploy Magda v0.0.57-0 or later +- Build the function + - Run `yarn build` +- Push docker image to minikube + - Run `eval $(minikube docker-env)` + - Run `yarn docker-build-local` +- Deploy function to Minikube + - Make sure `namespacePrefix` field in [`deploy/minikube-dev.yaml`](./deploy/minikube-dev.yaml) contains correct `magda-core` deploy namespace. By default, it's `default` and it works if you've deployed Magda to `default` namespace. + - Run `yarn deploy-local` +- Invoke your Function: + - Install [`faas-cli`](https://github.com/openfaas/faas-cli) + - Run `kubectl --namespace=[openfaas gateway namespace] port-forward svc/gateway 8080` to port-forward openfaas gateway + - Here, [openfaas gateway namespace] is `[magda-core namespace]-openfaas`. e.g. if magda is deployed to `default` namespace, `[openfaas gateway namespace]` would be `default-openfaas` + - Invoke by Run `echo "" | faas-cli faas-cli invoke magda-function-esri-url-processor` + - Alternatively, you can use [Postman](https://www.postman.com/) to send a HTTP Request (HTTP method doesn't matter here) to Magda gateway `/api/v0/openfaas/function/magda-function-esri-url-processor` + +### Deploy with Magda + +- Add as Magda dependencies: + +```yaml +- name: magda-function-esri-url-processor + version: 0.0.57-0 + repository: https://charts.magda.io + tags: + - all + - url-processors + - ckan-connector-functions +``` + +- Run `helm dep build` to pull the dependency +- Deploy Magda + +### Verify the function is deployed + +- Method One: + - Access Magda Gateway: `/api/v0/openfaas/system/function` with your web browser + - You might need Admin access to access this endpoint. However, you can disable the admin auth in Magda config. +- Method Two: + - Run `kubectl --namespace=[openfaas function namespace] get functions` + - Here, [openfaas function namespace] is `[magda-core namespace]-openfaas-fn`. e.g. if magda is deployed to `default` namespace, `[openfaas function namespace]` would be `default-openfaas-fn` + +> If the `Scale to Zero` option is set for the function (it's set to true by default), you won't see function pod in openfaas function namespace until you invoke the function + +### CI Setup + +This repo comes with script to build, test & release script to release docker image & helm chart to Magda repo. You need to setup the following Github action secrets to make it work: + +- `AWS_ACCESS_KEY_ID`: Magda helm chart repo S3 bucket access key +- `AWS_SECRET_ACCESS_KEY`: Magda helm chart repo S3 bucket access key secret +- `DOCKER_HUB_PASSWORD`: Magda docker hub bot password +- `GITHUB_ACCESS_TOKEN`: Magda github bot access token + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesHeader" . }} + +{{ template "chart.valuesTable" . }} \ No newline at end of file diff --git a/deploy/magda-function-esri-url-processor/Chart.lock b/deploy/magda-function-esri-url-processor/Chart.lock new file mode 100644 index 0000000..0355bf1 --- /dev/null +++ b/deploy/magda-function-esri-url-processor/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: magda-common + repository: https://charts.magda.io + version: 1.0.0-alpha.4 +digest: sha256:a5dcf2df16ca5a3972f92d91434c2e78be0ce411994d2a3cb89c674a711ddc24 +generated: "2021-10-11T22:02:04.486075+11:00" diff --git a/deploy/magda-function-esri-url-processor/Chart.yaml b/deploy/magda-function-esri-url-processor/Chart.yaml index d87411b..1f53e20 100644 --- a/deploy/magda-function-esri-url-processor/Chart.yaml +++ b/deploy/magda-function-esri-url-processor/Chart.yaml @@ -5,3 +5,9 @@ version: "0.0.57-0" kubeVersion: ">= 1.14.0-0" home: "https://github.com/magda-io/magda-function-esri-url-processor" sources: ["https://github.com/magda-io/magda-function-esri-url-processor"] +annotations: + magdaModuleType: "urlProcessor" +dependencies: + - name: magda-common + version: "1.0.0-alpha.4" + repository: "https://charts.magda.io" \ No newline at end of file diff --git a/deploy/magda-function-esri-url-processor/templates/function.yaml b/deploy/magda-function-esri-url-processor/templates/function.yaml index 74390b4..46c0028 100644 --- a/deploy/magda-function-esri-url-processor/templates/function.yaml +++ b/deploy/magda-function-esri-url-processor/templates/function.yaml @@ -12,7 +12,7 @@ metadata: spec: name: {{ .Chart.Name }} handler: node bootstrap.js - image: "{{ .Values.image.repository | default .Values.global.urlProcessors.image.repository | default .Values.global.image.repository | default .Values.defaultImage.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag | default .Values.global.urlProcessors.image.tag | default .Values.global.image.tag | default .Values.defaultImage.tag }}" + image: {{ include "magda.image" . | quote }} labels: com.openfaas.scale.zero: "true" # You can add any other labels to help the caller to tell the purpose of the functions @@ -24,23 +24,15 @@ spec: read_timeout: 65s write_timeout: 65s exec_timeout: 60s -{{- /* -See chart value file for details of the logic used to generate this setting value below.!!! -*/}} -{{- $imagePullSecret := (ne (.Values.image.imagePullSecret | typeOf) "") | ternary .Values.image.imagePullSecret ( (ne (.Values.global.urlProcessors.image.imagePullSecret | typeOf) "") | ternary .Values.global.urlProcessors.image.imagePullSecret ( (ne (.Values.global.image.imagePullSecret | typeOf) "") | ternary .Values.global.image.imagePullSecret .Values.defaultImage.imagePullSecret ) ) -}} -{{- if .Values.secrets }} - secrets: - {{- range .Values.secrets }} - - {{.}} - {{- end }} - {{- if ne ($imagePullSecret | toString) "false" }} - - {{ $imagePullSecret }} + {{- $pullSecrets := include "magda.image.getConsolidatedPullSecretList" . | mustFromJson }} + {{- if and (.Values.secrets | empty | not) (kindIs "slice" .Values.secrets) }} + {{- $pullSecrets = concat $pullSecrets .Values.secrets }} {{- end }} -{{- else }} - {{- if ne ($imagePullSecret | toString) "false" }} + {{- if (not (empty $pullSecrets)) }} secrets: - - {{ $imagePullSecret }} + {{- range $pullSecrets }} + - {{ . | quote }} + {{- end }} {{- end }} -{{- end }} {{ .Values.resources | toYaml | indent 2 -}} {{- end }}