forked from Yelp/detect-secrets
-
Notifications
You must be signed in to change notification settings - Fork 45
/
Makefile.ibm
202 lines (162 loc) · 9.2 KB
/
Makefile.ibm
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
SHELL := /bin/bash
MAKEFLAGS += --warn-undefined-variables
# Docker related param
DEBUG_IMAGE_TAG := $(TRAVIS_BRANCH)-$(TRAVIS_BUILD_NUMBER)-id-$(TRAVIS_BUILD_ID)-time-$(shell date +%s)
DOCKER_DOMAIN := git-defenders
DOCKER_REGISTRY_ICR := icr.io
DOCKER_USER_ICR := iamapikey
DOCKER_PASS_ICR := $(IBM_CLOUD_API_KEY)
# DOCKER_REGISTRY_ART := txo-toolbox-team-docker-local.artifactory.swg-devops.com
# DOCKER_USER_ART := $(ART_USER_ID)
# DOCKER_PASS_ART := $(ART_API_KEY)
DOCKER_IMAGES_TO_TAG := detect-secrets detect-secrets-hook
DOCKER_IMAGES_TO_SCAN := detect-secrets detect-secrets-hook detect-secrets:redhat-ubi detect-secrets:redhat-ubi-custom
DOCKER_IMAGES_TO_PUBLISH :=
DOCKER_REGISTRIES := $(DOCKER_REGISTRY_ICR)
IMAGE_NAME :=
DOCKER_REGISTRY :=
DOCKER_DOMAIN_LOCAL := $(DOCKER_DOMAIN)
# COS related param
COS_REGION := us-south
COS_BUCKET := detect-secrets-client-version
COS_FILE := version
# sync related param
# Do not know branch, then figure it out using git
TRAVIS_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
# Test related param
# Extra pytest options such as "-k keyword" to restrict the number of test cases to run
EXTRA_PYTEST_OPTIONS :=
# Trivy related
TRIVY ?= /tmp/trivy
TRIVY_VERSION := $(shell curl -s "https://api.github.com/repos/aquasecurity/trivy/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/')
TRIVY_OS := $(shell uname | sed 's/Darwin/macOS/' )
TRIVY_ARCH := $(shell uname -m | cut -d_ -f2 )
# Cosign related
COSIGN ?= /tmp/cosign
COSIGN_VERSION := $(shell curl -s "https://api.github.com/repos/sigstore/cosign/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/')
TRAVIS_COMMIT ?= head
setup-trivy:
curl -sSfL https://github.com/aquasecurity/trivy/releases/download/v$(TRIVY_VERSION)/trivy_$(TRIVY_VERSION)_$(TRIVY_OS)-$(TRIVY_ARCH)bit.tar.gz -o /tmp/trivy.tar.gz
tar zxvf /tmp/trivy.tar.gz -C $(dir $(TRIVY)) trivy
setup-cosign:
curl -sSfL https://github.com/sigstore/cosign/releases/download/v$(COSIGN_VERSION)/cosign-linux-amd64 -o $(COSIGN)
chmod +x $(COSIGN)
trivy-scan-python-vulnerabilities:
PIPENV_YES=True # If set, Pipenv automatically assumes “yes” at all prompts. This prevents the build from hanging.
PIPENV_IGNORE_VIRTUALENVS=True # Forces Pipenv to ignore the existing virtual environment and create its own instead
# Generate a Pipfile.lock, Trivy does not auto-detect requirements-dev.txt (https://aquasecurity.github.io/trivy/v0.28.1/docs/vulnerability/detection/language/)
./scripts/gen-pipfile.sh > Pipfile
pipenv --python `which python3`
pipenv lock
$(TRIVY) fs --exit-code 1 --ignore-unfixed --scanners vuln ./
docker-quality-images:
for image_name in $(DOCKER_IMAGES_TO_SCAN) ; do \
$(TRIVY) image --exit-code 1 --ignore-unfixed $(DOCKER_DOMAIN_LOCAL)/$*$${image_name}; \
done \
docker-test-images:
docker run -it $(DOCKER_DOMAIN_LOCAL)/detect-secrets --version
docker run -it $(DOCKER_DOMAIN_LOCAL)/detect-secrets-hook --version
docker-build-images:
for dockerfile in Dockerfiles/*/*.Dockerfile; do \
image_name=$$(echo -e $$(basename $${dockerfile}) | cut -d\. -f2); \
if [ "$${image_name}" == "detect-secrets-redhat-ubi" ]; then \
image_name="detect-secrets:redhat-ubi"; \
elif [ "$${image_name}" == "detect-secrets-redhat-ubi-custom" ]; then \
image_name="detect-secrets:redhat-ubi-custom"; \
fi ; \
docker build -f "$${dockerfile}" -t $(DOCKER_DOMAIN_LOCAL)/$${image_name} .; \
done
docker-login:
# @echo $(DOCKER_PASS_ART) | docker login -u $(DOCKER_USER_ART) --password-stdin $(DOCKER_REGISTRY_ART);
@echo $(DOCKER_PASS_ICR) | docker login -u $(DOCKER_USER_ICR) --password-stdin $(DOCKER_REGISTRY_ICR);
docker-publish-images: docker-login
# Tagged UBI images in special way (since they are pre-tagged); these will be the frozen versions for UBI images
if [ -n "$(TRAVIS_TAG)" ]; then \
docker tag $(DOCKER_DOMAIN_LOCAL)/detect-secrets:redhat-ubi $(DOCKER_DOMAIN_LOCAL)/detect-secrets:$(subst +,.,$(TRAVIS_TAG))-redhat-ubi ; \
docker tag $(DOCKER_DOMAIN_LOCAL)/detect-secrets:redhat-ubi-custom $(DOCKER_DOMAIN_LOCAL)/detect-secrets:$(subst +,.,$(TRAVIS_TAG))-redhat-ubi-custom ; \
fi
# Tagged non-UBI images with tags specificied within deploy target based each deploy type
for image_name in $(DOCKER_IMAGES_TO_TAG) ; do \
for image_tag in $(IMAGE_TAGS) ; do \
docker tag $(DOCKER_DOMAIN_LOCAL)/$${image_name} $(DOCKER_DOMAIN_LOCAL)/$${image_name}:$${image_tag} ; \
done \
done
# Publish images to the different Registries; publish list is built within deploy target
for image_name in $(DOCKER_IMAGES_TO_PUBLISH) ; do \
for registry in $(DOCKER_REGISTRIES) ; do \
$(MAKE) docker-publish-image \
IMAGE_NAME=$${image_name} DOCKER_REGISTRY=$${registry}; \
done \
done
docker-publish-image:
docker tag $(DOCKER_DOMAIN_LOCAL)/$(IMAGE_NAME) $(DOCKER_REGISTRY)/$(DOCKER_DOMAIN)/$(IMAGE_NAME); \
docker push $(DOCKER_REGISTRY)/$(DOCKER_DOMAIN)/$(IMAGE_NAME); \
@echo "Signing image $(DOCKER_REGISTRY)/$(DOCKER_DOMAIN)/$(IMAGE_NAME)"; \
$(COSIGN) sign --key env://COSIGN_PRIVATE_KEY --yes $(DOCKER_REGISTRY)/$(DOCKER_DOMAIN)/$(IMAGE_NAME); \
@echo "Verifying image $(DOCKER_REGISTRY)/$(DOCKER_DOMAIN)/$(IMAGE_NAME)"; \
$(COSIGN) verify --key env://COSIGN_PUBLIC_KEY "$(DOCKER_REGISTRY)/$(DOCKER_DOMAIN)/$(IMAGE_NAME)";
cosign-login:
# @echo $(DOCKER_PASS_ART) | $(COSIGN) login -u $(DOCKER_USER_ART) --password-stdin $(DOCKER_REGISTRY_ART); \
@echo $(DOCKER_PASS_ICR) | $(COSIGN) login -u $(DOCKER_USER_ICR) --password-stdin $(DOCKER_REGISTRY_ICR);
publish-cos:
pip install requests packaging; \
should_update=$$(python scripts/version_greater_than_in_cos.py "$(TRAVIS_TAG)"); \
if [ "$${should_update}" == "yes" ]; then \
curl -sL https://ibm.biz/idt-installer | bash; \
ibmcloud login --apikey $(IBM_CLOUD_API_KEY) -a https://cloud.ibm.com -r $(COS_REGION); \
echo $(TRAVIS_TAG) > version.txt; \
ibmcloud cos put-object --bucket $(COS_BUCKET) --key $(COS_FILE) --body version.txt --region $(COS_REGION); \
fi
release:
git fetch origin --tag
# Once a new tag is generated, it will trigger Travis to publish new image
TAG_VERSION=$$(git describe --tags --abbrev=0); \
NEW_VERSION=$$(grep VERSION detect_secrets/__init__.py | cut -d\' -f2 ); \
if [ $$TAG_VERSION != $$NEW_VERSION ]; then \
git tag $$NEW_VERSION $(TRAVIS_COMMIT); \
git push origin --tags; \
fi
deploy:
# TRAVIS_TAG, ex: 0.13.1+ibm.46.dss, the + needs to be replaced with . to avoid docker tag warning
if [ -n "$(TRAVIS_TAG)" ]; then \
$(MAKE) docker-publish-images push-tag publish-cos IMAGE_TAGS="$(subst +,.,$(TRAVIS_TAG))" \
DOCKER_IMAGES_TO_PUBLISH="detect-secrets:$(subst +,.,$(TRAVIS_TAG)) \
detect-secrets-hook:$(subst +,.,$(TRAVIS_TAG)) \
detect-secrets:$(subst +,.,$(TRAVIS_TAG))-redhat-ubi \
detect-secrets:$(subst +,.,$(TRAVIS_TAG))-redhat-ubi-custom" ; \
fi
# DEBUG_IMAGE_TAG, ex: master-420-id-248741968-time-1648742240
if [ "$(TRAVIS_BRANCH)" == "master" ]; then \
$(MAKE) docker-publish-images sync-branches IMAGE_TAGS="$(DEBUG_IMAGE_TAG)" \
DOCKER_IMAGES_TO_PUBLISH="detect-secrets detect-secrets-hook \
detect-secrets:redhat-ubi detect-secrets:redhat-ubi-custom \
detect-secrets:$(DEBUG_IMAGE_TAG) detect-secrets-hook:$(DEBUG_IMAGE_TAG)" ; \
$(MAKE) release; \
fi
# DEBUG_IMAGE_TAG, ex: dss-416-id-248693152-time-1648674570
if [ "$(TRAVIS_BRANCH)" == "dss" ]; then \
$(MAKE) docker-publish-images IMAGE_TAGS="dss-latest $(DEBUG_IMAGE_TAG)" \
DOCKER_IMAGES_TO_PUBLISH="detect-secrets:$(DEBUG_IMAGE_TAG) detect-secrets-hook:$(DEBUG_IMAGE_TAG) \
detect-secrets:dss-latest detect-secrets-hook:dss-latest" ; \
fi
push-tag:
git remote add github https://github.ibm.com/Whitewater/whitewater-detect-secrets.git
git fetch github
git push github $(TRAVIS_TAG)
sync-branches:
# sync to dss
git fetch origin --tag
git push origin $(TRAVIS_BRANCH):dss -f --tags
# sync to Whitewater/whitewater-detect-secrets
git remote add github https://github.ibm.com/Whitewater/whitewater-detect-secrets.git
git fetch github
git push github $(TRAVIS_BRANCH):master -f --tags
fix-dns:
# add public DNS name for public.dhe.ibm.com. The 9.x one is not serving traffic now
# public.dhe.ibm.com access is required for ibm-db pip package
echo "$$(dig @8.8.8.8 public.dhe.ibm.com +short | tail -n1) public.dhe.ibm.com" | sudo tee -a /etc/hosts
test-local:
coverage run -m pytest tests $(EXTRA_PYTEST_OPTIONS)
coverage report --show-missing --include=tests/* --fail-under 100
coverage report --show-missing --include=detect_secrets/* --fail-under 97
.PHONY: docker-test-images docker-build-images docker-login cosign-login docker-publish-images docker-publish-image publish-cos deploy push-tag sync-branches fix-dns test-local setup-trivy docker-quality-images release