Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use es-index-cleaner golang implementation #3204

Merged
merged 3 commits into from
Aug 19, 2021
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci-elasticsearch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,7 @@ jobs:
- name: Install tools
run: make install-ci

- uses: docker/setup-qemu-action@v1

- name: Run elasticsearch integration tests
run: bash scripts/es-integration-test.sh ${{ matrix.version.distribution }} ${{ matrix.version.image }}
15 changes: 11 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@ grpc-plugin-storage-integration-test:
.PHONY: test-compile-es-scripts
test-compile-es-scripts:
docker run --rm -v ${PWD}:/tmp/jaeger python:3-alpine3.11 /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esRollover.py
docker run --rm -v ${PWD}:/tmp/jaeger python:3-alpine3.11 /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esCleaner.py

.PHONY: index-cleaner-integration-test
index-cleaner-integration-test: docker-images-elastic
Expand Down Expand Up @@ -216,6 +215,10 @@ build-esmapping-generator:
build-esmapping-generator-linux:
GOOS=linux GOARCH=amd64 $(GOBUILD) -o ./plugin/storage/es/esmapping-generator ./cmd/esmapping-generator/main.go

.PHONY: build-es-index-cleaner
build-es-index-cleaner:
$(GOBUILD) -o ./cmd/es-index-cleaner/es-index-cleaner-$(GOOS)-$(GOARCH) ./cmd/es-index-cleaner/main.go

.PHONY: docker-hotrod
docker-hotrod:
GOOS=linux $(MAKE) build-examples
Expand Down Expand Up @@ -306,7 +309,8 @@ build-platform-binaries: build-agent \
build-examples \
build-tracegen \
build-anonymizer \
build-esmapping-generator
build-esmapping-generator \
build-es-index-cleaner

.PHONY: build-all-platforms
build-all-platforms: build-binaries-linux build-binaries-windows build-binaries-darwin build-binaries-s390x build-binaries-arm64 build-binaries-ppc64le
Expand All @@ -316,10 +320,13 @@ docker-images-cassandra:
docker build -t $(DOCKER_NAMESPACE)/jaeger-cassandra-schema:${DOCKER_TAG} plugin/storage/cassandra/
@echo "Finished building jaeger-cassandra-schema =============="

docker-images-elastic: TARGET = release
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought --target would be useful if a Dockerfile contained multiple build stages like debug and release. However, from what I see, there is only a release build stage in cmd/es-index-cleaner/Dockerfile. Would it be useful to still include the target param in this case?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

True, this can be removed. Done.


.PHONY: docker-images-elastic
docker-images-elastic:
docker-images-elastic: create-baseimg
GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-esmapping-generator
docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} plugin/storage/es
GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-es-index-cleaner
docker build --target $(TARGET) -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} --build-arg base_image=$(BASE_IMAGE) --build-arg TARGETARCH=$(GOARCH) cmd/es-index-cleaner
docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} plugin/storage/es -f plugin/storage/es/Dockerfile.rollover --build-arg TARGETARCH=$(GOARCH)
@echo "Finished building jaeger-es-indices-clean =============="

Expand Down
6 changes: 6 additions & 0 deletions cmd/es-index-cleaner/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
ARG base_image
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Ashmita152, are you interested in reviewing this?


FROM $base_image AS release
ARG TARGETARCH
COPY es-index-cleaner-linux-$TARGETARCH /go/bin/es-index-cleaner-linux
ENTRYPOINT ["/go/bin/es-index-cleaner-linux"]
2 changes: 1 addition & 1 deletion cmd/es-index-cleaner/app/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ type Config struct {
// AddFlags adds flags for TLS to the FlagSet.
func (c *Config) AddFlags(flags *flag.FlagSet) {
flags.String(indexPrefix, "", "Index prefix")
flags.Bool(archive, false, "Whether to remove archive indices")
flags.Bool(archive, false, "Whether to remove archive indices. It works only for rollover")
flags.Bool(rollover, false, "Whether to remove indices created by rollover")
flags.Int(timeout, 120, "Number of seconds to wait for master node response")
flags.String(indexDateSeparator, "-", "Index date separator")
Expand Down
15 changes: 5 additions & 10 deletions cmd/es-index-cleaner/app/index_filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,18 +42,13 @@ func (i *IndexFilter) Filter(indices []Index) []Index {

func (i *IndexFilter) filter(indices []Index) []Index {
var reg *regexp.Regexp
if !i.Rollover && !i.Archive {
// daily indices
reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service|dependencies)-\\d{4}%s\\d{2}%s\\d{2}", i.IndexPrefix, i.IndexDateSeparator, i.IndexDateSeparator))
} else if !i.Rollover && i.Archive {
// daily archive
reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive", i.IndexPrefix))
} else if i.Rollover && !i.Archive {
// rollover
if i.Archive {
// archive works only for rollover
reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive-\\d{6}", i.IndexPrefix))
pavolloffay marked this conversation as resolved.
Show resolved Hide resolved
} else if i.Rollover {
reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service)-\\d{6}", i.IndexPrefix))
} else {
// rollover archive
reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive-\\d{6}", i.IndexPrefix))
reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service|dependencies)-\\d{4}%s\\d{2}%s\\d{2}", i.IndexPrefix, i.IndexDateSeparator, i.IndexDateSeparator))
}

var filtered []Index
Expand Down
12 changes: 7 additions & 5 deletions cmd/es-index-cleaner/app/index_filter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -229,19 +229,21 @@ func testIndexFilter(t *testing.T, prefix string) {
},
},
{
name: "archive indices, remove older 2 days",
name: "archive indices, remove older 1 days - archive works only for rollover",
filter: &IndexFilter{
IndexPrefix: prefix,
IndexDateSeparator: "-",
Archive: true,
Rollover: false,
DeleteBeforeThisDate: time20200807.Add(-time.Hour * 24 * time.Duration(2)),
DeleteBeforeThisDate: time20200807.Add(-time.Hour * 24 * time.Duration(1)),
},
expected: []Index{
{
Index: prefix + "jaeger-span-archive",
CreationTime: time.Date(2020, time.August, 0, 15, 0, 0, 0, time.UTC),
Aliases: map[string]bool{},
Index: prefix + "jaeger-span-archive-000001",
CreationTime: time.Date(2020, time.August, 5, 15, 0, 0, 0, time.UTC),
Aliases: map[string]bool{
prefix + "jaeger-span-archive-read": true,
},
},
},
},
Expand Down
11 changes: 0 additions & 11 deletions plugin/storage/es/Dockerfile

This file was deleted.

117 changes: 0 additions & 117 deletions plugin/storage/es/esCleaner.py

This file was deleted.

4 changes: 2 additions & 2 deletions plugin/storage/integration/es_index_cleaner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,10 +114,10 @@ func TestIndexCleaner(t *testing.T) {
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("%s_no_prefix", test.name), func(t *testing.T) {
t.Run(fmt.Sprintf("%s_no_prefix, %s", test.name, test.envVars), func(t *testing.T) {
runIndexCleanerTest(t, client, "", test.expectedIndices, test.envVars)
})
t.Run(fmt.Sprintf("%s_prefix", test.name), func(t *testing.T) {
t.Run(fmt.Sprintf("%s_prefix, %s", test.name, test.envVars), func(t *testing.T) {
runIndexCleanerTest(t, client, indexPrefix, test.expectedIndices, append(test.envVars, "INDEX_PREFIX="+indexPrefix))
})
}
Expand Down
2 changes: 1 addition & 1 deletion scripts/build-upload-docker-images.sh
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ do
done

# build/upload images for jaeger-es-index-cleaner and jaeger-es-rollover
bash scripts/build-upload-a-docker-image.sh -c jaeger-es-index-cleaner -d plugin/storage/es -p "${platforms}"
bash scripts/build-upload-a-docker-image.sh -b -c jaeger-es-index-cleaner -d cmd/es-index-cleaner -p "${platforms}" -t release
bash scripts/build-upload-a-docker-image.sh -c jaeger-es-rollover -d plugin/storage/es -f Dockerfile.rollover -p "${platforms}"

# build/upload images for jaeger-tracegen and jaeger-anonymizer
Expand Down