increase bulk action retry to 5 (#148169)

## Summary

Increase retry count to 5 to help retry on agent doc version conflict.
It looks like 3 retries are not enough for 100k agents update tags.
#144161

This can be tested on an ECE high memory instance with 100k horde
agents.

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/fleet/server/services/agents/action_runner.ts

@@ -22,7 +22,7 @@ import { getAgentActions } from './actions';
 import { closePointInTime, getAgentsByKuery } from './crud';
 import type { BulkActionsResolver } from './bulk_actions_resolver';
 
-export const MAX_RETRY_COUNT = 3;
+export const MAX_RETRY_COUNT = 5;
 
 export interface ActionParams {
   kuery: string;
@@ -113,10 +113,10 @@ export abstract class ActionRunner {
               );
 
             if (this.retryParams.retryCount === MAX_RETRY_COUNT) {
-              const errorMessage = `Stopping after ${MAX_RETRY_COUNT}rd retry. Error: ${error.message}`;
+              const errorMessage = `Stopping after retry #${MAX_RETRY_COUNT}. Error: ${error.message}`;
               appContextService.getLogger().warn(errorMessage);
 
-              // clean up tasks after 3rd retry reached
+              // clean up tasks after last retry reached
               await Promise.all([
                 this.bulkActionsResolver!.removeIfExists(this.checkTaskId!),
                 this.bulkActionsResolver!.removeIfExists(this.retryParams.taskId!),

x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts

@@ -212,12 +212,12 @@ describe('update_agent_tags', () => {
           tagsToRemove: [],
           kuery: '',
           total: 100,
-          retryCount: 3,
+          retryCount: 5,
         }
       )
     ).rejects.toThrowError('version conflict of 100 agents');
     const errorResults = esClient.bulk.mock.calls[0][0] as any;
-    expect(errorResults.body[1].error).toEqual('version conflict on 3rd retry');
+    expect(errorResults.body[1].error).toEqual('version conflict on last retry');
   });
 
   it('should run add tags async when actioning more agents than batch size', async () => {

x-pack/plugins/fleet/server/services/agents/update_agent_tags_action_runner.ts

@@ -208,7 +208,7 @@ export async function updateTagsBatch(
         getUuidArray(res.version_conflicts!).map((id) => ({
           agentId: id,
           actionId,
-          error: 'version conflict on 3rd retry',
+          error: 'version conflict on last retry',
         }))
       );
     }