Skip to content

Commit

Permalink
changed variable scope to instance level (#114)
Browse files Browse the repository at this point in the history
Signed-off-by: Rishabh Singh <sngri@amazon.com>
  • Loading branch information
rishabh6788 committed May 3, 2022
1 parent 476dfae commit 83a6200
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 9 deletions.
4 changes: 3 additions & 1 deletion lib/ci-stack.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ export interface CIStackProps extends StackProps {
}

export class CIStack extends Stack {
public readonly monitoring: JenkinsMonitoring;

constructor(scope: Construct, id: string, props: CIStackProps) {
super(scope, id, props);

Expand Down Expand Up @@ -121,7 +123,7 @@ export class CIStack extends Stack {
useSsl,
});

const monitoring = new JenkinsMonitoring(this, externalLoadBalancer, mainJenkinsNode);
this.monitoring = new JenkinsMonitoring(this, externalLoadBalancer, mainJenkinsNode);

if (additionalCommandsContext.toString() !== 'undefined') {
new RunAdditionalCommands(this, additionalCommandsContext.toString());
Expand Down
2 changes: 1 addition & 1 deletion lib/compute/jenkins-main-node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,7 @@ export class JenkinsMainNode {

// Download jenkins-cli from the local machine
InitCommand.shellCommand('until $(curl --output /dev/null --silent --head --fail http://localhost:8080); do sleep 5; done &&'
+' wget -O "jenkins-cli.jar" http://localhost:8080/jnlpJars/jenkins-cli.jar'),
+ ' wget -O "jenkins-cli.jar" http://localhost:8080/jnlpJars/jenkins-cli.jar'),

InitFile.fromFileInline('/initial_jenkins.yaml', jenkinsyaml),

Expand Down
15 changes: 8 additions & 7 deletions lib/monitoring/ci-alarms.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,12 @@ import { JenkinsExternalLoadBalancer } from '../network/ci-external-load-balance
import { JenkinsMainNode } from '../compute/jenkins-main-node';

export class JenkinsMonitoring {
public readonly alarms: Alarm[] = [];

constructor(stack: Stack, externalLoadBalancer: JenkinsExternalLoadBalancer, mainNode: JenkinsMainNode) {
const dashboard = new Dashboard(stack, 'AlarmDashboard');

const alarms: Alarm[] = [];
alarms.push(new Alarm(stack, 'ExternalLoadBalancerUnhealthyHosts', {
this.alarms.push(new Alarm(stack, 'ExternalLoadBalancerUnhealthyHosts', {
alarmDescription: 'If any hosts behind the load balancer are unhealthy',
metric: externalLoadBalancer.targetGroup.metricUnhealthyHostCount(),
evaluationPeriods: 3,
Expand All @@ -27,7 +28,7 @@ export class JenkinsMonitoring {
treatMissingData: TreatMissingData.BREACHING,
}));

alarms.push(new Alarm(stack, 'MainNodeTooManyJenkinsProcessesFound', {
this.alarms.push(new Alarm(stack, 'MainNodeTooManyJenkinsProcessesFound', {
alarmDescription: 'Only one jenkins process should run at any given time on the main node, there might be a cloudwatch configuration issue',
metric: mainNode.ec2InstanceMetrics.foundJenkinsProcessCount.with({ statistic: 'max' }),
evaluationPeriods: 3,
Expand All @@ -36,7 +37,7 @@ export class JenkinsMonitoring {
treatMissingData: TreatMissingData.IGNORE,
}));

alarms.push(new Alarm(stack, 'MainNodeHighCpuUtilization', {
this.alarms.push(new Alarm(stack, 'MainNodeHighCpuUtilization', {
alarmDescription: 'The jenkins process is using much more CPU that expected, it should be investigated for a stuck process/job',
metric: mainNode.ec2InstanceMetrics.cpuTime.with({ statistic: 'max' }),
evaluationPeriods: 5,
Expand All @@ -45,7 +46,7 @@ export class JenkinsMonitoring {
treatMissingData: TreatMissingData.IGNORE,
}));

alarms.push(new Alarm(stack, 'MainNodeHighMemoryUtilization', {
this.alarms.push(new Alarm(stack, 'MainNodeHighMemoryUtilization', {
alarmDescription: 'The jenkins process is using more memory than expected, it should be investigated for a large number of jobs or heavy weight jobs',
metric: mainNode.ec2InstanceMetrics.memUsed.with({ statistic: 'max' }),
evaluationPeriods: 5,
Expand All @@ -54,7 +55,7 @@ export class JenkinsMonitoring {
treatMissingData: TreatMissingData.IGNORE,
}));

alarms.push(new Alarm(stack, 'MainNodeCloudwatchEvents', {
this.alarms.push(new Alarm(stack, 'MainNodeCloudwatchEvents', {
alarmDescription: `Cloudwatch events have stopped being received from the main node.
Use session manager to exam the host and the /opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log`,
metric: mainNode.ec2InstanceMetrics.memUsed.with({ statistic: 'n' }),
Expand All @@ -69,7 +70,7 @@ Use session manager to exam the host and the /opt/aws/amazon-cloudwatch-agent/lo
treatMissingData: TreatMissingData.MISSING,
}));

alarms
this.alarms
.map((alarm) => new AlarmWidget({ alarm }))
.forEach((widget) => dashboard.addWidgets(widget));
}
Expand Down

0 comments on commit 83a6200

Please sign in to comment.