Skip to content

Commit

Permalink
fix material views and mdl in env.yml
Browse files Browse the repository at this point in the history
  • Loading branch information
@aloftus23
aloftus23 committed Oct 3, 2024
1 parent a419c33 commit 1337696
Show file tree
Hide file tree
Showing 4 changed files with 48 additions and 39 deletions.
3 changes: 3 additions & 0 deletions backend/env.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,9 @@ staging:
REACT_APP_COGNITO_CLIENT_ID: 3vd67chnkk7dg3jt3nkoidf94e
REACT_APP_COGNITO_USER_POOL_ID: us-gov-west-1_N84vh2cq2
REACT_APP_COGNITO_CALLBACK_URL: https://staging.crossfeed.cyber.dhs.gov/okta-callback
MDL_USERNAME: ${ssm:/crossfeed/staging/MDL_USERNAME}
MDL_PASSWORD: ${ssm:/crossfeed/staging/MDL_PASSWORD}
MDL_NAME: ${ssm:/crossfeed/staging/MDL_NAME}

prod:
REGION: us-gov-east-1
Expand Down
34 changes: 16 additions & 18 deletions backend/src/api/stats.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ export const get = wrapHandler(async (event) => {
search.filters?.organizations.length > 0
) {
console.log('adding org filter -> ?');
qs.andWhere('domain."organizationId" IN (:...orgs)', {
qs.andWhere('stat."organizationId" IN (:...orgs)', {
orgs: search.filters?.organizations
});
}
Expand All @@ -100,21 +100,17 @@ export const get = wrapHandler(async (event) => {
}

if (search.filters?.regions && search.filters.regions.length > 0) {
qs.andWhere('"organization"."regionId" IN (:...regions)', {
qs.andWhere('"stat"."regionId" IN (:...regions)', {
regions: search.filters.regions
});
}

qs.andWhere(
'(domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true))'
);

// Handles the case where no orgs and no regions are set, and we pull stats for a region that will never exist
if (
search.filters?.organizations?.length === 0 &&
search.filters?.regions?.length === 0
) {
qs.andWhere('organization."regionId" IN (:...regions)', {
qs.andWhere('stat."regionId" IN (:...regions)', {
regions: ['FORCEEMPTY']
});
}
Expand All @@ -135,20 +131,22 @@ export const get = wrapHandler(async (event) => {

const services = await performQuery(
VwServiceStats.createQueryBuilder('stat')
.select('service as id, sum(stat.count) as value')
.groupBy('service')
.select('service as id, sum(stat.count) as value, stat."regionId"')
.groupBy('service, stat."regionId"')
.orderBy('value', 'DESC')
);
const ports = await performQuery(
VwPortsStats.createQueryBuilder('stat')
.select('port as id, sum(stat.count) as value')
.groupBy('port')
.select('port as id, sum(stat.count) as value, stat."regionId"')
.groupBy('port, stat."regionId"')
.orderBy('value', 'DESC')
);
const numVulnerabilities = await performQuery(
VwNumVulns.createQueryBuilder('stat')
.select('"domainSeverity" as id, sum(stat.count) as value')
.groupBy('"domainSeverity"')
.select(
'"domainSeverity" as id, sum(stat.count) as value, stat."regionId"'
)
.groupBy('"domainSeverity", stat."regionId"')
.orderBy('value', 'DESC')
.limit(MAX_RESULTS)
);
Expand All @@ -163,17 +161,17 @@ export const get = wrapHandler(async (event) => {
await filterQuery(
VwMostCommonVulns.createQueryBuilder('stat')
.select(
'stat.title, stat.description, stat.severity, sum(stat.count) as count'
'stat.title, stat.description, stat.severity, sum(stat.count) as count, stat."regionId"'
)
.groupBy('stat.title, stat.description, stat.severity')
.groupBy('stat.title, stat.description, stat.severity, stat."regionId"')
.orderBy('count', 'DESC')
.limit(MAX_RESULTS)
)
).getRawMany();
const severity = await performQuery(
VwSeverityStats.createQueryBuilder('stat')
.select('stat.severity as id, sum(stat.count) as value')
.groupBy('stat.severity')
.select('stat.severity as id, sum(stat.count) as value, stat."regionId"')
.groupBy('stat.severity, stat."regionId"')
.orderBy('stat.severity', 'ASC')
);
const total = await performQuery(
Expand All @@ -183,7 +181,7 @@ export const get = wrapHandler(async (event) => {
await (
await filterQuery(
VwOrgStats.createQueryBuilder('stat')
.select('stat.name as id, "orgId", count as value')
.select('stat.name as id, "orgId", count as value, stat."regionId"')
.orderBy('value', 'DESC')
)
).getRawMany()
Expand Down
46 changes: 27 additions & 19 deletions backend/src/models/material-views.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@ import { BaseEntity, ViewEntity, ViewColumn } from 'typeorm';
@ViewEntity({
name: 'vw_service_stats',
expression: `
SELECT o.acronym, o.id as "organizationId", s.service, COUNT(*) AS count
SELECT o.acronym, o.id as "organizationId", s.service, COUNT(*) AS count, o."regionId" as "regionId"
FROM service s
INNER JOIN domain ON s."domainId" = domain.id
join organization o on domain."organizationId" = o.id
WHERE s.service IS NOT NULL
GROUP BY o.acronym, o.id, s.service
WHERE s.service IS NOT NULL
AND (domain."isFceb" = true OR (domain."isFceb" = false AND domain."fromCidr" = true))
GROUP BY o.acronym, o.id, s.service, "regionId"
ORDER BY count DESC;
`,
materialized: true
Expand All @@ -30,11 +31,12 @@ export class VwServiceStats extends BaseEntity {
@ViewEntity({
name: 'vw_port_stats',
expression: `
SELECT o.acronym, o.id as "organizationId", s.port, COUNT(*) AS count
SELECT o.acronym, o.id as "organizationId", s.port, COUNT(*) AS count, o."regionId" as "regionId"
from "domain" d
join service s on s."domainId" = d.id
join organization o on d."organizationId" = o.id
group by o.acronym, o.id, s.port
WHERE (d."isFceb" = true OR (d."isFceb" = false AND d."fromCidr" = true))
group by o.acronym, o.id, s.port, "regionId"
ORDER BY count DESC;
`,
materialized: true
Expand All @@ -56,14 +58,15 @@ export class VwPortsStats extends BaseEntity {
@ViewEntity({
name: 'vw_num_vulns_stats',
expression: `
select acronym, "organizationId", "domainSeverity", "count"
select acronym, "organizationId", "domainSeverity", "count", "regionId" as "regionId"
from
(select o.acronym, o.id as "organizationId" , CONCAT(d.name, '|', v.severity) as "domainSeverity", count(*) as count, row_number() over (partition by o.acronym order by count(*) DESC) as row
(select o.acronym, o.id as "organizationId", o."regionId" as "regionId", CONCAT(d.name, '|', v.severity) as "domainSeverity", count(*) as count, row_number() over (partition by o.acronym order by count(*) DESC) as row
from "domain" d
join vulnerability v on v."domainId" = d.id
join organization o on d."organizationId" = o.id
where v.state = 'open'
group by o.acronym, o.id, d.name, v.severity
and (d."isFceb" = true OR (d."isFceb" = false AND d."fromCidr" = true))
group by o.acronym, o.id, d.name, v.severity, "regionId"
order by count desc) as ranked_vulns
where "row" <= 50;
`,
Expand All @@ -86,13 +89,14 @@ export class VwNumVulns extends BaseEntity {
@ViewEntity({
name: 'vw_latest_vulns_stats',
expression: `
select acronym, "organizationId", id, "createdAt","updatedAt", "lastSeen","title", "cve", "cwe", "cpe","description", "references","cvss","severity", "needsPopulation", "state", "substate","source", "notes","actions","structuredData","isKev","kevResults","domainId","serviceId"
select acronym, "organizationId", id, "createdAt","updatedAt", "lastSeen","title", "cve", "cwe", "cpe","description", "references","cvss","severity", "needsPopulation", "state", "substate","source", "notes","actions","structuredData","isKev","kevResults","domainId","serviceId", "regionId"
from
(select o.acronym, o.id as "organizationId", v.*, row_number() over (partition by o.acronym order by v."createdAt" desc) as row
(select o.acronym, o.id as "organizationId", o."regionId" as "regionId", v.*, row_number() over (partition by o.acronym order by v."createdAt" desc) as row
from vulnerability v
left join "domain" d on v."domainId" = d.id
join organization o on d."organizationId" = o.id
where v.state = 'open'
and (d."isFceb" = true OR (d."isFceb" = false AND d."fromCidr" = true))
order by v."createdAt" desc) as ranked_vulns
where "row" <= 50;
`,
Expand Down Expand Up @@ -178,15 +182,16 @@ export class VwLatestVulns extends BaseEntity {
@ViewEntity({
name: 'vw_most_common_vulns_stats',
expression: `
select acronym, "organizationId", title, description, severity, "count"
select acronym, "organizationId", title, description, severity, "count", "regionId"
from
(select o.acronym, o.id as "organizationId", v.title, v.description, v.severity, count(v.*) as "count",
(select o.acronym, o.id as "organizationId", o."regionId" as "regionId", v.title, v.description, v.severity, count(v.*) as "count",
row_number() over (partition by o.acronym order by count(*) desc, case severity when 'Critical' then 0 when 'High' then 1 when 'Medium' then 2 when 'Low' then 3 end) as row
from vulnerability v
left join "domain" d on d.id = v."domainId"
join organization o on d."organizationId" = o.id
where v.state = 'open'
group by o.acronym, o.id, v.title, v.description, v.severity
and (d."isFceb" = true OR (d."isFceb" = false AND d."fromCidr" = true))
group by o.acronym, o.id, v.title, v.description, v.severity, "regionId"
) as ranked_vulns
where row <=50;
`,
Expand Down Expand Up @@ -215,12 +220,13 @@ export class VwMostCommonVulns extends BaseEntity {
@ViewEntity({
name: 'vw_severity_stats',
expression: `
select o.acronym, o.id as "organizationId", v.severity, count(*) as "count"
select o.acronym, o.id as "organizationId", v.severity, count(*) as "count", o."regionId" as "regionId"
from vulnerability v
left join "domain" d on d.id = v."domainId"
join organization o on d."organizationId" = o.id
where v.state = 'open'
group by o.acronym, o.id, v.severity
and (d."isFceb" = true OR (d."isFceb" = false AND d."fromCidr" = true))
group by o.acronym, o.id, v.severity, "regionId"
order by v.severity asc;
`,
materialized: true
Expand All @@ -242,10 +248,11 @@ export class VwSeverityStats extends BaseEntity {
@ViewEntity({
name: 'vw_domain_stats',
expression: `
select o.acronym, o.id as "organizationId", count(*)
select o.acronym, o.id as "organizationId", count(*), o."regionId" as "regionId"
from "domain" d
join organization o on d."organizationId" = o.id
group by o.acronym, o.id;
where (d."isFceb" = true OR (d."isFceb" = false AND d."fromCidr" = true))
group by o.acronym, o.id, "regionId";
`,
materialized: true
})
Expand All @@ -263,12 +270,13 @@ export class VwDomainStats extends BaseEntity {
@ViewEntity({
name: 'vw_org_stats',
expression: `
select o.name, o.acronym, o.id as "organizationId", o.id as "orgId", count(*) as "count"
select o.name, o.acronym, o.id as "organizationId", o.id as "orgId", count(*) as "count", o."regionId" as "regionId"
from "domain" d
join organization o on d."organizationId" = o.id
join vulnerability v on d.id = v."domainId"
where v.state = 'open'
group by o.name, o.acronym, o.id
and (d."isFceb" = true OR (d."isFceb" = false AND d."fromCidr" = true))
group by o.name, o.acronym, o.id, "regionId"
order by "count" desc;
`,
materialized: true
Expand Down
4 changes: 2 additions & 2 deletions infrastructure/database.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ resource "aws_db_instance" "db" {
max_allocated_storage = 10000
storage_type = "gp2"
engine = "postgres"
engine_version = "15.5"
engine_version = "15.7"
allow_major_version_upgrade = true
skip_final_snapshot = true
availability_zone = data.aws_availability_zones.available.names[1]
availability_zone = data.aws_availability_zones.available.names[0]
multi_az = true
backup_retention_period = 35
storage_encrypted = true
Expand Down

0 comments on commit 1337696

Please sign in to comment.