Skip to content

Commit

Permalink
feat(utils): offer a better way to infer a file format from a service…
Browse files Browse the repository at this point in the history
… output, add GML and DXF formats

The WFS services sometimes offer otuput formats like SHAPE-ZIP or GML2,
and we need a more reliable way to infer a file format from them

Use this in the data service to make sure we output known mime types from
WFS and ESRI Rest
  • Loading branch information
jahow committed Feb 1, 2024
1 parent 61dce52 commit 91aad32
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 27 deletions.
8 changes: 4 additions & 4 deletions libs/feature/dataviz/src/lib/service/data.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { TestBed } from '@angular/core/testing'
import { DataService } from './data.service'
import { openDataset } from '@geonetwork-ui/data-fetcher'
import { PROXY_PATH } from '@geonetwork-ui/util/shared'
import { firstValueFrom, lastValueFrom } from 'rxjs'
import { lastValueFrom } from 'rxjs'

const newEndpointCall = jest.fn()

Expand Down Expand Up @@ -258,7 +258,7 @@ describe('DataService', () => {
},
{
description: 'Lieu de surveillance (ligne)',
mimeType: 'gml',
mimeType: 'application/gml+xml',
name: 'surval_parametre_ligne',
url: new URL(
'http://local/wfs?GetFeature&FeatureType=surval_parametre_ligne&format=gml'
Expand Down Expand Up @@ -301,7 +301,7 @@ describe('DataService', () => {
},
{
description: 'Lieu de surveillance (ligne)',
mimeType: 'gml',
mimeType: 'application/gml+xml',
name: 'nojson_type',
url: new URL(
'http://local/wfs?GetFeature&FeatureType=nojson_type&format=gml'
Expand Down Expand Up @@ -355,7 +355,7 @@ describe('DataService', () => {
},
{
description: 'Lieu de surveillance (ligne)',
mimeType: 'gml',
mimeType: 'application/gml+xml',
name: '',
url: new URL(
'http://unique-feature-type/wfs?GetFeature&FeatureType=myOnlyOne&format=gml'
Expand Down
8 changes: 5 additions & 3 deletions libs/feature/dataviz/src/lib/service/data.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import {
SupportedTypes,
} from '@geonetwork-ui/data-fetcher'
import {
extensionToFormat,
getFileFormat,
getFileFormatFromServiceOutput,
getMimeTypeForFormat,
ProxyService,
} from '@geonetwork-ui/util/shared'
Expand Down Expand Up @@ -140,7 +140,9 @@ export class DataService {
...wfsLink,
type: 'download',
url: new URL(urls[format]),
mimeType: getMimeTypeForFormat(extensionToFormat(format)) || format,
mimeType: getMimeTypeForFormat(
getFileFormatFromServiceOutput(format)
),
}))
)
)
Expand All @@ -154,7 +156,7 @@ export class DataService {
url: new URL(
this.getDownloadUrlFromEsriRest(esriRestLink.url.toString(), format)
),
mimeType: getMimeTypeForFormat(extensionToFormat(format)) || format,
mimeType: getMimeTypeForFormat(getFileFormatFromServiceOutput(format)),
}))
}

Expand Down
45 changes: 36 additions & 9 deletions libs/util/shared/src/lib/links/link-utils.spec.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import { LINK_FIXTURES } from '@geonetwork-ui/common/fixtures'
import {
checkFileFormat,
extensionToFormat,
FORMATS,
getBadgeColor,
getFileFormat,
getFileFormatFromServiceOutput,
getLinkLabel,
mimeTypeToFormat,
getLinkPriority,
mimeTypeToFormat,
} from './link-utils'
import { DatasetDownloadDistribution } from '@geonetwork-ui/common/domain/model/record'

Expand Down Expand Up @@ -50,7 +50,7 @@ describe('link utils', () => {
expect(getFileFormat(LINK_FIXTURES.geodataShp)).toEqual('shp')
})
})
describe('for a shapefile link withe MimeType', () => {
describe('for a shapefile link with MimeType', () => {
it('returns shp format', () => {
expect(getFileFormat(LINK_FIXTURES.geodataShpWithMimeType)).toEqual(
'shp'
Expand Down Expand Up @@ -165,11 +165,38 @@ describe('link utils', () => {
}
)
})
describe('#extensionToFormat for an XLS extension', () => {
it('returns excel format', () => {
expect(extensionToFormat('XLS')).toEqual('excel')
})

describe('#getFileFormatFromServiceOutput', () => {
// service output, recognized file format
const toTest = [
['SHAPE-ZIP', 'shp'],
['application/vnd.google-earth.kml xml', 'kml'],
['KML', 'kml'],
['excel2007', 'excel'],
['XLS', 'excel'],
['gml2', 'gml'],
['gml3', 'gml'],
['text/xml; subtype=gml/3.1.1', 'gml'],
['gml32', 'gml'],
['DXF', 'dxf'],
['DXF-ZIP', 'zip'],
['json', 'json'],
['geojson', 'geojson'],
['Acbd', null],
]

describe.each(toTest)(
'service output=%s, recognized file format=%s',
(serviceOutput, fileFormat) => {
it('returns the correct file format', () => {
expect(getFileFormatFromServiceOutput(serviceOutput)).toEqual(
fileFormat
)
})
}
)
})

describe('#getBadgeColor for format', () => {
it('returns #1e5180', () => {
expect(getBadgeColor('json')).toEqual('#1e5180')
Expand All @@ -190,15 +217,15 @@ describe('link utils', () => {
})
).toEqual(nFormats - 1)
})
it(`returns ${nFormats - 5}`, () => {
it(`returns ${nFormats - 6}`, () => {
expect(
getLinkPriority({
description: 'Data in KML format',
name: 'abc.kml',
url: new URL('https://my.server/files/abc.kml'),
type: 'download',
})
).toEqual(nFormats - 5)
).toEqual(nFormats - 6)
})
})
describe('#checkFileFormat', () => {
Expand Down
45 changes: 34 additions & 11 deletions libs/util/shared/src/lib/links/link-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,15 @@ export const FORMATS = {
color: '#328556',
mimeTypes: ['x-gis/x-shapefile'],
},
gml: {
extensions: ['gml'],
priority: 5,
color: '#c92bce',
mimeTypes: ['application/gml+xml', 'text/xml; subtype=gml'],
},
kml: {
extensions: ['kml', 'kmz'],
priority: 5,
priority: 6,
color: '#348009',
mimeTypes: [
'application/vnd.google-earth.kml+xml',
Expand All @@ -54,34 +60,40 @@ export const FORMATS = {
},
gpkg: {
extensions: ['gpkg', 'geopackage'],
priority: 6,
priority: 7,
color: '#ea79ba',
mimeTypes: ['application/geopackage+sqlite3'],
},
zip: {
extensions: ['zip', 'tar.gz'],
priority: 7,
priority: 8,
color: '#f2bb3a',
mimeTypes: ['application/zip', 'application/x-zip'],
},
pdf: {
extensions: ['pdf'],
priority: 8,
priority: 9,
color: '#db544a',
mimeTypes: ['application/pdf'],
},
jpg: {
extensions: ['jpg', 'jpeg', 'jfif', 'pjpeg', 'pjp'],
priority: 8,
priority: 9,
color: '#673ab7',
mimeTypes: ['image/jpg'],
},
svg: {
extensions: ['svg'],
priority: 9,
priority: 10,
color: '#d98294',
mimeTypes: ['image/svg+xml'],
},
dxf: {
extensions: ['dxf'],
priority: 11,
color: '#de630b',
mimeTypes: ['application/x-dxf', 'image/x-dxf'],
},
} as const

export type FileFormat = keyof typeof FORMATS
Expand All @@ -102,13 +114,24 @@ export function getLinkPriority(link: DatasetDistribution): number {
return getFormatPriority(getFileFormat(link))
}

export function extensionToFormat(extension: string): FileFormat {
for (const format in FORMATS) {
for (const alias of FORMATS[format].extensions) {
if (alias === extension.toLowerCase()) return format as FileFormat
export function getFileFormatFromServiceOutput(
serviceOutput: string
): FileFormat | null {
function formatMatcher(format: typeof FORMATS[FileFormat]): boolean {
const output = serviceOutput.toLowerCase()
return (
format.extensions.some((extension: string) =>
output.includes(extension)
) ||
format.mimeTypes.some((mimeType: string) => output.includes(mimeType))
)
}
for (const formatName in FORMATS) {
if (formatMatcher(FORMATS[formatName])) {
return formatName as FileFormat
}
}
return undefined
return null
}

export function getFileFormat(link: DatasetDistribution): FileFormat {
Expand Down

0 comments on commit 91aad32

Please sign in to comment.