Merge pull request #786 from geonetwork/fix-missing-shp-format-wfs

Data service: better detect file formats from WFS outputs

libs/common/domain/src/lib/model/record/metadata.model.ts

@@ -122,6 +122,7 @@ export interface DatasetDownloadDistribution {
   // textEncoding?: string
   name?: string
   description?: string
+  accessServiceProtocol?: ServiceProtocol
 }
 
 export interface OnlineLinkResource {

libs/feature/dataviz/src/lib/service/data.service.spec.ts

@@ -2,7 +2,7 @@ import { TestBed } from '@angular/core/testing'
 import { DataService } from './data.service'
 import { openDataset } from '@geonetwork-ui/data-fetcher'
 import { PROXY_PATH } from '@geonetwork-ui/util/shared'
-import { firstValueFrom, lastValueFrom } from 'rxjs'
+import { lastValueFrom } from 'rxjs'
 
 const newEndpointCall = jest.fn()
 
@@ -160,7 +160,7 @@ describe('DataService', () => {
         ),
         type: 'service',
         accessServiceProtocol: 'wfs',
-      }
+      } as const
       describe('WFS unreachable (CORS)', () => {
         it('throws a relevant error', async () => {
           try {
@@ -233,7 +233,7 @@ describe('DataService', () => {
               url: new URL(
                 'http://local/wfs?GetFeature&FeatureType=surval_parametre_ligne&format=csv'
               ),
-              type: 'service',
+              type: 'download',
               accessServiceProtocol: 'wfs',
             },
             {
@@ -243,7 +243,7 @@ describe('DataService', () => {
               url: new URL(
                 'http://local/wfs?GetFeature&FeatureType=surval_parametre_ligne&format=xls'
               ),
-              type: 'service',
+              type: 'download',
               accessServiceProtocol: 'wfs',
             },
             {
@@ -253,17 +253,17 @@ describe('DataService', () => {
               url: new URL(
                 'http://local/wfs?GetFeature&FeatureType=surval_parametre_ligne&format=json'
               ),
-              type: 'service',
+              type: 'download',
               accessServiceProtocol: 'wfs',
             },
             {
               description: 'Lieu de surveillance (ligne)',
-              mimeType: 'gml',
+              mimeType: 'application/gml+xml',
               name: 'surval_parametre_ligne',
               url: new URL(
                 'http://local/wfs?GetFeature&FeatureType=surval_parametre_ligne&format=gml'
               ),
-              type: 'service',
+              type: 'download',
               accessServiceProtocol: 'wfs',
             },
           ])
@@ -286,7 +286,7 @@ describe('DataService', () => {
               url: new URL(
                 'http://local/wfs?GetFeature&FeatureType=nojson_type&format=csv'
               ),
-              type: 'service',
+              type: 'download',
               accessServiceProtocol: 'wfs',
             },
             {
@@ -296,17 +296,17 @@ describe('DataService', () => {
               url: new URL(
                 'http://local/wfs?GetFeature&FeatureType=nojson_type&format=xls'
               ),
-              type: 'service',
+              type: 'download',
               accessServiceProtocol: 'wfs',
             },
             {
               description: 'Lieu de surveillance (ligne)',
-              mimeType: 'gml',
+              mimeType: 'application/gml+xml',
               name: 'nojson_type',
               url: new URL(
                 'http://local/wfs?GetFeature&FeatureType=nojson_type&format=gml'
               ),
-              type: 'service',
+              type: 'download',
               accessServiceProtocol: 'wfs',
             },
           ])
@@ -329,7 +329,7 @@ describe('DataService', () => {
               url: new URL(
                 'http://unique-feature-type/wfs?GetFeature&FeatureType=myOnlyOne&format=csv'
               ),
-              type: 'service',
+              type: 'download',
               accessServiceProtocol: 'wfs',
             },
             {
@@ -339,7 +339,7 @@ describe('DataService', () => {
               url: new URL(
                 'http://unique-feature-type/wfs?GetFeature&FeatureType=myOnlyOne&format=xls'
               ),
-              type: 'service',
+              type: 'download',
               accessServiceProtocol: 'wfs',
             },
             {
@@ -350,17 +350,17 @@ describe('DataService', () => {
                 'http://unique-feature-type/wfs?GetFeature&FeatureType=myOnlyOne&format=json'
               ),
 
-              type: 'service',
+              type: 'download',
               accessServiceProtocol: 'wfs',
             },
             {
               description: 'Lieu de surveillance (ligne)',
-              mimeType: 'gml',
+              mimeType: 'application/gml+xml',
               name: '',
               url: new URL(
                 'http://unique-feature-type/wfs?GetFeature&FeatureType=myOnlyOne&format=gml'
               ),
-              type: 'service',
+              type: 'download',
               accessServiceProtocol: 'wfs',
             },
           ])

libs/feature/dataviz/src/lib/service/data.service.ts

@@ -9,8 +9,8 @@ import {
   SupportedTypes,
 } from '@geonetwork-ui/data-fetcher'
 import {
-  extensionToFormat,
   getFileFormat,
+  getFileFormatFromServiceOutput,
   getMimeTypeForFormat,
   ProxyService,
 } from '@geonetwork-ui/util/shared'
@@ -45,7 +45,7 @@ interface WfsDownloadUrls {
 export class DataService {
   constructor(private proxy: ProxyService) {}
 
-  private getDownloadUrlsFromWfs(
+  getDownloadUrlsFromWfs(
     wfsUrl: string,
     featureTypeName: string
   ): Observable<WfsDownloadUrls> {
@@ -120,7 +120,7 @@ export class DataService {
     )
   }
 
-  private getDownloadUrlFromEsriRest(apiUrl: string, format: string): string {
+  getDownloadUrlFromEsriRest(apiUrl: string, format: string): string {
     return this.proxy.getProxiedUrl(
       `${apiUrl}/query?f=${format}&where=1=1&outFields=*`
     )
@@ -138,8 +138,11 @@ export class DataService {
       map((urls) =>
         Object.keys(urls).map((format) => ({
           ...wfsLink,
+          type: 'download',
           url: new URL(urls[format]),
-          mimeType: getMimeTypeForFormat(extensionToFormat(format)) || format,
+          mimeType: getMimeTypeForFormat(
+            getFileFormatFromServiceOutput(format)
+          ),
         }))
       )
     )
@@ -153,7 +156,7 @@ export class DataService {
       url: new URL(
         this.getDownloadUrlFromEsriRest(esriRestLink.url.toString(), format)
       ),
-      mimeType: getMimeTypeForFormat(extensionToFormat(format)) || format,
+      mimeType: getMimeTypeForFormat(getFileFormatFromServiceOutput(format)),
     }))
   }
 

libs/util/shared/src/lib/links/link-utils.spec.ts

@@ -1,13 +1,13 @@
 import { LINK_FIXTURES } from '@geonetwork-ui/common/fixtures'
 import {
   checkFileFormat,
-  extensionToFormat,
   FORMATS,
   getBadgeColor,
   getFileFormat,
+  getFileFormatFromServiceOutput,
   getLinkLabel,
-  mimeTypeToFormat,
   getLinkPriority,
+  mimeTypeToFormat,
 } from './link-utils'
 import { DatasetDownloadDistribution } from '@geonetwork-ui/common/domain/model/record'
 
@@ -50,7 +50,7 @@ describe('link utils', () => {
         expect(getFileFormat(LINK_FIXTURES.geodataShp)).toEqual('shp')
       })
     })
-    describe('for a shapefile link withe MimeType', () => {
+    describe('for a shapefile link with MimeType', () => {
       it('returns shp format', () => {
         expect(getFileFormat(LINK_FIXTURES.geodataShpWithMimeType)).toEqual(
           'shp'
@@ -165,11 +165,38 @@ describe('link utils', () => {
       }
     )
   })
-  describe('#extensionToFormat for an XLS extension', () => {
-    it('returns excel format', () => {
-      expect(extensionToFormat('XLS')).toEqual('excel')
-    })
+
+  describe('#getFileFormatFromServiceOutput', () => {
+    // service output, recognized file format
+    const toTest = [
+      ['SHAPE-ZIP', 'shp'],
+      ['application/vnd.google-earth.kml xml', 'kml'],
+      ['KML', 'kml'],
+      ['excel2007', 'excel'],
+      ['XLS', 'excel'],
+      ['gml2', 'gml'],
+      ['gml3', 'gml'],
+      ['text/xml; subtype=gml/3.1.1', 'gml'],
+      ['gml32', 'gml'],
+      ['DXF', 'dxf'],
+      ['DXF-ZIP', 'zip'],
+      ['json', 'json'],
+      ['geojson', 'geojson'],
+      ['Acbd', null],
+    ]
+
+    describe.each(toTest)(
+      'service output=%s, recognized file format=%s',
+      (serviceOutput, fileFormat) => {
+        it('returns the correct file format', () => {
+          expect(getFileFormatFromServiceOutput(serviceOutput)).toEqual(
+            fileFormat
+          )
+        })
+      }
+    )
   })
+
   describe('#getBadgeColor for format', () => {
     it('returns #1e5180', () => {
       expect(getBadgeColor('json')).toEqual('#1e5180')
@@ -190,15 +217,15 @@ describe('link utils', () => {
         })
       ).toEqual(nFormats - 1)
     })
-    it(`returns ${nFormats - 5}`, () => {
+    it(`returns ${nFormats - 6}`, () => {
       expect(
         getLinkPriority({
           description: 'Data in KML format',
           name: 'abc.kml',
           url: new URL('https://my.server/files/abc.kml'),
           type: 'download',
         })
-      ).toEqual(nFormats - 5)
+      ).toEqual(nFormats - 6)
     })
   })
   describe('#checkFileFormat', () => {

libs/util/shared/src/lib/links/link-utils.ts

@@ -43,9 +43,15 @@ export const FORMATS = {
     color: '#328556',
     mimeTypes: ['x-gis/x-shapefile'],
   },
+  gml: {
+    extensions: ['gml'],
+    priority: 5,
+    color: '#c92bce',
+    mimeTypes: ['application/gml+xml', 'text/xml; subtype=gml'],
+  },
   kml: {
     extensions: ['kml', 'kmz'],
-    priority: 5,
+    priority: 6,
     color: '#348009',
     mimeTypes: [
       'application/vnd.google-earth.kml+xml',
@@ -54,34 +60,40 @@ export const FORMATS = {
   },
   gpkg: {
     extensions: ['gpkg', 'geopackage'],
-    priority: 6,
+    priority: 7,
     color: '#ea79ba',
     mimeTypes: ['application/geopackage+sqlite3'],
   },
   zip: {
     extensions: ['zip', 'tar.gz'],
-    priority: 7,
+    priority: 8,
     color: '#f2bb3a',
     mimeTypes: ['application/zip', 'application/x-zip'],
   },
   pdf: {
     extensions: ['pdf'],
-    priority: 8,
+    priority: 9,
     color: '#db544a',
     mimeTypes: ['application/pdf'],
   },
   jpg: {
     extensions: ['jpg', 'jpeg', 'jfif', 'pjpeg', 'pjp'],
-    priority: 8,
+    priority: 9,
     color: '#673ab7',
     mimeTypes: ['image/jpg'],
   },
   svg: {
     extensions: ['svg'],
-    priority: 9,
+    priority: 10,
     color: '#d98294',
     mimeTypes: ['image/svg+xml'],
   },
+  dxf: {
+    extensions: ['dxf'],
+    priority: 11,
+    color: '#de630b',
+    mimeTypes: ['application/x-dxf', 'image/x-dxf'],
+  },
 } as const
 
 export type FileFormat = keyof typeof FORMATS
@@ -102,13 +114,24 @@ export function getLinkPriority(link: DatasetDistribution): number {
   return getFormatPriority(getFileFormat(link))
 }
 
-export function extensionToFormat(extension: string): FileFormat {
-  for (const format in FORMATS) {
-    for (const alias of FORMATS[format].extensions) {
-      if (alias === extension.toLowerCase()) return format as FileFormat
+export function getFileFormatFromServiceOutput(
+  serviceOutput: string
+): FileFormat | null {
+  function formatMatcher(format: typeof FORMATS[FileFormat]): boolean {
+    const output = serviceOutput.toLowerCase()
+    return (
+      format.extensions.some((extension: string) =>
+        output.includes(extension)
+      ) ||
+      format.mimeTypes.some((mimeType: string) => output.includes(mimeType))
+    )
+  }
+  for (const formatName in FORMATS) {
+    if (formatMatcher(FORMATS[formatName])) {
+      return formatName as FileFormat
     }
   }
-  return undefined
+  return null
 }
 
 export function getFileFormat(link: DatasetDistribution): FileFormat {