wip: move formatters to a module

This mainly allows to dynamically load the third party libraries.

In the same process, those libs have changed:

- tokml => switch to placemarkio fork, more up to date and available
  as ESM
- togpx => switch to geojson-to-gpx, more up to date and available as
  ESM (note: this lib does not export polygons, because they do not
  make sense in GPX world, while the previous was converting them as
  lines before)

package.json

@@ -37,6 +37,8 @@
   },
   "homepage": "http://wiki.openstreetmap.org/wiki/UMap",
   "dependencies": {
+    "@dwayneparton/geojson-to-gpx": "^0.2.0",
+    "@placemarkio/tokml": "^0.3.3",
     "@tmcw/togeojson": "^5.8.0",
     "colorbrewer": "^1.5.6",
     "csv2geojson": "5.1.1",
@@ -62,9 +64,7 @@
     "leaflet.path.drag": "0.0.6",
     "leaflet.photon": "0.9.1",
     "osmtogeojson": "^3.0.0-beta.3",
-    "simple-statistics": "^7.8.3",
+    "simple-statistics": "^7.8.3"
-    "togpx": "^0.5.4",
-    "tokml": "0.4.0"
   },
   "browserslist": [
     "> 0.5%, last 2 versions, Firefox ESR, not dead, not op_mini all"

scripts/vendorsjs.sh

@@ -19,16 +19,16 @@ mkdir -p umap/static/umap/vendors/formbuilder/ && cp -r node_modules/leaflet-for
 mkdir -p umap/static/umap/vendors/measurable/ && cp -r node_modules/leaflet-measurable/Leaflet.Measurable.* umap/static/umap/vendors/measurable/
 mkdir -p umap/static/umap/vendors/photon/ && cp -r node_modules/leaflet.photon/leaflet.photon.js umap/static/umap/vendors/photon/
 mkdir -p umap/static/umap/vendors/csv2geojson/ && cp -r node_modules/csv2geojson/csv2geojson.js umap/static/umap/vendors/csv2geojson/
-mkdir -p umap/static/umap/vendors/togeojson/ && cp -r node_modules/@tmcw/togeojson/dist/togeojson.umd.* umap/static/umap/vendors/togeojson/
+mkdir -p umap/static/umap/vendors/togeojson/ && cp node_modules/@tmcw/togeojson/dist/togeojson.es.mjs umap/static/umap/vendors/togeojson/togeojson.es.js
+mkdir -p umap/static/umap/vendors/tokml/ && cp node_modules/@placemarkio/tokml/dist/tokml.es.mjs umap/static/umap/vendors/tokml/tokml.es.js
 mkdir -p umap/static/umap/vendors/osmtogeojson/ && cp -r node_modules/osmtogeojson/osmtogeojson.js umap/static/umap/vendors/osmtogeojson/
 mkdir -p umap/static/umap/vendors/georsstogeojson/ && cp -r node_modules/georsstogeojson/GeoRSSToGeoJSON.js umap/static/umap/vendors/georsstogeojson/
-mkdir -p umap/static/umap/vendors/togpx/ && cp -r node_modules/togpx/togpx.js umap/static/umap/vendors/togpx/
-mkdir -p umap/static/umap/vendors/tokml && cp -r node_modules/tokml/tokml.js umap/static/umap/vendors/tokml
 mkdir -p umap/static/umap/vendors/locatecontrol/ && cp -r node_modules/leaflet.locatecontrol/dist/L.Control.Locate.min.* umap/static/umap/vendors/locatecontrol/
 mkdir -p umap/static/umap/vendors/dompurify/ && cp -r node_modules/dompurify/dist/purify.es.mjs umap/static/umap/vendors/dompurify/purify.es.js
 mkdir -p umap/static/umap/vendors/dompurify/ && cp -r node_modules/dompurify/dist/purify.es.mjs.map umap/static/umap/vendors/dompurify/purify.es.mjs.map
 mkdir -p umap/static/umap/vendors/colorbrewer/ && cp node_modules/colorbrewer/index.js umap/static/umap/vendors/colorbrewer/colorbrewer.js
 mkdir -p umap/static/umap/vendors/simple-statistics/ && cp node_modules/simple-statistics/dist/simple-statistics.min.* umap/static/umap/vendors/simple-statistics/
 mkdir -p umap/static/umap/vendors/iconlayers/ && cp node_modules/leaflet-iconlayers/dist/* umap/static/umap/vendors/iconlayers/
+mkdir -p umap/static/umap/vendors/geojson-to-gpx/ && cp node_modules/@dwayneparton/geojson-to-gpx/dist/index.js umap/static/umap/vendors/geojson-to-gpx/
 
 echo 'Done!'

umap/static/umap/js/modules/formatter.js

@@ -0,0 +1,132 @@
+/* Uses globals for: csv2geojson, osmtogeojson, GeoRSSToGeoJSON (not available as ESM) */
+import { translate } from './i18n.js'
+
+export default class Formatter {
+  async fromGPX(str) {
+    let togeojson
+    await import('../../vendors/togeojson/togeojson.es.js').then((module) => {
+      togeojson = module
+    })
+    return togeojson.gpx(this.toDom(str))
+  }
+
+  async fromKML(str) {
+    console.log(str)
+    let togeojson
+    await import('../../vendors/togeojson/togeojson.es.js').then((module) => {
+      togeojson = module
+    })
+    return togeojson.kml(this.toDom(str), {
+      skipNullGeometry: true,
+    })
+  }
+
+  async fromGeoJSON(str) {
+    try {
+      return JSON.parse(str)
+    } catch (err) {
+      U.Alert.error(`Invalid JSON file: ${err}`)
+    }
+  }
+
+  async fromOSM(str) {
+    let src
+    try {
+      src = JSON.parse(str)
+    } catch (e) {
+      src = this.toDom(str)
+    }
+    return osmtogeojson(src, { flatProperties: true })
+  }
+
+  fromCSV(str, callback) {
+    csv2geojson.csv2geojson(
+      str,
+      {
+        delimiter: 'auto',
+        includeLatLon: false,
+      },
+      (err, result) => {
+        // csv2geojson fallback to null geometries when it cannot determine
+        // lat or lon columns. This is valid geojson, but unwanted from a user
+        // point of view.
+        if (result?.features.length) {
+          if (result.features[0].geometry === null) {
+            err = {
+              type: 'Error',
+              message: translate('Cannot determine latitude and longitude columns.'),
+            }
+          }
+        }
+        if (err) {
+          let message
+          if (err.type === 'Error') {
+            message = err.message
+          } else {
+            message = translate('{count} errors during import: {message}', {
+              count: err.length,
+              message: err[0].message,
+            })
+          }
+          U.Alert.error(message, 10000)
+          console.error(err)
+        }
+        if (result?.features.length) {
+          callback(result)
+        }
+      }
+    )
+  }
+
+  async fromGeoRSS(str) {
+    return GeoRSSToGeoJSON(this.toDom(c))
+  }
+
+  toDom(x) {
+    const doc = new DOMParser().parseFromString(x, 'text/xml')
+    const errorNode = doc.querySelector('parsererror')
+    if (errorNode) {
+      U.Alert.error(translate('Cannot parse data'))
+    }
+    return doc
+  }
+
+  async parse(str, format) {
+    switch (format) {
+      case 'csv':
+        return new Promise((resolve, reject) => {
+          return this.fromCSV(str, (data) => resolve(data))
+        })
+      case 'gpx':
+        return await this.fromGPX(str)
+      case 'kml':
+        return await this.fromKML(str)
+      case 'osm':
+        return await this.fromOSM(str)
+      case 'georss':
+        return await this.fromGeoRSS(str)
+      case 'geojson':
+        return await this.fromGeoJSON(str)
+    }
+  }
+
+  async toGPX(geojson) {
+    let togpx
+    await import('../../vendors/geojson-to-gpx/index.js').then((module) => {
+      togpx = module
+    })
+    for (const feature of geojson.features) {
+      feature.properties.desc = feature.properties.description
+    }
+    const gpx = togpx.default(geojson)
+    return new XMLSerializer().serializeToString(gpx)
+  }
+
+  async toKML(geojson) {
+    let tokml
+    await import('../../vendors/tokml/tokml.es.js').then((module) => {
+      tokml = module
+    })
+    return tokml.toKML(geojson)
+  }
+}

umap/static/umap/js/modules/global.js

@@ -7,6 +7,7 @@ import { AjaxAutocomplete, AjaxAutocompleteMultiple } from './autocomplete.js'
 import Browser from './browser.js'
 import Caption from './caption.js'
 import Facets from './facets.js'
+import Formatter from './formatter.js'
 import Help from './help.js'
 import Importer from './importer.js'
 import Orderable from './orderable.js'
@@ -36,6 +37,7 @@ window.U = {
   Dialog,
   EditPanel,
   Facets,
+  Formatter,
   FullPanel,
   Help,
   HTTPError,

umap/static/umap/js/umap.js

@@ -116,6 +116,8 @@ U.Map = L.Map.extend({
     // Needed for actions labels
     this.help = new U.Help(this)
 
+    this.formatter = new U.Formatter(this)
+
     this.initControls()
     // Needs locate control and hash to exist
     this.initCenter()

umap/static/umap/js/umap.layer.js

@@ -792,11 +792,9 @@ U.DataLayer = L.Evented.extend({
     const response = await this.map.request.get(url)
     if (response?.ok) {
       this.clear()
-      this.rawToGeoJSON(
+      await this.map.formatter
-        await response.text(),
+        .parse(await response.text(), this.options.remoteData.format)
-        this.options.remoteData.format,
+        .then((geojson) => this.fromGeoJSON(geojson))
-        (geojson) => this.fromGeoJSON(geojson)
-      )
     }
   },
 
@@ -930,83 +928,6 @@ U.DataLayer = L.Evented.extend({
     }
   },
 
-  addRawData: function (c, type) {
-    this.rawToGeoJSON(c, type, (geojson) => this.addData(geojson))
-  },
-
-  rawToGeoJSON: (c, type, callback) => {
-    const toDom = (x) => {
-      const doc = new DOMParser().parseFromString(x, 'text/xml')
-      const errorNode = doc.querySelector('parsererror')
-      if (errorNode) {
-        U.Alert.error(L._('Cannot parse data'))
-      }
-      return doc
-    }
-
-    // TODO add a duck typing guessType
-    if (type === 'csv') {
-      csv2geojson.csv2geojson(
-        c,
-        {
-          delimiter: 'auto',
-          includeLatLon: false,
-        },
-        (err, result) => {
-          // csv2geojson fallback to null geometries when it cannot determine
-          // lat or lon columns. This is valid geojson, but unwanted from a user
-          // point of view.
-          if (result?.features.length) {
-            if (result.features[0].geometry === null) {
-              err = {
-                type: 'Error',
-                message: L._('Cannot determine latitude and longitude columns.'),
-              }
-            }
-          }
-          if (err) {
-            let message
-            if (err.type === 'Error') {
-              message = err.message
-            } else {
-              message = L._('{count} errors during import: {message}', {
-                count: err.length,
-                message: err[0].message,
-              })
-            }
-            U.Alert.error(message, 10000)
-            console.error(err)
-          }
-          if (result?.features.length) {
-            callback(result)
-          }
-        }
-      )
-    } else if (type === 'gpx') {
-      callback(toGeoJSON.gpx(toDom(c)))
-    } else if (type === 'georss') {
-      callback(GeoRSSToGeoJSON(toDom(c)))
-    } else if (type === 'kml') {
-      callback(toGeoJSON.kml(toDom(c)))
-    } else if (type === 'osm') {
-      let d
-      try {
-        d = JSON.parse(c)
-      } catch (e) {
-        d = toDom(c)
-      }
-      callback(osmtogeojson(d, { flatProperties: true }))
-    } else if (type === 'geojson') {
-      try {
-        const gj = JSON.parse(c)
-        callback(gj)
-      } catch (err) {
-        U.Alert.error(`Invalid JSON file: ${err}`)
-        return
-      }
-    }
-  },
-
   // The choice of the name is not ours, because it is required by Leaflet.
   // It is misleading, as the returned objects are uMap objects, and not
   // GeoJSON features.
@@ -1136,10 +1057,12 @@ U.DataLayer = L.Evented.extend({
     return new U.Polygon(this.map, latlngs, { geojson: geojson, datalayer: this }, id)
   },
 
-  importRaw: function (raw, type) {
+  importRaw: async function (raw, format) {
-    this.addRawData(raw, type)
+    await this.map.formatter
+      .parse(raw, format)
+      .then((geojson) => this.addData(geojson))
+      .then(() => this.zoomTo())
     this.isDirty = true
-    this.zoomTo()
   },
 
   importFromFiles: function (files, type) {

umap/static/umap/js/umap.permissions.js

@@ -116,7 +116,7 @@ U.MapPermissions = L.Class.extend({
         L._('Advanced actions')
       )
       const advancedButtons = L.DomUtil.create('div', 'button-bar', advancedActions)
-      const download = L.DomUtil.createButton(
+      L.DomUtil.createButton(
         'button',
         advancedButtons,
         L._('Attach the map to my account'),

umap/static/umap/js/umap.share.js

@@ -1,22 +1,22 @@
 U.Share = L.Class.extend({
   EXPORT_TYPES: {
     geojson: {
-      formatter: (map) => JSON.stringify(map.toGeoJSON(), null, 2),
+      formatter: async (map) => JSON.stringify(map.toGeoJSON(), null, 2),
       ext: '.geojson',
       filetype: 'application/json',
     },
     gpx: {
-      formatter: (map) => togpx(map.toGeoJSON()),
+      formatter: async (map) => await map.formatter.toGPX(map.toGeoJSON()),
       ext: '.gpx',
       filetype: 'application/gpx+xml',
     },
     kml: {
-      formatter: (map) => tokml(map.toGeoJSON()),
+      formatter: async (map) => await map.formatter.toKML(map.toGeoJSON()),
       ext: '.kml',
       filetype: 'application/vnd.google-earth.kml+xml',
     },
     csv: {
-      formatter: (map) => {
+      formatter: async (map) => {
         const table = []
         map.eachFeature((feature) => {
           const row = feature.toGeoJSON().properties
@@ -156,17 +156,17 @@ U.Share = L.Class.extend({
     this.map.panel.open({ content: this.container })
   },
 
-  format: function (mode) {
+  format: async function (mode) {
     const type = this.EXPORT_TYPES[mode]
-    const content = type.formatter(this.map)
+    const content = await type.formatter(this.map)
     let name = this.map.options.name || 'data'
     name = name.replace(/[^a-z0-9]/gi, '_').toLowerCase()
     const filename = name + type.ext
     return { content, filetype: type.filetype, filename }
   },
 
-  download: function (mode) {
+  download: async function (mode) {
-    const { content, filetype, filename } = this.format(mode)
+    const { content, filetype, filename } = await this.format(mode)
     const blob = new Blob([content], { type: filetype })
     window.URL = window.URL || window.webkitURL
     const el = document.createElement('a')

umap/templates/umap/js.html

@@ -19,7 +19,6 @@
 <script src="{% static 'umap/vendors/minimap/Control.MiniMap.min.js' %}"
         defer></script>
 <script src="{% static 'umap/vendors/csv2geojson/csv2geojson.js' %}" defer></script>
-<script src="{% static 'umap/vendors/togeojson/togeojson.umd.js' %}" defer></script>
 <script src="{% static 'umap/vendors/osmtogeojson/osmtogeojson.js' %}" defer></script>
 <script src="{% static 'umap/vendors/loading/Control.Loading.js' %}" defer></script>
 <script src="{% static 'umap/vendors/markercluster/leaflet.markercluster.js' %}"
@@ -37,9 +36,7 @@
         defer></script>
 <script src="{% static 'umap/vendors/measurable/Leaflet.Measurable.js' %}"
         defer></script>
-<script src="{% static 'umap/vendors/togpx/togpx.js' %}" defer></script>
 <script src="{% static 'umap/vendors/iconlayers/iconLayers.js' %}" defer></script>
-<script src="{% static 'umap/vendors/tokml/tokml.js' %}" defer></script>
 <script src="{% static 'umap/vendors/locatecontrol/L.Control.Locate.min.js' %}"
         defer></script>
 <script src="{% static 'umap/vendors/colorbrewer/colorbrewer.js' %}" defer></script>

umap/tests/integration/test_export_map.py

@@ -218,8 +218,7 @@ def test_gpx_export(map, live_server, bootstrap, page):
     download.save_as(path)
     assert (
         path.read_text()
-        == """<gpx xmlns="http://www.topografix.com/GPX/1/1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd" version="1.1" creator="togpx"><metadata/><wpt lat="52.57635" lon="-0.274658"><name>test</name><desc>name=test
+        == """<?xml version="1.0" encoding="UTF-8"?><gpx xmlns="http://www.topografix.com/GPX/1/1" version="1.1" creator="@dwayneparton/geojson-to-gpx"><wpt lat="52.57635" lon="-0.274658"><name>test</name><desc>Some description</desc></wpt><trk><name>test</name><trkseg><trkpt lat="54.476422" lon="-0.571289"/><trkpt lat="54.610255" lon="0.439453"/><trkpt lat="53.448807" lon="1.724854"/><trkpt lat="53.988395" lon="4.163818"/><trkpt lat="53.533778" lon="5.306396"/><trkpt lat="53.709714" lon="6.591797"/><trkpt lat="53.350551" lon="7.042236"/></trkseg></trk></gpx>"""
-description=Some description</desc></wpt><trk><name>name poly</name><desc>name=name poly</desc><trkseg><trkpt lat="53.585984" lon="11.25"/><trkpt lat="52.975108" lon="10.151367"/><trkpt lat="52.167194" lon="12.689209"/><trkpt lat="53.199452" lon="14.084473"/><trkpt lat="53.618579" lon="12.634277"/><trkpt lat="53.585984" lon="11.25"/><trkpt lat="53.585984" lon="11.25"/></trkseg></trk><trk><name>test</name><desc>name=test</desc><trkseg><trkpt lat="54.476422" lon="-0.571289"/><trkpt lat="54.610255" lon="0.439453"/><trkpt lat="53.448807" lon="1.724854"/><trkpt lat="53.988395" lon="4.163818"/><trkpt lat="53.533778" lon="5.306396"/><trkpt lat="53.709714" lon="6.591797"/><trkpt lat="53.350551" lon="7.042236"/></trkseg></trk></gpx>"""
     )
 
 
@@ -235,7 +234,7 @@ def test_kml_export(map, live_server, bootstrap, page):
     download.save_as(path)
     assert (
         path.read_text()
-        == """<?xml version="1.0" encoding="UTF-8"?><kml xmlns="http://www.opengis.net/kml/2.2"><Document><Placemark><name>name poly</name><ExtendedData><Data name="name"><value>name poly</value></Data></ExtendedData><Polygon><outerBoundaryIs><LinearRing><coordinates>11.25,53.585984 10.151367,52.975108 12.689209,52.167194 14.084473,53.199452 12.634277,53.618579 11.25,53.585984 11.25,53.585984</coordinates></LinearRing></outerBoundaryIs></Polygon></Placemark><Placemark><name>test</name><description>Some description</description><ExtendedData><Data name="_umap_options"><value>[object Object]</value></Data><Data name="name"><value>test</value></Data><Data name="description"><value>Some description</value></Data></ExtendedData><Point><coordinates>-0.274658,52.57635</coordinates></Point></Placemark><Placemark><name>test</name><ExtendedData><Data name="_umap_options"><value>[object Object]</value></Data><Data name="name"><value>test</value></Data></ExtendedData><LineString><coordinates>-0.571289,54.476422 0.439453,54.610255 1.724854,53.448807 4.163818,53.988395 5.306396,53.533778 6.591797,53.709714 7.042236,53.350551</coordinates></LineString></Placemark></Document></kml>"""
+        == """<kml xmlns="http://www.opengis.net/kml/2.2"><Document>\n<Placemark id="gyNzM">\n<name>name poly</name><ExtendedData></ExtendedData>\n  <Polygon>\n<outerBoundaryIs>\n  <LinearRing><coordinates>11.25,53.585984\n10.151367,52.975108\n12.689209,52.167194\n14.084473,53.199452\n12.634277,53.618579\n11.25,53.585984\n11.25,53.585984</coordinates></LinearRing></outerBoundaryIs></Polygon></Placemark>\n<Placemark id="QwNjg">\n<name>test</name><description>Some description</description><ExtendedData>\n  <Data name="_umap_options"><value>{"color":"OliveDrab"}</value></Data></ExtendedData>\n  <Point><coordinates>-0.274658,52.57635</coordinates></Point></Placemark>\n<Placemark id="YwMTM">\n<name>test</name><ExtendedData>\n  <Data name="_umap_options"><value>{"fill":false,"opacity":0.6}</value></Data></ExtendedData>\n  <LineString><coordinates>-0.571289,54.476422\n0.439453,54.610255\n1.724854,53.448807\n4.163818,53.988395\n5.306396,53.533778\n6.591797,53.709714\n7.042236,53.350551</coordinates></LineString></Placemark></Document></kml>"""
     )