chore: sync save state (#2487)

When a peer save the map, other peers in dirty state should not need to
save the map anymore.

That implementation uses the lastKnownHLC as a reference, but it changes
all dirty states at once. Another impementation could be to have each
object sync its dirty state, but in this case we do not have a HLC per
object as reference, and it also creates more messages.

ping @almet if you're around and wanna have a fresh look. :)

umap/models.py

@@ -523,17 +523,27 @@ class DataLayer(NamedModel):
     def metadata(self, request=None):
         # Retrocompat: minimal settings for maps not saved after settings property
         # has been introduced
-        obj = self.settings or {
+        metadata = self.settings
+        if not metadata:
+            # Fallback to file for old datalayers.
+            data = json.loads(self.geojson.read().decode())
+            metadata = data.get("_umap_options")
+            if not metadata:
+                metadata = {
                     "name": self.name,
                     "displayOnLoad": self.display_on_load,
                 }
+            # Save it to prevent file reading at each map load.
+            self.settings = metadata
+            # Do not update the modified_at.
+            self.save(update_fields=["settings"])
         if self.old_id:
-            obj["old_id"] = self.old_id
-        obj["id"] = self.pk
-        obj["permissions"] = {"edit_status": self.edit_status}
-        obj["editMode"] = "advanced" if self.can_edit(request) else "disabled"
-        obj["_referenceVersion"] = self.reference_version
-        return obj
+            metadata["old_id"] = self.old_id
+        metadata["id"] = self.pk
+        metadata["permissions"] = {"edit_status": self.edit_status}
+        metadata["editMode"] = "advanced" if self.can_edit(request) else "disabled"
+        metadata["_referenceVersion"] = self.reference_version
+        return metadata
 
     def clone(self, map_inst=None):
         new = self.__class__.objects.get(pk=self.pk)

umap/static/umap/js/modules/data/layer.js

@@ -45,8 +45,6 @@ export class DataLayer extends ServerStored {
     this._features = {}
     this._geojson = null
     this._propertiesIndex = []
-    this._loaded = false // Are layer metadata loaded
-    this._dataloaded = false // Are layer data loaded
 
     this._leafletMap = leafletMap
     this.parentPane = this._leafletMap.getPane('overlayPane')
@@ -85,6 +83,7 @@ export class DataLayer extends ServerStored {
     this.connectToMap()
     this.permissions = new DataLayerPermissions(this._umap, this)
 
+    this._needsFetch = this.createdOnServer
     if (!this.createdOnServer) {
       if (this.showAtLoad()) this.show()
     }
@@ -243,7 +242,7 @@ export class DataLayer extends ServerStored {
   }
 
   dataChanged() {
-    if (!this.hasDataLoaded()) return
+    if (!this.isLoaded()) return
     this._umap.onDataLayersChanged()
     this.layer.dataChanged()
   }
@@ -252,13 +251,13 @@ export class DataLayer extends ServerStored {
     if (!geojson) return []
     const features = this.addData(geojson, sync)
     this._geojson = geojson
+    this._needsFetch = false
     this.onDataLoaded()
     this.dataChanged()
     return features
   }
 
   onDataLoaded() {
-    this._dataloaded = true
     this.renderLegend()
   }
 
@@ -268,7 +267,6 @@ export class DataLayer extends ServerStored {
     if (geojson._umap_options) this.setOptions(geojson._umap_options)
     if (this.isRemoteLayer()) await this.fetchRemoteData()
     else this.fromGeoJSON(geojson, false)
-    this._loaded = true
   }
 
   clear() {
@@ -320,7 +318,7 @@ export class DataLayer extends ServerStored {
 
   async fetchRemoteData(force) {
     if (!this.isRemoteLayer()) return
-    if (!this.hasDynamicData() && this.hasDataLoaded() && !force) return
+    if (!this.hasDynamicData() && this.isLoaded() && !force) return
     if (!this.isVisible()) return
     // Keep non proxied url for later use in Alert.
     const remoteUrl = this._umap.renderUrl(this.options.remoteData.url)
@@ -345,11 +343,7 @@ export class DataLayer extends ServerStored {
   }
 
   isLoaded() {
-    return !this.createdOnServer || this._loaded
-  }
-
-  hasDataLoaded() {
-    return this._dataloaded
+    return !this._needsFetch
   }
 
   backupOptions() {
@@ -633,8 +627,6 @@ export class DataLayer extends ServerStored {
     this.propagateDelete()
     this._leaflet_events_bk = this._leaflet_events
     this.clear()
-    delete this._loaded
-    delete this._dataloaded
   }
 
   reset() {
@@ -652,7 +644,6 @@ export class DataLayer extends ServerStored {
     this.hide()
     if (this.isRemoteLayer()) this.fetchRemoteData()
     else if (this._geojson_bk) this.fromGeoJSON(this._geojson_bk)
-    this._loaded = true
     this.show()
     this.isDirty = false
   }
@@ -1108,9 +1099,7 @@ export class DataLayer extends ServerStored {
 
   async save() {
     if (this.isDeleted) return await this.saveDelete()
-    if (!this.isLoaded()) {
-      return
-    }
+    if (!this.isLoaded()) return
     const geojson = this.umapGeoJSON()
     const formData = new FormData()
     formData.append('name', this.options.name)
@@ -1172,7 +1161,6 @@ export class DataLayer extends ServerStored {
       this.backupOptions()
       this.backupData()
       this.connectToMap()
-      this._loaded = true
       this.redraw() // Needed for reordering features
       return true
     }

umap/static/umap/js/modules/rendering/layers/classified.js

@@ -75,7 +75,7 @@ const ClassifiedMixin = {
   },
 
   renderLegend: function (container) {
-    if (!this.datalayer.hasDataLoaded()) return
+    if (!this.datalayer.isLoaded()) return
     const parent = DomUtil.create('ul', '', container)
     const items = this.getLegendItems()
     for (const [color, label] of items) {

umap/static/umap/js/modules/saving.js

@@ -10,6 +10,11 @@ export async function save() {
   }
 }
 
+export function clear() {
+  _queue.clear()
+  onUpdate()
+}
+
 function add(obj) {
   _queue.add(obj)
   onUpdate()

umap/static/umap/js/modules/sync/engine.js

@@ -2,6 +2,7 @@ import * as Utils from '../utils.js'
 import { HybridLogicalClock } from './hlc.js'
 import { DataLayerUpdater, FeatureUpdater, MapUpdater } from './updaters.js'
 import { WebSocketTransport } from './websocket.js'
+import * as SaveManager from '../saving.js'
 
 // Start reconnecting after 2 seconds, then double the delay each time
 // maxing out at 32 seconds.
@@ -125,6 +126,16 @@ export class SyncEngine {
     this._send({ verb: 'delete', subject, metadata, key })
   }
 
+  saved() {
+    if (this.offline) return
+    if (this.transport) {
+      this.transport.send('SavedMessage', {
+        sender: this.peerId,
+        lastKnownHLC: this._operations.getLastKnownHLC(),
+      })
+    }
+  }
+
   _send(inputMessage) {
     const message = this._operations.addLocal(inputMessage)
 
@@ -168,6 +179,8 @@ export class SyncEngine {
       } else if (payload.message.verb === 'ListOperationsResponse') {
         this.onListOperationsResponse(payload)
       }
+    } else if (kind === 'SavedMessage') {
+      this.onSavedMessage(payload)
     } else {
       throw new Error(`Received unknown message from the websocket server: ${kind}`)
     }
@@ -280,6 +293,13 @@ export class SyncEngine {
     // Else: apply
   }
 
+  onSavedMessage({ sender, lastKnownHLC }) {
+    debug(`received saved message from peer ${sender}`, lastKnownHLC)
+    if (lastKnownHLC === this._operations.getLastKnownHLC() && SaveManager.isDirty) {
+      SaveManager.clear()
+    }
+  }
+
   /**
    * Send a message to another peer (via the transport layer)
    *
@@ -350,7 +370,7 @@ export class Operations {
   }
 
   /**
-   * Tick the clock and add store the passed message in the operations list.
+   * Tick the clock and store the passed message in the operations list.
    *
    * @param {*} inputMessage
    * @returns {*} clock-aware message

umap/static/umap/js/modules/sync/updaters.js

@@ -55,9 +55,6 @@ export class DataLayerUpdater extends BaseUpdater {
   upsert({ value }) {
     // Upsert only happens when a new datalayer is created.
     const datalayer = this._umap.createDataLayer(value, false)
-    // Prevent the layer to get data from the server, as it will get it
-    // from the sync.
-    datalayer._loaded = true
   }
 
   update({ key, metadata, value }) {
@@ -92,7 +89,7 @@ export class FeatureUpdater extends BaseUpdater {
   upsert({ metadata, value }) {
     const { id, layerId } = metadata
     const datalayer = this.getDataLayerFromID(layerId)
-    const feature = this.getFeatureFromMetadata(metadata, value)
+    const feature = this.getFeatureFromMetadata(metadata)
 
     if (feature) {
       feature.geometry = value.geometry
@@ -109,7 +106,7 @@ export class FeatureUpdater extends BaseUpdater {
       return
     }
     if (key === 'geometry') {
-      const feature = this.getFeatureFromMetadata(metadata, value)
+      const feature = this.getFeatureFromMetadata(metadata)
       feature.geometry = value
     } else {
       this.updateObjectValue(feature, key, value)

umap/static/umap/js/modules/umap.js

@@ -684,6 +684,7 @@ export default class Umap extends ServerStored {
         Alert.success(translate('Map has been saved!'))
       })
     }
+    this.sync.saved()
     this.fire('saved')
   }
 

umap/sync/app.py

@@ -14,6 +14,7 @@ from .payloads import (
     OperationMessage,
     PeerMessage,
     Request,
+    SavedMessage,
 )
 
 
@@ -165,6 +166,10 @@ class Peer:
                 case OperationMessage():
                     await self.broadcast(text_data)
 
+                # Broadcast the new map state to connected peers
+                case SavedMessage():
+                    await self.broadcast(text_data)
+
                 # Send peer messages to the proper peer
                 case PeerMessage():
                     await self.send_to(incoming.root.recipient, text_data)

umap/sync/payloads.py

@@ -30,10 +30,17 @@ class PeerMessage(BaseModel):
     message: dict
 
 
+class SavedMessage(BaseModel):
+    kind: Literal["SavedMessage"] = "SavedMessage"
+    lastKnownHLC: str
+
+
 class Request(RootModel):
     """Any message coming from the websocket should be one of these, and will be rejected otherwise."""
 
-    root: Union[PeerMessage, OperationMessage] = Field(discriminator="kind")
+    root: Union[PeerMessage, OperationMessage, SavedMessage] = Field(
+        discriminator="kind"
+    )
 
 
 class JoinResponse(BaseModel):

umap/tests/integration/test_websocket_sync.py

@@ -398,7 +398,9 @@ def test_should_sync_datalayers(new_page, asgi_live_server, tilelayer):
     peerA.locator("#map").click()
 
     # Make sure this new marker is in Layer 2 for peerB
-    expect(peerB.get_by_text("Layer 2")).to_be_visible()
+    # Show features for this layer in the brower.
+    peerB.get_by_role("heading", name="Layer 2").locator(".datalayer-name").click()
+    expect(peerB.locator("li").filter(has_text="Layer 2")).to_be_visible()
     peerB.locator(".panel.left").get_by_role("button", name="Show/hide layer").nth(
         1
     ).click()
@@ -421,9 +423,11 @@ def test_should_sync_datalayers(new_page, asgi_live_server, tilelayer):
         1
     ).click()
 
-    # Now peerA saves the layer 2 to the server
-    with peerA.expect_response(re.compile(".*/datalayer/update/.*")):
-        peerA.get_by_role("button", name="Save").click()
+    # Peer A should not be in dirty state
+    expect(peerA.locator("body")).not_to_have_class(re.compile(".*umap-is-dirty.*"))
+
+    # Peer A should only have two markers
+    expect(peerA.locator(".leaflet-marker-icon")).to_have_count(2)
 
     assert DataLayer.objects.count() == 2
 
@@ -557,3 +561,46 @@ def test_create_and_sync_map(new_page, asgi_live_server, tilelayer, login, user)
     peerA.get_by_role("button", name="Edit").click()
     expect(markersA).to_have_count(2)
     expect(markersB).to_have_count(2)
+
+
+@pytest.mark.xdist_group(name="websockets")
+def test_should_sync_saved_status(new_page, asgi_live_server, tilelayer):
+    map = MapFactory(name="sync", edit_status=Map.ANONYMOUS)
+    map.settings["properties"]["syncEnabled"] = True
+    map.save()
+
+    # Create two tabs
+    peerA = new_page("Page A")
+    peerA.goto(f"{asgi_live_server.url}{map.get_absolute_url()}?edit")
+    peerB = new_page("Page B")
+    peerB.goto(f"{asgi_live_server.url}{map.get_absolute_url()}?edit")
+
+    # Create a new marker from peerA
+    peerA.get_by_title("Draw a marker").click()
+    peerA.locator("#map").click(position={"x": 220, "y": 220})
+
+    # Peer A should be in dirty state
+    expect(peerA.locator("body")).to_have_class(re.compile(".*umap-is-dirty.*"))
+
+    # Peer B should not be in dirty state
+    expect(peerB.locator("body")).not_to_have_class(re.compile(".*umap-is-dirty.*"))
+
+    # Create a new marker from peerB
+    peerB.get_by_title("Draw a marker").click()
+    peerB.locator("#map").click(position={"x": 200, "y": 250})
+
+    # Peer B should be in dirty state
+    expect(peerB.locator("body")).to_have_class(re.compile(".*umap-is-dirty.*"))
+
+    # Peer A should still be in dirty state
+    expect(peerA.locator("body")).to_have_class(re.compile(".*umap-is-dirty.*"))
+
+    # Save layer to the server from peerA
+    with peerA.expect_response(re.compile(".*/datalayer/create/.*")):
+        peerA.get_by_role("button", name="Save").click()
+
+    # Peer B should not be in dirty state
+    expect(peerB.locator("body")).not_to_have_class(re.compile(".*umap-is-dirty.*"))
+
+    # Peer A should not be in dirty state
+    expect(peerA.locator("body")).not_to_have_class(re.compile(".*umap-is-dirty.*"))