Implement per-user message queues to prevent duplicate post syncing. Correclty initialize from a user or connect URL. Dont register the service worker when we\'re headless or an archive peer. Wrap sync logging with superlog

src/App.ts

@@ -46,6 +46,14 @@ interface StoragePost {
   data: Post;
 }
 
+
+
+interface SyncItem {
+  peerID: string;
+  postIDs: string[];
+
+}
+
 export class App {
   username: string = '';
   peername: string = '';
@@ -67,24 +75,27 @@ export class App {
   peerManager: PeerManager | null = null;
   sync: Sync = new Sync();
   renderTimer: number = 0;
-  postSyncQueue: any[] = [];
+  syncQueues: Map<string, SyncItem[]> = new Map();
-  postSyncPromise: any = null;
+  syncing: Set<string> = new Set();
 
-  async syncPostsInQueue() {
+  async processSyncQueue(userID: string) {
 
-    if (this.postSyncPromise) {
+    if (this.syncing.has(userID)) {
       return;
     }
 
-    while (this.postSyncQueue.length !== 0) {
+    let syncQueue = this.syncQueues.get(userID) as SyncItem[];
 
-      let queueItem = this.postSyncQueue.pop();
+    while (syncQueue.length !== 0) {
+      this.syncing.add(userID);
+      let syncItem = syncQueue.pop();
 
-      let userID = queueItem.userID;
+      if (!syncItem) {
-      let peerID = queueItem.peerID;
+        throw new Error();
-      let postIDs = queueItem.postIDs;
+      }
 
-      new Promise(async (resolve, reject) => {
+      let peerID = syncItem?.peerID;
+      let postIDs = syncItem?.postIDs;
       let neededPostIDs = await this.sync.checkPostIds(userID, peerID, postIDs);
 
       if (neededPostIDs.length > 0) {
@@ -94,23 +105,29 @@ export class App {
 
       }
       else {
-          console.log.apply(null, log(`[app] Don't need any posts for user ${logID(userID)} from peer ${logID(sendingPeerID)}`));
+        console.log.apply(null, log(`[app] Don't need any posts for user ${logID(userID)} from peer ${logID(peerID)}`));
+      }
     }
 
-      })
+    this.syncing.delete(userID);
-
-    }
-
-
-
-
-
   }
 
   addPostIDsToSyncQueue(userID: string, peerID: string, postIDs: string[]) {
-    this.postSyncQueue.push({ userID: userID, peerID: peerID, postIDs: postIDs });
+
+    let syncQueue = this.syncQueues.get(userID);
+
+    if (!syncQueue) {
+      let newArray: SyncItem[] = [];
+      this.syncQueues.set(userID, newArray);
+      syncQueue = newArray;
     }
 
+    syncQueue.push({ peerID: peerID, postIDs: postIDs });
+
+    this.processSyncQueue(userID);
+  }
+
+
   // To avoid reuesting the same posts from multiple peers:
   // 1. Add incoming IDs to queue
   // 2. Call a function that tests IDs and then gets posts.
@@ -135,11 +152,7 @@ export class App {
 
       let postIDs = await this.peerManager?.rpc.getPostIDsForUser(sendingPeerID, userID);
       console.log.apply(null, log(`[app] Got (${postIDs.length}) post IDs for user [${logID(userID)}] from peer [${logID(sendingPeerID)}]`));
-
-
       this.addPostIDsToSyncQueue(userID, sendingPeerID, postIDs);
-
-
     }
   }
 
@@ -167,9 +180,7 @@ export class App {
     });
 
     this.peerManager.addEventListener(PeerEventTypes.PEER_DISCONNECTED, async (event: any) => {
-      let peerID = event.peerID;
       console.log.apply(null, log(`[app]: peer disconnected:${event.peerID}`));
-      this.sync.deleteUserPeer(peerID);
     });
 
 
@@ -206,15 +217,17 @@ export class App {
       for (let post of posts) {
         console.log.apply(null, log(`[app] sendPostForUser sending post [${logID(post.post_id)}] to [${logID(requestingPeerID)}]`, userID, post.author, post.text));
 
-        this.peerManager?.rpc.sendPostForUser(requestingPeerID, userID, post);
+        await this.peerManager?.rpc.sendPostForUser(requestingPeerID, this.peerID, userID, post);
       }
+
+      return true;
       // return posts;
 
       // return postIDs;
     });
 
-    this.peerManager.registerRPC('sendPostForUser', async (userID: string, post: Post) => {
+    this.peerManager.registerRPC('sendPostForUser', async (sendingPeerID: string, userID: string, post: Post) => {
-      console.log.apply(null, log(`[app] sendPostForUser got post ${logID(userID)} author ${post.author} text ${post.text}`));
+      console.log.apply(null, log(`[app] sendPostForUser got post[${logID(post.post_id)}] from peer[${logID(sendingPeerID)}] for user[${logID(userID)}] author[${post.author}] text[${post.text}]`));
       // if (post.text === "image...") {
       //   debugger;
       // }
@@ -225,7 +238,9 @@ export class App {
         clearTimeout(this.renderTimer);
       }
 
-      this.renderTimer = setTimeout(() => { this.render() }, 200);
+      this.renderTimer = setTimeout(() => { this.render() }, 1000);
+
+      return true;
       // }
     });
 
@@ -794,7 +809,7 @@ export class App {
     return document.getElementById(elementName) as HTMLDivElement;
   }
 
-  initButtons(userID: string, posts: StoragePost[], registration: ServiceWorkerRegistration | undefined) {
+  initButtons(userID: string, posts: StoragePost[]) {
     // let font1Button = document.getElementById("button_font1") as HTMLButtonElement;
     // let font2Button = document.getElementById("button_font2") as HTMLButtonElement;
     // let importTweetsButton = document.getElementById("import_tweets") as HTMLButtonElement;
@@ -1067,13 +1082,7 @@ export class App {
       this.limitPosts = parseInt(limitPostsParam);
     }
 
-    this.peerID = this.getPeerID();
-    this.peername = this.getPeername();
-    this.userID = this.getUserID();
-    this.username = this.getUsername();
  
-    this.sync.setUserID(this.userID)
-    this.sync.setArchive(this.isArchivePeer);
 
     this.getRoute();
     if (this.router.route === App.Route.CONNECT) {
@@ -1082,6 +1091,15 @@ export class App {
       localStorage.removeItem("dandelion_username");
     }
 
+
+    this.peerID = this.getPeerID();
+    this.peername = this.getPeername();
+    this.userID = this.getUserID();
+    this.username = this.getUsername();
+
+    this.sync.setUserID(this.userID)
+    this.sync.setArchive(this.isArchivePeer);
+
     this.connect();
 
     await this.initDB();
@@ -1124,17 +1142,19 @@ export class App {
     //   let storageUsed = (await navigator?.storage?.estimate())?.usage/1024/1024
     // }
 
-    // if (urlParams.get("sw") === "true") {
     let registration;
+    let shouldRegisterServiceWorker = !(this.isBootstrapPeer || this.isArchivePeer || this.isHeadless);
+
+    if (shouldRegisterServiceWorker) {
       registration = await this.registerServiceWorker();
-    // }
+    }
 
     document.getElementById('username')!.innerText = `${this.username}`;
     document.getElementById('peername')!.innerText = `peername:${this.peername}`;
     document.getElementById('user_id')!.innerText = `user_id:${this.userID}`;
     document.getElementById('peer_id')!.innerText = `peer_id:${this.peerID}`;
 
-    this.initButtons(this.userID, this.posts, registration);
+    this.initButtons(this.userID, this.posts);
 
 
 

src/PeerManager.ts

@@ -714,28 +714,29 @@ class PeerConnection {
 
 
     while (this.dataChannel.bufferedAmount >= 8 * 1024 * 1024) {
-      await new Promise<void>((resolve, reject) => { setTimeout(()=> resolve(), 1000);
+      await new Promise<void>((resolve, reject) => {
+        setTimeout(() => resolve(), 1000);
       })
     }
 
     let messageJSON = JSON.stringify(message);
 
-this.messageSuperlog && console.log.apply(null, log(`[${logID(this.remotePeerID)}]<-datachannel[${logID(this.peerManager.peerID)}]:`, message.type, message, `message size:${messageJSON.length}`));
+    this.messageSuperlog && console.log.apply(null, log(`[${logID(this.remotePeerID)}]<-datachannel[${logID(this.peerManager.peerID)}]:`, message.type, message, `message size:${messageJSON.length}`));
 
-if (messageJSON.length > this.chunkSize) {
+    if (messageJSON.length > this.chunkSize) {
       this.messageSuperlog && console.log.apply(null, log(`[datachannel] sending long message: `, messageJSON.length));
       this.sendLongMessage(messageJSON);
       return;
-}
+    }
 
 
 
-try {
+    try {
       this.dataChannel?.send(messageJSON);
-} catch (e) {
+    } catch (e) {
       console.log.apply(null, log(e));
 
-}
+    }
 
     // this.onMessage(messageJSON);
   }
@@ -748,7 +749,7 @@ try {
     const hashArray = Array.from(new Uint8Array(hashBuffer));
     const hashHex = hashArray.map((b) => b.toString(16).padStart(2, "0")).join('');
     return hashHex;
-}
+  }
 
   async sendLongMessage(message: string) {
     // message = JSON.parse(message);
@@ -770,12 +771,12 @@ try {
       let netMessage = { type: 'chunk', message_id: messageID, hash: hash, chunk_index: i, total_chunks: chunks, chunk: chunk, chunk_hash: chunkHash };
       await this.send(netMessage);
     }
-}
+  }
 
 
 
 
-call(functionName: string, args: any) {
+  call(functionName: string, args: any) {
     let transactionID = generateID(); // make this faster as we will only ever have a small number of in-flight queries on a peer
     // Think about a timeout here to auto reject it after a while.
     let promise = new Promise((resolve, reject) => {
@@ -796,7 +797,7 @@ call(functionName: string, args: any) {
     this.send(message);
 
     return promise;
-}
+  }
 
   async onMessage(messageJSON: any) {
 
@@ -878,5 +879,5 @@ call(functionName: string, args: any) {
     }
 
     // this.peerManger.onMessage(this.remotePeerID, message);
-}
+  }
 }

src/Sync.ts

@@ -181,12 +181,12 @@ export class Sync {
     async checkPostIds(userID: string, peerID: string, postIDs: string[]) {
         let startTime = performance.now();
         let neededPostIds = await checkPostIds(userID, postIDs);
-        console.log.apply(null, log(`ID Check for user ${logID(userID)} took ${(performance.now() - startTime).toFixed(2)}ms`));
+        this.syncSuperlog && console.log.apply(null, log(`[sync] ID Check for user ${logID(userID)} with IDs from peer[${logID(peerID)}] took ${(performance.now() - startTime).toFixed(2)}ms`));
 
         if (neededPostIds.length > 0) {
-            console.log.apply(null, log(`Need posts (${neededPostIds.length}) for user ${logID(userID)} from peer ${logID(peerID)}`));;
+            this.syncSuperlog && console.log.apply(null, log(`[sync] Need posts (${neededPostIds.length}) for user[${logID(userID)}] from peer[${logID(peerID)}]`));;
         } else {
-            console.log.apply(null, log(`Don't need any posts for user ${logID(userID)} from peer ${logID(peerID)}`));;
+            this.syncSuperlog && console.log.apply(null, log(`[sync] Don't need any posts for user[${logID(userID)}] from peer[${logID(peerID)}]`));;
 
         }