bobbydigitales/dandelion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement per-user message queues to prevent duplicate post syncing. Correclty initialize from a user or connect URL. Dont register the service worker when we\'re headless or an archive peer. Wrap sync logging with superlog

Browse Source
This commit is contained in:
bobbydigitales
2025-05-27 00:01:52 -07:00
parent 71b047099b
commit 75b55104bd
3 changed files with 200 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
src/App.ts
View File

@@ -46,6 +46,14 @@ interface StoragePost {
data: Post;
}
interface SyncItem {
peerID: string;
postIDs: string[];
}
export class App {
username: string = '';
peername: string = '';
@@ -67,24 +75,27 @@ export class App {
peerManager: PeerManager | null = null;
sync: Sync = new Sync();
renderTimer: number = 0;
postSyncQueue: any[] = [];
postSyncPromise: any = null;
syncQueues: Map<string, SyncItem[]> = new Map();
syncing: Set<string> = new Set();
async syncPostsInQueue() {
async processSyncQueue(userID: string) {
if (this.postSyncPromise) {
if (this.syncing.has(userID)) {
return;
}
while (this.postSyncQueue.length !== 0) {
let syncQueue = this.syncQueues.get(userID) as SyncItem[];
let queueItem = this.postSyncQueue.pop();
while (syncQueue.length !== 0) {
this.syncing.add(userID);
let syncItem = syncQueue.pop();
let userID = queueItem.userID;
let peerID = queueItem.peerID;
let postIDs = queueItem.postIDs;
if (!syncItem) {
throw new Error();
}
new Promise(async (resolve, reject) => {
let peerID = syncItem?.peerID;
let postIDs = syncItem?.postIDs;
let neededPostIDs = await this.sync.checkPostIds(userID, peerID, postIDs);
if (neededPostIDs.length > 0) {
@@ -94,23 +105,29 @@ export class App {
}
else {
console.log.apply(null, log(`[app] Don't need any posts for user ${logID(userID)} from peer ${logID(sendingPeerID)}`));
console.log.apply(null, log(`[app] Don't need any posts for user ${logID(userID)} from peer ${logID(peerID)}`));
}
}
})
}
this.syncing.delete(userID);
}
addPostIDsToSyncQueue(userID: string, peerID: string, postIDs: string[]) {
this.postSyncQueue.push({ userID: userID, peerID: peerID, postIDs: postIDs });
let syncQueue = this.syncQueues.get(userID);
if (!syncQueue) {
let newArray: SyncItem[] = [];
this.syncQueues.set(userID, newArray);
syncQueue = newArray;
}
syncQueue.push({ peerID: peerID, postIDs: postIDs });
this.processSyncQueue(userID);
}
// To avoid reuesting the same posts from multiple peers:
// 1. Add incoming IDs to queue
// 2. Call a function that tests IDs and then gets posts.
@@ -135,11 +152,7 @@ export class App {
let postIDs = await this.peerManager?.rpc.getPostIDsForUser(sendingPeerID, userID);
console.log.apply(null, log(`[app] Got (${postIDs.length}) post IDs for user [${logID(userID)}] from peer [${logID(sendingPeerID)}]`));
this.addPostIDsToSyncQueue(userID, sendingPeerID, postIDs);
}
}
@@ -167,9 +180,7 @@ export class App {
});
this.peerManager.addEventListener(PeerEventTypes.PEER_DISCONNECTED, async (event: any) => {
let peerID = event.peerID;
console.log.apply(null, log(`[app]: peer disconnected:${event.peerID}`));
this.sync.deleteUserPeer(peerID);
});
@@ -206,15 +217,17 @@ export class App {
for (let post of posts) {
console.log.apply(null, log(`[app] sendPostForUser sending post [${logID(post.post_id)}] to [${logID(requestingPeerID)}]`, userID, post.author, post.text));
this.peerManager?.rpc.sendPostForUser(requestingPeerID, userID, post);
await this.peerManager?.rpc.sendPostForUser(requestingPeerID, this.peerID, userID, post);
}
return true;
// return posts;
// return postIDs;
});
this.peerManager.registerRPC('sendPostForUser', async (userID: string, post: Post) => {
console.log.apply(null, log(`[app] sendPostForUser got post ${logID(userID)} author ${post.author} text ${post.text}`));
this.peerManager.registerRPC('sendPostForUser', async (sendingPeerID: string, userID: string, post: Post) => {
console.log.apply(null, log(`[app] sendPostForUser got post[${logID(post.post_id)}] from peer[${logID(sendingPeerID)}] for user[${logID(userID)}] author[${post.author}] text[${post.text}]`));
// if (post.text === "image...") {
// debugger;
// }
@@ -225,7 +238,9 @@ export class App {
clearTimeout(this.renderTimer);
}
this.renderTimer = setTimeout(() => { this.render() }, 200);
this.renderTimer = setTimeout(() => { this.render() }, 1000);
return true;
// }
});
@@ -794,7 +809,7 @@ export class App {
return document.getElementById(elementName) as HTMLDivElement;
}
initButtons(userID: string, posts: StoragePost[], registration: ServiceWorkerRegistration | undefined) {
initButtons(userID: string, posts: StoragePost[]) {
// let font1Button = document.getElementById("button_font1") as HTMLButtonElement;
// let font2Button = document.getElementById("button_font2") as HTMLButtonElement;
// let importTweetsButton = document.getElementById("import_tweets") as HTMLButtonElement;
@@ -1067,13 +1082,7 @@ export class App {
this.limitPosts = parseInt(limitPostsParam);
}
this.peerID = this.getPeerID();
this.peername = this.getPeername();
this.userID = this.getUserID();
this.username = this.getUsername();
this.sync.setUserID(this.userID)
this.sync.setArchive(this.isArchivePeer);
this.getRoute();
if (this.router.route === App.Route.CONNECT) {
@@ -1082,6 +1091,15 @@ export class App {
localStorage.removeItem("dandelion_username");
}
this.peerID = this.getPeerID();
this.peername = this.getPeername();
this.userID = this.getUserID();
this.username = this.getUsername();
this.sync.setUserID(this.userID)
this.sync.setArchive(this.isArchivePeer);
this.connect();
await this.initDB();
@@ -1124,17 +1142,19 @@ export class App {
// let storageUsed = (await navigator?.storage?.estimate())?.usage/1024/1024
// }
// if (urlParams.get("sw") === "true") {
let registration;
let shouldRegisterServiceWorker = !(this.isBootstrapPeer || this.isArchivePeer || this.isHeadless);
if (shouldRegisterServiceWorker) {
registration = await this.registerServiceWorker();
// }
}
document.getElementById('username')!.innerText = `${this.username}`;
document.getElementById('peername')!.innerText = `peername:${this.peername}`;
document.getElementById('user_id')!.innerText = `user_id:${this.userID}`;
document.getElementById('peer_id')!.innerText = `peer_id:${this.peerID}`;
this.initButtons(this.userID, this.posts, registration);
this.initButtons(this.userID, this.posts);

3
src/PeerManager.ts
View File

@@ -714,7 +714,8 @@ class PeerConnection {
while (this.dataChannel.bufferedAmount >= 8 * 1024 * 1024) {
await new Promise<void>((resolve, reject) => { setTimeout(()=> resolve(), 1000);
await new Promise<void>((resolve, reject) => {
setTimeout(() => resolve(), 1000);
})
}

6
src/Sync.ts
View File

@@ -181,12 +181,12 @@ export class Sync {
async checkPostIds(userID: string, peerID: string, postIDs: string[]) {
let startTime = performance.now();
let neededPostIds = await checkPostIds(userID, postIDs);
console.log.apply(null, log(`ID Check for user ${logID(userID)} took ${(performance.now() - startTime).toFixed(2)}ms`));
this.syncSuperlog && console.log.apply(null, log(`[sync] ID Check for user ${logID(userID)} with IDs from peer[${logID(peerID)}] took ${(performance.now() - startTime).toFixed(2)}ms`));
if (neededPostIds.length > 0) {
console.log.apply(null, log(`Need posts (${neededPostIds.length}) for user ${logID(userID)} from peer ${logID(peerID)}`));;
this.syncSuperlog && console.log.apply(null, log(`[sync] Need posts (${neededPostIds.length}) for user[${logID(userID)}] from peer[${logID(peerID)}]`));;
} else {
console.log.apply(null, log(`Don't need any posts for user ${logID(userID)} from peer ${logID(peerID)}`));;
this.syncSuperlog && console.log.apply(null, log(`[sync] Don't need any posts for user[${logID(userID)}] from peer[${logID(peerID)}]`));;
}