import { openDatabase, getData, addData, addDataArray, clearData, deleteData, mergeDataArray, getAllData, checkPostIds, getAllIds, getPostsByIds } from "db"; import { log, logID } from "log"; async function bytesToBase64DataUrl(bytes: Uint8Array, type = "application/octet-stream") { return await new Promise((resolve, reject) => { const reader = Object.assign(new FileReader(), { onload: () => resolve(reader.result), onerror: () => reject(reader.error), }); reader.readAsDataURL(new File([bytes], "", { type })); }); } async function arrayBufferToBase64(buffer: ArrayBuffer) { var bytes = new Uint8Array(buffer); return (await bytesToBase64DataUrl(bytes) as string).replace("data:application/octet-stream;base64,", ""); } async function base64ToArrayBuffer(base64String: string) { let response = await fetch("data:application/octet-stream;base64," + base64String); let arrayBuffer = await response.arrayBuffer(); return arrayBuffer; } export class Sync { isArchivePeer: boolean = false; userID: string = ""; userPeers: Map> = new Map(); userIDsToSync: Set = new Set(); syncSuperlog: boolean = false; setArchive(isHeadless: boolean) { this.isArchivePeer = isHeadless; } setUserID(userID: string) { this.userIDsToSync = new Set(this.getFollowing(userID)); } shouldSyncUserID(userID: string) { let shouldSyncAllUsers = this.isArchivePeer; if (shouldSyncAllUsers) { return true; } return this.userIDsToSync.has(userID); } getPeersForUser(userID:string) { let peers = this.userPeers.get(userID); if (!peers) { return []; } return [...peers.keys()]; } addUserPeer(userID: string, peerID: string) { this.syncSuperlog && console.log.apply(null, log(`[sync] addUserPeer user:${logID(userID)} peer:${logID(peerID)}`));; if (!this.userPeers.has(userID)) { this.userPeers.set(userID, new Set()); } let peers = this.userPeers.get(userID) as Set; peers.add(peerID); this.syncSuperlog && console.log.apply(null, log(this.userPeers));; } deleteUserPeer(peerIDToDelete: string) { for (const peers of this.userPeers.values()) { for (const peerID of peers) { if (peerID === peerIDToDelete) { peers.delete(peerIDToDelete); } } } } // shouldSyncUserID(userID: string) { // if (app.isHeadless) { // return true; // } // return this.UserIDsTothis.has(userID); // } async getKnownUsers() { let knownUsers = [...(await indexedDB.databases())].map(db => db.name?.replace('user_', '')).filter(userID => userID !== undefined); knownUsers = knownUsers .filter(userID => this.shouldSyncUserID(userID)) .filter(userID => !this.userBlockList.has(userID)) .filter(async userID => (await getAllIds(userID)).length > 0); // TODO:EASYOPT getting all the IDs is unecessary, replace it with a test to get a single ID. return knownUsers; } userBlockList = new Set([ '5d63f0b2-a842-41bf-bf06-e0e4f6369271', '5f1b85c4-b14c-454c-8df1-2cacc93f8a77', // 'bba3ad24-9181-4e22-90c8-c265c80873ea' ]) postBlockList = new Set([ '1c71f53c-c467-48e4-bc8c-39005b37c0d5', '64203497-f77b-40d6-9e76-34d17372e72a', '243130d8-4a41-471e-8898-5075f1bd7aec', 'e01eff89-5100-4b35-af4c-1c1bcb007dd0', '194696a2-d850-4bb0-98f7-47416b3d1662', 'f6b21eb1-a0ff-435b-8efc-6a3dd70c0dca', 'dd1d92aa-aa24-4166-a925-94ba072a9048' ]); getFollowing(userID: string): string[] { // Rob if (userID === 'b38b623c-c3fa-4351-9cab-50233c99fa4e') { return [ 'b38b623c-c3fa-4351-9cab-50233c99fa4e', '6d774268-16cd-4e86-8bbe-847a0328893d', // Sean '05a495a0-0dd8-4186-94c3-b8309ba6fc4c', // Martin 'a0e42390-08b5-4b07-bc2b-787f8e5f1297', // BMO 'bba3ad24-9181-4e22-90c8-c265c80873ea', // Harry '8f6802be-c3b6-46c1-969c-5f90cbe01479', // Fiona ] } // Martin if (userID === '05a495a0-0dd8-4186-94c3-b8309ba6fc4c') { return [ 'b38b623c-c3fa-4351-9cab-50233c99fa4e', 'a0e42390-08b5-4b07-bc2b-787f8e5f1297', // BMO ] } // Fiona if (userID === '8f6802be-c3b6-46c1-969c-5f90cbe01479') { return [ 'b38b623c-c3fa-4351-9cab-50233c99fa4e', // Rob 'a0e42390-08b5-4b07-bc2b-787f8e5f1297', // BMO '05a495a0-0dd8-4186-94c3-b8309ba6fc4c', // Martin ] } return ['a0e42390-08b5-4b07-bc2b-787f8e5f1297']; // Follow BMO by default :) } async getPostIdsForUser(userID: string) { let postIds = await getAllIds(userID) ?? []; postIds = postIds.filter((postID: string) => !this.postBlockList.has(postID)); if (postIds.length === 0) { console.log.apply(null, log(`Net: I know about user ${logID(userID)} but I have 0 posts`)); return null; } return postIds; } async checkPostIds(userID:string, peerID:string, postIDs:string[]) { let startTime = performance.now(); let neededPostIds = await checkPostIds(userID, postIDs); console.log.apply(null, log(`ID Check for user ${logID(userID)} took ${(performance.now() - startTime) .toFixed(2)}ms`)); if (neededPostIds.length > 0) { console.log.apply(null, log(`Need posts (${neededPostIds.length}) for user ${logID(userID)} from peer ${logID(peerID)}`));; } else { console.log.apply(null, log(`Don't need any posts for user ${logID(userID)} from peer ${logID(peerID)}`));; } // if (postIds.length === 0) { // return []; // } return neededPostIds; } async getPostsForUser(userID:string, postIDs: string[]) { let posts = await getPostsByIds(userID, postIDs) ?? []; console.log.apply(null, log(`[sync] got ${posts.length} posts for user ${logID(userID)}`));; // app.timerStart(); let output = []; console.log.apply(null, log("Serializing images")); for (let post of posts) { let newPost = (post as any).data; if (newPost.image_data) { // let compressedData = await wsConnection.compressArrayBuffer(newPost.image_data); // console.log.apply(null, log((newPost.image_data.byteLength - compressedData.byteLength) / 1024 / 1024); // TODO don't do this, use Blobs direclty! // https://developer.chrome.com/blog/blob-support-for-Indexeddb-landed-on-chrome-dev newPost.image_data = await arrayBufferToBase64(newPost.image_data); } // let megs = JSON.stringify(newPost).length/1024/1024; // console.log.apply(null, log(`getPostsForUserHandler id:${newPost.post_id} post length:${megs}`); output.push(newPost); } return output; // console.log.apply(null, log(`getPostsForUser`,output)); } async writePostForUser(userID:string, post:any) { // HACK: Some posts have insanely large images, so I'm gonna skip them. // Once we support delete then we we could delete these posts in a sensible way. if (this.postBlockList.has(post.post_id)) { console.log.apply(null, log(`Skipping blocked post: ${post.post_id}`));; return; } // HACK - some posts had the wrong author ID if (userID === this.userID) { post.author_id = this.userID; } post.post_timestamp = new Date(post.post_timestamp); if (post.image_data) { post.image_data = await base64ToArrayBuffer(post.image_data); } console.log.apply(null, log(`Merging same user peer posts...`)); await mergeDataArray(userID, [post]); } // async getPostIdsForUserHandler(data: any) { // let message = data.message; // let postIds = await getAllIds(message.user_id) ?? []; // postIds = postIds.filter((postID: string) => !this.postBlockList.has(postID)); // if (postIds.length === 0) { // console.log.apply(null, log(`Net: I know about user ${logID(message.user_id)} but I have 0 posts, so I'm not sending any to to peer ${logID(data.from)}`));; // return; // } // console.log.apply(null, log(`Net: Sending ${postIds.length} post Ids for user ${logID(message.user_id)} to peer ${logID(data.from)}`)); // let responseMessage = { type: "peer_message", from: app.peerID, to: data.from, from_username: app.username, from_peername: app.peername, message: { type: "get_post_ids_for_user_response", post_ids: postIds, user_id: message.user_id } } // this.send(responseMessage); // } }