Merge pull request 'Export all posts when you export, also store the user id and username of the user exporting.' (#5) from bobbyd-export-all into main

Reviewed-on: #5

deno.json

@@ -1,6 +1,7 @@
 {
   "tasks": {
     "build": "deno run -A npm:typescript/bin/tsc",
-    "watch": "deno run -A npm:typescript/bin/tsc --watch"
+    "watch": "deno run -A npm:typescript/bin/tsc --watch",
+    "test": "deno test --allow-net src/App.test.ts"
   }
 }

deno.lock

@@ -3,8 +3,10 @@
   "specifiers": {
     "jsr:@deno-library/compress@*": "0.5.6",
     "jsr:@deno-library/crc32@1.0.2": "1.0.2",
+    "jsr:@std/assert@*": "1.0.19",
     "jsr:@std/bytes@^1.0.2": "1.0.6",
     "jsr:@std/fs@1.0.5": "1.0.5",
+    "jsr:@std/internal@^1.0.12": "1.0.12",
     "jsr:@std/io@0.225.0": "0.225.0",
     "jsr:@std/path@1.0.8": "1.0.8",
     "jsr:@std/path@^1.0.7": "1.0.8",
@@ -29,6 +31,12 @@
     "@deno-library/crc32@1.0.2": {
       "integrity": "d2061bfee30c87c97f285dfca0fdc4458e632dc072a33ecfc73ca5177a5a39a0"
     },
+    "@std/assert@1.0.19": {
+      "integrity": "eaada96ee120cb980bc47e040f82814d786fe8162ecc53c91d8df60b8755991e",
+      "dependencies": [
+        "jsr:@std/internal"
+      ]
+    },
     "@std/bytes@1.0.6": {
       "integrity": "f6ac6adbd8ccd99314045f5703e23af0a68d7f7e58364b47d2c7f408aeb5820a"
     },
@@ -38,6 +46,9 @@
         "jsr:@std/path@^1.0.7"
       ]
     },
+    "@std/internal@1.0.12": {
+      "integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027"
+    },
     "@std/io@0.225.0": {
       "integrity": "c1db7c5e5a231629b32d64b9a53139445b2ca640d828c26bf23e1c55f8c079b3",
       "dependencies": [

setup.sh

@@ -9,15 +9,6 @@ if ! command -v deno &>/dev/null; then
     export PATH="$DENO_INSTALL/bin:$PATH"
 fi
 
-# Install node if needed
-if ! command -v node &>/dev/null; then
-    echo "Installing node..."
-    brew install node
-fi
-
-# Install TypeScript dependencies
-npm install
-
 # Install tmux if needed
 if ! command -v tmux &>/dev/null; then
     echo "Installing tmux..."

src/App.ts

@@ -528,42 +528,105 @@ export class App {
     globalThis.URL.revokeObjectURL(url);
   }
 
-  async importPostsForUser(userID: string, buffer: ArrayBuffer) {
+  async importPostsForUser(buffer: ArrayBuffer) {
+    const startTime = performance.now();
     console.log.apply(null, log("Importing posts"));
     const json = await decompressBuffer(buffer);
-    const posts = JSON.parse(json);
+    const data = JSON.parse(json);
 
-    for (let post of posts) {
-      if (post.image_data && typeof post.image_data === 'string') {
-        post.image_data = await base64ToArrayBuffer(post.image_data);
-      }
-      if (post.post_timestamp && typeof post.post_timestamp === 'string') {
-        post.post_timestamp = new Date(post.post_timestamp);
-      }
+    let postsByUser: { [userID: string]: any[] };
+    let username = this.username;
+    let userID = this.userID;
+
+    if (Array.isArray(data)) {
+      console.log.apply(null, log("Detected old export format"));
+      postsByUser = { [this.userID]: data };
+    } else {
+      console.log.apply(null, log("Detected new export format"));
+      const { username: importedUsername, userID: importedUserID, posts } = data;
+      username = importedUsername;
+      userID = importedUserID;
+      postsByUser = posts;
+      localStorage.setItem("dandelion_username", username);
+      localStorage.setItem("dandelion_id", userID);
     }
 
-    await mergeDataArray(userID, posts);
-    console.log.apply(null, log(`Imported ${posts.length} posts`));
+    let totalPostsImported = 0;
+    const userTimings: { [userID: string]: number } = {};
+
+    for (const [sourceUserID, posts] of Object.entries(postsByUser)) {
+      const userStartTime = performance.now();
+      const postList = posts as any[];
+
+      for (let post of postList) {
+        if (post.image_data && typeof post.image_data === 'string') {
+          post.image_data = await base64ToArrayBuffer(post.image_data);
+        }
+        if (post.post_timestamp && typeof post.post_timestamp === 'string') {
+          post.post_timestamp = new Date(post.post_timestamp);
+        }
+      }
+
+      await mergeDataArray(sourceUserID, postList);
+      totalPostsImported += postList.length;
+      userTimings[sourceUserID] = performance.now() - userStartTime;
+    }
+
+    const totalTime = performance.now() - startTime;
+    const userTimingsLog = Object.entries(userTimings)
+      .map(([uid, ms]) => `${logID(uid)}: ${ms.toFixed(2)}ms`)
+      .join(', ');
+
+    console.log.apply(null, log(`Imported ${totalPostsImported} posts from ${Object.keys(postsByUser).length} users in ${totalTime.toFixed(2)}ms (${userTimingsLog})`));
   }
 
-  async exportPostsForUser(userID: string) {
+  async exportPostsForUser() {
+    console.log.apply(null, log("Exporting all posts for all users"));
+    const exportStartTime = performance.now();
 
-    let posts = await getAllData(userID);
+    const knownUsers = [...(await indexedDB.databases())]
+      .map(db => db.name?.replace('user_', ''))
+      .filter((userID): userID is string => userID !== undefined);
 
-    let output = [];
+    const postsByUser: { [userID: string]: any[] } = {};
+    const userTimings: { [userID: string]: number } = {};
 
-    console.log.apply(null, log("Serializing images"));
-    for (let post of posts) {
-      let newPost = (post as any).data;
+    for (const userID of knownUsers) {
+      const userStartTime = performance.now();
+      const posts = await getAllData(userID);
+      const output = [];
 
-      if (newPost.image_data) {
-        newPost.image_data = await arrayBufferToBase64(newPost.image_data);
+      for (let post of posts) {
+        let newPost = (post as any).data;
+
+        if (newPost.image_data) {
+          newPost.image_data = await arrayBufferToBase64(newPost.image_data);
+        }
+
+        output.push(newPost);
       }
 
-      output.push(newPost);
+      if (output.length > 0) {
+        postsByUser[userID] = output;
+      }
+
+      userTimings[userID] = performance.now() - userStartTime;
     }
 
-    let compressedData = await compressString(JSON.stringify(output));
+    const totalTime = performance.now() - exportStartTime;
+    const userTimingsLog = Object.entries(userTimings)
+      .map(([uid, ms]) => `${logID(uid)}: ${ms.toFixed(2)}ms`)
+      .join(', ');
+
+    console.log.apply(null, log(`Exported ${Object.keys(postsByUser).length} users in ${totalTime.toFixed(2)}ms (${userTimingsLog})`));
+
+    const exportData = {
+      username: this.username,
+      userID: this.userID,
+      posts: postsByUser
+    };
+
+    let compressedData = await compressString(JSON.stringify(exportData));
 
     const d = new Date();
     const timestamp = `${d.getFullYear()
@@ -573,8 +636,7 @@ export class App {
       }_${String(d.getMinutes()).padStart(2, '0')
       }_${String(d.getSeconds()).padStart(2, '0')}`;
 
-
-    this.downloadBinary(compressedData, `ddln_${this.username}_export_${timestamp}.json.gz`);
+    this.downloadBinary(compressedData, `ddln_export_${timestamp}.json.gz`);
   }
 
   async importTweetArchive(userID: string, tweetArchive: any[]) {
@@ -1013,15 +1075,16 @@ export class App {
       const file = importFilePicker.files?.[0];
       if (!file) return;
       const buffer = await file.arrayBuffer();
-      await this.importPostsForUser(this.userID, buffer);
+      await this.importPostsForUser(buffer);
       importFilePicker.value = '';
+      this.userID = localStorage.getItem("dandelion_id") || this.userID;
+      this.username = localStorage.getItem("dandelion_username") || this.username;
       this.render();
     });
 
     let exportButton = this.button("export-button");
     exportButton.addEventListener('click', async e => {
-
-      await this.exportPostsForUser(this.userID)
+      await this.exportPostsForUser()
     });
 
     let composeButton = this.div('compose-button');

static/App.js

@@ -384,36 +384,88 @@ export class App {
         document.body.removeChild(link);
         globalThis.URL.revokeObjectURL(url);
     }
-    async importPostsForUser(userID, buffer) {
+    async importPostsForUser(buffer) {
+        const startTime = performance.now();
         console.log.apply(null, log("Importing posts"));
         const json = await decompressBuffer(buffer);
-        const posts = JSON.parse(json);
-        for (let post of posts) {
-            if (post.image_data && typeof post.image_data === 'string') {
-                post.image_data = await base64ToArrayBuffer(post.image_data);
-            }
-            if (post.post_timestamp && typeof post.post_timestamp === 'string') {
-                post.post_timestamp = new Date(post.post_timestamp);
-            }
+        const data = JSON.parse(json);
+        let postsByUser;
+        let username = this.username;
+        let userID = this.userID;
+        if (Array.isArray(data)) {
+            console.log.apply(null, log("Detected old export format"));
+            postsByUser = { [this.userID]: data };
         }
-        await mergeDataArray(userID, posts);
-        console.log.apply(null, log(`Imported ${posts.length} posts`));
+        else {
+            console.log.apply(null, log("Detected new export format"));
+            const { username: importedUsername, userID: importedUserID, posts } = data;
+            username = importedUsername;
+            userID = importedUserID;
+            postsByUser = posts;
+            localStorage.setItem("dandelion_username", username);
+            localStorage.setItem("dandelion_id", userID);
+        }
+        let totalPostsImported = 0;
+        const userTimings = {};
+        for (const [sourceUserID, posts] of Object.entries(postsByUser)) {
+            const userStartTime = performance.now();
+            const postList = posts;
+            for (let post of postList) {
+                if (post.image_data && typeof post.image_data === 'string') {
+                    post.image_data = await base64ToArrayBuffer(post.image_data);
+                }
+                if (post.post_timestamp && typeof post.post_timestamp === 'string') {
+                    post.post_timestamp = new Date(post.post_timestamp);
+                }
+            }
+            await mergeDataArray(sourceUserID, postList);
+            totalPostsImported += postList.length;
+            userTimings[sourceUserID] = performance.now() - userStartTime;
+        }
+        const totalTime = performance.now() - startTime;
+        const userTimingsLog = Object.entries(userTimings)
+            .map(([uid, ms]) => `${logID(uid)}: ${ms.toFixed(2)}ms`)
+            .join(', ');
+        console.log.apply(null, log(`Imported ${totalPostsImported} posts from ${Object.keys(postsByUser).length} users in ${totalTime.toFixed(2)}ms (${userTimingsLog})`));
     }
-    async exportPostsForUser(userID) {
-        let posts = await getAllData(userID);
-        let output = [];
-        console.log.apply(null, log("Serializing images"));
-        for (let post of posts) {
-            let newPost = post.data;
-            if (newPost.image_data) {
-                newPost.image_data = await arrayBufferToBase64(newPost.image_data);
+    async exportPostsForUser() {
+        console.log.apply(null, log("Exporting all posts for all users"));
+        const exportStartTime = performance.now();
+        const knownUsers = [...(await indexedDB.databases())]
+            .map(db => db.name?.replace('user_', ''))
+            .filter((userID) => userID !== undefined);
+        const postsByUser = {};
+        const userTimings = {};
+        for (const userID of knownUsers) {
+            const userStartTime = performance.now();
+            const posts = await getAllData(userID);
+            const output = [];
+            for (let post of posts) {
+                let newPost = post.data;
+                if (newPost.image_data) {
+                    newPost.image_data = await arrayBufferToBase64(newPost.image_data);
+                }
+                output.push(newPost);
             }
-            output.push(newPost);
+            if (output.length > 0) {
+                postsByUser[userID] = output;
+            }
+            userTimings[userID] = performance.now() - userStartTime;
         }
-        let compressedData = await compressString(JSON.stringify(output));
+        const totalTime = performance.now() - exportStartTime;
+        const userTimingsLog = Object.entries(userTimings)
+            .map(([uid, ms]) => `${logID(uid)}: ${ms.toFixed(2)}ms`)
+            .join(', ');
+        console.log.apply(null, log(`Exported ${Object.keys(postsByUser).length} users in ${totalTime.toFixed(2)}ms (${userTimingsLog})`));
+        const exportData = {
+            username: this.username,
+            userID: this.userID,
+            posts: postsByUser
+        };
+        let compressedData = await compressString(JSON.stringify(exportData));
         const d = new Date();
         const timestamp = `${d.getFullYear()}_${String(d.getMonth() + 1).padStart(2, '0')}_${String(d.getDate()).padStart(2, '0')}_${String(d.getHours()).padStart(2, '0')}_${String(d.getMinutes()).padStart(2, '0')}_${String(d.getSeconds()).padStart(2, '0')}`;
-        this.downloadBinary(compressedData, `ddln_${this.username}_export_${timestamp}.json.gz`);
+        this.downloadBinary(compressedData, `ddln_export_${timestamp}.json.gz`);
     }
     async importTweetArchive(userID, tweetArchive) {
         console.log.apply(null, log("Importing tweet archive"));
@@ -760,13 +812,15 @@ export class App {
             if (!file)
                 return;
             const buffer = await file.arrayBuffer();
-            await this.importPostsForUser(this.userID, buffer);
+            await this.importPostsForUser(buffer);
             importFilePicker.value = '';
+            this.userID = localStorage.getItem("dandelion_id") || this.userID;
+            this.username = localStorage.getItem("dandelion_username") || this.username;
             this.render();
         });
         let exportButton = this.button("export-button");
         exportButton.addEventListener('click', async (e) => {
-            await this.exportPostsForUser(this.userID);
+            await this.exportPostsForUser();
         });
         let composeButton = this.div('compose-button');
         composeButton.addEventListener('click', e => {