converting to IndexedDB

index.html

@@ -8,6 +8,8 @@
   <title>Dandelion</title>
   <script type="module" src="main.js"></script>
   <script src="marked.min.js"></script>
+  <script src="purify.min.js"></script>
+
 
   <style>
     @font-face {

main.go

@@ -3,6 +3,8 @@ package main
 import (
 	"log"
 	"net/http"
+	"os"
+	"path/filepath"
 	"strconv"
 
 	"github.com/gorilla/websocket"
@@ -50,6 +52,18 @@ func (lh *LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	lh.handler.ServeHTTP(w, r)
 }
 
+// noDirListing wraps an http.FileServer handler to prevent directory listings
+func noDirListing(h http.Handler, root string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		path := filepath.Join(root, r.URL.Path)
+		if info, err := os.Stat(path); err == nil && info.IsDir() {
+			http.NotFound(w, r) // Always return 404 for directories
+			return
+		}
+		h.ServeHTTP(w, r)
+	}
+}
+
 func main() {
 	dir := "./"
 	port := 80
@@ -59,8 +73,10 @@ func main() {
 
 	// Set up file server and WebSocket endpoint
 	fs := http.FileServer(http.Dir(dir))
-	loggingHandler := &LoggingHandler{handler: fs}
+	// loggingHandler := &LoggingHandler{handler: fs}
-	http.Handle("/", loggingHandler)
+	// http.Handle("/", loggingHandler)
+	http.Handle("/", noDirListing(fs, dir))
+
 	http.HandleFunc("/ws", handleWebSocket)
 
 	// Configure and start the HTTP server

src/db.ts

@@ -34,7 +34,9 @@ export function openDatabase(userID:string): Promise<IDBDatabase> {
         request.onupgradeneeded = (event: IDBVersionChangeEvent) => {
             const db: IDBDatabase = (event.target as IDBOpenDBRequest).result;
             if (!db.objectStoreNames.contains(storeName)) {
-                db.createObjectStore(storeName, { keyPath: "id", autoIncrement: true });
+                let store = db.createObjectStore(storeName, { keyPath: "id", autoIncrement: true });
+                store.createIndex("datetimeIndex", "post_timestamp", { unique: false });
+
             }
         };
 
@@ -54,7 +56,7 @@ export async function addData(userID: string, data: any): Promise<void> {
         const transaction = db.transaction(storeName, "readwrite");
         const store = transaction.objectStore(storeName);
 
-        const addRequest = store.add(data);
+        const addRequest = store.add({post_timestamp: data.post_timestamp, data:data});
 
         addRequest.onsuccess = (e: Event) => {
             // console.log('Data has been added:', (e.target as IDBRequest).result);
@@ -82,7 +84,7 @@ export async function addDataArray(userID: string, array: any[]): Promise<void>
         array.reverse();
 
         for (let data of array) {
-            const addRequest = store.add(data);
+            const addRequest = store.add({post_timestamp: data.post_timestamp, data:data});
             addRequest.onsuccess = (e: Event) => {
                 // console.log('Data has been added:', (e.target as IDBRequest).result);
             };
@@ -95,9 +97,9 @@ export async function addDataArray(userID: string, array: any[]): Promise<void>
 
             count++;
 
-            if (count % 100 === 0) {
+            // if (count % 100 === 0) {
-                console.log(`Added ${count} posts...`);
+            //     console.log(`Added ${count} posts...`);
-            }
+            // }
         }
 
        
@@ -106,7 +108,7 @@ export async function addDataArray(userID: string, array: any[]): Promise<void>
     }
 }
 
-export async function getData(userID:string, lowerID:number, upperID:number): Promise<any | undefined> {
+export async function getData(userID:string, lowerID:Date, upperID:Date): Promise<any | undefined> {
     const storeName = `${storeNameBase}_${userID}`;
     const db = await openDatabase(userID);
     const transaction = db.transaction(storeName, "readonly");
@@ -117,12 +119,15 @@ export async function getData(userID:string, lowerID:number, upperID:number): Pr
 
         const records: any[] = [];
 
-        const cursorRequest = store.openCursor(keyRangeValue);
+        const index = store.index("datetimeIndex");
+
+
+        const cursorRequest = index.openCursor(keyRangeValue);
 
         cursorRequest.onsuccess = (event: Event) => {
             const cursor = (event.target as IDBRequest).result as IDBCursorWithValue;
             if (cursor) {
-                records.push(cursor.value); // Collect the record
+                records.push(cursor.value.data); // Collect the record
                 cursor.continue(); // Move to the next item in the range
             } else {
                 // No more entries in the range

src/main.ts

@@ -1,51 +1,79 @@
-import { openDatabase, getData, addData, addDataArray, getAllData } from "./db.js"
+import { openDatabase, getData, addData, addDataArray } from "./db.js"
 
-declare let marked:any;
+declare let marked: any;
+declare let DOMPurify: any;
 // let posts:any;
 // let keyBase = "dandelion_posts_v1_"
 // let key:string = "";
 
-interface PostTimestamp {
+// interface PostTimestamp {
-  year: number,
+//   year: number,
-  month: number,
+//   month: number,
-  day: number,
+//   day: number,
-  hour: number,
+//   hour: number,
-  minute: number,
+//   minute: number,
-  second: number,
+//   second: number,
-}
+// }
 
 class Post {
-  post_timestamp:PostTimestamp;
+  post_timestamp: Date;
-  author:string;
+  author: string;
-  author_id:string;
+  author_id: string;
-  text:string;
+  text: string;
-  // format
+  image_data: ArrayBuffer | null;
-  constructor(author: string, author_id:string, text: string, post_timestamp: PostTimestamp, format = null) {
+
+
+  importedFrom: "twitter" | null;
+  importSource: any;
+
+  constructor(author: string, author_id: string, text: string, post_timestamp: Date, imageData: ArrayBuffer | null = null, importedFrom: "twitter" | null = null, importSource: any = null) {
     this.post_timestamp = post_timestamp;
+
     this.author = author;
     this.author_id = author_id;
     this.text = text;
-    // this.format = format;
+    this.image_data = imageData;
+
+    this.importedFrom = importedFrom;
+    this.importSource = importSource;
   }
 }
 
-function timestampToString(t:PostTimestamp) {
 
-return `${t.year}/${t.month}/${t.day}-${t.hour}:${t.minute}:${t.second}`
+function initMarkdown() {
-}
+  const renderer = new marked.Renderer();
-
+  renderer.link = (href: any, title: string, text: string) => {
-function getCurrentTimestamp() {
+    return `<a href="${href}" target="_blank"${title ? ` title="${title}"` : ''}>${text}</a>`;
-  let date = new Date();
-  return  {
-    year: date.getFullYear(),
-    month: date.getMonth()+1,
-    day: date.getDate(),
-    hour: date.getHours(),
-    minute: date.getMinutes(),
-    second: date.getSeconds(),
   };
+  marked.setOptions({ renderer: renderer });
 }
 
+function waitMs(durationMs: number) {
+  return new Promise(resolve => setTimeout(resolve, durationMs));
+}
+
+
+// function getTimestampFromDate(date: Date) {
+//   return {
+//     year: date.getFullYear(),
+//     month: date.getMonth() + 1,
+//     day: date.getDate(),
+//     hour: date.getHours(),
+//     minute: date.getMinutes(),
+//     second: date.getSeconds(),
+//   };
+// }
+
+// function getTimestampFromString(dateString: string) {
+//   let date = new Date(dateString);
+//   return getTimestampFromDate(date);
+// }
+
+// function getCurrentTimestamp() {
+//   let date = new Date();
+//   return getTimestampFromDate(date);
+// }
+
 function uuidv4() {
   return "10000000-1000-4000-8000-100000000000".replace(/[018]/g, (c: any) =>
     (c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> c / 4).toString(16)
@@ -105,7 +133,66 @@ function timerDelta() {
   return performance.now() - time;
 }
 
-async function createTestData2() {
+async function getFixedTweetText(entry: any) {
+
+
+  let fullText = entry.tweet.full_text;
+
+  let linkMarkdown = "";
+  for (const url of entry.tweet.entities.urls) {
+    linkMarkdown = `[${url.display_url}](${url.expanded_url})`;
+    fullText = fullText.replace(url.url, linkMarkdown);
+  }
+
+  return fullText
+
+  // for (let url of )
+  // const regex = /https:\/\/t\.co\/[\w-]+/g;
+
+  // let returnText = tweetText;
+
+  // if (tweetText.includes("t.co")) {
+  //   // console.log(tweetText);
+  // }
+
+  // let matches = returnText.match(regex);
+  // if (!matches) {
+  //   return returnText;
+  // }
+
+  // for (const match of matches) {
+  //   // console.log(match);
+  //   let fetchedData = "";
+  //   try {
+  //     fetchedData = await (await fetch(match)).text();
+  //     await waitMs(100);
+  //   } catch (e) {
+  //     console.log(e);
+  //   }
+
+  //   if (fetchedData.includes('http-equiv="refresh"')) {
+  //     // console.log(fetchedData);
+
+  //     // let urlRegex = /URL=(http.:\/\/.+)\"/;
+  //     let urlRegex = new RegExp(`URL=(http[s]?://.+?)"`);
+  //     let urlMatch = fetchedData.match(urlRegex);
+  //     if (urlMatch?.length == 2) {
+  //       // console.log(urlMatch[1]);
+  //       returnText = returnText.replace(match, urlMatch[1]);
+  //       // URL=https://youtu.be/yGci-Lb87zs"
+  //     } else {
+  //       throw new Error(fetchedData);
+  //     }
+  //   }
+  // }
+
+  // console.log(returnText);
+
+  // return returnText;
+}
+
+async function createTestData2(userID: string) {
+  log("Importing tweet archive")
   let postsTestData: any[] = [];
 
   let response = await fetch("./tweets.js");
@@ -116,6 +203,7 @@ async function createTestData2() {
 
 
   // let tweets = JSON.parse(tweetJSON);
+  let count = 0;
 
   for (let entry of (window as any).tweetData) {
     // if (entry.tweet.hasOwnProperty("in_reply_to_screen_name") || entry.tweet.retweeted || entry.tweet.full_text.startsWith("RT")) {
@@ -128,48 +216,63 @@ async function createTestData2() {
       isImage = mediaURL.includes('jpg');
     }
 
-    let imageData;
+    let imageData = null;
-    let encodedImage = null;
+    // if (isImage) {
-    if (isImage) {
+    //   try {
-      try {
+    //     let response = await fetch(mediaURL);
-        imageData = await (await fetch(mediaURL)).arrayBuffer();
+    //     await waitMs(100);
-        encodedImage = await arrayBufferToBase64(imageData);
+    //     if (response.status === 200) {
-      } catch (e) {
+    //       imageData = await response.arrayBuffer();
-        console.log(e);
+    //     }
-      }
+    //     console.log(imageData);
+    //   } catch (e) {
+    //     console.log(e);
+    //   }
 
+    // }
+
+    let timeStamp = new Date(entry.tweet.created_at);
+
+    // let tweetText = entry.tweet.full_text;
+    let tweetText = await getFixedTweetText(entry);
+
+    let newPost = new Post('bobbydigitales', userID, tweetText, timeStamp, imageData, 'twitter', entry);
+
+    postsTestData.push(newPost);
+
+    // postsTestData.push({
+    //   post_timestamp: timeStamp,
+    //   author: `bobbydigitales`,
+    //   text: tweetText,
+    //   image: imageData
+    // });
+
+    count++;
+    if (count % 100 === 0) {
+      log(`Imported ${count} posts...`);
+      // render(postsTestData);
     }
 
+    // if (count == 100-1) {
+    //   break;
+    // }
 
-    postsTestData.push({
-      post_timestamp: {
-        year: 2023,
-        month: 10,
-        day: 19,
-        hour: 14,
-        minute: 53,
-        second: 0,
-      },
-      author: `bobbydigitales`,
-      text: entry.tweet.full_text,
-      image: encodedImage
-    });
   }
 
-  let rant = await (await fetch('ranting.md')).text();;
+  // let rant = await (await fetch('ranting.md')).text();;
 
-  postsTestData.unshift({
+  // postsTestData.unshift({
-    post_timestamp: {
+  //   post_timestamp: {
-      year: 2023,
+  //     year: 2023,
-      month: 10,
+  //     month: 10,
-      day: 19,
+  //     day: 19,
-      hour: 14,
+  //     hour: 14,
-      minute: 53,
+  //     minute: 53,
-      second: 0,
+  //     second: 0,
-    },
+  //   },
-    author: `bobbydigitales`,
+  //   author: `bobbydigitales`,
-    text: rant
+  //   text: rant
-  })
+  // })
 
   return postsTestData;
 }
@@ -209,11 +312,11 @@ async function registerServiceWorker() {
 
 function addPost(userID: string, posts: Post[], postText: string) {
   if ((typeof postText !== "string") || postText.length === 0) {
-    log ("Not posting an empty string...")
+    log("Not posting an empty string...")
     return;
   }
 
-  let post = new Post(`bobbydigitales`, userID, postText, getCurrentTimestamp());
+  let post = new Post(`bobbydigitales`, userID, postText, new Date());
 
   posts.push(post);
   // localStorage.setItem(key, JSON.stringify(posts));
@@ -309,17 +412,17 @@ function initButtons(userID: string, posts: Post[]) {
 }
 
 async function loadPosts(userID: string) {
- 
+
   timerStart();
-  let posts = await getData(userID, 1, 1000);
+  let posts: any = await getData(userID, new Date(2021, 1), new Date());
-  
+
   if (posts.length > 0) {
     log(`Loaded ${posts.length} posts in ${timerDelta().toFixed(2)}ms`);
     return posts;
   }
 
-  posts = await createTestData3(userID);
+  posts = await createTestData2(userID);
-  
+
   log("Adding test data...");
   addDataArray(userID, posts);
 
@@ -335,8 +438,8 @@ async function loadPosts(userID: string) {
   // log("Finished!");
 
 
-  return await getData(userID, 1, 1000);
+  return await getData(userID, new Date(2022, 1), new Date());
-  
+
 
   // debugger;
 
@@ -386,8 +489,6 @@ async function main() {
   }
 
   let userID = getUserID();
-
-
   log(`Your user ID is: ${userID}`);
 
   if (navigator.storage && navigator.storage.persist && !navigator.storage.persisted) {
@@ -395,6 +496,8 @@ async function main() {
     console.log(`Persisted storage granted: ${isPersisted}`);
   }
 
+  initMarkdown();
+
   // let main = await fetch("/main.js");
   // let code = await main.text();
   //   console.log(code);
@@ -424,30 +527,22 @@ function render(posts: Post[]) {
   }
   contentDiv.innerHTML = "";
   let count = 0;
-  
 
-  for (let i=posts.length -1; i>0; i--) {
+
+  for (let i = posts.length - 1; i >= 0; i--) {
     let postData = posts[i];
 
     let post = renderPost(postData);
 
-    fragment.appendChild(post);
+    if (post) {
-    count++;
+      fragment.appendChild(post);
-    // if (count > 500) {
+      count++;
-    //   break;
+    }
-    // }
+    if (count > 500) {
+      break;
+    }
   }
 
-  // for (const postData of posts) {
-    // let post = renderPost(postData);
-
-    // fragment.appendChild(post);
-
-    // count++;
-    // if (count > 500) {
-    //   break;
-    // }
-  // }
 
   if (!contentDiv) {
     throw new Error("Couldn't get content div!");
@@ -466,22 +561,35 @@ function renderPost(post: Post) {
   let timestampDiv = document.createElement("div");
   let hr = document.createElement("hr");
 
-  
+
   textDiv.innerHTML = marked.parse(post.text);
-  timestampDiv.innerText = timestampToString(post.post_timestamp);
+  // textDiv.innerHTML = DOMPurify.sanitize(marked.parse(post.text));
+
+  timestampDiv.innerText = post.post_timestamp.toLocaleDateString();
   containerDiv.appendChild(hr);
   containerDiv.appendChild(textDiv);
-  containerDiv.appendChild(timestampDiv);
 
   if (!("image" in post && post.image)) {
+    containerDiv.appendChild(timestampDiv);
     return containerDiv;
+    // return null;
   }
 
   let image = document.createElement("img");
-  image.src = image.src = "data:image/png;base64," + post.image;
+  const blob = new Blob([post.image as ArrayBuffer], { type: 'image/jpg' });
+  const url = URL.createObjectURL(blob);
+  image.onload = () => {
+    URL.revokeObjectURL(url);
+  };
+
+
+  image.src = url;
+  // image.src = image.src = "data:image/png;base64," + post.image;
   image.className = "postImage";
 
   containerDiv.appendChild(image);
+  containerDiv.appendChild(timestampDiv);
+
   return containerDiv;
 }