bobbydigitales/dandelion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

converting to IndexedDB

Browse Source
This commit is contained in:
“bobbydigitales”
2023-11-12 23:31:07 -08:00
parent 6cd5a04cc1
commit 41054a5dbf
5 changed files with 240 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
index.html
View File

@@ -8,6 +8,8 @@
<title>Dandelion</title>
<script type="module" src="main.js"></script>
<script src="marked.min.js"></script>
<script src="purify.min.js"></script>
<style>
@font-face {

BIN
main
View File

Binary file not shown.

20
main.go
View File

@@ -3,6 +3,8 @@ package main
import (
"log"
"net/http"
"os"
"path/filepath"
"strconv"
"github.com/gorilla/websocket"
@@ -50,6 +52,18 @@ func (lh *LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
lh.handler.ServeHTTP(w, r)
}
// noDirListing wraps an http.FileServer handler to prevent directory listings
func noDirListing(h http.Handler, root string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
path := filepath.Join(root, r.URL.Path)
if info, err := os.Stat(path); err == nil && info.IsDir() {
http.NotFound(w, r) // Always return 404 for directories
return
}
h.ServeHTTP(w, r)
}
}
func main() {
dir := "./"
port := 80
@@ -59,8 +73,10 @@ func main() {
// Set up file server and WebSocket endpoint
fs := http.FileServer(http.Dir(dir))
loggingHandler := &LoggingHandler{handler: fs}
http.Handle("/", loggingHandler)
// loggingHandler := &LoggingHandler{handler: fs}
// http.Handle("/", loggingHandler)
http.Handle("/", noDirListing(fs, dir))
http.HandleFunc("/ws", handleWebSocket)
// Configure and start the HTTP server

23
src/db.ts
View File

@@ -34,7 +34,9 @@ export function openDatabase(userID:string): Promise<IDBDatabase> {
request.onupgradeneeded = (event: IDBVersionChangeEvent) => {
const db: IDBDatabase = (event.target as IDBOpenDBRequest).result;
if (!db.objectStoreNames.contains(storeName)) {
db.createObjectStore(storeName, { keyPath: "id", autoIncrement: true });
let store = db.createObjectStore(storeName, { keyPath: "id", autoIncrement: true });
store.createIndex("datetimeIndex", "post_timestamp", { unique: false });
}
};
@@ -54,7 +56,7 @@ export async function addData(userID: string, data: any): Promise<void> {
const transaction = db.transaction(storeName, "readwrite");
const store = transaction.objectStore(storeName);
const addRequest = store.add(data);
const addRequest = store.add({post_timestamp: data.post_timestamp, data:data});
addRequest.onsuccess = (e: Event) => {
// console.log('Data has been added:', (e.target as IDBRequest).result);
@@ -82,7 +84,7 @@ export async function addDataArray(userID: string, array: any[]): Promise<void>
array.reverse();
for (let data of array) {
const addRequest = store.add(data);
const addRequest = store.add({post_timestamp: data.post_timestamp, data:data});
addRequest.onsuccess = (e: Event) => {
// console.log('Data has been added:', (e.target as IDBRequest).result);
};
@@ -95,9 +97,9 @@ export async function addDataArray(userID: string, array: any[]): Promise<void>
count++;
if (count % 100 === 0) {
console.log(`Added ${count} posts...`);
}
// if (count % 100 === 0) {
// console.log(`Added ${count} posts...`);
// }
}
@@ -106,7 +108,7 @@ export async function addDataArray(userID: string, array: any[]): Promise<void>
}
}
export async function getData(userID:string, lowerID:number, upperID:number): Promise<any | undefined> {
export async function getData(userID:string, lowerID:Date, upperID:Date): Promise<any | undefined> {
const storeName = `${storeNameBase}_${userID}`;
const db = await openDatabase(userID);
const transaction = db.transaction(storeName, "readonly");
@@ -117,12 +119,15 @@ export async function getData(userID:string, lowerID:number, upperID:number): Pr
const records: any[] = [];
const cursorRequest = store.openCursor(keyRangeValue);
const index = store.index("datetimeIndex");
const cursorRequest = index.openCursor(keyRangeValue);
cursorRequest.onsuccess = (event: Event) => {
const cursor = (event.target as IDBRequest).result as IDBCursorWithValue;
if (cursor) {
records.push(cursor.value); // Collect the record
records.push(cursor.value.data); // Collect the record
cursor.continue(); // Move to the next item in the range
} else {
// No more entries in the range

304
src/main.ts
View File

@@ -1,51 +1,79 @@
import { openDatabase, getData, addData, addDataArray, getAllData } from "./db.js"
import { openDatabase, getData, addData, addDataArray } from "./db.js"
declare let marked:any;
declare let marked: any;
declare let DOMPurify: any;
// let posts:any;
// let keyBase = "dandelion_posts_v1_"
// let key:string = "";
interface PostTimestamp {
year: number,
month: number,
day: number,
hour: number,
minute: number,
second: number,
}
// interface PostTimestamp {
// year: number,
// month: number,
// day: number,
// hour: number,
// minute: number,
// second: number,
// }
class Post {
post_timestamp:PostTimestamp;
author:string;
author_id:string;
text:string;
// format
constructor(author: string, author_id:string, text: string, post_timestamp: PostTimestamp, format = null) {
post_timestamp: Date;
author: string;
author_id: string;
text: string;
image_data: ArrayBuffer | null;
importedFrom: "twitter" | null;
importSource: any;
constructor(author: string, author_id: string, text: string, post_timestamp: Date, imageData: ArrayBuffer | null = null, importedFrom: "twitter" | null = null, importSource: any = null) {
this.post_timestamp = post_timestamp;
this.author = author;
this.author_id = author_id;
this.text = text;
// this.format = format;
this.image_data = imageData;
this.importedFrom = importedFrom;
this.importSource = importSource;
}
}
function timestampToString(t:PostTimestamp) {
return `${t.year}/${t.month}/${t.day}-${t.hour}:${t.minute}:${t.second}`
}
function getCurrentTimestamp() {
let date = new Date();
return {
year: date.getFullYear(),
month: date.getMonth()+1,
day: date.getDate(),
hour: date.getHours(),
minute: date.getMinutes(),
second: date.getSeconds(),
function initMarkdown() {
const renderer = new marked.Renderer();
renderer.link = (href: any, title: string, text: string) => {
return `<a href="${href}" target="_blank"${title ? ` title="${title}"` : ''}>${text}</a>`;
};
marked.setOptions({ renderer: renderer });
}
function waitMs(durationMs: number) {
return new Promise(resolve => setTimeout(resolve, durationMs));
}
// function getTimestampFromDate(date: Date) {
// return {
// year: date.getFullYear(),
// month: date.getMonth() + 1,
// day: date.getDate(),
// hour: date.getHours(),
// minute: date.getMinutes(),
// second: date.getSeconds(),
// };
// }
// function getTimestampFromString(dateString: string) {
// let date = new Date(dateString);
// return getTimestampFromDate(date);
// }
// function getCurrentTimestamp() {
// let date = new Date();
// return getTimestampFromDate(date);
// }
function uuidv4() {
return "10000000-1000-4000-8000-100000000000".replace(/[018]/g, (c: any) =>
(c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> c / 4).toString(16)
@@ -105,7 +133,66 @@ function timerDelta() {
return performance.now() - time;
}
async function createTestData2() {
async function getFixedTweetText(entry: any) {
let fullText = entry.tweet.full_text;
let linkMarkdown = "";
for (const url of entry.tweet.entities.urls) {
linkMarkdown = `[${url.display_url}](${url.expanded_url})`;
fullText = fullText.replace(url.url, linkMarkdown);
}
return fullText
// for (let url of )
// const regex = /https:\/\/t\.co\/[\w-]+/g;
// let returnText = tweetText;
// if (tweetText.includes("t.co")) {
// // console.log(tweetText);
// }
// let matches = returnText.match(regex);
// if (!matches) {
// return returnText;
// }
// for (const match of matches) {
// // console.log(match);
// let fetchedData = "";
// try {
// fetchedData = await (await fetch(match)).text();
// await waitMs(100);
// } catch (e) {
// console.log(e);
// }
// if (fetchedData.includes('http-equiv="refresh"')) {
// // console.log(fetchedData);
// // let urlRegex = /URL=(http.:\/\/.+)\"/;
// let urlRegex = new RegExp(`URL=(http[s]?://.+?)"`);
// let urlMatch = fetchedData.match(urlRegex);
// if (urlMatch?.length == 2) {
// // console.log(urlMatch[1]);
// returnText = returnText.replace(match, urlMatch[1]);
// // URL=https://youtu.be/yGci-Lb87zs"
// } else {
// throw new Error(fetchedData);
// }
// }
// }
// console.log(returnText);
// return returnText;
}
async function createTestData2(userID: string) {
log("Importing tweet archive")
let postsTestData: any[] = [];
let response = await fetch("./tweets.js");
@@ -116,6 +203,7 @@ async function createTestData2() {
// let tweets = JSON.parse(tweetJSON);
let count = 0;
for (let entry of (window as any).tweetData) {
// if (entry.tweet.hasOwnProperty("in_reply_to_screen_name") || entry.tweet.retweeted || entry.tweet.full_text.startsWith("RT")) {
@@ -128,48 +216,63 @@ async function createTestData2() {
isImage = mediaURL.includes('jpg');
}
let imageData;
let encodedImage = null;
if (isImage) {
try {
imageData = await (await fetch(mediaURL)).arrayBuffer();
encodedImage = await arrayBufferToBase64(imageData);
} catch (e) {
console.log(e);
}
let imageData = null;
// if (isImage) {
// try {
// let response = await fetch(mediaURL);
// await waitMs(100);
// if (response.status === 200) {
// imageData = await response.arrayBuffer();
// }
// console.log(imageData);
// } catch (e) {
// console.log(e);
// }
// }
let timeStamp = new Date(entry.tweet.created_at);
// let tweetText = entry.tweet.full_text;
let tweetText = await getFixedTweetText(entry);
let newPost = new Post('bobbydigitales', userID, tweetText, timeStamp, imageData, 'twitter', entry);
postsTestData.push(newPost);
// postsTestData.push({
// post_timestamp: timeStamp,
// author: `bobbydigitales`,
// text: tweetText,
// image: imageData
// });
count++;
if (count % 100 === 0) {
log(`Imported ${count} posts...`);
// render(postsTestData);
}
// if (count == 100-1) {
// break;
// }
postsTestData.push({
post_timestamp: {
year: 2023,
month: 10,
day: 19,
hour: 14,
minute: 53,
second: 0,
},
author: `bobbydigitales`,
text: entry.tweet.full_text,
image: encodedImage
});
}
let rant = await (await fetch('ranting.md')).text();;
// let rant = await (await fetch('ranting.md')).text();;
postsTestData.unshift({
post_timestamp: {
year: 2023,
month: 10,
day: 19,
hour: 14,
minute: 53,
second: 0,
},
author: `bobbydigitales`,
text: rant
})
// postsTestData.unshift({
// post_timestamp: {
// year: 2023,
// month: 10,
// day: 19,
// hour: 14,
// minute: 53,
// second: 0,
// },
// author: `bobbydigitales`,
// text: rant
// })
return postsTestData;
}
@@ -209,11 +312,11 @@ async function registerServiceWorker() {
function addPost(userID: string, posts: Post[], postText: string) {
if ((typeof postText !== "string") || postText.length === 0) {
log ("Not posting an empty string...")
log("Not posting an empty string...")
return;
}
let post = new Post(`bobbydigitales`, userID, postText, getCurrentTimestamp());
let post = new Post(`bobbydigitales`, userID, postText, new Date());
posts.push(post);
// localStorage.setItem(key, JSON.stringify(posts));
@@ -309,17 +412,17 @@ function initButtons(userID: string, posts: Post[]) {
}
async function loadPosts(userID: string) {
timerStart();
let posts = await getData(userID, 1, 1000);
let posts: any = await getData(userID, new Date(2021, 1), new Date());
if (posts.length > 0) {
log(`Loaded ${posts.length} posts in ${timerDelta().toFixed(2)}ms`);
return posts;
}
posts = await createTestData3(userID);
posts = await createTestData2(userID);
log("Adding test data...");
addDataArray(userID, posts);
@@ -335,8 +438,8 @@ async function loadPosts(userID: string) {
// log("Finished!");
return await getData(userID, 1, 1000);
return await getData(userID, new Date(2022, 1), new Date());
// debugger;
@@ -386,8 +489,6 @@ async function main() {
}
let userID = getUserID();
log(`Your user ID is: ${userID}`);
if (navigator.storage && navigator.storage.persist && !navigator.storage.persisted) {
@@ -395,6 +496,8 @@ async function main() {
console.log(`Persisted storage granted: ${isPersisted}`);
}
initMarkdown();
// let main = await fetch("/main.js");
// let code = await main.text();
// console.log(code);
@@ -424,30 +527,22 @@ function render(posts: Post[]) {
}
contentDiv.innerHTML = "";
let count = 0;
for (let i=posts.length -1; i>0; i--) {
for (let i = posts.length - 1; i >= 0; i--) {
let postData = posts[i];
let post = renderPost(postData);
fragment.appendChild(post);
count++;
// if (count > 500) {
// break;
// }
if (post) {
fragment.appendChild(post);
count++;
}
if (count > 500) {
break;
}
}
// for (const postData of posts) {
// let post = renderPost(postData);
// fragment.appendChild(post);
// count++;
// if (count > 500) {
// break;
// }
// }
if (!contentDiv) {
throw new Error("Couldn't get content div!");
@@ -466,22 +561,35 @@ function renderPost(post: Post) {
let timestampDiv = document.createElement("div");
let hr = document.createElement("hr");
textDiv.innerHTML = marked.parse(post.text);
timestampDiv.innerText = timestampToString(post.post_timestamp);
// textDiv.innerHTML = DOMPurify.sanitize(marked.parse(post.text));
timestampDiv.innerText = post.post_timestamp.toLocaleDateString();
containerDiv.appendChild(hr);
containerDiv.appendChild(textDiv);
containerDiv.appendChild(timestampDiv);
if (!("image" in post && post.image)) {
containerDiv.appendChild(timestampDiv);
return containerDiv;
// return null;
}
let image = document.createElement("img");
image.src = image.src = "data:image/png;base64," + post.image;
const blob = new Blob([post.image as ArrayBuffer], { type: 'image/jpg' });
const url = URL.createObjectURL(blob);
image.onload = () => {
URL.revokeObjectURL(url);
};
image.src = url;
// image.src = image.src = "data:image/png;base64," + post.image;
image.className = "postImage";
containerDiv.appendChild(image);
containerDiv.appendChild(timestampDiv);
return containerDiv;
}