bobbydigitales/dandelion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

converting to IndexedDB

Browse Source
This commit is contained in:
“bobbydigitales”
2023-11-12 23:31:07 -08:00
parent 6cd5a04cc1
commit 41054a5dbf
5 changed files with 240 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

304
src/main.ts
View File

@@ -1,51 +1,79 @@
import { openDatabase, getData, addData, addDataArray, getAllData } from "./db.js"
import { openDatabase, getData, addData, addDataArray } from "./db.js"
declare let marked:any;
declare let marked: any;
declare let DOMPurify: any;
// let posts:any;
// let keyBase = "dandelion_posts_v1_"
// let key:string = "";
interface PostTimestamp {
year: number,
month: number,
day: number,
hour: number,
minute: number,
second: number,
}
// interface PostTimestamp {
// year: number,
// month: number,
// day: number,
// hour: number,
// minute: number,
// second: number,
// }
class Post {
post_timestamp:PostTimestamp;
author:string;
author_id:string;
text:string;
// format
constructor(author: string, author_id:string, text: string, post_timestamp: PostTimestamp, format = null) {
post_timestamp: Date;
author: string;
author_id: string;
text: string;
image_data: ArrayBuffer | null;
importedFrom: "twitter" | null;
importSource: any;
constructor(author: string, author_id: string, text: string, post_timestamp: Date, imageData: ArrayBuffer | null = null, importedFrom: "twitter" | null = null, importSource: any = null) {
this.post_timestamp = post_timestamp;
this.author = author;
this.author_id = author_id;
this.text = text;
// this.format = format;
this.image_data = imageData;
this.importedFrom = importedFrom;
this.importSource = importSource;
}
}
function timestampToString(t:PostTimestamp) {
return `${t.year}/${t.month}/${t.day}-${t.hour}:${t.minute}:${t.second}`
}
function getCurrentTimestamp() {
let date = new Date();
return {
year: date.getFullYear(),
month: date.getMonth()+1,
day: date.getDate(),
hour: date.getHours(),
minute: date.getMinutes(),
second: date.getSeconds(),
function initMarkdown() {
const renderer = new marked.Renderer();
renderer.link = (href: any, title: string, text: string) => {
return `<a href="${href}" target="_blank"${title ? ` title="${title}"` : ''}>${text}</a>`;
};
marked.setOptions({ renderer: renderer });
}
function waitMs(durationMs: number) {
return new Promise(resolve => setTimeout(resolve, durationMs));
}
// function getTimestampFromDate(date: Date) {
// return {
// year: date.getFullYear(),
// month: date.getMonth() + 1,
// day: date.getDate(),
// hour: date.getHours(),
// minute: date.getMinutes(),
// second: date.getSeconds(),
// };
// }
// function getTimestampFromString(dateString: string) {
// let date = new Date(dateString);
// return getTimestampFromDate(date);
// }
// function getCurrentTimestamp() {
// let date = new Date();
// return getTimestampFromDate(date);
// }
function uuidv4() {
return "10000000-1000-4000-8000-100000000000".replace(/[018]/g, (c: any) =>
(c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> c / 4).toString(16)
@@ -105,7 +133,66 @@ function timerDelta() {
return performance.now() - time;
}
async function createTestData2() {
async function getFixedTweetText(entry: any) {
let fullText = entry.tweet.full_text;
let linkMarkdown = "";
for (const url of entry.tweet.entities.urls) {
linkMarkdown = `[${url.display_url}](${url.expanded_url})`;
fullText = fullText.replace(url.url, linkMarkdown);
}
return fullText
// for (let url of )
// const regex = /https:\/\/t\.co\/[\w-]+/g;
// let returnText = tweetText;
// if (tweetText.includes("t.co")) {
// // console.log(tweetText);
// }
// let matches = returnText.match(regex);
// if (!matches) {
// return returnText;
// }
// for (const match of matches) {
// // console.log(match);
// let fetchedData = "";
// try {
// fetchedData = await (await fetch(match)).text();
// await waitMs(100);
// } catch (e) {
// console.log(e);
// }
// if (fetchedData.includes('http-equiv="refresh"')) {
// // console.log(fetchedData);
// // let urlRegex = /URL=(http.:\/\/.+)\"/;
// let urlRegex = new RegExp(`URL=(http[s]?://.+?)"`);
// let urlMatch = fetchedData.match(urlRegex);
// if (urlMatch?.length == 2) {
// // console.log(urlMatch[1]);
// returnText = returnText.replace(match, urlMatch[1]);
// // URL=https://youtu.be/yGci-Lb87zs"
// } else {
// throw new Error(fetchedData);
// }
// }
// }
// console.log(returnText);
// return returnText;
}
async function createTestData2(userID: string) {
log("Importing tweet archive")
let postsTestData: any[] = [];
let response = await fetch("./tweets.js");
@@ -116,6 +203,7 @@ async function createTestData2() {
// let tweets = JSON.parse(tweetJSON);
let count = 0;
for (let entry of (window as any).tweetData) {
// if (entry.tweet.hasOwnProperty("in_reply_to_screen_name") || entry.tweet.retweeted || entry.tweet.full_text.startsWith("RT")) {
@@ -128,48 +216,63 @@ async function createTestData2() {
isImage = mediaURL.includes('jpg');
}
let imageData;
let encodedImage = null;
if (isImage) {
try {
imageData = await (await fetch(mediaURL)).arrayBuffer();
encodedImage = await arrayBufferToBase64(imageData);
} catch (e) {
console.log(e);
}
let imageData = null;
// if (isImage) {
// try {
// let response = await fetch(mediaURL);
// await waitMs(100);
// if (response.status === 200) {
// imageData = await response.arrayBuffer();
// }
// console.log(imageData);
// } catch (e) {
// console.log(e);
// }
// }
let timeStamp = new Date(entry.tweet.created_at);
// let tweetText = entry.tweet.full_text;
let tweetText = await getFixedTweetText(entry);
let newPost = new Post('bobbydigitales', userID, tweetText, timeStamp, imageData, 'twitter', entry);
postsTestData.push(newPost);
// postsTestData.push({
// post_timestamp: timeStamp,
// author: `bobbydigitales`,
// text: tweetText,
// image: imageData
// });
count++;
if (count % 100 === 0) {
log(`Imported ${count} posts...`);
// render(postsTestData);
}
// if (count == 100-1) {
// break;
// }
postsTestData.push({
post_timestamp: {
year: 2023,
month: 10,
day: 19,
hour: 14,
minute: 53,
second: 0,
},
author: `bobbydigitales`,
text: entry.tweet.full_text,
image: encodedImage
});
}
let rant = await (await fetch('ranting.md')).text();;
// let rant = await (await fetch('ranting.md')).text();;
postsTestData.unshift({
post_timestamp: {
year: 2023,
month: 10,
day: 19,
hour: 14,
minute: 53,
second: 0,
},
author: `bobbydigitales`,
text: rant
})
// postsTestData.unshift({
// post_timestamp: {
// year: 2023,
// month: 10,
// day: 19,
// hour: 14,
// minute: 53,
// second: 0,
// },
// author: `bobbydigitales`,
// text: rant
// })
return postsTestData;
}
@@ -209,11 +312,11 @@ async function registerServiceWorker() {
function addPost(userID: string, posts: Post[], postText: string) {
if ((typeof postText !== "string") || postText.length === 0) {
log ("Not posting an empty string...")
log("Not posting an empty string...")
return;
}
let post = new Post(`bobbydigitales`, userID, postText, getCurrentTimestamp());
let post = new Post(`bobbydigitales`, userID, postText, new Date());
posts.push(post);
// localStorage.setItem(key, JSON.stringify(posts));
@@ -309,17 +412,17 @@ function initButtons(userID: string, posts: Post[]) {
}
async function loadPosts(userID: string) {
timerStart();
let posts = await getData(userID, 1, 1000);
let posts: any = await getData(userID, new Date(2021, 1), new Date());
if (posts.length > 0) {
log(`Loaded ${posts.length} posts in ${timerDelta().toFixed(2)}ms`);
return posts;
}
posts = await createTestData3(userID);
posts = await createTestData2(userID);
log("Adding test data...");
addDataArray(userID, posts);
@@ -335,8 +438,8 @@ async function loadPosts(userID: string) {
// log("Finished!");
return await getData(userID, 1, 1000);
return await getData(userID, new Date(2022, 1), new Date());
// debugger;
@@ -386,8 +489,6 @@ async function main() {
}
let userID = getUserID();
log(`Your user ID is: ${userID}`);
if (navigator.storage && navigator.storage.persist && !navigator.storage.persisted) {
@@ -395,6 +496,8 @@ async function main() {
console.log(`Persisted storage granted: ${isPersisted}`);
}
initMarkdown();
// let main = await fetch("/main.js");
// let code = await main.text();
// console.log(code);
@@ -424,30 +527,22 @@ function render(posts: Post[]) {
}
contentDiv.innerHTML = "";
let count = 0;
for (let i=posts.length -1; i>0; i--) {
for (let i = posts.length - 1; i >= 0; i--) {
let postData = posts[i];
let post = renderPost(postData);
fragment.appendChild(post);
count++;
// if (count > 500) {
// break;
// }
if (post) {
fragment.appendChild(post);
count++;
}
if (count > 500) {
break;
}
}
// for (const postData of posts) {
// let post = renderPost(postData);
// fragment.appendChild(post);
// count++;
// if (count > 500) {
// break;
// }
// }
if (!contentDiv) {
throw new Error("Couldn't get content div!");
@@ -466,22 +561,35 @@ function renderPost(post: Post) {
let timestampDiv = document.createElement("div");
let hr = document.createElement("hr");
textDiv.innerHTML = marked.parse(post.text);
timestampDiv.innerText = timestampToString(post.post_timestamp);
// textDiv.innerHTML = DOMPurify.sanitize(marked.parse(post.text));
timestampDiv.innerText = post.post_timestamp.toLocaleDateString();
containerDiv.appendChild(hr);
containerDiv.appendChild(textDiv);
containerDiv.appendChild(timestampDiv);
if (!("image" in post && post.image)) {
containerDiv.appendChild(timestampDiv);
return containerDiv;
// return null;
}
let image = document.createElement("img");
image.src = image.src = "data:image/png;base64," + post.image;
const blob = new Blob([post.image as ArrayBuffer], { type: 'image/jpg' });
const url = URL.createObjectURL(blob);
image.onload = () => {
URL.revokeObjectURL(url);
};
image.src = url;
// image.src = image.src = "data:image/png;base64," + post.image;
image.className = "postImage";
containerDiv.appendChild(image);
containerDiv.appendChild(timestampDiv);
return containerDiv;
}