From e5ce2b355492db0193965062076be865d100a88b Mon Sep 17 00:00:00 2001 From: Elliot Braem Date: Fri, 20 Dec 2024 09:21:21 -0600 Subject: [PATCH] better moderation, categories, and see approval/rejection --- backend/src/services/db/index.ts | 206 +++++++++++++++------ backend/src/services/twitter/client.ts | 34 +++- backend/src/types/twitter.ts | 3 +- frontend/src/components/SubmissionList.tsx | 147 +++++++++++---- 4 files changed, 295 insertions(+), 95 deletions(-) diff --git a/backend/src/services/db/index.ts b/backend/src/services/db/index.ts index 8fd7fef..adecd8a 100644 --- a/backend/src/services/db/index.ts +++ b/backend/src/services/db/index.ts @@ -4,6 +4,7 @@ import { mkdir } from "node:fs/promises"; import { join, dirname } from "node:path"; import { existsSync } from "node:fs"; import { broadcastUpdate } from "../../index"; +import { logger } from "utils/logger"; export class DatabaseService { private db: Database; @@ -40,9 +41,8 @@ export class DatabaseService { user_id TEXT NOT NULL, username TEXT NOT NULL, content TEXT NOT NULL, - hashtags TEXT NOT NULL, - category TEXT, description TEXT, + categories TEXT, status TEXT NOT NULL DEFAULT 'pending', acknowledgment_tweet_id TEXT, moderation_response_tweet_id TEXT, @@ -50,17 +50,49 @@ export class DatabaseService { ) `); - // Create moderation_history table - this.db.run(` - CREATE TABLE IF NOT EXISTS moderation_history ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - tweet_id TEXT NOT NULL, - admin_id TEXT NOT NULL, - action TEXT NOT NULL, - timestamp DATETIME NOT NULL, - FOREIGN KEY (tweet_id) REFERENCES submissions(tweet_id) - ) - `); + // Handle moderation_history table migration + try { + // Backup existing data + this.db.run(` + CREATE TABLE IF NOT EXISTS moderation_history_backup AS + SELECT * FROM moderation_history; + `); + + // Drop and recreate with correct schema + this.db.run(`DROP TABLE IF EXISTS moderation_history`); + this.db.run(` + CREATE TABLE moderation_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tweet_id TEXT NOT NULL, + admin_id TEXT NOT NULL, + action TEXT NOT NULL, + timestamp DATETIME NOT NULL, + FOREIGN KEY (tweet_id) REFERENCES submissions(tweet_id) + ) + `); + + // Restore data if backup exists + this.db.run(` + INSERT INTO moderation_history (tweet_id, admin_id, action, timestamp) + SELECT tweet_id, admin_id, action, timestamp + FROM moderation_history_backup; + `); + + // Clean up backup + this.db.run(`DROP TABLE IF EXISTS moderation_history_backup`); + } catch (e) { + // If no existing table, just create new one + this.db.run(` + CREATE TABLE IF NOT EXISTS moderation_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tweet_id TEXT NOT NULL, + admin_id TEXT NOT NULL, + action TEXT NOT NULL, + timestamp DATETIME NOT NULL, + FOREIGN KEY (tweet_id) REFERENCES submissions(tweet_id) + ) + `); + } // Create submission_counts table for rate limiting this.db.run(` @@ -91,14 +123,70 @@ export class DatabaseService { } catch (e) { // Column might already exist } + + try { + this.db.run( + `ALTER TABLE submissions ADD COLUMN categories TEXT` + ); + } catch (e) { + // Column might already exist + } + + // Remove old columns + try { + // SQLite doesn't support DROP COLUMN before version 3.35.0 + // Instead, we need to recreate the table without those columns + this.db.run(` + CREATE TABLE IF NOT EXISTS submissions_new ( + tweet_id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + username TEXT NOT NULL, + content TEXT NOT NULL, + description TEXT, + categories TEXT, + status TEXT NOT NULL DEFAULT 'pending', + acknowledgment_tweet_id TEXT, + moderation_response_tweet_id TEXT, + created_at TEXT NOT NULL + ) + `); + + this.db.run(` + INSERT OR REPLACE INTO submissions_new + SELECT + tweet_id, + user_id, + username, + content, + description, + categories, + status, + acknowledgment_tweet_id, + moderation_response_tweet_id, + created_at + FROM submissions + `); + + this.db.run(`DROP TABLE IF EXISTS submissions`); + this.db.run(`ALTER TABLE submissions_new RENAME TO submissions`); + + // Recreate indexes + this.db.run(` + CREATE INDEX IF NOT EXISTS idx_acknowledgment_tweet_id + ON submissions(acknowledgment_tweet_id) + `); + } catch (e) { + // Table might not exist or other error + logger.error("Error updating table structure:", e); + } } saveSubmission(submission: TwitterSubmission): void { const stmt = this.db.prepare(` INSERT INTO submissions ( - tweet_id, user_id, username, content, hashtags, category, description, status, + tweet_id, user_id, username, content, description, categories, status, acknowledgment_tweet_id, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) `); stmt.run( @@ -106,9 +194,8 @@ export class DatabaseService { submission.userId, submission.username, submission.content, - JSON.stringify(submission.hashtags), - submission.category || null, submission.description || null, + submission.categories ? JSON.stringify(submission.categories) : null, submission.status, submission.acknowledgmentTweetId || null, submission.createdAt, @@ -157,7 +244,7 @@ export class DatabaseService { const submission = this.db .prepare( ` - SELECT s.*, GROUP_CONCAT( + SELECT s.*, json_group_array( json_object( 'adminId', m.admin_id, 'action', m.action, @@ -180,9 +267,8 @@ export class DatabaseService { userId: submission.user_id, username: submission.username, content: submission.content, - hashtags: JSON.parse(submission.hashtags), - category: submission.category, description: submission.description, + categories: submission.categories ? JSON.parse(submission.categories) : [], status: submission.status, acknowledgmentTweetId: submission.acknowledgment_tweet_id, moderationResponseTweetId: submission.moderation_response_tweet_id, @@ -202,13 +288,16 @@ export class DatabaseService { const submission = this.db .prepare( ` - SELECT s.*, GROUP_CONCAT( - json_object( - 'adminId', m.admin_id, - 'action', m.action, - 'timestamp', m.timestamp, - 'tweetId', m.tweet_id - ) + SELECT s.*, json_group_array( + CASE + WHEN m.admin_id IS NULL THEN NULL + ELSE json_object( + 'adminId', m.admin_id, + 'action', m.action, + 'timestamp', m.timestamp, + 'tweetId', m.tweet_id + ) + END ) as moderation_history FROM submissions s LEFT JOIN moderation_history m ON s.tweet_id = m.tweet_id @@ -225,9 +314,8 @@ export class DatabaseService { userId: submission.user_id, username: submission.username, content: submission.content, - hashtags: JSON.parse(submission.hashtags), - category: submission.category, description: submission.description, + categories: submission.categories ? JSON.parse(submission.categories) : [], status: submission.status, acknowledgmentTweetId: submission.acknowledgment_tweet_id, moderationResponseTweetId: submission.moderation_response_tweet_id, @@ -245,13 +333,16 @@ export class DatabaseService { const submissions = this.db .prepare( ` - SELECT s.*, GROUP_CONCAT( - json_object( - 'adminId', m.admin_id, - 'action', m.action, - 'timestamp', m.timestamp, - 'tweetId', m.tweet_id - ) + SELECT s.*, json_group_array( + CASE + WHEN m.admin_id IS NULL THEN NULL + ELSE json_object( + 'adminId', m.admin_id, + 'action', m.action, + 'timestamp', m.timestamp, + 'tweetId', m.tweet_id + ) + END ) as moderation_history FROM submissions s LEFT JOIN moderation_history m ON s.tweet_id = m.tweet_id @@ -265,18 +356,19 @@ export class DatabaseService { userId: submission.user_id, username: submission.username, content: submission.content, - hashtags: JSON.parse(submission.hashtags), - category: submission.category, description: submission.description, + categories: submission.categories ? JSON.parse(submission.categories) : [], status: submission.status, acknowledgmentTweetId: submission.acknowledgment_tweet_id, moderationResponseTweetId: submission.moderation_response_tweet_id, createdAt: submission.created_at, moderationHistory: submission.moderation_history - ? JSON.parse(`[${submission.moderation_history}]`).map((m: any) => ({ - ...m, - timestamp: new Date(m.timestamp), - })) + ? JSON.parse(submission.moderation_history) + .filter((m: any) => m !== null) + .map((m: any) => ({ + ...m, + timestamp: new Date(m.timestamp), + })) : [], })); } @@ -287,13 +379,16 @@ export class DatabaseService { const submissions = this.db .prepare( ` - SELECT s.*, GROUP_CONCAT( - json_object( - 'adminId', m.admin_id, - 'action', m.action, - 'timestamp', m.timestamp, - 'tweetId', m.tweet_id - ) + SELECT s.*, json_group_array( + CASE + WHEN m.admin_id IS NULL THEN NULL + ELSE json_object( + 'adminId', m.admin_id, + 'action', m.action, + 'timestamp', m.timestamp, + 'tweetId', m.tweet_id + ) + END ) as moderation_history FROM submissions s LEFT JOIN moderation_history m ON s.tweet_id = m.tweet_id @@ -308,18 +403,19 @@ export class DatabaseService { userId: submission.user_id, username: submission.username, content: submission.content, - hashtags: JSON.parse(submission.hashtags), - category: submission.category, description: submission.description, + categories: submission.categories ? JSON.parse(submission.categories) : [], status: submission.status, acknowledgmentTweetId: submission.acknowledgment_tweet_id, moderationResponseTweetId: submission.moderation_response_tweet_id, createdAt: submission.created_at, moderationHistory: submission.moderation_history - ? JSON.parse(`[${submission.moderation_history}]`).map((m: any) => ({ - ...m, - timestamp: new Date(m.timestamp), - })) + ? JSON.parse(submission.moderation_history) + .filter((m: any) => m !== null) + .map((m: any) => ({ + ...m, + timestamp: new Date(m.timestamp), + })) : [], })); } diff --git a/backend/src/services/twitter/client.ts b/backend/src/services/twitter/client.ts index 3b018a5..8c7b882 100644 --- a/backend/src/services/twitter/client.ts +++ b/backend/src/services/twitter/client.ts @@ -259,13 +259,32 @@ export class TwitterService { return; } - // Create submission using the original tweet's content + // Extract curator handle from submission tweet + const submissionMatch = tweet.text?.match(/!submit\s+@(\w+)/i); + if (!submissionMatch) { + logger.error(`Invalid submission format in tweet ${tweet.id}`); + return; + } + + // Extract categories from hashtags in submission tweet (excluding command hashtags) + const categories = (tweet.hashtags || []).filter(tag => + !['submit', 'approve', 'reject'].includes(tag.toLowerCase()) + ); + + // Extract description: everything after !submit @handle that's not a hashtag + const description = tweet.text + ?.replace(/!submit\s+@\w+/i, '') // Remove command + .replace(/#\w+/g, '') // Remove hashtags + .trim() || undefined; + + // Create submission using the original tweet's content and submission metadata const submission: TwitterSubmission = { tweetId: originalTweet.id!, // The tweet being submitted userId: originalTweet.userId!, username: originalTweet.username!, content: originalTweet.text || "", - hashtags: originalTweet.hashtags || [], + categories: categories, + description: description || undefined, status: "pending", moderationHistory: [], createdAt: @@ -331,11 +350,17 @@ export class TwitterService { if (!action) return; // Add moderation to database + const adminUsername = this.adminIdCache.get(userId); + if (!adminUsername) { + logger.error(`Could not find username for admin ID ${userId}`); + return; + } + const moderation: Moderation = { - adminId: userId, + adminId: adminUsername, action: action, timestamp: tweet.timeParsed || new Date(), - tweetId: tweet.id, + tweetId: submission.tweetId, // Use the original submission's tweetId }; db.saveModerationAction(moderation); @@ -375,6 +400,7 @@ export class TwitterService { tweet: Tweet, submission: TwitterSubmission, ): Promise { + // TODO: Add NEAR integration here for rejected submissions const responseTweetId = await this.replyToTweet( tweet.id!, "Your submission has been reviewed and was not accepted for the public goods news feed.", diff --git a/backend/src/types/twitter.ts b/backend/src/types/twitter.ts index d6735d7..34eac86 100644 --- a/backend/src/types/twitter.ts +++ b/backend/src/types/twitter.ts @@ -3,9 +3,8 @@ export interface TwitterSubmission { userId: string; username: string; content: string; - hashtags: Array; - category?: string; description?: string; + categories?: string[]; status: "pending" | "approved" | "rejected"; moderationHistory: Moderation[]; acknowledgmentTweetId?: string; diff --git a/frontend/src/components/SubmissionList.tsx b/frontend/src/components/SubmissionList.tsx index 217f07e..d69e790 100644 --- a/frontend/src/components/SubmissionList.tsx +++ b/frontend/src/components/SubmissionList.tsx @@ -2,7 +2,9 @@ import { useEffect, useState } from "react"; import axios from "axios"; import { TwitterSubmission } from "../types/twitter"; import { useLiveUpdates } from "../contexts/LiveUpdateContext"; -import { ExternalLink } from "lucide-react"; +import { ExternalLink, Eye } from "lucide-react"; + +const BOT_ID = "test_curation"; const StatusBadge = ({ status }: { status: TwitterSubmission["status"] }) => { const baseClasses = "px-2 py-1 rounded-full text-sm font-semibold border"; @@ -20,20 +22,38 @@ const SubmissionList = () => { const [submissions, setSubmissions] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); - const [filter, setFilter] = useState( - "all", - ); + const [statusFilter, setStatusFilter] = useState< + TwitterSubmission["status"] | "all" + >("all"); + const [selectedCategories, setSelectedCategories] = useState([]); + + // Get unique categories across all submissions + const allCategories = [ + ...new Set(submissions.flatMap((s) => s.categories || [])), + ].sort(); const { lastUpdate } = useLiveUpdates(); const fetchSubmissions = async () => { try { setLoading(true); const url = - filter === "all" + statusFilter === "all" ? "/api/submissions" - : `/api/submissions?status=${filter}`; + : `/api/submissions?status=${statusFilter}`; const response = await axios.get(url); - setSubmissions([...response.data].reverse()); + const data = [...response.data].reverse(); + + // Filter by selected categories if any are selected + const filteredData = + selectedCategories.length > 0 + ? data.filter((submission) => + selectedCategories.every((category) => + submission.categories?.includes(category), + ), + ) + : data; + + setSubmissions(filteredData); setError(null); } catch (err) { setError("Failed to fetch submissions"); @@ -46,18 +66,28 @@ const SubmissionList = () => { // Initial fetch useEffect(() => { fetchSubmissions(); - }, [filter]); + }, [statusFilter, selectedCategories]); // Handle live updates useEffect(() => { if (lastUpdate?.type === "update") { - const updatedSubmissions = - filter === "all" + const filteredByStatus = + statusFilter === "all" ? lastUpdate.data - : lastUpdate.data.filter((s) => s.status === filter); - setSubmissions([...updatedSubmissions].reverse()); + : lastUpdate.data.filter((s) => s.status === statusFilter); + + const filteredByCategories = + selectedCategories.length > 0 + ? filteredByStatus.filter((submission) => + selectedCategories.every((category) => + submission.categories?.includes(category), + ), + ) + : filteredByStatus; + + setSubmissions([...filteredByCategories].reverse()); } - }, [lastUpdate, filter]); + }, [lastUpdate, statusFilter, selectedCategories]); const getTweetUrl = (tweetId: string, username: string) => { return `https://x.com/${username}/status/${tweetId}`; @@ -92,15 +122,16 @@ const SubmissionList = () => { return (
-
+
+ {/* Status filters */}
{(["all", "pending", "approved", "rejected"] as const).map( (status) => (
+ + {/* Category filters */} + {allCategories.length > 0 && ( +
+

+ Filter by Categories: +

+
+ {allCategories.map((category) => ( + + ))} +
+
+ )}
@@ -168,29 +229,47 @@ const SubmissionList = () => {

{submission.content}

-
- {submission.hashtags.map((tag) => ( - - #{tag} - - ))} -
- +
+ + + + +
- {submission.category && ( -

- Category:{" "} - {submission.category} -

+ {submission.description && ( +
+

+ Curator's Notes: +

+

{submission.description}

+
)} - {submission.description && ( -

- Description:{" "} - {submission.description} -

+ {submission.categories && submission.categories.length > 0 && ( +
+ {submission.categories.map((category) => ( + { + if (!selectedCategories.includes(category)) { + setSelectedCategories((prev) => [...prev, category]); + } + }} + > + {category} + + ))} +
)}
))}