Skip to content

Commit

Permalink
better moderation, categories, and see approval/rejection
Browse files Browse the repository at this point in the history
  • Loading branch information
@elliotBraem
elliotBraem committed Dec 20, 2024
1 parent 354e6ac commit e5ce2b3
Show file tree
Hide file tree
Showing 4 changed files with 295 additions and 95 deletions.
206 changes: 151 additions & 55 deletions backend/src/services/db/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { mkdir } from "node:fs/promises";
import { join, dirname } from "node:path";
import { existsSync } from "node:fs";
import { broadcastUpdate } from "../../index";
import { logger } from "utils/logger";

export class DatabaseService {
private db: Database;
Expand Down Expand Up @@ -40,27 +41,58 @@ export class DatabaseService {
user_id TEXT NOT NULL,
username TEXT NOT NULL,
content TEXT NOT NULL,
hashtags TEXT NOT NULL,
category TEXT,
description TEXT,
categories TEXT,
status TEXT NOT NULL DEFAULT 'pending',
acknowledgment_tweet_id TEXT,
moderation_response_tweet_id TEXT,
created_at TEXT NOT NULL
)
`);

// Create moderation_history table
this.db.run(`
CREATE TABLE IF NOT EXISTS moderation_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tweet_id TEXT NOT NULL,
admin_id TEXT NOT NULL,
action TEXT NOT NULL,
timestamp DATETIME NOT NULL,
FOREIGN KEY (tweet_id) REFERENCES submissions(tweet_id)
)
`);
// Handle moderation_history table migration
try {
// Backup existing data
this.db.run(`
CREATE TABLE IF NOT EXISTS moderation_history_backup AS
SELECT * FROM moderation_history;
`);

// Drop and recreate with correct schema
this.db.run(`DROP TABLE IF EXISTS moderation_history`);
this.db.run(`
CREATE TABLE moderation_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tweet_id TEXT NOT NULL,
admin_id TEXT NOT NULL,
action TEXT NOT NULL,
timestamp DATETIME NOT NULL,
FOREIGN KEY (tweet_id) REFERENCES submissions(tweet_id)
)
`);

// Restore data if backup exists
this.db.run(`
INSERT INTO moderation_history (tweet_id, admin_id, action, timestamp)
SELECT tweet_id, admin_id, action, timestamp
FROM moderation_history_backup;
`);

// Clean up backup
this.db.run(`DROP TABLE IF EXISTS moderation_history_backup`);
} catch (e) {
// If no existing table, just create new one
this.db.run(`
CREATE TABLE IF NOT EXISTS moderation_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tweet_id TEXT NOT NULL,
admin_id TEXT NOT NULL,
action TEXT NOT NULL,
timestamp DATETIME NOT NULL,
FOREIGN KEY (tweet_id) REFERENCES submissions(tweet_id)
)
`);
}

// Create submission_counts table for rate limiting
this.db.run(`
Expand Down Expand Up @@ -91,24 +123,79 @@ export class DatabaseService {
} catch (e) {
// Column might already exist
}

try {
this.db.run(
`ALTER TABLE submissions ADD COLUMN categories TEXT`
);
} catch (e) {
// Column might already exist
}

// Remove old columns
try {
// SQLite doesn't support DROP COLUMN before version 3.35.0
// Instead, we need to recreate the table without those columns
this.db.run(`
CREATE TABLE IF NOT EXISTS submissions_new (
tweet_id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
username TEXT NOT NULL,
content TEXT NOT NULL,
description TEXT,
categories TEXT,
status TEXT NOT NULL DEFAULT 'pending',
acknowledgment_tweet_id TEXT,
moderation_response_tweet_id TEXT,
created_at TEXT NOT NULL
)
`);

this.db.run(`
INSERT OR REPLACE INTO submissions_new
SELECT
tweet_id,
user_id,
username,
content,
description,
categories,
status,
acknowledgment_tweet_id,
moderation_response_tweet_id,
created_at
FROM submissions
`);

this.db.run(`DROP TABLE IF EXISTS submissions`);
this.db.run(`ALTER TABLE submissions_new RENAME TO submissions`);

// Recreate indexes
this.db.run(`
CREATE INDEX IF NOT EXISTS idx_acknowledgment_tweet_id
ON submissions(acknowledgment_tweet_id)
`);
} catch (e) {
// Table might not exist or other error
logger.error("Error updating table structure:", e);
}
}

saveSubmission(submission: TwitterSubmission): void {
const stmt = this.db.prepare(`
INSERT INTO submissions (
tweet_id, user_id, username, content, hashtags, category, description, status,
tweet_id, user_id, username, content, description, categories, status,
acknowledgment_tweet_id, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`);

stmt.run(
submission.tweetId,
submission.userId,
submission.username,
submission.content,
JSON.stringify(submission.hashtags),
submission.category || null,
submission.description || null,
submission.categories ? JSON.stringify(submission.categories) : null,
submission.status,
submission.acknowledgmentTweetId || null,
submission.createdAt,
Expand Down Expand Up @@ -157,7 +244,7 @@ export class DatabaseService {
const submission = this.db
.prepare(
`
SELECT s.*, GROUP_CONCAT(
SELECT s.*, json_group_array(
json_object(
'adminId', m.admin_id,
'action', m.action,
Expand All @@ -180,9 +267,8 @@ export class DatabaseService {
userId: submission.user_id,
username: submission.username,
content: submission.content,
hashtags: JSON.parse(submission.hashtags),
category: submission.category,
description: submission.description,
categories: submission.categories ? JSON.parse(submission.categories) : [],
status: submission.status,
acknowledgmentTweetId: submission.acknowledgment_tweet_id,
moderationResponseTweetId: submission.moderation_response_tweet_id,
Expand All @@ -202,13 +288,16 @@ export class DatabaseService {
const submission = this.db
.prepare(
`
SELECT s.*, GROUP_CONCAT(
json_object(
'adminId', m.admin_id,
'action', m.action,
'timestamp', m.timestamp,
'tweetId', m.tweet_id
)
SELECT s.*, json_group_array(
CASE
WHEN m.admin_id IS NULL THEN NULL
ELSE json_object(
'adminId', m.admin_id,
'action', m.action,
'timestamp', m.timestamp,
'tweetId', m.tweet_id
)
END
) as moderation_history
FROM submissions s
LEFT JOIN moderation_history m ON s.tweet_id = m.tweet_id
Expand All @@ -225,9 +314,8 @@ export class DatabaseService {
userId: submission.user_id,
username: submission.username,
content: submission.content,
hashtags: JSON.parse(submission.hashtags),
category: submission.category,
description: submission.description,
categories: submission.categories ? JSON.parse(submission.categories) : [],
status: submission.status,
acknowledgmentTweetId: submission.acknowledgment_tweet_id,
moderationResponseTweetId: submission.moderation_response_tweet_id,
Expand All @@ -245,13 +333,16 @@ export class DatabaseService {
const submissions = this.db
.prepare(
`
SELECT s.*, GROUP_CONCAT(
json_object(
'adminId', m.admin_id,
'action', m.action,
'timestamp', m.timestamp,
'tweetId', m.tweet_id
)
SELECT s.*, json_group_array(
CASE
WHEN m.admin_id IS NULL THEN NULL
ELSE json_object(
'adminId', m.admin_id,
'action', m.action,
'timestamp', m.timestamp,
'tweetId', m.tweet_id
)
END
) as moderation_history
FROM submissions s
LEFT JOIN moderation_history m ON s.tweet_id = m.tweet_id
Expand All @@ -265,18 +356,19 @@ export class DatabaseService {
userId: submission.user_id,
username: submission.username,
content: submission.content,
hashtags: JSON.parse(submission.hashtags),
category: submission.category,
description: submission.description,
categories: submission.categories ? JSON.parse(submission.categories) : [],
status: submission.status,
acknowledgmentTweetId: submission.acknowledgment_tweet_id,
moderationResponseTweetId: submission.moderation_response_tweet_id,
createdAt: submission.created_at,
moderationHistory: submission.moderation_history
? JSON.parse(`[${submission.moderation_history}]`).map((m: any) => ({
...m,
timestamp: new Date(m.timestamp),
}))
? JSON.parse(submission.moderation_history)
.filter((m: any) => m !== null)
.map((m: any) => ({
...m,
timestamp: new Date(m.timestamp),
}))
: [],
}));
}
Expand All @@ -287,13 +379,16 @@ export class DatabaseService {
const submissions = this.db
.prepare(
`
SELECT s.*, GROUP_CONCAT(
json_object(
'adminId', m.admin_id,
'action', m.action,
'timestamp', m.timestamp,
'tweetId', m.tweet_id
)
SELECT s.*, json_group_array(
CASE
WHEN m.admin_id IS NULL THEN NULL
ELSE json_object(
'adminId', m.admin_id,
'action', m.action,
'timestamp', m.timestamp,
'tweetId', m.tweet_id
)
END
) as moderation_history
FROM submissions s
LEFT JOIN moderation_history m ON s.tweet_id = m.tweet_id
Expand All @@ -308,18 +403,19 @@ export class DatabaseService {
userId: submission.user_id,
username: submission.username,
content: submission.content,
hashtags: JSON.parse(submission.hashtags),
category: submission.category,
description: submission.description,
categories: submission.categories ? JSON.parse(submission.categories) : [],
status: submission.status,
acknowledgmentTweetId: submission.acknowledgment_tweet_id,
moderationResponseTweetId: submission.moderation_response_tweet_id,
createdAt: submission.created_at,
moderationHistory: submission.moderation_history
? JSON.parse(`[${submission.moderation_history}]`).map((m: any) => ({
...m,
timestamp: new Date(m.timestamp),
}))
? JSON.parse(submission.moderation_history)
.filter((m: any) => m !== null)
.map((m: any) => ({
...m,
timestamp: new Date(m.timestamp),
}))
: [],
}));
}
Expand Down
34 changes: 30 additions & 4 deletions backend/src/services/twitter/client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,13 +259,32 @@ export class TwitterService {
return;
}

// Create submission using the original tweet's content
// Extract curator handle from submission tweet
const submissionMatch = tweet.text?.match(/!submit\s+@(\w+)/i);
if (!submissionMatch) {
logger.error(`Invalid submission format in tweet ${tweet.id}`);
return;
}

// Extract categories from hashtags in submission tweet (excluding command hashtags)
const categories = (tweet.hashtags || []).filter(tag =>
!['submit', 'approve', 'reject'].includes(tag.toLowerCase())
);

// Extract description: everything after !submit @handle that's not a hashtag
const description = tweet.text
?.replace(/!submit\s+@\w+/i, '') // Remove command
.replace(/#\w+/g, '') // Remove hashtags
.trim() || undefined;

// Create submission using the original tweet's content and submission metadata
const submission: TwitterSubmission = {
tweetId: originalTweet.id!, // The tweet being submitted
userId: originalTweet.userId!,
username: originalTweet.username!,
content: originalTweet.text || "",
hashtags: originalTweet.hashtags || [],
categories: categories,
description: description || undefined,
status: "pending",
moderationHistory: [],
createdAt:
Expand Down Expand Up @@ -331,11 +350,17 @@ export class TwitterService {
if (!action) return;

// Add moderation to database
const adminUsername = this.adminIdCache.get(userId);
if (!adminUsername) {
logger.error(`Could not find username for admin ID ${userId}`);
return;
}

const moderation: Moderation = {
adminId: userId,
adminId: adminUsername,
action: action,
timestamp: tweet.timeParsed || new Date(),
tweetId: tweet.id,
tweetId: submission.tweetId, // Use the original submission's tweetId
};
db.saveModerationAction(moderation);

Expand Down Expand Up @@ -375,6 +400,7 @@ export class TwitterService {
tweet: Tweet,
submission: TwitterSubmission,
): Promise<void> {
// TODO: Add NEAR integration here for rejected submissions
const responseTweetId = await this.replyToTweet(
tweet.id!,
"Your submission has been reviewed and was not accepted for the public goods news feed.",
Expand Down
3 changes: 1 addition & 2 deletions backend/src/types/twitter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@ export interface TwitterSubmission {
userId: string;
username: string;
content: string;
hashtags: Array<string>;
category?: string;
description?: string;
categories?: string[];
status: "pending" | "approved" | "rejected";
moderationHistory: Moderation[];
acknowledgmentTweetId?: string;
Expand Down
Loading

0 comments on commit e5ce2b3

Please sign in to comment.