Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds Notion Plugin #7

Closed
wants to merge 28 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ RUN cd backend && bun install
# Copy backend source code
COPY backend ./backend

ENV NODE_ENV="production"

# Build backend
RUN cd backend && bun run build

Expand All @@ -53,6 +55,10 @@ RUN mkdir -p /litefs /var/lib/litefs && \
chown -R bun:bun /litefs /var/lib/litefs

# Create volume mount points
# Set environment variables first
ENV DATABASE_URL="file:/litefs/db"
ENV FRONTEND_DIST_PATH="/app/frontend/dist"

# Copy only necessary files from builders
COPY --from=backend-builder --chown=bun:bun /app/package.json ./
COPY --chown=bun:bun curate.config.json ./
Expand All @@ -62,11 +68,6 @@ COPY --from=backend-builder --chown=bun:bun /app/backend ./backend

RUN cd backend && bun install

# Set environment variables
ENV DATABASE_URL="file:/litefs/db"
ENV NODE_ENV="production"
ENV FRONTEND_DIST_PATH="/app/frontend/dist"

# Expose the port
EXPOSE 3000

Expand Down
10 changes: 5 additions & 5 deletions backend/src/__tests__/mocks/distribution-service.mock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ export class MockDistributionService {
submissionId: string;
}> = [];

async processStreamOutput(
feedId: string,
submissionId: string,
): Promise<void> {
this.processedSubmissions.push({ feedId, submissionId });
async processStreamOutput(feedId: string, submission: any): Promise<void> {
this.processedSubmissions.push({
feedId,
submissionId: submission.tweetId,
});
}

async processRecapOutput(): Promise<void> {
Expand Down
166 changes: 145 additions & 21 deletions backend/src/__tests__/mocks/twitter-service.mock.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,100 @@
import { Tweet } from "agent-twitter-client";
import { SearchMode, Tweet } from "agent-twitter-client";
import { TwitterService } from "../../services/twitter/client";
import { logger } from "../../utils/logger";

export class MockTwitterService {
export class MockTwitterService extends TwitterService {
private mockTweets: Tweet[] = [];
private mockUserIds: Map<string, string> = new Map();
private lastCheckedTweetId: string | null = null;
private tweetIdCounter: bigint = BigInt(Date.now());
private testLastCheckedTweetId: string | null = null;

public addMockTweet(tweet: Tweet) {
this.mockTweets.push(tweet);
constructor() {
// Pass config with the bot's username so mentions are found
super({
username: "test_bot",
password: "mock_pass",
email: "[email protected]",
});
// Override the client with a basic mock
(this as any).client = {
isLoggedIn: async () => true,
login: async () => {},
logout: async () => {},
getCookies: async () => [],
setCookies: async () => {},
fetchSearchTweets: async (
query: string,
count: number,
mode: SearchMode,
) => {
// Filter tweets that match the query (mentions @test_bot)
const matchingTweets = this.mockTweets.filter((tweet) =>
tweet.text?.includes("@test_bot"),
);

// Sort by ID descending (newest first) to match Twitter search behavior
const sortedTweets = [...matchingTweets].sort((a, b) => {
const aId = BigInt(a.id);
const bId = BigInt(b.id);
return bId > aId ? 1 : bId < aId ? -1 : 0;
});

return {
tweets: sortedTweets.slice(0, count),
};
},
likeTweet: async (tweetId: string) => {
logger.info(`Mock: Liked tweet ${tweetId}`);
return true;
},
sendTweet: async (message: string, replyToId?: string) => {
const newTweet = this.addMockTweet({
text: message,
username: "test_bot",
inReplyToStatusId: replyToId,
});
return {
json: async () => ({
data: {
create_tweet: {
tweet_results: {
result: {
rest_id: newTweet.id,
},
},
},
},
}),
} as Response;
},
};
}

private getNextTweetId(): string {
this.tweetIdCounter = this.tweetIdCounter + BigInt(1);
return this.tweetIdCounter.toString();
}

public addMockTweet(tweet: Partial<Tweet> & { inReplyToStatusId?: string }) {
const fullTweet: Tweet = {
id: tweet.id || this.getNextTweetId(),
text: tweet.text || "",
username: tweet.username || "test_user",
userId: tweet.userId || `mock-user-id-${tweet.username || "test_user"}`,
timeParsed: tweet.timeParsed || new Date(),
hashtags: tweet.hashtags || [],
mentions: tweet.mentions || [],
photos: tweet.photos || [],
urls: tweet.urls || [],
videos: tweet.videos || [],
thread: [],
inReplyToStatusId: tweet.inReplyToStatusId,
};
this.mockTweets.push(fullTweet);
logger.info(
`Mock: Added tweet "${fullTweet.text}" from @${fullTweet.username}${tweet.inReplyToStatusId ? ` as reply to ${tweet.inReplyToStatusId}` : ""}`,
);
return fullTweet;
}

public addMockUserId(username: string, userId: string) {
Expand All @@ -15,41 +103,77 @@ export class MockTwitterService {

public clearMockTweets() {
this.mockTweets = [];
logger.info("Mock: Cleared all tweets");
}

async initialize(): Promise<void> {
// No-op for mock
logger.info("Mock Twitter service initialized");
}

async stop(): Promise<void> {
// No-op for mock
logger.info("Mock Twitter service stopped");
}

async getUserIdByScreenName(screenName: string): Promise<string> {
return this.mockUserIds.get(screenName) || `mock-user-id-${screenName}`;
}

async fetchAllNewMentions(): Promise<Tweet[]> {
return this.mockTweets;
}
const BATCH_SIZE = 200;

async getTweet(tweetId: string): Promise<Tweet | null> {
return this.mockTweets.find((t) => t.id === tweetId) || null;
}
// Get the last tweet ID we processed
const lastCheckedId = this.testLastCheckedTweetId
? BigInt(this.testLastCheckedTweetId)
: null;

async replyToTweet(tweetId: string, message: string): Promise<string | null> {
return `mock-reply-${Date.now()}`;
}
// Get latest tweets first (up to batch size), excluding already checked tweets
const latestTweets = [...this.mockTweets]
.sort((a, b) => {
const aId = BigInt(a.id);
const bId = BigInt(b.id);
return bId > aId ? -1 : bId < aId ? 1 : 0; // Descending order (newest first)
})
.filter((tweet) => {
const tweetId = BigInt(tweet.id);
return !lastCheckedId || tweetId > lastCheckedId;
})
.slice(0, BATCH_SIZE);

if (latestTweets.length === 0) {
logger.info("No tweets found");
return [];
}

// Filter for mentions
const newMentions = latestTweets.filter(
(tweet) =>
tweet.text?.includes("@test_bot") ||
tweet.mentions?.some((m) => m.username === "test_bot"),
);

async setLastCheckedTweetId(tweetId: string): Promise<void> {
this.lastCheckedTweetId = tweetId;
// Sort chronologically (oldest to newest) to match real service
newMentions.sort((a, b) => {
const aId = BigInt(a.id);
const bId = BigInt(b.id);
return aId > bId ? 1 : aId < bId ? -1 : 0;
});

// Update last checked ID if we found new tweets
if (latestTweets.length > 0) {
// Use the first tweet from latestTweets since it's the newest (they're in descending order)
const highestId = latestTweets[0].id;
await this.setLastCheckedTweetId(highestId);
}

return newMentions;
}

async likeTweet(tweetId: string): Promise<void> {
return;
async setLastCheckedTweetId(tweetId: string) {
this.testLastCheckedTweetId = tweetId;
logger.info(`Last checked tweet ID updated to: ${tweetId}`);
}

getLastCheckedTweetId(): string | null {
return this.lastCheckedTweetId;
async getTweet(tweetId: string): Promise<Tweet | null> {
return this.mockTweets.find((t) => t.id === tweetId) || null;
}
}
23 changes: 0 additions & 23 deletions backend/src/config/config.ts

This file was deleted.

5 changes: 3 additions & 2 deletions backend/src/external/gpt-transform.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { TwitterSubmission } from "types/twitter";
import { TransformerPlugin } from "../types/plugin";

interface Message {
Expand Down Expand Up @@ -29,11 +30,11 @@ export default class GPTTransformer implements TransformerPlugin {
this.apiKey = config.apiKey;
}

async transform(content: string): Promise<string> {
async transform(submission: TwitterSubmission): Promise<string> {
try {
const messages: Message[] = [
{ role: "system", content: this.prompt },
{ role: "user", content },
{ role: "user", content: submission.content },
];

const response = await fetch(
Expand Down
102 changes: 102 additions & 0 deletions backend/src/external/notion/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
import { Client } from "@notionhq/client";
import { DistributorPlugin } from "types/plugin";
import { TwitterSubmission } from "types/twitter";

export default class NotionPlugin implements DistributorPlugin {
name = "notion";
private client: Client | null = null;
private databaseId: string | null = null;

async initialize(
feedId: string,
config: Record<string, string>,
): Promise<void> {
// Validate required config
if (!config.token) {
throw new Error("Notion plugin requires token");
}
if (!config.databaseId) {
throw new Error("Notion plugin requires databaseId");
}

this.client = new Client({ auth: config.token });
this.databaseId = config.databaseId;

try {
// Validate credentials by attempting to query the database
await this.client.databases.retrieve({
database_id: this.databaseId,
});
} catch (error) {
console.error("Failed to initialize Notion plugin:", error);
throw new Error("Failed to validate Notion credentials");
}
}

async distribute(
feedId: string,
submission: TwitterSubmission,
): Promise<void> {
if (!this.client || !this.databaseId) {
throw new Error("Notion plugin not initialized");
}

try {
await this.createDatabaseRow(submission);
} catch (error) {
console.error("Failed to create Notion database row:", error);
throw error;
}
}

private async createDatabaseRow(
submission: TwitterSubmission,
): Promise<void> {
if (!this.client || !this.databaseId) {
throw new Error("Notion plugin not initialized");
}

await this.client.pages.create({
parent: {
database_id: this.databaseId,
},
properties: {
tweetId: {
rich_text: [{ text: { content: submission.tweetId } }],
},
userId: {
rich_text: [{ text: { content: submission.userId } }],
},
username: {
rich_text: [{ text: { content: submission.username } }],
},
curatorId: {
rich_text: [{ text: { content: submission.curatorId } }],
},
curatorUsername: {
rich_text: [{ text: { content: submission.curatorUsername } }],
},
content: {
rich_text: [{ text: { content: submission.content.slice(0, 2000) } }],
},
curatorNotes: {
rich_text: submission.curatorNotes
? [{ text: { content: submission.curatorNotes } }]
: [],
},
curatorTweetId: {
rich_text: [{ text: { content: submission.curatorTweetId } }],
},
createdAt: {
rich_text: [{ text: { content: submission.createdAt } }],
},
submittedAt: {
rich_text: [{ text: { content: submission.submittedAt || "" } }],
},
status: {
rich_text: [{ text: { content: submission.status || "pending" } }],
},
},
});
}
}
Loading