Get forum channels from Discord (#405)

backend/src/serverless/integrations/services/integrations/discordIntegrationService.ts

@@ -5,6 +5,8 @@ import {
   DiscordMembers,
   DiscordMention,
   DiscordStreamProcessResult,
+  ProcessedChannel,
+  ProcessedChannels,
 } from '../../types/discordTypes'
 import { DISCORD_CONFIG } from '../../../../config'
 import { DiscordMemberAttributes } from '../../../../database/attributes/member/discord'
@@ -20,7 +22,6 @@ import { IntegrationType, PlatformType } from '../../../../types/integrationEnum
 import { timeout } from '../../../../utils/timing'
 import Operations from '../../../dbOperations/operations'
 import { DiscordGrid } from '../../grid/discordGrid'
-import { Channels } from '../../types/regularTypes'
 import getChannels from '../../usecases/discord/getChannels'
 import getMembers from '../../usecases/discord/getMembers'
 import getMessages from '../../usecases/discord/getMessages'
@@ -29,6 +30,7 @@ import { sendNodeWorkerMessage } from '../../../utils/nodeWorkerSQS'
 import { NodeWorkerIntegrationProcessMessage } from '../../../../types/mq/nodeWorkerIntegrationProcessMessage'
 import { AddActivitiesSingle } from '../../types/messageTypes'
 import { singleOrDefault } from '../../../../utils/arrays'
+import getThreads from '../../usecases/discord/getThreads'
 
 /* eslint class-methods-use-this: 0 */
 
@@ -83,18 +85,24 @@ export class DiscordIntegrationService extends IntegrationServiceBase {
   async preprocess(context: IStepContext): Promise<void> {
     const guildId = context.integration.integrationIdentifier
 
-    let channelsFromDiscordAPI: Channels = await getChannels(
+    const fromDiscordApi: ProcessedChannels = await getChannels(
       {
         guildId,
         token: this.getToken(context),
       },
       this.logger(context),
     )
 
+    let channelsFromDiscordAPI: ProcessedChannel[] = fromDiscordApi.channels
+
     const channels = context.integration.settings.channels
       ? context.integration.settings.channels
       : []
 
+    const forumChannels = context.integration.settings.forumChannels
+      ? context.integration.settings.forumChannels
+      : []
+
     // Add bool new property to new channels
     channelsFromDiscordAPI = channelsFromDiscordAPI.map((c) => {
       if (channels.filter((a) => a.id === c.id).length <= 0) {
@@ -103,16 +111,46 @@ export class DiscordIntegrationService extends IntegrationServiceBase {
       return c
     })
 
+    const threads = await getThreads(
+      {
+        guildId,
+        token: this.getToken(context),
+      },
+      this.logger(context),
+    )
+
+    const forumChannelsFromDiscordAPi = []
+
+    for (const thread of threads) {
+      const forumChannel: any = lodash.find(fromDiscordApi.forumChannels, { id: thread.parentId })
+      if (forumChannel) {
+        forumChannelsFromDiscordAPi.push({
+          ...forumChannel,
+          threadId: thread.id,
+          new: forumChannels.filter((c) => c.id === forumChannel.id).length <= 0,
+          threadName: thread.name,
+        })
+      }
+    }
+
     context.pipelineData = {
       settingsChannels: channels,
       channels: channelsFromDiscordAPI,
+      forumChannels: forumChannelsFromDiscordAPi,
       channelsInfo: channelsFromDiscordAPI.reduce((acc, channel) => {
         acc[channel.id] = {
           name: channel.name,
           new: !!channel.new,
         }
         return acc
       }, {}),
+      forumChannelsInfo: forumChannelsFromDiscordAPi.reduce((acc, forumChannel) => {
+        acc[forumChannel.id] = {
+          name: forumChannel.name,
+          new: !!forumChannel.new,
+        }
+        return acc
+      }, {}),
       guildId: context.integration.integrationIdentifier,
     }
   }
@@ -132,15 +170,27 @@ export class DiscordIntegrationService extends IntegrationServiceBase {
       },
     ]
 
-    return predefined.concat(
-      context.pipelineData.channels.map((c) => ({
-        value: 'channel',
-        metadata: {
-          id: c.id,
-          page: '',
-        },
-      })),
-    )
+    return predefined
+      .concat(
+        context.pipelineData.channels.map((c) => ({
+          value: 'channel',
+          metadata: {
+            id: c.id,
+            page: '',
+          },
+        })),
+      )
+      .concat(
+        context.pipelineData.forumChannels.map((c) => ({
+          value: 'forumChannel',
+          metadata: {
+            id: c.threadId,
+            page: '',
+            forumChannelId: c.id,
+            threadName: c.threadName,
+          },
+        })),
+      )
   }
 
   async processStream(
@@ -260,6 +310,15 @@ export class DiscordIntegrationService extends IntegrationServiceBase {
       const { new: _, ...raw } = ch
       return raw
     })
+
+    context.integration.settings.forumChannels = lodash.uniqBy(
+      context.pipelineData.forumChannels.map((ch) => {
+        const { new: _, ...raw } = ch
+        delete raw.threadId
+        return raw
+      }),
+      (ch: any) => ch.id,
+    )
   }
 
   parseActivities(
@@ -330,7 +389,12 @@ export class DiscordIntegrationService extends IntegrationServiceBase {
     const activities: AddActivitiesSingle[] = records.reduce((acc, record) => {
       let parent = ''
 
-      const channelInfo = context.pipelineData.channelsInfo[stream.metadata.id]
+      const isForum = stream.metadata.forumChannelId !== undefined
+
+      let channelInfo = context.pipelineData.channelsInfo[stream.metadata.id]
+      if (isForum) {
+        channelInfo = context.pipelineData.forumChannelsInfo[stream.metadata.forumChannelId]
+      }
 
       if (!channelInfo) {
         const log = this.logger(context)
@@ -352,6 +416,7 @@ export class DiscordIntegrationService extends IntegrationServiceBase {
           value: 'thread',
           metadata: {
             id: record.thread.id,
+            forumChannelId: stream.metadata.forumChannelId,
           },
         })
 
@@ -371,6 +436,8 @@ export class DiscordIntegrationService extends IntegrationServiceBase {
       // record.parentId means that it's a reply
       else if (record.message_reference && record.message_reference.message_id) {
         parent = record.message_reference.message_id
+      } else if (stream.value === 'forumChannel') {
+        parent = stream.metadata.id
       }
 
       let avatarUrl: string | boolean = false
@@ -382,19 +449,25 @@ export class DiscordIntegrationService extends IntegrationServiceBase {
         const activityObject = {
           tenant: context.integration.tenantId,
           platform: PlatformType.DISCORD,
-          type: 'message',
+          type: isForum && record.id === parent ? 'thread_started' : 'message',
           sourceId: record.id,
           sourceParentId: parent,
           timestamp: moment(record.timestamp).utc().toDate(),
+          ...(stream.value === 'forumChannel' &&
+            record.id === parent && { title: stream.metadata.threadName }),
           body: record.content
             ? DiscordIntegrationService.replaceMentions(record.content, record.mentions)
             : '',
           url: `https://discordapp.com/channels/${context.pipelineData.guildId}/${stream.metadata.id}/${record.id}`,
           channel: channelInfo.name,
           attributes: {
-            thread: record.thread !== undefined || stream.value === 'thread',
+            thread:
+              record.thread !== undefined ||
+              stream.value === 'thread' ||
+              stream.value === 'forumChannel',
             reactions: record.reactions ? record.reactions : [],
             attachments: record.attachments ? record.attachments : [],
+            forum: isForum,
           },
           member: {
             username: record.author.username,
@@ -465,6 +538,7 @@ export class DiscordIntegrationService extends IntegrationServiceBase {
         return { fn: getMembers, arg: { guildId } }
       case 'channel':
       case 'thread':
+      case 'forumChannel':
         return { fn: getMessages, arg: { channelId: stream.metadata.id } }
       default:
         throw new Error(`Unknown stream ${stream.value}!`)

backend/src/serverless/integrations/types/discordTypes.ts

@@ -22,13 +22,20 @@ export interface DiscordGetMembersInput {
 }
 
 export interface DiscordChannel {
+  parentId?: string
   id: string
   name: string
   thread?: boolean
+  type?: number
 }
 
 export type DiscordChannels = DiscordChannel[]
 
+export interface DiscordChannelsOut {
+  channels: DiscordChannels
+  forumChannels: DiscordChannels
+}
+
 export interface DiscordAuthor {
   id: string
   username: string
@@ -90,3 +97,15 @@ export interface DiscordGetMessagesOutput extends DiscordParsedReponse {
 export interface DiscordGetMembersOutput extends DiscordParsedReponse {
   records: DiscordMembers | []
 }
+
+export type ProcessedChannel = {
+  id: string
+  name: string
+  thread?: boolean
+  new?: boolean
+}
+
+export interface ProcessedChannels {
+  channels: Array<ProcessedChannel>
+  forumChannels: Array<ProcessedChannel>
+}

backend/src/serverless/integrations/types/regularTypes.ts

@@ -8,15 +8,6 @@ export type Repo = {
 
 export type Repos = Array<Repo>
 
-export type Channel = {
-  id: string
-  name: string
-  thread?: boolean
-  new?: boolean
-}
-
-export type Channels = Array<Channel>
-
 export type Endpoint = string
 
 export type Endpoints = Array<Endpoint>

backend/src/serverless/integrations/usecases/discord/getChannels.ts

@@ -2,6 +2,7 @@ import axios from 'axios'
 import {
   DiscordChannel,
   DiscordChannels,
+  DiscordChannelsOut,
   DiscordGetChannelsInput,
   DiscordGetMessagesInput,
 } from '../../types/discordTypes'
@@ -31,7 +32,7 @@ async function getChannels(
   input: DiscordGetChannelsInput,
   logger: Logger,
   tryChannels = true,
-): Promise<DiscordChannels> {
+): Promise<DiscordChannelsOut> {
   try {
     const config = {
       method: 'get',
@@ -44,6 +45,14 @@ async function getChannels(
     const response = await axios(config)
     const result: DiscordChannels = response.data
 
+    const forumChannels = result
+      .filter((c) => c.type === 15)
+      .map((c) => ({
+        name: c.name,
+        id: c.id,
+        thread: true,
+      }))
+
     if (tryChannels) {
       const out: DiscordChannels = []
       for (const channel of result) {
@@ -67,13 +76,21 @@ async function getChannels(
           }
         }
       }
-      return out
+      return {
+        channels: out,
+        forumChannels,
+      }
     }
 
-    return result.map((c) => ({
+    const channelsOut = result.map((c) => ({
       name: c.name,
       id: c.id,
     }))
+
+    return {
+      channels: channelsOut,
+      forumChannels,
+    }
   } catch (err) {
     logger.error({ err, input }, 'Error while getting channels from Discord')
     throw err

backend/src/serverless/integrations/usecases/discord/getThreads.ts

@@ -21,6 +21,7 @@ async function getThreads(
     return result.map((c) => ({
       name: c.name,
       id: c.id,
+      parentId: c.parent_id,
       thread: true,
     }))
   } catch (err) {

frontend/src/i18n/en.js

@@ -349,6 +349,7 @@ const en = {
         message: 'sent a message',
         replied: 'replied to a message',
         replied_thread: 'replied to a thread',
+        started_thread: 'started a new thread',
         joined_guild: 'joined server',
         left_guild: 'left server'
       },

frontend/src/integrations/discord/components/activity/discord-activity-content.vue

@@ -1,7 +1,9 @@
 <template>
   <div>
     <blockquote
-      v-if="activity.parent && displayThread"
+      v-if="
+        activity.parent && displayThread && !isParentInForum
+      "
       class="relative px-3 border-l-4 text-gray-500 border-gray-200 text-xs leading-5 mb-4 parsed-body"
       v-html="$sanitize($marked(activity.parent.body))"
     />
@@ -38,6 +40,11 @@ export default {
       required: false,
       default: true
     }
+  },
+  computed: {
+    isParentInForum() {
+      return this.activity.type === 'thread_started'
+    }
   }
 }
 </script>