diff --git a/.env.template b/.env.template index 3f7f570d1..7b0b0d830 100644 --- a/.env.template +++ b/.env.template @@ -6,6 +6,11 @@ NODE_ENV=development MONGODB_URI=mongodb://mongodb:27017/bt REDIS_URI=redis://redis:6379 +SIS_CLASS_APP_ID=_ +SIS_CLASS_APP_KEY=_ +SIS_COURSE_APP_ID=_ +SIS_COURSE_APP_KEY=_ + GOOGLE_CLIENT_ID=_ GOOGLE_CLIENT_SECRET=_ SESSION_SECRET=_ diff --git a/backend/src/config.ts b/backend/src/config.ts index 5bb932f40..1d0d9d569 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -21,6 +21,12 @@ export interface Config { mongoDB: { uri: string; }; + sis: { + CLASS_APP_ID: string; + CLASS_APP_KEY: string; + COURSE_APP_ID: string; + COURSE_APP_KEY: string; + }; SESSION_SECRET: string; GOOGLE_CLIENT_ID: string; GOOGLE_CLIENT_SECRET: string; @@ -36,6 +42,12 @@ export const config: Config = { mongoDB: { uri: env("MONGODB_URI"), }, + sis: { + CLASS_APP_ID: env("SIS_CLASS_APP_ID"), + CLASS_APP_KEY: env("SIS_CLASS_APP_KEY"), + COURSE_APP_ID: env("SIS_COURSE_APP_ID"), + COURSE_APP_KEY: env("SIS_COURSE_APP_KEY"), + }, SESSION_SECRET: env("SESSION_SECRET"), GOOGLE_CLIENT_ID: env("GOOGLE_CLIENT_ID"), GOOGLE_CLIENT_SECRET: env("GOOGLE_CLIENT_SECRET"), diff --git a/backend/src/scripts/update-catalog.ts b/backend/src/scripts/update-catalog.ts new file mode 100644 index 000000000..a8df30b99 --- /dev/null +++ b/backend/src/scripts/update-catalog.ts @@ -0,0 +1,183 @@ +import mongooseLoader from '../bootstrap/loaders/mongoose'; +import { CourseModel, CourseType } from '../models/course'; +import { SemesterModel, SemesterType } from '../models/semester'; +import { config } from "../config"; + +import axios, { AxiosResponse } from 'axios'; +import { SISResponse } from '../utils/sis'; +import { MongooseBulkWriteOptions } from 'mongoose'; +import { ClassModel, ClassType } from '../models/class'; +import { SectionModel, SectionType } from '../models/section'; + +const SIS_COURSE_URL = 'https://gateway.api.berkeley.edu/sis/v4/courses'; +const SIS_CLASS_URL = 'https://gateway.api.berkeley.edu/sis/v1/classes'; +const SIS_SECTION_URL = 'https://gateway.api.berkeley.edu/sis/v1/classes/sections'; + +const semToTermId = (s: SemesterType) => { + // term-id is computed by dropping the century digit of the year, then adding the term code + const termMap: { [key: string] : number} = {'Fall': 8, 'Spring': 2, 'Summer': 5} + return `${Math.floor(s.year / 1000)}${s.year % 100}${termMap[s.term]}` +} + +const queryPages = async (url: string, params: any, headers: any, field: string, retries: number = 3) => { + let page = 1 + const values: T[] = [] + + console.log("Querying SIS API pages...") + console.log(`URL: ${url}`) + console.log(`Params: ${JSON.stringify(params)}`) + console.log(`Headers: ${JSON.stringify(headers)}`) + while (true) { + let resp: AxiosResponse>; + + try { + resp = await axios.get(url, { params: { 'page-number': page, ...params}, headers }); + } catch (err) { + if (axios.isAxiosError(err) && err.response?.status === 404) { + break; + } else { + console.log(`Unexpected err querying SIS API. Error: ${err}.`) + + if (retries > 0) { + retries--; + console.log(`Retrying...`) + continue; + } else { + console.log(`Too many errors querying SIS API for courses. Terminating update...`) + throw err; + } + } + } + + values.push(...resp.data.apiResponse.response[field]); + page++; + } + + console.log(`Completed querying SIS API. Received ${values.length} objects in ${page - 1} pages.`) + + return values; +} + +const updateCourses = async () => { + const headers = { + 'app_id': config.sis.COURSE_APP_ID, + 'app_key': config.sis.COURSE_APP_KEY, + } + const params = { + 'status-code': 'ACTIVE', + 'page-size': 100, + } + + const courses = await queryPages(SIS_COURSE_URL, params, headers, 'courses'); + + console.log("Updating database with new course data...") + + const bulkOps = courses.map(c => ({ + replaceOne: { + filter: { classDisplayName: c.classDisplayName }, + replacement: c, + upsert: true, + } + })); + + const options = { strict: 'throw' } as MongooseBulkWriteOptions; + + const res = await CourseModel.bulkWrite(bulkOps, options); + + console.log(`Completed updating database with new course data. Created ${res.upsertedCount} and updated ${res.modifiedCount} course objects.`) +} + +const updateClasses = async () => { + const headers = { + 'app_id': config.sis.CLASS_APP_ID, + 'app_key': config.sis.CLASS_APP_KEY, + } + + const activeSemesters = await SemesterModel.find({ active: true }).lean(); + const classes: ClassType[] = []; + + for (const s of activeSemesters) { + console.log(`Updating classses for ${s.term} ${s.year}...`) + + const params = { + 'term-id': semToTermId(s), + 'page-size': 100, + } + + const semesterClasses = await queryPages(SIS_CLASS_URL, params, headers, 'classes'); + classes.push(...semesterClasses); + } + + console.log("Updating database with new class data...") + const bulkOps = classes.map(c => ({ + replaceOne: { + filter: { displayName: c.displayName }, + replacement: c, + upsert: true, + } + })); + + const options = { strict: 'throw' } as MongooseBulkWriteOptions; + + const res = await ClassModel.bulkWrite(bulkOps, options); + + console.log(`Completed updating database with new class data. Created ${res.upsertedCount} and updated ${res.modifiedCount} class objects.`) +} + +const updateSections = async () => { + const headers = { + 'app_id': config.sis.CLASS_APP_ID, + 'app_key': config.sis.CLASS_APP_KEY, + } + + const activeSemesters = await SemesterModel.find({ active: true }).lean(); + const sections: SectionType[] = []; + + for (const s of activeSemesters) { + console.log(`Updating sections for ${s.term} ${s.year}...`) + + const params = { + 'term-id': semToTermId(s), + 'page-size': 100, + } + + const semesterClasses = await queryPages(SIS_SECTION_URL, params, headers, 'classSections'); + sections.push(...semesterClasses); + } + + console.log("Updating database with new section data...") + const bulkOps = sections.map(s => ({ + replaceOne: { + filter: { displayName: s.displayName }, + replacement: s, + upsert: true, + } + })); + + const options = { strict: 'throw' } as MongooseBulkWriteOptions; + + const res = await SectionModel.bulkWrite(bulkOps, options); + + console.log(`Completed updating database with new section data. Created ${res.upsertedCount} and updated ${res.modifiedCount} section objects.`) + +} + +(async () => { + try { + await mongooseLoader(); + + console.log("\n=== UPDATE COURSES ===") + await updateCourses(); + + console.log("\n=== UPDATE CLASSES ===") + await updateClasses(); + + console.log("\n=== UPDATE SECTIONS ===") + await updateSections(); + } catch (err) { + console.error(err); + process.exit(1); + } + + process.exit(0); +})(); \ No newline at end of file