From 592eeb53b93ed887e9e4bcf0d2829e83ee3ababc Mon Sep 17 00:00:00 2001 From: Benji Marshall Date: Fri, 20 Nov 2020 13:07:00 +0000 Subject: [PATCH] Add batches of responses to Google Sheets (#51) * Implement adding batches of responses * Implement clearing batches of responses --- .../SheetsServiceParsingHelper.cs | 46 +++- .../SheetsServiceRequestsHelper.cs | 120 +++++++++ .../UnsafeEventsSheetsService.cs | 255 +++++++++++++----- 3 files changed, 354 insertions(+), 67 deletions(-) diff --git a/DiscordBot.DataAccess/SheetsServiceParsingHelper.cs b/DiscordBot.DataAccess/SheetsServiceParsingHelper.cs index 4a0f368..9f88f43 100644 --- a/DiscordBot.DataAccess/SheetsServiceParsingHelper.cs +++ b/DiscordBot.DataAccess/SheetsServiceParsingHelper.cs @@ -1,6 +1,9 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; +using DiscordBot.DataAccess.Exceptions; using DiscordBot.DataAccess.Models; +using Google.Apis.Sheets.v4.Data; namespace DiscordBot.DataAccess { @@ -27,10 +30,15 @@ internal static class SheetsServiceParsingHelper return ulong.Parse(cellContents); } - public static IEnumerable ParseResponseHeaders(IList> sheetsResponse) + public static IEnumerable ParseResponseHeaders(ValueRange sheetsResponse, int eventKey) { - return sheetsResponse[0] - .Zip(sheetsResponse[1]) + if (sheetsResponse == null || sheetsResponse.Values.Count < 2) + { + throw new EventsSheetsInitialisationException($"Event sheet {eventKey} is empty"); + } + + return sheetsResponse.Values[0] + .Zip(sheetsResponse.Values[1]) .Skip(1) // Skip titles column .Select<(object emoji, object name), EventResponse>(response => new EventResponse((string)response.emoji, (string)response.name) @@ -51,5 +59,35 @@ IList row ) .Select(pair => (pair.response, userId)); } + + public static int? FindRowNumberOfKey( + ValueRange response, + SheetsColumn keyColumn, + int numberOfHeaderRows, + ulong key + ) + { + if (response == null || response.Values.Count < numberOfHeaderRows) + { + throw new EventNotFoundException($"Event key {key} not recognised"); + } + + try + { + var rowNumber = response.Values + .Skip(numberOfHeaderRows) + .Select((values, index) => (values, index)) + .First(row => ulong.Parse((string)row.values[keyColumn.Index]) == key); + + // Extract row number, plus a correction factor: + // Correct for skipping the header + // These lists are 0 indexed, but Sheets index from 1 + return rowNumber.index + numberOfHeaderRows + 1; + } + catch (InvalidOperationException) + { + return null; + } + } } } diff --git a/DiscordBot.DataAccess/SheetsServiceRequestsHelper.cs b/DiscordBot.DataAccess/SheetsServiceRequestsHelper.cs index 29eaa74..910aad0 100644 --- a/DiscordBot.DataAccess/SheetsServiceRequestsHelper.cs +++ b/DiscordBot.DataAccess/SheetsServiceRequestsHelper.cs @@ -127,6 +127,101 @@ internal static ValueRange MakeCellUpdate(string range, object value) }; } + internal static IEnumerable AddResponseRow( + int responseSheetId, + IList responsesOptions, + ulong userId, + IEnumerable responses + ) + { + var newRowValues = responsesOptions + .Select(response => responses.Contains(response.Emoji) ? 1.0 : 0) + .ToList(); + + // If none of the user's emoji are recognised, don't add a user response row + if (newRowValues.All(responseBit => responseBit == 0)) + { + return Enumerable.Empty(); + } + + var newRow = newRowValues + .Select(MakeCellData) + .Prepend(MakeCellData(userId.ToString())); + + var appendCellsRequest = new AppendCellsRequest() + { + Rows = new[] + { + new RowData() + { + Values = newRow.ToList() + } + }, + SheetId = responseSheetId, + Fields = "*" + }; + + return new[] { new Request() { AppendCells = appendCellsRequest } }; + } + + internal static IEnumerable UpdateResponseRow( + int responseSheetId, + IList responsesOptions, + int responseRow, + IEnumerable responses + ) + { + var responseColumnList = responsesOptions.ToList(); + var indices = responses + .Select(emoji => + responseColumnList.FindIndex(eventResponse => eventResponse.Emoji == emoji) + ) + .Where(index => index != -1); // If the emoji wasn't found, drop this update + + var cellData = MakeCellData(1); + return indices.Select(index => UpdateCell(responseSheetId, responseRow - 1, index + 1, cellData)); + } + + internal static IEnumerable AddUserResponsesRequests( + int sheetId, + IList responsesOptions, + ValueRange userIdColumn, + ulong userId, + IEnumerable responses + ) + { + var userRow = SheetsServiceParsingHelper.FindRowNumberOfKey( + userIdColumn, + UnsafeEventsSheetsService.UserIdColumn, + 2, + userId + ); + + if (userRow == null) + { + return AddResponseRow(sheetId, responsesOptions, userId, responses); + } + + return UpdateResponseRow(sheetId, responsesOptions, userRow.Value, responses); + } + + internal static IEnumerable ClearUserResponsesRequests( + int sheetId, + ValueRange userIdColumn, + IEnumerable users + ) + { + return users + .Select(user => SheetsServiceParsingHelper.FindRowNumberOfKey( + userIdColumn, + UnsafeEventsSheetsService.UserIdColumn, + 2, + user + )) + .Where(userRow => userRow != null) + .Select(userRow => RemoveRow(sheetId, userRow!.Value)); + } + internal static T ExecuteRequestsWithRetries(ClientServiceRequest request) => ExecuteRequestsWithRetriesAsync(request).Result; @@ -170,5 +265,30 @@ private static CellData MakeCellData(string stringValue) UserEnteredValue = new ExtendedValue() { StringValue = stringValue } }; } + + private static Request UpdateCell(int sheetId, int row, int col, CellData value) + { + var updateCellsRequest = new UpdateCellsRequest() + { + Range = new GridRange() + { + StartRowIndex = row, + EndRowIndex = row + 1, + StartColumnIndex = col, + EndColumnIndex = col + 1, + SheetId = sheetId + }, + Rows = new[] + { + new RowData() + { + Values = new[] { value } + } + }, + Fields = "*" + }; + + return new Request() { UpdateCells = updateCellsRequest }; + } } } diff --git a/DiscordBot.DataAccess/UnsafeEventsSheetsService.cs b/DiscordBot.DataAccess/UnsafeEventsSheetsService.cs index d56e33c..48f4093 100644 --- a/DiscordBot.DataAccess/UnsafeEventsSheetsService.cs +++ b/DiscordBot.DataAccess/UnsafeEventsSheetsService.cs @@ -262,15 +262,137 @@ public async Task AddResponseForUserAsync(int eventKey, ulong userId, string res } } -#pragma warning disable 1998 // Turn off compiler warning for synchronous unimplemented methods public async Task AddResponseBatchAsync(IEnumerable reactions) { + // Get the sheets to find their IDs when updating + var sheets = await GetSheets(); + + // Rearrange the list of reactions into a dictionary where: + // reactionDictionary[eventKey][userId] = list of that user's reactions to that event + var eventList = await ListEventsAsync(); + var reactionDictionary = CreateReactionDictionary(eventList, reactions); + var eventKeyList = reactionDictionary.Keys.ToList(); + + // If there are no reactions to recognised events, there is no work to be done + if (!eventKeyList.Any()) + { + return; + } + + // Make the requests to find which rows users' already have response information on, + // and the set of response options for each event. + // Each event sheet has two requests in the batch, for these two bits of information, grouped by event key + var responseSheetsRequest = sheetsService.Spreadsheets.Values.BatchGet(spreadsheetId); + responseSheetsRequest.Ranges = eventKeyList.SelectMany(eventKey => + new[] + { + $"{eventKey}!{UserIdColumn.Letter}:{UserIdColumn.Letter}", + $"{eventKey}!1:2" + } + ).ToList(); + responseSheetsRequest.ValueRenderOption = BatchGetRequest.ValueRenderOptionEnum.FORMATTEDVALUE; + var responseSheetsResponse = + await SheetsServiceRequestsHelper.ExecuteRequestsWithRetriesAsync(responseSheetsRequest); + + // Make the update requests, zipping in the pair of relevant responses for each event's response sheet + var updateRequests = eventKeyList + .Zip( + responseSheetsResponse.ValueRanges.Where((userIdColumn, i) => i % 2 == 0), + (eventKey, userIdColumn) => new { eventKey, userIdColumn } + ) + .Zip( + responseSheetsResponse.ValueRanges.Where((userIdColumn, i) => i % 2 == 1), + (pair, responsesOptions) => new { pair.eventKey, pair.userIdColumn, responsesOptions } + ) + .SelectMany(triple => + { + var eventKey = triple.eventKey; + var sheetId = FindSheetId(sheets, eventKey.ToString()); + + var responsesOptions = + SheetsServiceParsingHelper.ParseResponseHeaders(triple.responsesOptions, eventKey).ToList(); + + var eventUpdates = reactionDictionary[eventKey]; + + return eventUpdates.SelectMany(entry => SheetsServiceRequestsHelper.AddUserResponsesRequests( + sheetId, + responsesOptions, + triple.userIdColumn, + entry.Key, // User ID + entry.Value // User's responses + )); + }); + + var batchUpdateRequest = new BatchUpdateSpreadsheetRequest() + { + Requests = updateRequests.ToList() + }; + // If there are no updates to be made, don't send a request to Google Sheets + if (batchUpdateRequest.Requests.Count == 0) + { + return; + } + + var batchRequest = sheetsService.Spreadsheets.BatchUpdate(batchUpdateRequest, spreadsheetId); + await SheetsServiceRequestsHelper.ExecuteRequestsWithRetriesAsync(batchRequest); } public async Task ClearResponseBatchAsync(IEnumerable reactions) { + // Get the sheets to find their IDs when updating + var sheets = await GetSheets(); + + // Rearrange the list of reactions into a dictionary where: + // usersToClearDictionary[eventKey] = list of users to have their responses cleared for that event + // where that list only contains user IDs at most once (ie has distinct elements) + var eventList = await ListEventsAsync(); + var usersToClearDictionary = CreateReactionDictionary(eventList, reactions).ToDictionary( + entry => entry.Key, + entry => entry.Value.Keys.Distinct() + ); + var eventKeyList = usersToClearDictionary.Keys.ToList(); + + // If there are no reactions to recognised events, there is no work to be done + if (!eventKeyList.Any()) + { + return; + } + + // Make the requests to find which rows the users' response information are on + // Each event sheet has one request in the batch + var responseSheetsRequest = sheetsService.Spreadsheets.Values.BatchGet(spreadsheetId); + responseSheetsRequest.Ranges = eventKeyList.Select(eventKey => + $"{eventKey}!{UserIdColumn.Letter}:{UserIdColumn.Letter}" + ).ToList(); + responseSheetsRequest.ValueRenderOption = BatchGetRequest.ValueRenderOptionEnum.FORMATTEDVALUE; + var responseSheetsResponse = + await SheetsServiceRequestsHelper.ExecuteRequestsWithRetriesAsync(responseSheetsRequest); + + // Make the update requests, zipping in the column of IDs for each event's response sheet + var updateRequests = eventKeyList + .Zip( + responseSheetsResponse.ValueRanges, + (eventKey, userIdColumn) => new { eventKey, userIdColumn } + ) + .SelectMany(pair => SheetsServiceRequestsHelper.ClearUserResponsesRequests( + FindSheetId(sheets, pair.eventKey.ToString()), + pair.userIdColumn, + usersToClearDictionary[pair.eventKey] + )); + + var batchUpdateRequest = new BatchUpdateSpreadsheetRequest() + { + Requests = updateRequests.ToList() + }; + // If there are no updates to be made, don't send a request to Google Sheets + if (batchUpdateRequest.Requests.Count == 0) + { + return; + } + + var batchRequest = sheetsService.Spreadsheets.BatchUpdate(batchUpdateRequest, spreadsheetId); + await SheetsServiceRequestsHelper.ExecuteRequestsWithRetriesAsync(batchRequest); } -#pragma warning restore 1998 public async Task ClearResponsesForUserAsync(int eventKey, ulong userId) { @@ -302,12 +424,7 @@ public async Task>> GetSignupsByRes request.ValueRenderOption = GetRequest.ValueRenderOptionEnum.FORMATTEDVALUE; var response = await SheetsServiceRequestsHelper.ExecuteRequestsWithRetriesAsync(request); - if (response == null || response.Values.Count < 2) - { - throw new EventInitialisationException("Sign up sheet is empty"); - } - - var responseColumns = SheetsServiceParsingHelper.ParseResponseHeaders(response.Values).ToList(); + var responseColumns = SheetsServiceParsingHelper.ParseResponseHeaders(response, eventId).ToList(); var result = responseColumns.ToDictionary( eventResponse => eventResponse, @@ -375,17 +492,18 @@ private int GetLargestKey() } } + private async Task> GetSheets() + { + var sheetListRequest = sheetsService.Spreadsheets.Get(spreadsheetId); + var spreadsheet = await SheetsServiceRequestsHelper.ExecuteRequestsWithRetriesAsync(sheetListRequest); + return spreadsheet.Sheets; + } + private int GetSheetIdFromTitle(string title) => GetSheetIdFromTitleAsync(title).Result; - private async Task GetSheetIdFromTitleAsync(string title) - { - var request = sheetsService.Spreadsheets.Get(spreadsheetId); - var spreadsheet = await SheetsServiceRequestsHelper.ExecuteRequestsWithRetriesAsync(request); - var sheets = spreadsheet.Sheets; - - return FindSheetId(sheets, title); - } + private async Task GetSheetIdFromTitleAsync(string title) => + FindSheetId(await GetSheets(), title); private int FindSheetId(IEnumerable sheets, string title) { @@ -418,33 +536,13 @@ private int GetMetadataSheetId() ulong key ) { - try - { - var request = sheetsService.Spreadsheets.Values.Get( - spreadsheetId, - $"{sheetName}!{keyColumn.Letter}:{keyColumn.Letter}" - ); - var response = await SheetsServiceRequestsHelper.ExecuteRequestsWithRetriesAsync(request); - - if (response == null || response.Values.Count < numberOfHeaderRows) - { - throw new EventNotFoundException($"Event key {key} not recognised"); - } - - var rowNumber = response.Values - .Skip(numberOfHeaderRows) - .Select((values, index) => (values, index)) - .First(row => ulong.Parse((string)row.values[keyColumn.Index]) == key); + var request = sheetsService.Spreadsheets.Values.Get( + spreadsheetId, + $"{sheetName}!{keyColumn.Letter}:{keyColumn.Letter}" + ); + var response = await SheetsServiceRequestsHelper.ExecuteRequestsWithRetriesAsync(request); - // Extract row number, plus a correction factor: - // Correct for skipping the header - // These lists are 0 indexed, but Sheets index from 1 - return rowNumber.index + numberOfHeaderRows + 1; - } - catch (InvalidOperationException) - { - return null; - } + return SheetsServiceParsingHelper.FindRowNumberOfKey(response, keyColumn, numberOfHeaderRows, key); } private async Task GetEventRowNumberAsync(int eventKey) @@ -475,29 +573,15 @@ private async Task GetEventRowNumberAsync(int eventKey) private async Task> GetEventResponseOptionsAsync(int eventKey) { - try - { - var request = sheetsService.Spreadsheets.Values.Get( - spreadsheetId, - $"{eventKey}!1:2" - ); - request.ValueRenderOption = GetRequest.ValueRenderOptionEnum.FORMATTEDVALUE; + var request = sheetsService.Spreadsheets.Values.Get( + spreadsheetId, + $"{eventKey}!1:2" + ); + request.ValueRenderOption = GetRequest.ValueRenderOptionEnum.FORMATTEDVALUE; - var sheetsResponse = await SheetsServiceRequestsHelper.ExecuteRequestsWithRetriesAsync(request); + var sheetsResponse = await SheetsServiceRequestsHelper.ExecuteRequestsWithRetriesAsync(request); - if (sheetsResponse == null || sheetsResponse.Values.Count < 2) - { - throw new EventsSheetsInitialisationException($"Event sheet {eventKey} is empty"); - } - - return SheetsServiceParsingHelper.ParseResponseHeaders(sheetsResponse.Values); - } - catch (GoogleApiException) - { - throw new EventInitialisationException( - $"Could not find event responses for event {eventKey}. Has it been published yet?" - ); - } + return SheetsServiceParsingHelper.ParseResponseHeaders(sheetsResponse, eventKey); } private async Task AddResponseForNewUserAsync( @@ -545,5 +629,50 @@ IEnumerable responseColumns await SheetsServiceRequestsHelper.ExecuteRequestsWithRetriesAsync(request); } + + private Dictionary>> CreateReactionDictionary( + IEnumerable eventList, + IEnumerable reactions + ) + { + var reactionList = reactions.ToList(); + var messageIds = reactionList.Select(reaction => reaction.MessageId).Distinct(); + + var messageIdDictionary = eventList + .Where(discordEvent => + discordEvent.MessageId != null && messageIds.Contains(discordEvent.MessageId.Value) + ) + .ToDictionary( + discordEvent => discordEvent.MessageId!.Value, + discordEvent => discordEvent.Key + ); + + var eventKeyList = messageIdDictionary.Values.ToList(); + + var reactionDictionary = + eventKeyList.ToDictionary( + eventKey => eventKey, + _ => new Dictionary>() + ); + + reactionList.ForEach(reaction => + { + // If message ID is not known, ignore this reaction + if (!messageIdDictionary.ContainsKey(reaction.MessageId)) + { + return; + } + + var eventResponses = reactionDictionary[messageIdDictionary[reaction.MessageId]]; + + if (!eventResponses.ContainsKey(reaction.UserId)) + { + eventResponses[reaction.UserId] = new List(); + } + eventResponses[reaction.UserId].Add(reaction.Emoji); + }); + + return reactionDictionary; + } } }