Skip to content

Commit

Permalink
Corrects logic for getting precondition path matches + fixes overlap …
Browse files Browse the repository at this point in the history 
…considerations in shrinking bounds
  • Loading branch information
@nAmKcAz
nAmKcAz committed Dec 13, 2024
1 parent 52f05ec commit cd521da
Show file tree
Hide file tree
Showing 3 changed files with 92 additions and 62 deletions.
2 changes: 2 additions & 0 deletions ...ts/Runtime/Scripts/StateRecorder/BotSegments/Models/BotActions/InputPlaybackActionData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,8 @@ public bool ProcessAction(int segmentNumber, Dictionary<long, ObjectStatus> curr
{
// send event
result = true;

//TODO (REG-2237) : Replace with this finding the object and sending the raw position mouse event
MouseEventSender.SendMouseEvent(segmentNumber, replayMouseInputEntry, null, null, currentTransforms, currentEntities);
replayMouseInputEntry.Replay_IsDone = true;
}
Expand Down
150 changes: 88 additions & 62 deletions ...cripts/StateRecorder/BotSegments/Models/BotActions/KeyMoments/KeyMomentMouseActionData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public class KeyMomentMouseActionData : IBotActionData, IKeyMomentExploration, I
// These are all used to track the un-click work across multiple Update calls
private double _unClickTime = 0d;
private Vector2? _lastClickPosition = null;
private List<ObjectStatus> _previousOverlappingObjects = new();
private HashSet<ObjectStatus> _previousOverlappingObjects = new();
private MouseInputActionData _unClickAction = null;

public bool IsCompleted()
Expand Down Expand Up @@ -174,7 +174,7 @@ private bool HandleUnClickAction(int segmentNumber, Dictionary<long, ObjectStatu
Vector2 clickPosition;
if (unClickPreconditionMatches.Length > 0)
{
var bestUnClickMatchResult = FindBestMatch(segmentNumber, _unClickAction, unClickPreconditionMatches, out _);
var bestUnClickMatchResult = FindBestMatch(segmentNumber, _unClickAction, unClickPreconditionMatches);

if (bestUnClickMatchResult.HasValue)
{
Expand Down Expand Up @@ -206,7 +206,7 @@ private bool HandleUnClickAction(int segmentNumber, Dictionary<long, ObjectStatu
// isn't important.. we want to un-click exactly where we clicked
if (_previousOverlappingObjects.Any(a => a == bestUnClickMatchResult.Value.Item1))
{
RGDebug.LogDebug($"({segmentNumber}) Leaving mouse un-click action at previous action position: ({(int)_lastClickPosition.Value.x}, {(int)_lastClickPosition.Value.y}) based on object path existing at time of click: {bestUnClickMatchResult.Value.Item1.NormalizedPath}");
RGDebug.LogDebug($"({segmentNumber}) Leaving mouse un-click action at previous action position: ({(int)_lastClickPosition.Value.x}, {(int)_lastClickPosition.Value.y}) based on object target being at the point of the original click: {bestUnClickMatchResult.Value.Item1.NormalizedPath}");
clickPosition = _lastClickPosition.Value;
validateObjectMatches = false;
}
Expand Down Expand Up @@ -340,7 +340,7 @@ public bool ProcessAction(int segmentNumber, Dictionary<long, ObjectStatus> curr
var clickPaths = _preconditionNormalizedPaths[1];

var preconditionMatches = BuildPreconditions(segmentNumber, currentTransforms, currentEntities, clickPaths);
var bestMatchResult = FindBestMatch(segmentNumber, mouseActions[1], preconditionMatches, out var overlappingObjects);
var bestMatchResult = FindBestMatch(segmentNumber, mouseActions[1], preconditionMatches);
if (bestMatchResult.HasValue)
{

Expand All @@ -351,7 +351,7 @@ public bool ProcessAction(int segmentNumber, Dictionary<long, ObjectStatus> curr
return false;
}

_previousOverlappingObjects = overlappingObjects;
_previousOverlappingObjects = bestMatchResult.Value.Item3;
// on the 'click'.. save the position
_lastClickPosition = clickPosition;

Expand All @@ -370,7 +370,7 @@ public bool ProcessAction(int segmentNumber, Dictionary<long, ObjectStatus> curr
return false;
}

private Vector2 GetClickPositionForMatch((ObjectStatus, (float,float,float,float)) bestMatchResult)
private Vector2 GetClickPositionForMatch((ObjectStatus, (float,float,float,float), HashSet<ObjectStatus>) bestMatchResult)
{
// we started with clicking the center.. but this was limiting
//clickPosition = new Vector2(minX + (maxX - minX) / 2, minY + (maxY - minY) / 2);
Expand Down Expand Up @@ -410,7 +410,7 @@ private Vector2 GetClickPositionForMatch((ObjectStatus, (float,float,float,float
{
possibleTransformToClick.TokenizedObjectPath ??= PreconditionNormalizedPathData.TokenizeObjectPath(possibleTransformToClick.NormalizedPath);

// this should nearly always be smaller than the # of possibleTransformToClick
// this should nearly always be smaller than the # of possibleTransformToClick as far as optimizing the nested loops goes
for (var j = 0; j < preconditionsLength; j++)
{
var precondition = preConditionNormalizedPaths[j];
Expand All @@ -426,15 +426,15 @@ private Vector2 GetClickPositionForMatch((ObjectStatus, (float,float,float,float
}

preconditionMatches[j].Add((possibleTransformToClick, (null, null)));
break; // the for
break; // the for - we got an exact match.. stop processing this index
}
else
{
// tokenized matching logic
if (preconditionMatches[j].Count > 0 && preconditionMatches[j][0].Item2.Item1 == null)
{
// we already have exact matches.. ignore the tokenized ones
break; // the for
// we already have exact matches.. ignore the tokenized ones for this index
continue; // the for - this still might be an exact match for another index
}

var tokenMatches = EvaluateTokenMatches(precondition.tokenData, possibleTransformToClick.TokenizedObjectPath);
Expand Down Expand Up @@ -464,7 +464,7 @@ private Vector2 GetClickPositionForMatch((ObjectStatus, (float,float,float,float
preconditionMatches[j].Add((possibleTransformToClick, tokenMatches));
}

break; // the for
continue; // the for - might still be an exact match for another index
}
}
}
Expand Down Expand Up @@ -564,24 +564,27 @@ private Vector2 GetClickPositionForMatch((ObjectStatus, (float,float,float,float
return result;
}

private (ObjectStatus, (float,float,float,float))? FindBestMatch(int segmentNumber, MouseInputActionData mouseAction, List<(ObjectStatus, (int[], int[]))>[] preconditionMatches, out List<ObjectStatus> overlappingObjects)
/**
* Finds the best object on which to click given the precondition matches for the path names + the mouse location.
* Returns a tuple of (objectStatus, (minX,minY,maxY,maxY) bounds of that object, HashSet<ObjectStatus> of objects found to be overlapping that object)
*/
private (ObjectStatus, (float,float,float,float), HashSet<ObjectStatus>)? FindBestMatch(int segmentNumber, MouseInputActionData mouseAction, List<(ObjectStatus, (int[], int[]))>[] preconditionMatches)
{
overlappingObjects = new List<ObjectStatus>();

// now that we have all the precondition matches mapped out, let's see if we have the leftmost object..
if (preconditionMatches.Length > 0 && preconditionMatches[0].Count > 0)
{
// yay.. we found something(s).. figure out which one of them is the 'best' based on number of overlapping matches

// for each of the left most preconditionMatches[0] .. go through each of other precondition matches
// we will compute the 'smallest' click bounds for these overlaps as we go
var matchResults = new List<(ObjectStatus, (float, float, float, float))>(preconditionMatches[0].Count);
// for each of the objects matching the left most preconditionMatches[0] .. go through each of other precondition matches in the list and evaluate which of them have an overlap
// use all of those information to find the best match based on proximity to the original click + bounds area
var matchResults = new List<(ObjectStatus, (float, float, float, float), HashSet<ObjectStatus>)>(preconditionMatches[0].Count);

var screenWidth = Screen.width;
var screenHeight = Screen.height;

foreach (var preconditionMatch0 in preconditionMatches[0])
{
var myOverlappingObjects = new HashSet<ObjectStatus>();
var isPreconditionMatch0Interactable = true;
// now that we have a candidate match ... first let's make sure it's ready to be clicked
// visible (already true by the time we get here)/active-enabled(we already know that from a canvas perspective this is visible, but need to check UI component info)
Expand Down Expand Up @@ -642,7 +645,14 @@ private Vector2 GetClickPositionForMatch((ObjectStatus, (float,float,float,float
var preconditionMatchesI = preconditionMatches[i];
if (preconditionMatchesI.Count > 0)
{
// TODO: Currently this narrows the bounds.. but can cause you to actually get further from the original click position... i think we need to consider the 'most' overlapping match with the original for similar objects before choosing to narrow
// consider the 'most' overlapping match with the original before processing bounds narrowing
// this allows us to prefer shrinking the bounds based on the most overlapping objects with the current object... this keeps us picking the best objects to narrow with in cases where game UIs
// have large degree of canvas or bounds overlaps

// list of ObjectStatus, overlap-area, (minX,minY,maxX,maxY)
var sortedOverlappingPreconditionMatchesI = new List<(ObjectStatus, float, (float,float,float,float))>();

// sort the objects by those that overlap the most
foreach (var preconditionMatchI in preconditionMatchesI)
{
var isInteractable = true;
Expand All @@ -657,17 +667,18 @@ private Vector2 GetClickPositionForMatch((ObjectStatus, (float,float,float,float
var selectables = theTransform.GetComponents<Selectable>();
if (selectables.Length > 0)
{
// make sure 1 is interactable - if none leave isInteratable == true
// make sure 1 is interactable - if none leave isInteractable == true
isInteractable = selectables.Any(a => a.interactable);
}
}
}

if (isInteractable)
{
var pmIDidOverlap = false;
// ReSharper disable once PossibleInvalidOperationException - already filtered in BuildPreconditions to only have entries with valid visible bounds
var newBounds = preconditionMatchI.Item1.screenSpaceBounds.Value;

// for world space object, narrow the bounds to its collider
if (preconditionMatchI.Item1.worldSpaceBounds != null && preconditionMatchI.Item1 is TransformStatus preconditionMatchITs)
{
var preconditionMatchITransform = preconditionMatchITs.Transform;
Expand All @@ -685,67 +696,82 @@ private Vector2 GetClickPositionForMatch((ObjectStatus, (float,float,float,float
}
}

// adjust in for the min to ensure we don't miss a click
var newMinX = newBounds.min.x;
var newMinY = newBounds.min.y;
// adjust in for the max to ensure we don't miss a click
var newMaxX = newBounds.max.x;
var newMaxY = newBounds.max.y;
var areaMinX = Mathf.Max(newBounds.min.x, originalMinX);
var areaMaxX = Mathf.Min(newBounds.max.x, originalMaxX);
var areaMinY = Mathf.Max(newBounds.min.y, originalMinY);
var areaMaxY = Mathf.Min(newBounds.max.y, originalMaxY);

if (newMinX >= minX && newMinX <= maxX)
var areaOfOverlap = (areaMaxX - areaMinX) * (areaMaxY - areaMinY);
if (areaOfOverlap > 0)
{
minX = newMinX;
sortedOverlappingPreconditionMatchesI.Add((preconditionMatchI.Item1, areaOfOverlap, (areaMinX, areaMinY, areaMaxX, areaMaxY)));
}

if (newMinY >= minY && newMinY <= maxY)
else
{
minY = newMinY;
RGDebug.LogVerbose($"Excluding object with negative area of overlap");
}
}
}

if (newMaxX >= minX && newMaxX <= maxX)
{
maxX = newMaxX;
}
sortedOverlappingPreconditionMatchesI.Sort((a, b) =>
{
// sort biggest overlap numbers to the front
if (a.Item2 < b.Item2)
{
return 1;
}

if (newMaxY >= minY && newMaxY <= maxY)
{
maxY = newMaxY;
}
// we're comparing multiplied floats.. don't really need to care about the zillionth of a percent chance they are equal based on what we're doing with the data
return -1;
});

// then process those results narrowing bounds until we miss, then stop
foreach (var sortedOverlappingPreconditionMatch in sortedOverlappingPreconditionMatchesI)
{
// need to add all the overlaps
myOverlappingObjects.Add(sortedOverlappingPreconditionMatch.Item1);

if (newMinX >= originalMinX && newMinX <= originalMaxX)
{
pmIDidOverlap = true;
}
var didShrinkBounds = false;
var newMinX = sortedOverlappingPreconditionMatch.Item3.Item1;
var newMaxX = sortedOverlappingPreconditionMatch.Item3.Item3;

if (newMinY >= originalMinY && newMinY <= originalMaxY)
{
pmIDidOverlap = true;
}
var newMinY = sortedOverlappingPreconditionMatch.Item3.Item2;
var newMaxY = sortedOverlappingPreconditionMatch.Item3.Item4;

if (newMaxX >= originalMinX && newMaxX <= originalMaxX)
{
pmIDidOverlap = true;
}
if (newMinX > minX && newMinX < maxX)
{
didShrinkBounds = true;
minX = newMinX;
}

if (newMaxY >= originalMinY && newMaxY <= originalMaxY)
{
pmIDidOverlap = true;
}
if (newMinY > minY && newMinY < maxY)
{
didShrinkBounds = true;
minY = newMinY;
}

if (pmIDidOverlap)
{
overlappingObjects.Add(preconditionMatchI.Item1);
RGDebug.LogDebug($"({segmentNumber}) Tightened bounds: ({(int)minX}, {(int)minY}),({(int)maxX}, {(int)maxY}) for object path: {mouseAction.clickedObjectNormalizedPaths[0]} - overlap with object path [{i}]: {preconditionMatchI.Item1.NormalizedPath}");
}
if (newMaxX > minX && newMaxX < maxX)
{
didShrinkBounds = true;
maxX = newMaxX;
}

if (newMaxY > minY && newMaxY < maxY)
{
didShrinkBounds = true;
maxY = newMaxY;
}
}

if (didShrinkBounds)
{
RGDebug.LogDebug($"({segmentNumber}) Tightened bounds: ({(int)minX}, {(int)minY}),({(int)maxX}, {(int)maxY}) for object path: {mouseAction.clickedObjectNormalizedPaths[0]} - overlap with object path [{i}]: {sortedOverlappingPreconditionMatch.Item1.NormalizedPath}");
}

}
}
}

matchResults.Add((preconditionMatch0.Item1, (minX, minY, maxX, maxY)));
matchResults.Add((preconditionMatch0.Item1, (minX, minY, maxX, maxY), myOverlappingObjects));
}
}

Expand Down
Loading

0 comments on commit cd521da

Please sign in to comment.