Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix for restoration job to find agent endpoints when there are multiple replicas/stateful sets. #338

Closed
wants to merge 9 commits into from
Closed

Fix for restoration job to find agent endpoints when there are multiple replicas/stateful sets. #338

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
8a78c5c
add condition to check for replica set and resolve with first IP
hsubramanianaks Nov 9, 2023
d4c44d8
add logic to handle multi pods for replicaset and statefulset
hsubramanianaks Nov 14, 2023
9c3c0ee
change dict to tuple and add UT's
hsubramanianaks Nov 14, 2023
3cafa39
nit correct spelling
hsubramanianaks Nov 14, 2023
0e47e91
change foreach to any predicate
hsubramanianaks Nov 14, 2023
c7b871a
update image tag to push to acr for e2e
hsubramanianaks Nov 14, 2023
d6a82f8
cover UT's and complex scenarios
hsubramanianaks Nov 14, 2023
57b4f31
remove commented lines
hsubramanianaks Nov 15, 2023
1765d12
add owning objects as constants
hsubramanianaks Nov 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion deployment/settings/services/imagetag.setting
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
MINDARO_DEVHOSTAGENT_TAG=1.3.4
MINDARO_DEVHOSTAGENT_RESTORATIONJOB_TAG=1.3.4
MINDARO_DEVHOSTAGENT_RESTORATIONJOB_TAG=issue337
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is temporary changes for testing , I will revert them back to right versions.

MINDARO_ROUTINGMANAGER_TAG=1.2.3
MINDARO_LOCALAGENT_TAG=1.1.0
215 changes: 198 additions & 17 deletions src/devhostAgent.restorationjob.tests/RestorationJobAppTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ public void EnsureFailedPingsExit()
this.ConfigureHttpCall(GetFailedPingResult());

// Verify failure to retrieve agent endpoint
int exitCode = _app.Execute(Array.Empty<string>(), default(CancellationToken));
int exitCode = _app.Execute(Array.Empty<string>(), default);

Assert.Equal((int)Constants.ExitCode.Fail, exitCode);
A.CallTo(() => _autoFake.Resolve<ILog>().Error(A<string>.That.Contains("Failed to ping agent 3 times"), A<object[]>._)).MustHaveHappenedOnceExactly();
Expand All @@ -119,7 +119,7 @@ public void EnsureFailedPingsExit()
Fake.ClearRecordedCalls(_autoFake.Resolve<ILog>());
this.DeploymentPatch_Helper(true);

exitCode = _app.Execute(Array.Empty<string>(), default(CancellationToken));
exitCode = _app.Execute(Array.Empty<string>(), default);

Assert.Equal((int)Constants.ExitCode.Fail, exitCode);
A.CallTo(() => _autoFake.Resolve<ILog>().Error(A<string>.That.Contains("Failed to ping agent 3 times"), A<object[]>._)).MustHaveHappenedOnceExactly();
Expand Down Expand Up @@ -162,7 +162,7 @@ public void ExecutionTestWithShortWait()
.Then
.ReturnsLazily(GetSuccessPingResult(0));

int exitCode = _app.Execute(Array.Empty<string>(), default(CancellationToken));
int exitCode = _app.Execute(Array.Empty<string>(), default);

Assert.Equal(0, exitCode);
A.CallTo(_fakeDelegatingHandler).MustHaveHappenedTwiceOrMore();
Expand All @@ -174,27 +174,50 @@ public void ExecutionTestWithShortWait()
[InlineData("DeploymentPatch.json", nameof(DeploymentPatch_Helper))]
[InlineData("PodPatch.json", nameof(PodPatch_Helper))]
[InlineData("PodDeployment.json", nameof(PodDeployment_Helper))]
[InlineData("StatefulSetPatch.json", nameof(StatefulSetPatch_Helper))]
public void ExecutionTest(string patchStateFile, string testHelper)
{
string patchStateJson = File.ReadAllText(Path.Combine("TestData", patchStateFile));
A.CallTo(() => _autoFake.Resolve<IFileSystem>().ReadAllTextFromFile(DevHostConstants.DevHostRestorationJob.PatchStateFullPath, A<int>._)).Returns(patchStateJson);

// Setup callbacks
var method = typeof(RestorationJobAppTests).GetMethod(testHelper, BindingFlags.NonPublic | BindingFlags.Instance);
method.Invoke(this, new object[] { true });
switch (patchStateFile)
{
case "DeploymentPatch.json":
method.Invoke(this, new object[] { true, false, false });
break;
case "StatefulSetPatch.json":
method.Invoke(this, new object[] { true, false });
break;
default:
method.Invoke(this, new object[] { true });
break;
}
this.ConfigureHttpCall(GetSuccessPingResult(1))
.NumberOfTimes(3)
.Then
.ReturnsLazily(GetSuccessPingResult(0));

// Execute
int exitCode = _app.Execute(Array.Empty<string>(), default(CancellationToken));
int exitCode = _app.Execute(Array.Empty<string>(), default);

// Verify behavior
Assert.Equal(0, exitCode);
A.CallTo(_fakeDelegatingHandler).MustHaveHappened(4, Times.Exactly);
A.CallTo(() => _autoFake.Resolve<IRemoteRestoreJobCleaner>().CleanupRemoteRestoreJobByInstanceLabelAsync(A<string>._, A<string>._, A<CancellationToken>._)).MustHaveHappenedOnceExactly();
method.Invoke(this, new object[] { false });
switch (patchStateFile)
{
case "DeploymentPatch.json":
method.Invoke(this, new object[] { false, false, false });
break;
case "StatefulSetPatch.json":
method.Invoke(this, new object[] { false, false });
break;
default:
method.Invoke(this, new object[] { false });
break;
}
}

[Fact]
Expand All @@ -207,7 +230,7 @@ public void EnsureRestoresIflastPingWithSessionsIsNullAndRestoreTimeExceeded()

this.ConfigureHttpCall(GetSuccessPingResult(0)).NumberOfTimes(1);

int exitCode = _app.Execute(Array.Empty<string>(), default(CancellationToken));
int exitCode = _app.Execute(Array.Empty<string>(), default);
Assert.Equal(0, exitCode);
A.CallTo(_fakeDelegatingHandler).MustHaveHappenedOnceExactly();
A.CallTo(() => _autoFake.Resolve<IRemoteRestoreJobCleaner>().CleanupRemoteRestoreJobByInstanceLabelAsync(A<string>._, A<string>._, A<CancellationToken>._)).MustHaveHappenedOnceExactly();
Expand All @@ -223,7 +246,7 @@ public void EnsureNoRestoreIfRestoreTimeIsNotExceeded()

this.ConfigureHttpCall(GetSuccessPingResult(0)).NumberOfTimes(1);

int exitCode = _app.Execute(Array.Empty<string>(), default(CancellationToken));
int exitCode = _app.Execute(Array.Empty<string>(), default);
Assert.Equal(1, exitCode);
A.CallTo(_fakeDelegatingHandler).MustHaveHappenedTwiceExactly();
A.CallTo(() => _autoFake.Resolve<IRemoteRestoreJobCleaner>().CleanupRemoteRestoreJobByInstanceLabelAsync(A<string>._, A<string>._, A<CancellationToken>._)).MustNotHaveHappened();
Expand All @@ -239,15 +262,109 @@ public void EnsureNoRestoreIflastPingWithSessionsIsNotNull()
this.ConfigureHttpCall(GetSuccessPingResult(3))
.NumberOfTimes(1);

int exitCode = _app.Execute(Array.Empty<string>(), default(CancellationToken));
int exitCode = _app.Execute(Array.Empty<string>(), default);
Assert.Equal(1, exitCode);
A.CallTo(_fakeDelegatingHandler).MustHaveHappenedTwiceExactly();
A.CallTo(() => _autoFake.Resolve<IRemoteRestoreJobCleaner>().CleanupRemoteRestoreJobByInstanceLabelAsync(A<string>._, A<string>._, A<CancellationToken>._)).MustNotHaveHappened();
}

[Fact]
public void ExecuteForReplicaSet()
{
string patchStateJson = File.ReadAllText(Path.Combine("TestData", "DeploymentPatch.json"));
A.CallTo(() => _autoFake.Resolve<IFileSystem>().ReadAllTextFromFile(DevHostConstants.DevHostRestorationJob.PatchStateFullPath, A<int>._)).Returns(patchStateJson);
DeploymentPatch_Helper(false, true);
ConfigureHttpCall(GetSuccessPingResult(3))
.NumberOfTimes(3)
.Then
.ReturnsLazily(GetSuccessPingResult(0));

int exitCode = _app.Execute(Array.Empty<string>(), default);
Assert.Equal(0, exitCode);
A.CallTo(_fakeDelegatingHandler).MustHaveHappened(6, Times.Exactly);
A.CallTo(() => _autoFake.Resolve<IRemoteRestoreJobCleaner>().CleanupRemoteRestoreJobByInstanceLabelAsync(A<string>._, A<string>._, A<CancellationToken>._)).MustHaveHappened();

}

[Fact]
public void ExecuteForStatefulSet() {
string patchStateJson = File.ReadAllText(Path.Combine("TestData", "StatefulSetPatch.json"));
A.CallTo(() => _autoFake.Resolve<IFileSystem>().ReadAllTextFromFile(DevHostConstants.DevHostRestorationJob.PatchStateFullPath, A<int>._)).Returns(patchStateJson);
StatefulSetPatch_Helper(false, true);
ConfigureHttpCall(GetSuccessPingResult(3))
.NumberOfTimes(3)
.Then
.ReturnsLazily(GetSuccessPingResult(0));

int exitCode = _app.Execute(Array.Empty<string>(), default);
Assert.Equal(0, exitCode);
A.CallTo(_fakeDelegatingHandler).MustHaveHappened(8, Times.Exactly);
A.CallTo(() => _autoFake.Resolve<IRemoteRestoreJobCleaner>().CleanupRemoteRestoreJobByInstanceLabelAsync(A<string>._, A<string>._, A<CancellationToken>._)).MustHaveHappened();
}

[Fact]
public void ExecuteForReplicaSetWithAlternativeConnections() {
string patchStateJson = File.ReadAllText(Path.Combine("TestData", "DeploymentPatch.json"));
A.CallTo(() => _autoFake.Resolve<IFileSystem>().ReadAllTextFromFile(DevHostConstants.DevHostRestorationJob.PatchStateFullPath, A<int>._)).Returns(patchStateJson);
DeploymentPatch_Helper(false, true);
ConfigureHttpCall(GetSuccessPingResult(0))
.NumberOfTimes(1)
.Then
.ReturnsLazily(GetSuccessPingResult(2))
.NumberOfTimes(1)
.Then
.ReturnsLazily(GetSuccessPingResult(0));

int exitCode = _app.Execute(Array.Empty<string>(), default);
Assert.Equal(0, exitCode);
A.CallTo(_fakeDelegatingHandler).MustHaveHappened(6, Times.Exactly);
A.CallTo(() => _autoFake.Resolve<IRemoteRestoreJobCleaner>().CleanupRemoteRestoreJobByInstanceLabelAsync(A<string>._, A<string>._, A<CancellationToken>._)).MustHaveHappened();
DeploymentPatch_Helper(false, false);
}

[Fact]
public void ExecuteForReplicaSetWhenResultIsNull()
{
string patchStateJson = File.ReadAllText(Path.Combine("TestData", "DeploymentPatch.json"));
A.CallTo(() => _autoFake.Resolve<IFileSystem>().ReadAllTextFromFile(DevHostConstants.DevHostRestorationJob.PatchStateFullPath, A<int>._)).Returns(patchStateJson);
DeploymentPatch_Helper(false, true);
ConfigureHttpCall(GetSuccessPingResult(0), true /*isError true*/)
.NumberOfTimes(1)
.Then
.ReturnsLazily(GetSuccessPingResult(2))
.NumberOfTimes(1)
.Then
.ReturnsLazily(GetSuccessPingResult(0));
int exitCode = _app.Execute(Array.Empty<string>(), default);
Assert.Equal(0, exitCode);
A.CallTo(_fakeDelegatingHandler).MustHaveHappened(6, Times.Exactly);
A.CallTo(() => _autoFake.Resolve<IRemoteRestoreJobCleaner>().CleanupRemoteRestoreJobByInstanceLabelAsync(A<string>._, A<string>._, A<CancellationToken>._)).MustHaveHappened();
DeploymentPatch_Helper(false, false);
}

[Fact]
public void ExecuteForReplicaSetWhenSomePodsIsTerminating()
{
string patchStateJson = File.ReadAllText(Path.Combine("TestData", "DeploymentPatch.json"));
A.CallTo(() => _autoFake.Resolve<IFileSystem>().ReadAllTextFromFile(DevHostConstants.DevHostRestorationJob.PatchStateFullPath, A<int>._)).Returns(patchStateJson);
DeploymentPatch_Helper(false, true, true /*addPodsinTerminatingState true*/);
ConfigureHttpCall(GetSuccessPingResult(0), true /*isError true*/)
.NumberOfTimes(1)
.Then
.ReturnsLazily(GetSuccessPingResult(2))
.NumberOfTimes(1)
.Then
.ReturnsLazily(GetSuccessPingResult(0));
int exitCode = _app.Execute(Array.Empty<string>(), default);
Assert.Equal(0, exitCode);
A.CallTo(_fakeDelegatingHandler).MustHaveHappened(4, Times.Exactly);
A.CallTo(() => _autoFake.Resolve<IRemoteRestoreJobCleaner>().CleanupRemoteRestoreJobByInstanceLabelAsync(A<string>._, A<string>._, A<CancellationToken>._)).MustHaveHappened();
DeploymentPatch_Helper(false, false, false);
}

#region Test helpers

private void DeploymentPatch_Helper(bool isSetup)
private void DeploymentPatch_Helper(bool isSetup, bool isMultiReplicaSet = false, bool addPodsinTerminatingState = false)
{
if (isSetup)
{
Expand All @@ -256,6 +373,20 @@ private void DeploymentPatch_Helper(bool isSetup)
{
Items = new V1Pod[] { _CreateDevHostPod() }
});
} else if (isMultiReplicaSet) {
V1PodList podList = new()
{
Items = new List<V1Pod>()
};
for (int i = 0; i < 3; i++) {
if (i == 0 && addPodsinTerminatingState) {
podList.Items.Add(_CreateDevHostPod("ReplicaSet", "bikes-13ytg", "Terminating", false));
} else {
podList.Items.Add(_CreateDevHostPod());
}
}
A.CallTo(() => _autoFake.Resolve<IKubernetesClient>().ListPodsForDeploymentAsync(A<string>._, A<string>._, A<CancellationToken>._))
.Returns(podList);
}
else
{
Expand All @@ -265,6 +396,36 @@ private void DeploymentPatch_Helper(bool isSetup)
}
}

private void StatefulSetPatch_Helper(bool isSetup, bool isStatefulSet = false) {
if (isSetup)
{
A.CallTo(() => _autoFake.Resolve<IKubernetesClient>().ListPodsForStatefulSetAsync(A<string>._, A<string>._, A<CancellationToken>._))
.Returns(new V1PodList
{
Items = new V1Pod[] { _CreateDevHostPod() }
});
} else if (isStatefulSet) {
V1PodList podList = new()
{
Items = new List<V1Pod>()
};
for (int i = 0; i < 3; i++) {
if (i == 0) {
podList.Items.Add(_CreateDevHostPod("StatefulSet", "bikes-pqwrt"));
}
podList.Items.Add(_CreateDevHostPod("StatefulSet", "bikes-iutyr"));
}
A.CallTo(() => _autoFake.Resolve<IKubernetesClient>().ListPodsForStatefulSetAsync(A<string>._, A<string>._, A<CancellationToken>._))
.Returns(podList);
}
else
{
// Verify
A.CallTo(() => _autoFake.Resolve<IWorkloadRestorationService>().RestoreStatefulSetPatchAsync(A<StatefulSetPatch>._, A<CancellationToken>._, A<Action<ProgressMessage>>._, A<bool>._))
.MustHaveHappenedOnceExactly();
}
}

private void PodPatch_Helper(bool isSetup)
{
if (isSetup)
Expand Down Expand Up @@ -297,36 +458,56 @@ private void PodDeployment_Helper(bool isSetup)

#endregion Test helpers

private V1Pod _CreateDevHostPod()
private V1Pod _CreateDevHostPod(string kind = "ReplicaSet", string name="bikes-pqwrt", string phase = "Running", bool ready = true)
{
var pod = new V1Pod
{
Metadata = new V1ObjectMeta(namespaceProperty: "mynamespace", name: "mypod"),
Metadata = new V1ObjectMeta(namespaceProperty: "mynamespace", name: "mypod", ownerReferences: new V1OwnerReference[] {
new() {
ApiVersion = "apps/v1",
Kind = kind,
Name = name,
Uid = "1234"
}
}),
Spec = new V1PodSpec
{
Containers = new V1Container[]
{
new V1Container
{
new() {
Image = "mindaro/devhostagent:1234"
}
}
},
Status = new V1PodStatus
{
PodIP = "1.2.3.4"
PodIP = "1.2.3.4",
Phase = phase,
ContainerStatuses = new V1ContainerStatus[]
{
new() {
Ready = ready
}
}
}
};

return pod;
}

private IAfterCallConfiguredWithOutAndRefParametersConfiguration<IReturnValueConfiguration<Task<HttpResponseMessage>>> ConfigureHttpCall(Func<HttpResponseMessage> response)
private IAfterCallConfiguredWithOutAndRefParametersConfiguration<IReturnValueConfiguration<Task<HttpResponseMessage>>> ConfigureHttpCall(Func<HttpResponseMessage> response, bool isError = false)
{
return A.CallTo(_fakeDelegatingHandler)
if (!isError) {
return A.CallTo(_fakeDelegatingHandler)
.Where(x => x.Method.Name == "SendAsync")
.WithReturnType<Task<HttpResponseMessage>>()
.ReturnsLazily(response);
}

return (IAfterCallConfiguredWithOutAndRefParametersConfiguration<IReturnValueConfiguration<Task<HttpResponseMessage>>>) A.CallTo(_fakeDelegatingHandler)
.Where(x => x.Method.Name == "SendAsync")
.WithReturnType<Task<HttpResponseMessage>>()
.Throws(new HttpRequestException());
}

private Func<HttpResponseMessage> GetSuccessPingResult(int numSessions)
Expand Down
Loading