Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Superbenchmark executor: online inference config file and custom repo link #369

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Superbenchmark executor: online inference config file and custom repo link #369

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2c2a5c5
support for online config file
LiYueqian-James Aug 21, 2024
7540f57
support for custom sb repo link
LiYueqian-James Aug 21, 2024
65e428d
SB custom github link
Aug 23, 2024
850fa4c
bug fix
Aug 23, 2024
85baff6
inf file donwload
Aug 26, 2024
1f7591b
init different repo
Aug 26, 2024
e6eca5b
Merge branch 'main' into yueqianli/SBConfigAndRepo
LiYueqian-James Aug 26, 2024
8a0ca65
Merge branch 'main' into yueqianli/SBConfigAndRepo
LiYueqian-James Sep 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 55 additions & 12 deletions src/VirtualClient/VirtualClient.Actions/SuperBenchmark/SuperBenchmarkExecutor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ namespace VirtualClient.Actions
using System.Collections.Generic;
using System.IO;
using System.IO.Abstractions;
using System.Runtime.InteropServices;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis;
using Microsoft.Extensions.DependencyInjection;
using Polly;
using VirtualClient.Common;
using VirtualClient.Common.Extensions;
using VirtualClient.Common.Platform;
using VirtualClient.Common.Telemetry;
using VirtualClient.Contracts;
using VirtualClient.Contracts.Metadata;
Expand All @@ -26,6 +26,9 @@ namespace VirtualClient.Actions
public class SuperBenchmarkExecutor : VirtualClientComponent
{
private const string SuperBenchmarkRunShell = "RunSuperBenchmark.sh";
private const string DefaultSBRepoLink = "https://github.com/microsoft/superbenchmark";
private string configFileFullPath;
private string repositoryName;

private IFileSystem fileSystem;
private IPackageManager packageManager;
Expand Down Expand Up @@ -82,6 +85,17 @@ public string ConfigurationFile
}
}

/// <summary>
/// Link to the superbench repo.
/// </summary>
public string RepositoryLink
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to change the repository link? Even if we need to support extra config?

{
get
{
return this.Parameters.GetValue<string>(nameof(SuperBenchmarkExecutor.RepositoryLink), DefaultSBRepoLink);
}
}

/// <summary>
/// The username to execute superbench, required.
/// </summary>
Expand All @@ -106,7 +120,7 @@ public string SuperBenchmarkDirectory
{
get
{
return this.PlatformSpecifics.Combine(this.PlatformSpecifics.PackagesDirectory, "superbenchmark");
return this.PlatformSpecifics.Combine(this.PlatformSpecifics.PackagesDirectory, this.repositoryName);
}
}

Expand Down Expand Up @@ -151,18 +165,47 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel
/// </summary>
protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken)
{
SuperBenchmarkState state = await this.stateManager.GetStateAsync<SuperBenchmarkState>($"{nameof(SuperBenchmarkState)}", cancellationToken)
var repositoryLinkUri = new Uri(this.RepositoryLink);
this.repositoryName = Path.GetFileName(repositoryLinkUri.AbsolutePath);

// download config file if a link is provided
if (this.ConfigurationFile.StartsWith("http"))
{
var configFileUri = new Uri(this.ConfigurationFile);
string configFileName = Path.GetFileName(configFileUri.AbsolutePath);
string configFullPath = this.PlatformSpecifics.Combine(this.SuperBenchmarkDirectory, configFileName);

using (var client = new HttpClient())
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's not add un-unit testable code here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are cmd line based methods preferred? like wget or curl?

{
await Policy.Handle<Exception>().WaitAndRetryAsync(5, (retries) => TimeSpan.FromSeconds(retries * 2)).ExecuteAsync(async () =>
{
var response = await client.GetAsync(configFileUri);
using (var fs = new FileStream(configFullPath, FileMode.Create, FileAccess.Write, FileShare.Write))
{
await response.Content.CopyToAsync(fs);
}
});
}

this.configFileFullPath = configFullPath;
}
else
{
this.configFileFullPath = this.ConfigurationFile;
}

SuperBenchmarkState state = await this.stateManager.GetStateAsync<SuperBenchmarkState>(this.repositoryName, cancellationToken)
?? new SuperBenchmarkState();

if (!state.SuperBenchmarkInitialized)
{
// This is to grant directory folders for
await this.systemManager.MakeFilesExecutableAsync(this.PlatformSpecifics.CurrentDirectory, this.Platform, cancellationToken);

string cloneDir = this.PlatformSpecifics.Combine(this.PlatformSpecifics.PackagesDirectory, "superbenchmark");
string cloneDir = this.PlatformSpecifics.Combine(this.PlatformSpecifics.PackagesDirectory, this.repositoryName);
if (!this.fileSystem.Directory.Exists(cloneDir))
{
await this.ExecuteSbCommandAsync("git", $"clone -b v{this.Version} https://github.com/microsoft/superbenchmark", this.PlatformSpecifics.PackagesDirectory, telemetryContext, cancellationToken, true);
await this.ExecuteSbCommandAsync("git", $"clone -b v{this.Version} {this.RepositoryLink}", this.PlatformSpecifics.PackagesDirectory, telemetryContext, cancellationToken, true);
}

foreach (string file in this.fileSystem.Directory.GetFiles(this.PlatformSpecifics.GetScriptPath("superbenchmark")))
Expand All @@ -179,7 +222,7 @@ protected override async Task InitializeAsync(EventContext telemetryContext, Can
state.SuperBenchmarkInitialized = true;
}

await this.stateManager.SaveStateAsync<SuperBenchmarkState>($"{nameof(SuperBenchmarkState)}", state, cancellationToken);
await this.stateManager.SaveStateAsync<SuperBenchmarkState>(this.repositoryName, state, cancellationToken);
}

private async Task ExecuteSbCommandAsync(string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken, bool runElevated)
Expand All @@ -198,7 +241,7 @@ private async Task CaptureMetricsAsync(IProcessProxy process, string commandArgu
if (!cancellationToken.IsCancellationRequested)
{
this.MetadataContract.AddForScenario(
"SuperBenchmark",
this.repositoryName,
process.FullCommand(),
toolVersion: null);

Expand All @@ -215,12 +258,12 @@ private async Task CaptureMetricsAsync(IProcessProxy process, string commandArgu
IList<Metric> metrics = parser.Parse();

this.Logger.LogMetrics(
toolName: "SuperBenchmark",
scenarioName: "SuperBenchmark",
toolName: this.repositoryName,
scenarioName: this.repositoryName,
process.StartTime,
process.ExitTime,
metrics,
metricCategorization: $"{this.ConfigurationFile}",
metricCategorization: $"{this.configFileFullPath}",
scenarioArguments: commandArguments,
this.Tags,
telemetryContext);
Expand All @@ -232,7 +275,7 @@ private async Task CaptureMetricsAsync(IProcessProxy process, string commandArgu

private string GetCommandLineArguments()
{
return @$"run --host-list localhost -c {this.ConfigurationFile}";
return @$"run --host-list localhost -c {this.configFileFullPath}";
}

internal class SuperBenchmarkState : State
Expand Down
Loading