Skip to content

Commit

Permalink
Merge pull request #34 from Manuss20/main
Browse files Browse the repository at this point in the history 
Update VideoAnalyzer
  • Loading branch information
@rag2111
rag2111 authored Apr 21, 2024
2 parents 80e4962 + 9568bbc commit 6a8da3c
Show file tree
Hide file tree
Showing 7 changed files with 462 additions and 2 deletions.
277 changes: 277 additions & 0 deletions src/AIHub/Controllers/VideoAnalyzerController.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,277 @@
using System.ComponentModel.DataAnnotations;
using System;
using System.IO;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using System.Text.RegularExpressions;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Linq.Expressions;


namespace MVCWeb.Controllers;

public class VideoAnalyzerController : Controller
{
private string AOAIendpoint;
private string Visionendpoint;
private string AOAIsubscriptionKey;
private string VisionKey;
private string storageconnstring;
private string AOAIDeploymentName;
private readonly BlobContainerClient containerClient;
private readonly IEnumerable<BlobItem> blobs;
private Uri sasUri;
private VideoAnalyzerModel model;
private HttpClient httpClient;

public VideoAnalyzerController(IConfiguration config, IHttpClientFactory clientFactory)
{
AOAIendpoint = config.GetValue<string>("VideoAnalyzer:OpenAIEndpoint") ?? throw new ArgumentNullException("OpenAIEndpoint");
AOAIsubscriptionKey = config.GetValue<string>("VideoAnalyzer:OpenAISubscriptionKey") ?? throw new ArgumentNullException("OpenAISubscriptionKey");
storageconnstring = config.GetValue<string>("Storage:ConnectionString") ?? throw new ArgumentNullException("ConnectionString");
BlobServiceClient blobServiceClient = new BlobServiceClient(storageconnstring);
containerClient = blobServiceClient.GetBlobContainerClient(config.GetValue<string>("VideoAnalyzer:ContainerName"));
sasUri = containerClient.GenerateSasUri(Azure.Storage.Sas.BlobContainerSasPermissions.Read, DateTimeOffset.UtcNow.AddHours(1));
AOAIDeploymentName = config.GetValue<string>("VideoAnalyzer:DeploymentName") ?? throw new ArgumentNullException("DeploymentName");
Visionendpoint = config.GetValue<string>("VideoAnalyzer:VisionEndpoint") ?? throw new ArgumentNullException("VisionEndpoint");
VisionKey = config.GetValue<string>("VideoAnalyzer:VisionSubscriptionKey") ?? throw new ArgumentNullException("VisionSubscriptionKey");

// Obtain the blobs list in the container
blobs = containerClient.GetBlobs();
httpClient = clientFactory.CreateClient();
model = new VideoAnalyzerModel();
}

const string VIDEO_DOCUMENT_ID = "AOAIChatDocument";

public class acvDocumentIdWrapper
{
[JsonPropertyName("acv-document-id")]
public string? AcvDocumentId { get; set; }
}

static async Task<HttpResponseMessage> CreateVideoIndex(string visionApiEndpoint, string visionApiKey, string indexName)
{
using var client = new HttpClient();
string url = $"{visionApiEndpoint}/retrieval/indexes/{indexName}?api-version=2023-05-01-preview";
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", visionApiKey);
var data = new { features = new[] { new { name = "vision", domain = "surveillance" } } };
var content = new StringContent(JsonSerializer.Serialize(data), Encoding.UTF8, "application/json");
var response = await client.PutAsync(url, content);
return response;
}

static async Task<HttpResponseMessage> AddVideoToIndex(string visionApiEndpoint, string visionApiKey, string indexName, string videoUrl, string videoId)
{
using var client = new HttpClient();
string url = $"{visionApiEndpoint}/retrieval/indexes/{indexName}/ingestions/my-ingestion?api-version=2023-05-01-preview";
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", visionApiKey);
var data = new { videos = new[] { new { mode = "add", documentId = videoId, documentUrl = videoUrl } } };
var content = new StringContent(JsonSerializer.Serialize(data), Encoding.UTF8, "application/json");
var response = await client.PutAsync(url, content);
return response;
}

static async Task<bool> WaitForIngestionCompletion(string visionApiEndpoint, string visionApiKey, string indexName, int maxRetries = 30)
{
using var client = new HttpClient();
string url = $"{visionApiEndpoint}/retrieval/indexes/{indexName}/ingestions?api-version=2023-05-01-preview";
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", visionApiKey);
int retries = 0;
while (retries < maxRetries)
{
await Task.Delay(10000);
var response = await client.GetAsync(url);
if (response.IsSuccessStatusCode)
{
var stateData = JsonSerializer.Deserialize<dynamic>(await response.Content.ReadAsStringAsync());
if (stateData?.GetProperty("value").GetArrayLength() > 0 && stateData?.GetProperty("value")[0].GetProperty("state").GetString() == "Completed")
{
Console.WriteLine(stateData);
Console.WriteLine("Ingestion completed.");
return true;
}
else if (stateData?.GetProperty("value").GetArrayLength() > 0 && stateData?.GetProperty("value")[0].GetProperty("state").GetString() == "Failed")
{
Console.WriteLine(stateData);
Console.WriteLine("Ingestion failed.");
return false;
}
}
retries++;
}
return false;
}

public IActionResult VideoAnalyzer()
{
return View(new VideoAnalyzerModel());
}

[HttpPost]
public async Task<IActionResult> DenseCaptionVideo(string video_url, string prompt)
{
string GPT4V_ENDPOINT = $"{AOAIendpoint}openai/deployments/{AOAIDeploymentName}/extensions/chat/completions?api-version=2023-07-01-preview"; //2024-02-15-preview";
string VISION_API_ENDPOINT = $"{Visionendpoint}computervision";
string VISION_API_KEY = VisionKey;
string VIDEO_INDEX_NAME = Regex.Replace(video_url.Split("/").Last().Split(".").First().GetHashCode().ToString(), "[^a-zA-Z0-9]", "");



string VIDEO_FILE_SAS_URL = video_url + sasUri.Query;

// Step 1: Create an Index
var response = await CreateVideoIndex(VISION_API_ENDPOINT, VISION_API_KEY, VIDEO_INDEX_NAME);
Console.WriteLine(response.StatusCode);
Console.WriteLine(await response.Content.ReadAsStringAsync());

// Step 2: Add a video file to the index
response = await AddVideoToIndex(VISION_API_ENDPOINT, VISION_API_KEY, VIDEO_INDEX_NAME, VIDEO_FILE_SAS_URL, VIDEO_DOCUMENT_ID);
Console.WriteLine(response.StatusCode);
Console.WriteLine(await response.Content.ReadAsStringAsync());

// Step 3: Wait for ingestion to complete
if (!await WaitForIngestionCompletion(VISION_API_ENDPOINT, VISION_API_KEY, VIDEO_INDEX_NAME))
{
Console.WriteLine("Ingestion did not complete within the expected time.");
}

if (string.IsNullOrEmpty(AOAIsubscriptionKey))
{
var credential = new DefaultAzureCredential();
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", credential.GetToken(new TokenRequestContext(["https://cognitiveservices.azure.com/.default"])).Token);
}
else
{
httpClient.DefaultRequestHeaders.Add("api-key", AOAIsubscriptionKey);
}
var payload = new
{
dataSources = new[]
{
new
{
type = "AzureComputerVisionVideoIndex",
parameters = new
{
computerVisionBaseUrl = VISION_API_ENDPOINT,
computerVisionApiKey = VisionKey,
indexName = Regex.Replace(video_url.Split("/").Last().Split(".").First().GetHashCode().ToString(), "[^a-zA-Z0-9]", ""),
videoUrls = new[] { VIDEO_FILE_SAS_URL }
}
}
},
enhancements = new
{
video = new { enabled = true }
},
messages = new object[]
{
new {
role = "system",
content = new object[]
{
"You are an AI assistant that helps people find information."
}
},
new {
role = "user",
content = new object[]
{
new acvDocumentIdWrapper() {AcvDocumentId = VIDEO_DOCUMENT_ID},
prompt
},
}
},
temperature = 0.7,
top_p = 0.95,
max_tokens = 800
};

try
{
var chatResponse = await httpClient.PostAsync(GPT4V_ENDPOINT, new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json"));
chatResponse.EnsureSuccessStatusCode();
var responseContent = JsonSerializer.Deserialize<dynamic>(await chatResponse.Content.ReadAsStringAsync());
Console.WriteLine(responseContent);

model.Message = responseContent; //responseContent!.choices[0].message.content;
model.Video = VIDEO_FILE_SAS_URL;

}
catch
{
Console.WriteLine($"Error after GPT4V: {response.StatusCode}, {response.ReasonPhrase}");
}

return View("VideoAnalyzer", model);
}

// Upload a file to my azure storage account
[HttpPost]
public async Task<IActionResult> UploadFile(IFormFile videoFile)
{
// Check no video
if (CheckNullValues(videoFile))
{
ViewBag.Message = "You must upload an video";
return View("VideoAnalyzer");
}
if (string.IsNullOrEmpty(HttpContext.Request.Form["text"]))
{
ViewBag.Message = "You must enter a prompt to evaluate";
return View("VideoAnalyzer", model);
}
model.Prompt = HttpContext.Request.Form["text"];
// Upload file to azure storage account
string url = videoFile.FileName.ToString();
Console.WriteLine(url);

BlobClient blobClient = containerClient.GetBlobClient(url);
await blobClient.UploadAsync(videoFile.OpenReadStream(), true);

// Get the url of the file
Uri blobUrl = blobClient.Uri;

if (CheckVideoExtension(blobUrl.ToString()))
{
ViewBag.Message = "You must upload an video with .mp4 extension";
return View("VideoAnalyzer");
}

// Call EvaluateVideo with the url
Console.WriteLine(blobUrl.ToString());
await DenseCaptionVideo(blobUrl.ToString(), model.Prompt!);
ViewBag.Waiting = null;

return Ok(model);
}

[ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)]
public IActionResult Error()
{
return View(new ErrorViewModel { RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier });
}

private bool CheckNullValues(IFormFile videoFile)
{
if (videoFile == null)
{
return true;
}
return false;
}

private bool CheckVideoExtension(string blobUri)
{
string mp4 = ".mp4";
string uri_lower = blobUri;
if (uri_lower.Contains(mp4, StringComparison.OrdinalIgnoreCase))
{
return false;
}
return true;
}
}
9 changes: 9 additions & 0 deletions src/AIHub/Models/VideoAnalyzerModel.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
namespace MVCWeb.Models;

public class VideoAnalyzerModel
{
public string? Video { get; set; }
public string? Message { get; set; }
public string? Prompt { get; set; }

}
4 changes: 2 additions & 2 deletions src/AIHub/Views/Shared/_Layout.cshtml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,10 +282,10 @@
</a>
</li>
<li class="side-nav-item">
<a class="side-nav-link">@*href="@Url.Action("VideoAnalyzer", "VideoAnalyzer")">*@
<a class="side-nav-link" href="@Url.Action("VideoAnalyzer", "VideoAnalyzer")">
<i class=" uil-video"></i>
<span> Video Analyzer </span>
<span class="badge badge-outline-secondary rounded-pill float-end">Coming soon</span>
<span class="badge badge-outline-success rounded-pill float-end">New</span>
</a>
</li>

Expand Down
5 changes: 5 additions & 0 deletions src/AIHub/Views/Shared/_VideoAnalyzerScriptsPartial.cshtml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@

<link rel="stylesheet" href="/css/dropzonecustom.css">

<script src="/vendor/dropzone/dropzone.min.js"></script>
<script src="/js/ui/videoanalyzer.js"></script>
Loading

0 comments on commit 6a8da3c

Please sign in to comment.