From 950cf734278428261c9f4f24f21371cee3cadeb4 Mon Sep 17 00:00:00 2001 From: Patchouli Date: Thu, 7 Nov 2024 12:33:27 -0500 Subject: [PATCH] Add API client code and exception(s) --- source/Firecrawl/FirecrawlException.cs | 67 ++++ source/Firecrawl/FirecrawlService.cs | 482 +++++++++++++++++++++++++ source/Firecrawl/IFirecrawlService.cs | 82 +++++ 3 files changed, 631 insertions(+) create mode 100644 source/Firecrawl/FirecrawlException.cs create mode 100644 source/Firecrawl/FirecrawlService.cs create mode 100644 source/Firecrawl/IFirecrawlService.cs diff --git a/source/Firecrawl/FirecrawlException.cs b/source/Firecrawl/FirecrawlException.cs new file mode 100644 index 0000000..ec6f3f2 --- /dev/null +++ b/source/Firecrawl/FirecrawlException.cs @@ -0,0 +1,67 @@ +using System; +using System.Net; + +namespace Firecrawl +{ + /// + /// + /// + public class FirecrawlException : + Exception + { + /// + /// + /// + public HttpStatusCode? StatusCode { get; private init; } + + /// + /// + /// + public FirecrawlException() : + this( + default) + { } + + /// + /// + /// + /// + public FirecrawlException( + HttpStatusCode statusCode) : + this( + statusCode, + default) + { } + + /// + /// + /// + /// + /// + public FirecrawlException( + HttpStatusCode statusCode, + string message) : + this( + statusCode, + message, + default) + { } + + /// + /// + /// + /// + /// + /// + public FirecrawlException( + HttpStatusCode statusCode, + string message, + Exception innerException) : + base( + message, + innerException) + { + StatusCode = statusCode; + } + } +} diff --git a/source/Firecrawl/FirecrawlService.cs b/source/Firecrawl/FirecrawlService.cs new file mode 100644 index 0000000..2c9209f --- /dev/null +++ b/source/Firecrawl/FirecrawlService.cs @@ -0,0 +1,482 @@ +using System; +using System.Net.Http.Headers; +using System.Net.Http; +using System.Text.Json; +using System.Net.Http.Json; +using System.Net; +using System.Threading.Tasks; +using System.Threading; +using System.Globalization; + +namespace Firecrawl +{ + /// + /// + /// + public sealed class FirecrawlService : + IFirecrawlService + { + /// + /// Gets the default base URL for the Firecrawl API. + /// + public static Uri DefaultBaseUrl { get; } = + new Uri( + "https://api.firecrawl.dev/v1/"); + + /// + /// + /// + private static JsonSerializerOptions JsonSerializerOptions { get; } = + new JsonSerializerOptions() + { + + }; + + /// + /// + /// + /// + /// + /// + private static HttpClient CreateClient( + Uri baseUrl, + string apiKey) + { + ArgumentNullException + .ThrowIfNull( + baseUrl, + nameof(baseUrl)); + + ArgumentException + .ThrowIfNullOrEmpty( + apiKey, + nameof(apiKey)); + + var client = + new HttpClient(); + + client.BaseAddress = + baseUrl; + + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue( + "Bearer", + apiKey); + + return client; + } + + /// + /// + /// + /// + /// + /// + /// + /// + private static async Task ProcessResponseAsync( + HttpResponseMessage response, + CancellationToken cancellationToken = default) + { + if (response.StatusCode == HttpStatusCode.OK) + { + var result = + await response.Content + .ReadFromJsonAsync( + JsonSerializerOptions, + cancellationToken); + + return result; + } + else + { + var errorResult = + await response.Content + .ReadFromJsonAsync( + JsonSerializerOptions, + cancellationToken); + + throw new FirecrawlException( + response.StatusCode, + errorResult?.Message); + } + } + + private readonly HttpClient httpClient; + + /// + /// + /// + public Uri BaseUrl => + this.httpClient + .BaseAddress; + + /// + /// + /// + public AuthenticationHeaderValue AuthorizationHeaderValue => + this.httpClient + .DefaultRequestHeaders + .Authorization; + + /// + /// + /// + /// + public FirecrawlService( + string apiKey) : + this( + DefaultBaseUrl, + apiKey) + { } + + /// + /// + /// + /// + /// + public FirecrawlService( + Uri baseUrl, + string apiKey) : + this( + CreateClient( + baseUrl, + apiKey)) + { } + + /// + /// + /// + /// + public FirecrawlService( + HttpClient httpClient) + { + ArgumentNullException + .ThrowIfNull( + httpClient, + nameof(httpClient)); + + this.httpClient = httpClient; + } + + /// + /// + /// + /// + /// + /// + public async Task ScrapeAsync( + FirecrawlScrapeOptions options, + CancellationToken cancellationToken = default) + { + ArgumentNullException + .ThrowIfNull( + options, + nameof(options)); + + if (!TryCreateScrapeUrl( + out var requestUri)) + { + throw new InvalidOperationException( + $"Failed to create a valid URL for the request."); + } + + var response = + await this.httpClient + .PostAsJsonAsync( + requestUri, + options, + cancellationToken); + + return await ProcessResponseAsync( + response, + cancellationToken); + } + + /// + /// + /// + /// + /// + /// + public async Task CrawlAsync( + FirecrawlCrawlOptions options, + CancellationToken cancellationToken = default) + { + ArgumentNullException + .ThrowIfNull( + options, + nameof(options)); + + if (!TryCreateCrawlUrl( + out var requestUri)) + { + throw new InvalidOperationException( + $"Failed to create a valid URL for the request."); + } + + var response = + await this.httpClient + .PostAsJsonAsync( + requestUri, + options, + cancellationToken); + + return await ProcessResponseAsync( + response, + cancellationToken); + } + + /// + /// + /// + /// + /// + /// + public async Task GetCrawlStatusAsync( + string id, + CancellationToken cancellationToken = default) + { + ArgumentException + .ThrowIfNullOrEmpty( + id, + nameof(id)); + + if (!TryCreateCrawlResourceUrl( + id, + out var requestUri)) + { + throw new InvalidOperationException( + $"Failed to create a valid URL for the request."); + } + + var response = + await this.httpClient + .GetAsync( + requestUri, + cancellationToken); + + return await ProcessResponseAsync( + response, + cancellationToken); + } + + /// + /// + /// + /// + /// + /// + public async Task CancelCrawlAsync( + string id, + CancellationToken cancellationToken = default) + { + ArgumentException + .ThrowIfNullOrEmpty( + id, + nameof(id)); + + if (!TryCreateCrawlResourceUrl( + id, + out var requestUri)) + { + throw new InvalidOperationException( + $"Failed to create a valid URL for the request."); + } + + var response = + await this.httpClient + .DeleteAsync( + requestUri, + cancellationToken); + + return await ProcessResponseAsync( + response, + cancellationToken); + } + + /// + /// + /// + /// + /// + /// + public async Task MapAsync( + FirecrawlMapOptions options, + CancellationToken cancellationToken = default) + { + ArgumentNullException + .ThrowIfNull( + options, + nameof(options)); + + if (!TryCreateMapUrl( + out var requestUri)) + { + throw new InvalidOperationException( + $"Failed to create a valid URL for the request."); + } + + var response = + await this.httpClient + .PostAsJsonAsync( + requestUri, + options, + cancellationToken); + + return await ProcessResponseAsync( + response, + cancellationToken); + } + + /// + /// + /// + /// + /// + /// + public async Task BatchScrapeAsync( + FirecrawlBatchScrapeOptions options, + CancellationToken cancellationToken = default) + { + ArgumentNullException + .ThrowIfNull( + options, + nameof(options)); + + if (!TryCreateBatchScrapeUrl( + out var requestUri)) + { + throw new InvalidOperationException( + $"Failed to create a valid URL for the request."); + } + + var response = + await this.httpClient + .PostAsJsonAsync( + requestUri, + options, + cancellationToken); + + return await ProcessResponseAsync( + response, + cancellationToken); + } + + /// + /// + /// + /// + /// + /// + public async Task GetBatchScrapeStatusAsync( + string id, + CancellationToken cancellationToken = default) + { + ArgumentException + .ThrowIfNullOrEmpty( + id, + nameof(id)); + + if (!TryCreateBatchScrapeResourceUrl( + id, + out var requestUri)) + { + throw new InvalidOperationException( + $"Failed to create a valid URL for the request."); + } + + var response = + await this.httpClient + .GetAsync( + requestUri, + cancellationToken); + + return await ProcessResponseAsync( + response, + cancellationToken); + } + + /// + /// + /// + /// + /// + private bool TryCreateScrapeUrl( + out Uri result) => + Uri.TryCreate( + BaseUrl, + "scrape", + out result); + + /// + /// + /// + /// + /// + private bool TryCreateCrawlUrl( + out Uri result) => + Uri.TryCreate( + BaseUrl, + "crawl", + out result); + + /// + /// + /// + /// + /// + /// + private bool TryCreateCrawlResourceUrl( + string id, + out Uri result) => + Uri.TryCreate( + BaseUrl, + String.Format( + CultureInfo.InvariantCulture, + "crawl/{0}", + id), + out result); + + /// + /// + /// + /// + /// + private bool TryCreateMapUrl( + out Uri result) => + Uri.TryCreate( + BaseUrl, + "map", + out result); + + /// + /// + /// + /// + /// + private bool TryCreateBatchScrapeUrl( + out Uri result) => + Uri.TryCreate( + BaseUrl, + "batch/scrape", + out result); + + /// + /// + /// + /// + /// + /// + private bool TryCreateBatchScrapeResourceUrl( + string id, + out Uri result) => + Uri.TryCreate( + BaseUrl, + String.Format( + CultureInfo.InvariantCulture, + "batch/scrape/{0}", + id), + out result); + } +} diff --git a/source/Firecrawl/IFirecrawlService.cs b/source/Firecrawl/IFirecrawlService.cs new file mode 100644 index 0000000..858d193 --- /dev/null +++ b/source/Firecrawl/IFirecrawlService.cs @@ -0,0 +1,82 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Firecrawl +{ + /// + /// + /// + public interface IFirecrawlService + { + /// + /// + /// + /// + /// + /// + public Task ScrapeAsync( + FirecrawlScrapeOptions options, + CancellationToken cancellationToken = default); + + /// + /// + /// + /// + /// + /// + public Task CrawlAsync( + FirecrawlCrawlOptions options, + CancellationToken cancellationToken = default); + + /// + /// + /// + /// + /// + /// + public Task GetCrawlStatusAsync( + string id, + CancellationToken cancellationToken = default); + + /// + /// + /// + /// + /// + /// + public Task CancelCrawlAsync( + string id, + CancellationToken cancellationToken = default); + + /// + /// + /// + /// + /// + /// + public Task MapAsync( + FirecrawlMapOptions options, + CancellationToken cancellationToken = default); + + /// + /// + /// + /// + /// + /// + public Task BatchScrapeAsync( + FirecrawlBatchScrapeOptions options, + CancellationToken cancellationToken = default); + + /// + /// + /// + /// + /// + /// + public Task GetBatchScrapeStatusAsync( + string id, + CancellationToken cancellationToken = default); + } +}