diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a98b6f..acee918 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 0.1.2 + +- Bumped `ApifyClientVersion.ApiSpecVersion` to the Apify OpenAPI spec `v2-2026-07-08T143931Z` and the + project version to `0.1.2`. +- Aligned the `User-Agent` OS token with the reference client's Node `os.platform()` values (`win32`, + `darwin`, `linux`, `android`, `freebsd`, and — for platforms without a dedicated .NET helper — + `openbsd`, `netbsd`, `sunos`, `aix`) instead of `windows`. +- Request bodies of at least 1024 bytes are compressed before sending. The compression algorithm is + selectable via the new `ApifyClientOptions.RequestCompression` option (`RequestCompression` enum): + brotli (`Content-Encoding: br`) by default, or gzip (`Content-Encoding: gzip`). + ## 0.1.1 - Bumped `ApifyClientVersion.ApiSpecVersion` to the Apify OpenAPI spec `v2-2026-07-07T132551Z` and the diff --git a/README.md b/README.md index 6613541..cc74149 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,8 @@ var run = await client.Actor("apify/hello-world").CallAsync(null, null, null); // Read items from the run's default dataset. var items = await client.Dataset(run.DefaultDatasetId!).ListItemsAsync(); -Console.WriteLine("Item count: " + items.Count); +// Count is the number of items in THIS page; Total is the dataset's full count across all pages. +Console.WriteLine($"Items on this page: {items.Count} (of {items.Total} total)"); ``` `new ApifyClient("my-api-token")` takes the token as an explicit argument — it does **not** read diff --git a/docs/README.md b/docs/README.md index 19d1219..fc0784f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -52,12 +52,14 @@ var run = await client.Actor("apify/hello-world").CallAsync(null, null, null); // Read items from the run's default dataset. var items = await client.Dataset(run.DefaultDatasetId!).ListItemsAsync(); -Console.WriteLine("Item count: " + items.Count); +// Count is the number of items in THIS page; Total is the dataset's full count across all pages. +Console.WriteLine($"Items on this page: {items.Count} (of {items.Total} total)"); ``` `new ApifyClient("my-api-token")` takes the token as an explicit argument — it does **not** read `APIFY_TOKEN` (or any other environment variable) automatically. Read it yourself if you want that, -e.g. `new ApifyClient(Environment.GetEnvironmentVariable("APIFY_TOKEN"))`. +e.g. `new ApifyClient(Environment.GetEnvironmentVariable("APIFY_TOKEN") ?? throw new InvalidOperationException("Set APIFY_TOKEN"))` +(the null-coalescing throw keeps the call null-safe when the variable is unset). Get your API token from the [Apify Console → Settings → API & Integrations](https://console.apify.com/settings/integrations). @@ -70,11 +72,12 @@ the `using` directives for whichever ones a file references: | Namespace | What lives here | |---|---| | `Apify.Client` | The entry point (`ApifyClient`), `ApifyClientOptions`, `ApifyClientVersion`, and the log-redirection helper `StreamedLog`. | -| `Apify.Client.Resources` | Every resource client the entry point returns — `ActorClient`, `RunClient`, `BuildClient`, `DatasetClient`, `KeyValueStoreClient`, `RequestQueueClient`, `TaskClient`, `ScheduleClient`, `LogClient`, `UserClient`, the `…CollectionClient` types (including `NestedWebhookCollectionClient` and `WebhookDispatchCollectionClient`), etc. | +| `Apify.Client.Resources` | Every resource client the entry point returns — `ActorClient`, `RunClient`, `BuildClient`, `DatasetClient`, `KeyValueStoreClient`, `RequestQueueClient`, `TaskClient`, `ScheduleClient`, `LogClient`, `UserClient`, `ActorVersionClient`, `ActorEnvVarClient`, the `…CollectionClient` types (including `ActorVersionCollectionClient`, `ActorEnvVarCollectionClient`, `NestedWebhookCollectionClient`, and `WebhookDispatchCollectionClient`), etc. | | `Apify.Client.Models` | Data models returned by the clients — `Actor`, `ActorRun`, `Build`, `Dataset`, `RequestQueueRequest`, `ActorEnvVar`, `PaginationList`, and so on. | | `Apify.Client.Options` | The option/request objects passed into methods — `ActorStartOptions`, `DatasetListItemsOptions`, `DownloadItemsFormat`, `ListOptions`, `SetRecordOptions`, `StorageListOptions`, etc. | | `Apify.Client.Exceptions` | `ApifyApiException` and `ApifyTransportException`. | | `Apify.Client.Http` | The replaceable transport: `IHttpTransport` and the default `HttpClientTransport`. | +| `System.Text.Json.Nodes` (BCL) | Not an Apify namespace, but required whenever you name the JSON escape-hatch types the client returns/accepts — `JsonObject`/`JsonNode` (e.g. dataset items, `GetInputAsync`/`UpdateInputAsync`, `GetStatisticsAsync`, `GetOpenApiDefinitionAsync`, `MonthlyUsageAsync`/`LimitsAsync`, `RequestQueueRequest.UserData`). Add `using System.Text.Json.Nodes;`. | Fluent chains such as `client.Actor("id").Builds()` compile with only `using Apify.Client;` because the intermediate types are inferred. You only need `using Apify.Client.Resources;` when you name a resource @@ -116,11 +119,15 @@ var configured = new ApifyClient(new ApifyClientOptions | `MaxDelayBetweenRetriesMillis` | request timeout | Upper bound on the growing inter-retry delay. | | `TimeoutSecs` | `360` | Overall per-request timeout. | | `UserAgentSuffix` | `null` | Custom suffix appended to the `User-Agent` header. | +| `RequestCompression` | `RequestCompression.Brotli` | Algorithm used to compress request bodies ≥ 1024 bytes: `Brotli` (`Content-Encoding: br`) or `Gzip` (`Content-Encoding: gzip`). | | `HttpTransport` | `HttpClientTransport` | The replaceable transport (`Apify.Client.Http.IHttpTransport`). | Requests are retried on network errors, HTTP 429 (rate limit) and 5xx responses, with exponential backoff and jitter. 4xx responses (other than 429) are thrown immediately as `ApifyApiException`. +Request bodies of at least 1024 bytes are compressed before sending. Brotli is used by default; set +`RequestCompression = RequestCompression.Gzip` to send gzip-compressed bodies instead. + ### Replaceable HTTP transport The transport is `Apify.Client.Http.IHttpTransport`. The default is `HttpClientTransport`, which wraps diff --git a/docs/actors.md b/docs/actors.md index 83e7a2b..e1f1e65 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -48,7 +48,7 @@ foreach (var actor in page.Items) - `DefaultBuildAsync(int? waitForFinish = null)` → `BuildClient`. - `LastRun(LastRunOptions? options = null)` → `RunClient` (filter by `Status`/`Origin`). - `Builds()` → `BuildCollectionClient`; `Runs()` → `RunCollectionClient`. -- `Version(string versionNumber)` / `Versions()` — Actor versions. +- `Version(string versionNumber)` → `ActorVersionClient`; `Versions()` → `ActorVersionCollectionClient` — a single Actor version and the version collection. - `Webhooks()` → read-only `NestedWebhookCollectionClient`. `ActorStartOptions` fields: @@ -87,6 +87,51 @@ against) and `ContentType` (`string?`, content type of the input; defaults to `a ## Versions and environment variables +Manage an Actor's versions with `client.Actor(id).Versions()` (the whole collection) and +`client.Actor(id).Version(versionNumber)` (one version). Each version in turn owns a collection of +environment variables, reached with `.EnvVars()` / `.EnvVar(name)`. + +### Version collection — `client.Actor(id).Versions()` → `ActorVersionCollectionClient` + +- `ListAsync(ListOptions? options = null)` — list the Actor's versions (one page). Returns + `PaginationList`. +- `IterateAsync(ListOptions? options = null)` → `IAsyncEnumerable` — lazily iterate every + version across pages, fetching each page on demand. +- `CreateAsync(object version)` — create a version from any JSON-serializable definition. Returns + `ActorVersion`. + +`ListOptions` fields: `Offset` (`int?`, items to skip), `Limit` (`int?`, page size), `Desc` (`bool?`, +newest-first when `true`). + +### Single version — `client.Actor(id).Version(versionNumber)` → `ActorVersionClient` + +`versionNumber` is the version identifier (e.g. `0.1`). + +- `GetAsync()` → `ActorVersion?` (null if not found). +- `UpdateAsync(object newFields)` → `ActorVersion` — update with any JSON-serializable set of fields. +- `DeleteAsync()`. +- `EnvVars()` → `ActorEnvVarCollectionClient` — this version's environment-variable collection. +- `EnvVar(string name)` → `ActorEnvVarClient` — a single environment variable of this version. + +### Env-var collection — `Version(versionNumber).EnvVars()` → `ActorEnvVarCollectionClient` + +- `ListAsync()` — list the version's environment variables (the endpoint returns them in a single page). + Returns `PaginationList`. +- `IterateAsync()` → `IAsyncEnumerable` — iterate the variables; provided for parity with + the other collection iterators (yields the single page's items). +- `CreateAsync(ActorEnvVar envVar)` → `ActorEnvVar` — create an environment variable. + +### Single env-var — `Version(versionNumber).EnvVar(name)` → `ActorEnvVarClient` + +`name` is the environment variable's name. + +- `GetAsync()` → `ActorEnvVar?` (null if not found). +- `UpdateAsync(ActorEnvVar envVar)` → `ActorEnvVar`. +- `DeleteAsync()`. + +See [`ActorVersion`](models.md#actorversion) and [`ActorEnvVar`](models.md#actorenvvar) for the returned +models and the `ActorEnvVar` constructor used below. + ```csharp using Apify.Client; using Apify.Client.Models; diff --git a/docs/builds.md b/docs/builds.md index 8c06330..865a0f9 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -17,7 +17,7 @@ Access the account-wide build collection with `client.Builds()`, an Actor's buil - `WaitForFinishAsync(int? waitSecs = null)` → `Build` — client-side polling until terminal (`null` waits indefinitely). - `GetOpenApiDefinitionAsync()` → `JsonObject?`. -- `Log()` → `LogClient`. +- `Log()` → `LogClient` (its `GetAsync`/`StreamAsync` methods are documented under [Logs in misc.md](misc.md#logs--clientlogbuildorrunid)). Builds are created with `client.Actor(id).BuildAsync(string versionNumber, ActorBuildOptions? options = null)`. `ActorBuildOptions` fields: diff --git a/docs/examples.md b/docs/examples.md index 56f24b1..ecfa8db 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,9 +1,11 @@ # Examples -Each example below is a complete, runnable scenario. The canonical, compiled versions live in -[`tests/Apify.Client.Tests/Examples`](../tests/Apify.Client.Tests/Examples) and are executed -end-to-end against the live API by the **Test examples** CI step (they require an `APIFY_TOKEN`), so -the snippets here are guaranteed to stay valid and working. +Each example below is a complete, runnable scenario. The canonical, compiled versions live in the +client's source repository under +[`tests/Apify.Client.Tests/Examples`](https://github.com/apify/apify-client-dotnet/tree/master/tests/Apify.Client.Tests/Examples) +— a repo-internal path that is not part of the published NuGet package — and are executed end-to-end against the live API by the +**Test examples** CI step (they require an `APIFY_TOKEN`), so the snippets here are guaranteed to stay +valid and working. Every snippet runs inside an `async` context and assumes the following `using` directives appear at the top of the file, **before** any top-level statements (a `using` after the first statement is a `CS1529` @@ -16,7 +18,9 @@ using Apify.Client; using Apify.Client.Models; using Apify.Client.Options; -var client = new ApifyClient(Environment.GetEnvironmentVariable("APIFY_TOKEN")); +var client = new ApifyClient( + Environment.GetEnvironmentVariable("APIFY_TOKEN") + ?? throw new InvalidOperationException("Set the APIFY_TOKEN environment variable.")); ``` ## Run a store Actor and read its dataset @@ -25,7 +29,8 @@ var client = new ApifyClient(Environment.GetEnvironmentVariable("APIFY_TOKEN")); // The third argument bounds the wait in seconds (120 here); pass null to wait indefinitely. var run = await client.Actor("apify/hello-world").CallAsync(null, null, 120); var items = await client.Dataset(run.DefaultDatasetId!).ListItemsAsync(new DatasetListItemsOptions()); -Console.WriteLine("Item count: " + items.Count); +// Count is the number of items in THIS page; Total is the dataset's full count across all pages. +Console.WriteLine($"Items on this page: {items.Count} (of {items.Total} total)"); ``` ## Each storage: create, push, read diff --git a/docs/misc.md b/docs/misc.md index 890a071..c111320 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -54,6 +54,10 @@ var client = new ApifyClient("my-api-token"); var me = await client.Me().GetAsync(); Console.WriteLine(me?.Username); var usage = await client.Me().MonthlyUsageAsync(); + +// Any user's public profile by id (no private fields, and no account-only methods). +var otherUser = await client.User("some-user-id").GetAsync(); +Console.WriteLine(otherUser?.Username); ``` ## Logs — `client.Log(buildOrRunId)` @@ -66,10 +70,16 @@ var usage = await client.Me().MonthlyUsageAsync(); ```csharp using System; +using System.IO; using Apify.Client; using Apify.Client.Options; var client = new ApifyClient("my-api-token"); var log = await client.Log("some-run-id").GetAsync(new LogOptions { Raw = true }); Console.WriteLine(log); + +// Read a run's raw live log as a stream (the low-level alternative to GetStreamedLog's sink redirection). +await using var logStream = await client.Run("some-run-id").GetStreamedLogAsync(); +using var reader = new StreamReader(logStream); +Console.WriteLine(await reader.ReadToEndAsync()); ``` diff --git a/docs/runs.md b/docs/runs.md index 70b5958..59ab8c9 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -26,7 +26,7 @@ collections). - `ChargeAsync(RunChargeOptions options)` → `Task` — record pay-per-event charges (idempotent). - `WaitForFinishAsync(int? waitSecs = null)` → `ActorRun`. - `Dataset()`, `KeyValueStore()`, `RequestQueue()` — the run's default storages. -- `Log()` → `LogClient`; `GetStreamedLogAsync()` → `Stream` (live raw log). +- `Log()` → `LogClient` (its `GetAsync`/`StreamAsync` methods are documented under [Logs in misc.md](misc.md#logs--clientlogbuildorrunid)); `GetStreamedLogAsync()` → `Stream` (live raw log). - `GetStreamedLog(Action toLog, bool fromStart = true)` → `StreamedLog` — redirects the run's live log to `toLog` one complete message at a time. Call `Start()` to begin and `StopAsync()` (or dispose) to end. `fromStart: false` skips messages older than the helper's creation. diff --git a/docs/storages.md b/docs/storages.md index 4a8c56d..d45965c 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -33,7 +33,7 @@ reached from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQue `byte[]` (raw bytes, so binary formats like `Xlsx` are not corrupted; decode text formats yourself). - `PushItemsAsync(object items)` — push one object or an array of objects. - `GetStatisticsAsync()` → `JsonObject?`. -- `CreateItemsPublicUrlAsync(DatasetListItemsOptions? options = null, int? expiresInSecs = null)` → signed public URL. +- `CreateItemsPublicUrlAsync(DatasetListItemsOptions? options = null, int? expiresInSecs = null)` → `string` (a signed public URL). `DatasetListItemsOptions` selects and reshapes items: `Offset`/`Limit` (pagination), `Desc` (reverse order), `Fields`/`OutputFields`/`Omit` (choose columns), `Unwind`/`Flatten` (restructure nested @@ -96,8 +96,8 @@ Console.WriteLine(Encoding.UTF8.GetString(csvBytes)); // CSV is text; decode the - `SetRecordAsync(string key, byte[] value, string contentType, SetRecordOptions? options = null)` and `SetRecordJsonAsync(string key, object? value)` (serializes `value` to JSON bytes). - `DeleteRecordAsync(string key)`. -- `GetRecordPublicUrlAsync(string key)` and - `CreateKeysPublicUrlAsync(ListKeysOptions? options = null, int? expiresInSecs = null)`. +- `GetRecordPublicUrlAsync(string key)` → `string` and + `CreateKeysPublicUrlAsync(ListKeysOptions? options = null, int? expiresInSecs = null)` → `string` (both signed public URLs). `ListKeysOptions` fields: `Limit` (page size), `ExclusiveStartKey` (start after this key), `Prefix` (only keys with this prefix), `Collection` (a named record collection), and `Signature` @@ -136,8 +136,9 @@ options set a stable `ClientKey` (required to manage locks the client created) a - `GetAsync()` → `RequestQueue?`; `UpdateAsync(object newFields)` → `RequestQueue`; `DeleteAsync()`. - `AddRequestAsync(RequestQueueRequest request, bool forefront = false)` → `RequestQueueOperationInfo`. -- `GetRequestAsync(string id)`, `UpdateRequestAsync(RequestQueueRequest request, bool forefront = false)`, - `DeleteRequestAsync(string id)`. +- `GetRequestAsync(string id)` → `RequestQueueRequest?` (null if not found); + `UpdateRequestAsync(RequestQueueRequest request, bool forefront = false)` → `RequestQueueOperationInfo`; + `DeleteRequestAsync(string id)` (no return value). - `ListHeadAsync(int? limit = null)` → `RequestQueueHead`; `ListAndLockHeadAsync(int lockSecs, int? limit = null)`. - `BatchAddRequestsAsync(IReadOnlyList requests, bool forefront = false, BatchAddRequestsOptions? options = null)` diff --git a/src/Apify.Client/Apify.Client.csproj b/src/Apify.Client/Apify.Client.csproj index 0fc2184..a88b3af 100644 --- a/src/Apify.Client/Apify.Client.csproj +++ b/src/Apify.Client/Apify.Client.csproj @@ -6,7 +6,7 @@ Apify.Client - 0.1.1 + 0.1.2 Apify Apify Apify API client for .NET diff --git a/src/Apify.Client/ApifyClient.cs b/src/Apify.Client/ApifyClient.cs index 19ba06e..bf5f09c 100644 --- a/src/Apify.Client/ApifyClient.cs +++ b/src/Apify.Client/ApifyClient.cs @@ -1,5 +1,6 @@ using System; using System.Globalization; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; using System.Text.Json.Nodes; @@ -81,7 +82,7 @@ public ApifyClient(ApifyClientOptions options) options.TimeoutSecs); var userAgent = BuildUserAgent(options.UserAgentSuffix, options.IsAtHome ?? DefaultIsAtHome); - _http = new HttpClientCore(transport, options.Token, userAgent, retry); + _http = new HttpClientCore(transport, options.Token, userAgent, retry, options.RequestCompression); _baseUrl = TrimTrailingSlash(options.BaseUrl) + "/v2"; var publicSource = options.PublicBaseUrl ?? options.BaseUrl; @@ -266,24 +267,87 @@ private static string BuildUserAgent(string? suffix, Func isAtHomeFn) return ua; } - /// The lowercase operating-system family name, matching the reference clients' convention. - private static string CurrentOs() + /// + /// Additional Node os.platform() tokens for Unix platforms that .NET has no dedicated + /// helper for, matched via . + /// Solaris and illumos both report as sunos, matching Node. + /// + /// + /// These entries are deliberately kept even though .NET has no officially supported runtime on these + /// platforms today: the requirement is that every Apify client emit the exact same OS token as the + /// reference client's os.platform(), so should .NET ever run there the token stays aligned + /// rather than degrading to unknown. They are a forward-compatible superset, not live branches. + /// + private static readonly (OSPlatform Platform, string Token)[] ExtendedOsTokens = { - if (OperatingSystem.IsWindows()) + (OSPlatform.Create("OPENBSD"), "openbsd"), + (OSPlatform.Create("NETBSD"), "netbsd"), + (OSPlatform.Create("SOLARIS"), "sunos"), + (OSPlatform.Create("ILLUMOS"), "sunos"), + (OSPlatform.Create("AIX"), "aix"), + }; + + /// Resolves the current platform to its User-Agent OS token (see ). + private static string CurrentOs() => ResolveOsToken( + OperatingSystem.IsWindows(), + OperatingSystem.IsMacOS(), + OperatingSystem.IsAndroid(), + OperatingSystem.IsLinux(), + OperatingSystem.IsFreeBSD(), + RuntimeInformation.IsOSPlatform); + + /// + /// Maps the detected runtime platform to the short, lowercase token used in the User-Agent OS + /// field. The tokens are exactly the reference JS client's Node os.platform() values + /// (win32, darwin, android, linux, freebsd, openbsd, + /// netbsd, sunos, aix), so the token is identical across Apify clients. Android is + /// checked before Linux because an Android runtime is also Linux-based. The five platforms .NET exposes + /// dedicated helpers for are matched first; the remaining Unix platforms + /// Node names but .NET has no helper for are matched via . Platforms the + /// reference's Node runtime cannot run on at all (e.g. iOS, the browser) have no reference token and so + /// fall back to unknown. + /// + internal static string ResolveOsToken( + bool isWindows, + bool isMacOs, + bool isAndroid, + bool isLinux, + bool isFreeBsd, + Func isOsPlatform) + { + if (isWindows) { - return "windows"; + return "win32"; } - if (OperatingSystem.IsMacOS()) + if (isMacOs) { return "darwin"; } - if (OperatingSystem.IsLinux()) + if (isAndroid) + { + return "android"; + } + + if (isLinux) { return "linux"; } + if (isFreeBsd) + { + return "freebsd"; + } + + foreach (var (platform, token) in ExtendedOsTokens) + { + if (isOsPlatform(platform)) + { + return token; + } + } + return "unknown"; } } diff --git a/src/Apify.Client/ApifyClientOptions.cs b/src/Apify.Client/ApifyClientOptions.cs index ce89c42..2b19d08 100644 --- a/src/Apify.Client/ApifyClientOptions.cs +++ b/src/Apify.Client/ApifyClientOptions.cs @@ -33,6 +33,12 @@ public sealed class ApifyClientOptions /// Custom suffix appended to the User-Agent header. public string? UserAgentSuffix { get; set; } + /// + /// Algorithm used to compress large request bodies (default ). + /// Set it to to send gzip-compressed bodies instead. + /// + public RequestCompression RequestCompression { get; set; } = RequestCompression.Brotli; + /// Replaces the default transport (). public IHttpTransport? HttpTransport { get; set; } diff --git a/src/Apify.Client/ApifyClientVersion.cs b/src/Apify.Client/ApifyClientVersion.cs index bb519dd..f433471 100644 --- a/src/Apify.Client/ApifyClientVersion.cs +++ b/src/Apify.Client/ApifyClientVersion.cs @@ -14,11 +14,11 @@ public static class ApifyClientVersion /// The semantic version of this client library (see https://semver.org/). Changes to the public /// interface other than additive ones are considered breaking changes. /// - public const string ClientVersion = "0.1.1"; + public const string ClientVersion = "0.1.2"; /// /// The version of the Apify OpenAPI specification this client was generated and verified against. /// Corresponds to the info.version field of the Apify OpenAPI document. /// - public const string ApiSpecVersion = "v2-2026-07-07T132551Z"; + public const string ApiSpecVersion = "v2-2026-07-08T143931Z"; } diff --git a/src/Apify.Client/Internal/HttpClientCore.cs b/src/Apify.Client/Internal/HttpClientCore.cs index 8334ca8..f25f745 100644 --- a/src/Apify.Client/Internal/HttpClientCore.cs +++ b/src/Apify.Client/Internal/HttpClientCore.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.IO; +using System.IO.Compression; using System.Net.Http; using System.Net.Http.Headers; using System.Text; @@ -31,16 +33,35 @@ internal sealed class HttpClientCore private const int NotFound = 404; + /// + /// Request bodies whose size in bytes is at or above this threshold are compressed before sending, + /// matching the reference client's minimum-compression size. + /// + private const int MinCompressBytes = 1024; + + /// The Content-Encoding token used for brotli-compressed request bodies. + private const string BrotliEncoding = "br"; + + /// The Content-Encoding token used for gzip-compressed request bodies. + private const string GzipEncoding = "gzip"; + private readonly IHttpTransport _transport; private readonly string? _token; private readonly RetryConfig _retry; - - public HttpClientCore(IHttpTransport transport, string? token, string userAgent, RetryConfig retry) + private readonly RequestCompression _compression; + + public HttpClientCore( + IHttpTransport transport, + string? token, + string userAgent, + RetryConfig retry, + RequestCompression compression) { _transport = transport; _token = token; UserAgent = userAgent; _retry = retry; + _compression = compression; } /// The User-Agent header value this client sends. @@ -68,6 +89,9 @@ public async Task CallAsync( var maxAttempts = _retry.MaxRetries + 1; var path = ExtractPath(url); var baseTimeout = timeout ?? TimeSpan.FromSeconds(_retry.TimeoutSecs); + // Normalize (and, when large enough, compress) the body once up front so retries reuse the same + // prepared payload instead of re-encoding and re-compressing on every attempt. + var prepared = PrepareBody(body, bodyBytes, contentType); Exception? lastError = null; for (var attempt = 1; attempt <= maxAttempts; attempt++) @@ -76,7 +100,7 @@ public async Task CallAsync( try { var response = await SendOnceAsync( - method, url, body, bodyBytes, contentType, extraHeaders, + method, url, prepared, extraHeaders, AttemptTimeout(baseTimeout, attempt), cancellationToken).ConfigureAwait(false); var status = (int)response.StatusCode; @@ -114,21 +138,19 @@ await Task.Delay(TimeSpan.FromMilliseconds(RandomizedDelayMillis(delayMillis)), /// Opens a live streaming response (single attempt, no retry). Used by log streaming. public Task StreamAsync(string url, CancellationToken cancellationToken) { - var request = BuildRequest(HttpMethod.Get, url, null, null, string.Empty, null); + var request = BuildRequest(HttpMethod.Get, url, default, null); return _transport.SendAsync(request, TimeSpan.FromSeconds(_retry.TimeoutSecs), streaming: true, cancellationToken); } private async Task SendOnceAsync( HttpMethod method, string url, - string? body, - byte[]? bodyBytes, - string contentType, + PreparedBody prepared, IReadOnlyDictionary? extraHeaders, TimeSpan timeout, CancellationToken cancellationToken) { - using var request = BuildRequest(method, url, body, bodyBytes, contentType, extraHeaders); + using var request = BuildRequest(method, url, prepared, extraHeaders); return await _transport.SendAsync(request, timeout, streaming: false, cancellationToken).ConfigureAwait(false); } @@ -136,9 +158,7 @@ private async Task SendOnceAsync( private HttpRequestMessage BuildRequest( HttpMethod method, string url, - string? body, - byte[]? bodyBytes, - string contentType, + PreparedBody prepared, IReadOnlyDictionary? extraHeaders) { var request = new HttpRequestMessage(method, url); @@ -156,22 +176,86 @@ private HttpRequestMessage BuildRequest( } } - // Raw bytes take precedence so binary records (e.g. images, gzip) are sent verbatim; a string body - // is UTF-8 encoded. Setting the content type verbatim (no charset appended unless the caller added one). - HttpContent? content = bodyBytes is not null - ? new ByteArrayContent(bodyBytes) - : body is not null ? new StringContent(body, Encoding.UTF8) : null; - if (content is not null) + if (prepared.Bytes is not null) { - content.Headers.ContentType = string.IsNullOrEmpty(contentType) + var content = new ByteArrayContent(prepared.Bytes); + // Set the content type verbatim (no charset appended unless the caller added one). + content.Headers.ContentType = string.IsNullOrEmpty(prepared.ContentType) ? null - : MediaTypeHeaderValue.Parse(contentType); + : MediaTypeHeaderValue.Parse(prepared.ContentType); + if (prepared.ContentEncoding is not null) + { + content.Headers.ContentEncoding.Add(prepared.ContentEncoding); + } + request.Content = content; } return request; } + /// + /// Normalizes a request body to bytes and, when it is large enough, compresses it. Raw bytes take + /// precedence so binary records (e.g. images) are used as-is rather than re-encoded through a UTF-8 + /// string; a string body is UTF-8 encoded. Either kind of payload is then compressed once it reaches + /// the size threshold (see the remarks). + /// + /// + /// Bodies at or above are compressed with the configured + /// algorithm: brotli (Content-Encoding: br) by default, matching + /// the reference client's preference, or gzip (Content-Encoding: gzip) when + /// is selected. Both encodings match the reference client's, though + /// the reference only reaches gzip when brotli is unavailable; since .NET always ships brotli, gzip here + /// is a .NET-only opt-in. + /// + private PreparedBody PrepareBody(string? body, byte[]? bodyBytes, string contentType) + { + var raw = bodyBytes ?? (body is not null ? Encoding.UTF8.GetBytes(body) : null); + if (raw is null) + { + return default; + } + + if (raw.Length < MinCompressBytes) + { + return new PreparedBody(raw, contentType, null); + } + + return _compression == RequestCompression.Gzip + ? new PreparedBody(GzipCompress(raw), contentType, GzipEncoding) + : new PreparedBody(BrotliCompress(raw), contentType, BrotliEncoding); + } + + /// Brotli-compresses a payload into a self-contained byte array. + private static byte[] BrotliCompress(byte[] data) + { + using var output = new MemoryStream(); + using (var brotli = new BrotliStream(output, CompressionMode.Compress)) + { + brotli.Write(data, 0, data.Length); + } + + return output.ToArray(); + } + + /// Gzip-compresses a payload into a self-contained byte array. + private static byte[] GzipCompress(byte[] data) + { + using var output = new MemoryStream(); + using (var gzip = new GZipStream(output, CompressionMode.Compress)) + { + gzip.Write(data, 0, data.Length); + } + + return output.ToArray(); + } + + /// + /// A request body normalized to bytes, together with the content type and optional + /// Content-Encoding to send. A value carries no body. + /// + private readonly record struct PreparedBody(byte[]? Bytes, string ContentType, string? ContentEncoding); + /// /// Returns min(overall, base * 2^(attempt-1)): the first attempt uses the base timeout; each /// retry doubles it (a slow-but-progressing connection gets more time) while never exceeding the diff --git a/src/Apify.Client/RequestCompression.cs b/src/Apify.Client/RequestCompression.cs new file mode 100644 index 0000000..df20def --- /dev/null +++ b/src/Apify.Client/RequestCompression.cs @@ -0,0 +1,19 @@ +namespace Apify.Client; + +/// +/// Algorithm used to compress large request bodies before sending them. +/// +/// +/// The reference client prefers brotli and only falls back to gzip on runtimes where brotli is +/// unavailable. .NET always ships brotli, so instead of an automatic fallback this option lets callers +/// choose the algorithm explicitly, keeping both code paths genuinely selectable. Regardless of the +/// choice, only string/byte payloads at or above the compression threshold are compressed. +/// +public enum RequestCompression +{ + /// Compress with brotli (Content-Encoding: br). This is the default and matches the reference client's preference. + Brotli, + + /// Compress with gzip (Content-Encoding: gzip). + Gzip, +} diff --git a/tests/Apify.Client.Tests/Examples/RunStoreActorExample.cs b/tests/Apify.Client.Tests/Examples/RunStoreActorExample.cs index 61d6db5..74d45f1 100644 --- a/tests/Apify.Client.Tests/Examples/RunStoreActorExample.cs +++ b/tests/Apify.Client.Tests/Examples/RunStoreActorExample.cs @@ -12,6 +12,7 @@ public static async Task RunAsync(ApifyClient client) { var run = await client.Actor("apify/hello-world").CallAsync(null, null, 120); var items = await client.Dataset(run.DefaultDatasetId!).ListItemsAsync(new DatasetListItemsOptions()); - Console.WriteLine("Item count: " + items.Count); + // Count is the number of items in THIS page; Total is the dataset's full count across all pages. + Console.WriteLine($"Items on this page: {items.Count} (of {items.Total} total)"); } } diff --git a/tests/Apify.Client.Tests/Unit/ConfigTests.cs b/tests/Apify.Client.Tests/Unit/ConfigTests.cs index f5acf81..30c99c2 100644 --- a/tests/Apify.Client.Tests/Unit/ConfigTests.cs +++ b/tests/Apify.Client.Tests/Unit/ConfigTests.cs @@ -1,3 +1,6 @@ +using System; +using System.Linq; +using System.Runtime.InteropServices; using System.Text.RegularExpressions; using Xunit; @@ -37,6 +40,76 @@ public void UserAgentIsAtHomeTrueAndSuffix() Assert.EndsWith("; my-suffix", client.UserAgent, System.StringComparison.Ordinal); } + // The exact set of Node `os.platform()` tokens the reference JS client can emit. Every Apify client + // must report one of these (plus "unknown" for platforms Node cannot run on at all), so the OS token + // is identical across clients. + private static readonly string[] ReferenceOsTokens = + { + "win32", "darwin", "linux", "android", "freebsd", "openbsd", "netbsd", "sunos", "aix", + }; + + [Fact] + public void UserAgentOsTokenUsesShortLowercasePlatformIdentifier() + { + // The OS token must be a short, lowercase platform identifier aligned with the other Apify + // clients' Node `os.platform()` values, not a uname-style name (e.g. "win32", never "windows"). + var client = new ApifyClient(new ApifyClientOptions + { + Token = "t", + HttpTransport = new MockTransport(), + }); + + var osToken = Regex.Match(client.UserAgent, @"\(([^;]+);").Groups[1].Value; + // The emitted token must be a reference os.platform() token (or "unknown" on non-Node platforms). + Assert.Contains(osToken, ReferenceOsTokens.Append("unknown")); + Assert.DoesNotContain("windows", client.UserAgent, System.StringComparison.Ordinal); + } + + [Theory] + // .NET's OperatingSystem helpers map exactly to the reference Node os.platform() tokens. macOS must be + // "darwin" (never "osx"/"macos") and Windows must be "win32" (never "windows"). + [InlineData(true, false, false, false, false, "win32")] + [InlineData(false, true, false, false, false, "darwin")] + // Android is Linux-based, so it must win over the Linux check and report "android". + [InlineData(false, false, true, true, false, "android")] + [InlineData(false, false, false, true, false, "linux")] + [InlineData(false, false, false, false, true, "freebsd")] + public void ResolveOsTokenMapsHelperPlatformsToReferenceTokens( + bool isWindows, bool isMacOs, bool isAndroid, bool isLinux, bool isFreeBsd, string expected) + { + // No extended platform matches for these cases; the helper booleans decide the token. + var token = ApifyClient.ResolveOsToken(isWindows, isMacOs, isAndroid, isLinux, isFreeBsd, _ => false); + + Assert.Equal(expected, token); + } + + [Theory] + // Unix platforms .NET has no dedicated helper for, matched via RuntimeInformation.IsOSPlatform. Both + // Solaris and illumos report as "sunos", matching Node. + [InlineData("OPENBSD", "openbsd")] + [InlineData("NETBSD", "netbsd")] + [InlineData("SOLARIS", "sunos")] + [InlineData("ILLUMOS", "sunos")] + [InlineData("AIX", "aix")] + public void ResolveOsTokenMapsExtendedPlatformsToReferenceTokens(string osPlatform, string expected) + { + var target = OSPlatform.Create(osPlatform); + Func isOsPlatform = platform => platform == target; + + // No helper platform matches, so resolution falls through to the extended-platform lookup. + var token = ApifyClient.ResolveOsToken(false, false, false, false, false, isOsPlatform); + + Assert.Equal(expected, token); + } + + [Fact] + public void ResolveOsTokenFallsBackToUnknownForNonNodePlatforms() + { + // A platform with no OperatingSystem helper match and no extended-platform match (e.g. iOS or the + // browser, where the reference's Node runtime cannot run) has no reference token and reports "unknown". + Assert.Equal("unknown", ApifyClient.ResolveOsToken(false, false, false, false, false, _ => false)); + } + [Fact] public void ApiBaseUrlAppendsV2() { diff --git a/tests/Apify.Client.Tests/Unit/MockTransport.cs b/tests/Apify.Client.Tests/Unit/MockTransport.cs index 7c1d47d..d4ff35b 100644 --- a/tests/Apify.Client.Tests/Unit/MockTransport.cs +++ b/tests/Apify.Client.Tests/Unit/MockTransport.cs @@ -102,10 +102,15 @@ public async Task SendAsync(HttpRequestMessage request, Tim headers[header.Key] = string.Join(",", header.Value); } - // Read the raw bytes so binary bodies can be asserted verbatim; keep a UTF-8 decode for the - // string-body assertions (equivalent to ReadAsStringAsync for text content). + // Read the raw bytes so binary (and compressed) bodies can be asserted verbatim; keep a UTF-8 + // decode for the string-body assertions (equivalent to ReadAsStringAsync for text content). + // Mimic a server by decompressing the body per Content-Encoding before decoding it to text, so + // the logical JSON payload is recovered even when the client compressed a large request body. bodyBytes = await request.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); - body = System.Text.Encoding.UTF8.GetString(bodyBytes); + var encoding = request.Content.Headers.ContentEncoding.Count > 0 + ? request.Content.Headers.ContentEncoding.ToString() + : null; + body = System.Text.Encoding.UTF8.GetString(Decompress(bodyBytes, encoding)); } lock (_lock) @@ -170,6 +175,27 @@ public async Task SendAsync(HttpRequestMessage request, Tim } } + /// Decompresses a request body per its Content-Encoding (br/gzip), or returns it unchanged. + private static byte[] Decompress(byte[] bytes, string? encoding) + { + using var input = new System.IO.MemoryStream(bytes); + using System.IO.Stream? decompressor = encoding switch + { + "br" => new System.IO.Compression.BrotliStream(input, System.IO.Compression.CompressionMode.Decompress), + "gzip" => new System.IO.Compression.GZipStream(input, System.IO.Compression.CompressionMode.Decompress), + _ => null, + }; + + if (decompressor is null) + { + return bytes; + } + + using var output = new System.IO.MemoryStream(); + decompressor.CopyTo(output); + return output.ToArray(); + } + /// Builds a 200 batch response echoing each request-body uniqueKey as processed. private static HttpResponseMessage BuildEchoResponse(string requestBody) { diff --git a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs index 3e399f6..0dbc956 100644 --- a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs +++ b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs @@ -269,6 +269,92 @@ public async Task MaxTotalChargeUsdIsFormattedWithInvariantCulture() } } + [Fact] + public async Task LargeRequestBodyIsBrotliCompressed() + { + // A JSON body at or above the 1 KiB threshold is sent brotli-compressed: the transport sees the + // "br" Content-Encoding, and brotli-decompressing the raw bytes recovers the original JSON. + var transport = new MockTransport().QueueResponse(200, string.Empty); + var bigValue = new string('x', 4096); + await Client(transport).Dataset("ds1").PushItemsAsync(new { blob = bigValue }); + + var request = transport.LastRequest; + Assert.Equal("br", request.Header("Content-Encoding")); + + using var input = new System.IO.MemoryStream(request.BodyBytes); + using var brotli = new System.IO.Compression.BrotliStream(input, System.IO.Compression.CompressionMode.Decompress); + using var output = new System.IO.MemoryStream(); + await brotli.CopyToAsync(output); + var decoded = System.Text.Encoding.UTF8.GetString(output.ToArray()); + Assert.Equal(bigValue, JsonNode.Parse(decoded)!["blob"]!.GetValue()); + } + + [Fact] + public async Task LargeRequestBodyIsGzipCompressedWhenGzipSelected() + { + // With RequestCompression.Gzip selected, a body at or above the 1 KiB threshold is sent + // gzip-compressed: the transport sees the "gzip" Content-Encoding, and gzip-decompressing the raw + // bytes recovers the original JSON. This exercises the gzip code path end to end. + var transport = new MockTransport().QueueResponse(200, string.Empty); + var client = new ApifyClient(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + RequestCompression = RequestCompression.Gzip, + }); + var bigValue = new string('x', 4096); + await client.Dataset("ds1").PushItemsAsync(new { blob = bigValue }); + + var request = transport.LastRequest; + Assert.Equal("gzip", request.Header("Content-Encoding")); + + using var input = new System.IO.MemoryStream(request.BodyBytes); + using var gzip = new System.IO.Compression.GZipStream(input, System.IO.Compression.CompressionMode.Decompress); + using var output = new System.IO.MemoryStream(); + await gzip.CopyToAsync(output); + var decoded = System.Text.Encoding.UTF8.GetString(output.ToArray()); + Assert.Equal(bigValue, JsonNode.Parse(decoded)!["blob"]!.GetValue()); + } + + [Fact] + public async Task LargeRequestBodyUsesBrotliByDefault() + { + // The default RequestCompression is brotli, so a large body is brotli-compressed even without any + // explicit option, keeping the brotli path the reference-preferred default. + var transport = new MockTransport().QueueResponse(200, string.Empty); + await Client(transport).Dataset("ds1").PushItemsAsync(new { blob = new string('y', 4096) }); + + Assert.Equal("br", transport.LastRequest.Header("Content-Encoding")); + } + + [Theory] + // The threshold (MinCompressBytes = 1024) is inclusive: exactly 1024 bytes is compressed, 1023 is not. + // A raw byte payload gives exact control over the body size (no JSON framing to account for). + [InlineData(1024, "br")] + [InlineData(1023, "")] + public async Task CompressionThresholdIsInclusiveAt1024Bytes(int size, string expectedEncoding) + { + var transport = new MockTransport().QueueResponse(200, string.Empty); + var payload = new byte[size]; // zero-filled: size is exact and it compresses when over the threshold + await Client(transport).KeyValueStore("s1").SetRecordAsync("OUTPUT", payload, "application/octet-stream"); + + Assert.Equal(expectedEncoding, transport.LastRequest.Header("Content-Encoding")); + } + + [Fact] + public async Task SmallRequestBodyIsNotCompressed() + { + // A body below the 1 KiB threshold is sent verbatim with no Content-Encoding header. + var transport = new MockTransport().QueueResponse(200, string.Empty); + await Client(transport).Dataset("ds1").PushItemsAsync(new { blob = "small" }); + + var request = transport.LastRequest; + Assert.Equal(string.Empty, request.Header("Content-Encoding")); + Assert.Equal("small", JsonNode.Parse(request.Body)!["blob"]!.GetValue()); + } + [Fact] public async Task SetRecordSendsRawBytesWithVerbatimContentType() {