From f583c14b37e6c4a455611c3d16e62335d30c7c1d Mon Sep 17 00:00:00 2001 From: HadenSmith Date: Wed, 8 Jul 2026 14:25:00 -0600 Subject: [PATCH 1/6] Fix time series downloads on restricted networks --- .../Time Series/Support/TimeSeriesDownload.cs | 278 ++++++++++++++---- Numerics/Numerics.csproj | 6 +- Numerics/Properties/AssemblyInfo.cs | 3 + .../Time Series/Test_TimeSeriesDownload.cs | 190 ++++++++++++ 4 files changed, 416 insertions(+), 61 deletions(-) create mode 100644 Numerics/Properties/AssemblyInfo.cs diff --git a/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs b/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs index 6c294aa1..1528cf1f 100644 --- a/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs +++ b/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs @@ -25,30 +25,136 @@ namespace Numerics.Data /// public class TimeSeriesDownload { - private static readonly HttpClient _defaultClient = new HttpClient() { Timeout = TimeSpan.FromSeconds(60) }; - private static readonly HttpClient _decompressClient = new HttpClient(new HttpClientHandler - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate - }) - { Timeout = TimeSpan.FromSeconds(60) }; + private static HttpClient _defaultClient = CreateDefaultClient(); + private static HttpClient _decompressClient = CreateDecompressClient(); private const string UserAgent = "USACE-Numerics/2.0"; /// - /// GET with up to three attempts and exponential backoff (0.5s, 1s, 2s - /// + jitter) on transient failures. Retries 5xx, 408, 429, network + /// Maximum time allowed for one provider request attempt before retry/failure handling runs. + /// + /// + /// A short per-attempt timeout prevents blocked government-network routes from making UI + /// callers wait for the default minute-long HTTP timeout on each retry. + /// + private static readonly TimeSpan ProviderRequestTimeout = TimeSpan.FromSeconds(10); + + /// + /// Maximum time allowed for one explicit connectivity probe. + /// + /// + /// Connectivity probes are advisory only and must remain faster than real provider downloads. + /// + private static readonly TimeSpan ConnectivityProbeTimeout = TimeSpan.FromSeconds(2); + + /// + /// Default number of attempts for provider requests made through the retry helper. + /// + /// + /// Two attempts are enough to smooth over one transient outage without turning blocked + /// network paths into multi-minute waits. + /// + private const int DefaultMaxDownloadAttempts = 2; + + /// + /// Provider root endpoints used for the public Internet connectivity check. + /// + /// + /// The downloader no longer uses these URLs as a precondition for data requests. They are + /// retained only for callers that explicitly ask whether at least one supported provider + /// endpoint is reachable. + /// + private static readonly string[] InternetProbeUrls = + { + "https://waterservices.usgs.gov/nwis/", + "https://www.ncei.noaa.gov/", + "https://wateroffice.ec.gc.ca/", + "https://www.bom.gov.au/waterdata/" + }; + + /// + /// Creates the default HTTP client used for uncompressed provider requests. + /// + /// A configured HTTP client with the standard downloader timeout. + /// + /// The client is held as a static instance in production to avoid socket churn. + /// + private static HttpClient CreateDefaultClient() + { + return new HttpClient() { Timeout = TimeSpan.FromSeconds(60) }; + } + + /// + /// Creates the HTTP client used for provider requests that may return compressed payloads. + /// + /// A configured HTTP client with automatic GZip and Deflate decompression. + /// + /// The client is held as a static instance in production to avoid socket churn. + /// + private static HttpClient CreateDecompressClient() + { + return new HttpClient(new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }) + { Timeout = TimeSpan.FromSeconds(60) }; + } + + /// + /// Replaces the static HTTP clients used by the downloader for deterministic tests. + /// + /// The client to use for ordinary provider requests. + /// The client to use for compressed provider requests. + /// Thrown when either client is null. + /// + /// This method is internal so tests can verify request routing without adding public API. + /// + internal static void SetHttpClientsForTesting(HttpClient defaultClient, HttpClient decompressClient) + { + _defaultClient = defaultClient ?? throw new ArgumentNullException(nameof(defaultClient)); + _decompressClient = decompressClient ?? throw new ArgumentNullException(nameof(decompressClient)); + } + + /// + /// Restores the production HTTP clients after a deterministic test. + /// + /// + /// Tests call this method in a finally block so later tests do not inherit fake handlers. + /// + internal static void ResetHttpClientsForTesting() + { + _defaultClient = CreateDefaultClient(); + _decompressClient = CreateDecompressClient(); + } + + /// + /// GET with bounded retry and exponential backoff on transient failures. Retries 5xx, 408, 429, network /// exceptions, and KiWIS DatasourceError bodies. Does NOT retry 4xx — /// those mean the request itself is wrong. /// + /// The HTTP client used to issue the request. + /// The request URL. + /// The maximum number of request attempts. + /// Token used to cancel the request and retry delay. + /// The final HTTP status code and response body. + /// Thrown when all attempts fail with a network error. + /// Thrown when is canceled. + /// Thrown when all attempts exceed the per-attempt timeout. + /// + /// The helper bounds each attempt separately so blocked network paths fail quickly while + /// still allowing one retry for transient service errors. + /// private static async Task<(HttpStatusCode status, string body)> GetWithRetryAsync( - HttpClient client, string url, int maxAttempts = 3) + HttpClient client, string url, int maxAttempts = DefaultMaxDownloadAttempts, CancellationToken cancellationToken = default) { Exception? lastNet = null; for (int attempt = 1; attempt <= maxAttempts; attempt++) { try { - using var resp = await client.GetAsync(url); + using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + attemptCts.CancelAfter(ProviderRequestTimeout); + using var resp = await client.GetAsync(url, attemptCts.Token); string body = await resp.Content.ReadAsStringAsync(); bool transient = (int)resp.StatusCode >= 500 || @@ -59,27 +165,87 @@ public class TimeSeriesDownload if (!transient || attempt == maxAttempts) return (resp.StatusCode, body); } catch (HttpRequestException ex) { lastNet = ex; if (attempt == maxAttempts) throw; } - catch (TaskCanceledException ex) { lastNet = ex; if (attempt == maxAttempts) throw; } + catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } + catch (TaskCanceledException ex) + { + lastNet = CreateDownloadTimeoutException(url, ex); + if (attempt == maxAttempts) throw lastNet; + } - await Task.Delay(500 * (int)Math.Pow(2, attempt - 1)); + await Task.Delay(500 * (int)Math.Pow(2, attempt - 1), cancellationToken); } throw lastNet ?? new Exception("GetWithRetryAsync exhausted retries"); } /// - /// Checks if there is an Internet connection by probing Google's - /// connectivity-check endpoint (HTTP 204 No Content). + /// Creates a timeout exception for a provider request that did not complete quickly enough. /// + /// The URL that timed out. + /// The cancellation exception raised by the HTTP stack. + /// An exception with a user-facing timeout message. + /// + /// The message names the endpoint so application callers can distinguish a blocked provider + /// request from the old generic Internet connectivity failure. + /// + private static TimeoutException CreateDownloadTimeoutException(string url, Exception innerException) + { + return new TimeoutException( + $"The request to '{url}' did not complete within {ProviderRequestTimeout.TotalSeconds:0} seconds.", + innerException); + } + + /// + /// Checks if there is an Internet connection by probing provider endpoints used by the downloader. + /// + /// True when at least one supported provider endpoint completes an HTTP request; otherwise false. + /// + /// This method is not used as a preflight gate for provider downloads because restricted + /// networks may allow one data provider while blocking unrelated endpoints. + /// public static async Task IsConnectedToInternet() + { + using var cts = new CancellationTokenSource(); + var probes = InternetProbeUrls.Select(url => CanReachEndpoint(url, cts.Token)).ToList(); + + while (probes.Count > 0) + { + Task completed = await Task.WhenAny(probes); + probes.Remove(completed); + if (await completed) + { + cts.Cancel(); + return true; + } + } + + return false; + } + + /// + /// Determines whether an endpoint can complete an HTTP request. + /// + /// The endpoint URL to probe. + /// Token used to cancel the probe. + /// True when the request completes with any HTTP response; otherwise false. + /// + /// Any HTTP response means the network path exists. HTTP status codes are intentionally not + /// interpreted because security appliances and provider roots may return redirects or errors + /// while still proving transport reachability. + /// + private static async Task CanReachEndpoint(string url, CancellationToken cancellationToken = default) { try { - using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(ConnectivityProbeTimeout); + using (await _defaultClient.GetAsync( + url, + HttpCompletionOption.ResponseHeadersRead, + cts.Token)) { - var resp = await _defaultClient.GetAsync( - "https://www.google.com/generate_204", cts.Token); - return resp.IsSuccessStatusCode; } + + return true; } catch { @@ -191,8 +357,9 @@ public enum HeightUnit /// The station identification code. /// The time series type. Default = Daily precipitation. /// The depth unit. Default = inches. + /// Token used to cancel the download. /// A downloaded time series. - public static async Task FromGHCN(string siteNumber, TimeSeriesType timeSeriesType = TimeSeriesType.DailyPrecipitation, DepthUnit unit = DepthUnit.Inches) + public static async Task FromGHCN(string siteNumber, TimeSeriesType timeSeriesType = TimeSeriesType.DailyPrecipitation, DepthUnit unit = DepthUnit.Inches, CancellationToken cancellationToken = default) { @@ -212,12 +379,6 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType DateTime? previousDate = null; string tempFilePath = Path.Combine(Path.GetTempPath(), $"{Path.GetRandomFileName()}.dly"); - // Check internet connection - if (!await IsConnectedToInternet()) - { - throw new InvalidOperationException("No internet connection."); - } - try { @@ -230,20 +391,29 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType // directly (it buffers the whole body). Inline the same retry shape on the // headers-only GetAsync, then stream from the successful response. HttpResponseMessage? response = null; - for (int attempt = 1; attempt <= 3; attempt++) + for (int attempt = 1; attempt <= DefaultMaxDownloadAttempts; attempt++) { try { - response = await client.GetAsync(stationFileUrl, HttpCompletionOption.ResponseHeadersRead); + using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + attemptCts.CancelAfter(ProviderRequestTimeout); + response = await client.GetAsync(stationFileUrl, HttpCompletionOption.ResponseHeadersRead, attemptCts.Token); if ((int)response.StatusCode < 500 && response.StatusCode != HttpStatusCode.RequestTimeout && (int)response.StatusCode != 429) break; - if (attempt == 3) break; + if (attempt == DefaultMaxDownloadAttempts) break; response.Dispose(); } - catch (HttpRequestException) when (attempt < 3) { /* retry */ } - catch (TaskCanceledException) when (attempt < 3) { /* retry */ } - await Task.Delay(500 * (int)Math.Pow(2, attempt - 1)); + catch (HttpRequestException) when (attempt < DefaultMaxDownloadAttempts) { /* retry */ } + catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } + catch (TaskCanceledException ex) + { + if (attempt == DefaultMaxDownloadAttempts) + { + throw CreateDownloadTimeoutException(stationFileUrl, ex); + } + } + await Task.Delay(500 * (int)Math.Pow(2, attempt - 1), cancellationToken); } using (response) { @@ -251,7 +421,7 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType using (Stream stream = await response.Content.ReadAsStreamAsync()) using (FileStream fs = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 81920, useAsync: true)) { - await stream.CopyToAsync(fs); + await stream.CopyToAsync(fs, 81920, cancellationToken); } } } @@ -395,7 +565,8 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType /// /// USGS site number. /// The time series type. - public static async Task<(TimeSeries TimeSeries, string RawText)> FromUSGS(string siteNumber, TimeSeriesType timeSeriesType = TimeSeriesType.DailyDischarge) + /// Token used to cancel the download. + public static async Task<(TimeSeries TimeSeries, string RawText)> FromUSGS(string siteNumber, TimeSeriesType timeSeriesType = TimeSeriesType.DailyDischarge, CancellationToken cancellationToken = default) { @@ -411,12 +582,6 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType throw new ArgumentException("The time series type cannot be daily precipitation or daily snow.", nameof(timeSeriesType)); } - // Check internet connection - if (!await IsConnectedToInternet()) - { - throw new InvalidOperationException("No internet connection."); - } - var timeSeries = (timeSeriesType == TimeSeriesType.MeasuredDischarge || timeSeriesType == TimeSeriesType.MeasuredStage || timeSeriesType == TimeSeriesType.InstantaneousDischarge || timeSeriesType == TimeSeriesType.InstantaneousStage) ? new TimeSeries(TimeInterval.Irregular) @@ -441,7 +606,7 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType { var client = _decompressClient; - var (status, body) = await GetWithRetryAsync(client, url); + var (status, body) = await GetWithRetryAsync(client, url, cancellationToken: cancellationToken); if ((int)status >= 400) { string snippet = body.Length > 500 ? body.Substring(0, 500) + "…" : body; @@ -508,7 +673,7 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType else if (timeSeriesType == TimeSeriesType.PeakDischarge || timeSeriesType == TimeSeriesType.PeakStage) { { - var (peakStatus, peakBody) = await GetWithRetryAsync(_defaultClient, url); + var (peakStatus, peakBody) = await GetWithRetryAsync(_defaultClient, url, cancellationToken: cancellationToken); if ((int)peakStatus >= 400) { string snippet = peakBody.Length > 500 ? peakBody.Substring(0, 500) + "…" : peakBody; @@ -573,7 +738,7 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType { { var rawText = new System.Text.StringBuilder(); - await ParseUSGSOgcApiPages(_decompressClient, url, timeSeries, rawText); + await ParseUSGSOgcApiPages(_decompressClient, url, timeSeries, rawText, cancellationToken); textDownload = rawText.ToString(); } @@ -592,13 +757,14 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType /// The initial API URL. /// TimeSeries to populate with parsed data. /// Optional StringBuilder to accumulate raw JSON responses. - private static async Task ParseUSGSOgcApiPages(HttpClient client, string initialUrl, TimeSeries timeSeries, System.Text.StringBuilder? rawText = null) + /// Token used to cancel pagination requests. + private static async Task ParseUSGSOgcApiPages(HttpClient client, string initialUrl, TimeSeries timeSeries, System.Text.StringBuilder? rawText = null, CancellationToken cancellationToken = default) { string? nextUrl = initialUrl; while (nextUrl != null) { - var (status, json) = await GetWithRetryAsync(client, nextUrl); + var (status, json) = await GetWithRetryAsync(client, nextUrl, cancellationToken: cancellationToken); if ((int)status >= 400) { string snippet = json.Length > 500 ? json.Substring(0, 500) + "…" : json; @@ -651,7 +817,7 @@ private static async Task ParseUSGSOgcApiPages(HttpClient client, string initial nextUrl = candidateUrl; } // Small delay between pages to avoid rate limiting - await System.Threading.Tasks.Task.Delay(200); + await System.Threading.Tasks.Task.Delay(200, cancellationToken); break; } } @@ -682,6 +848,7 @@ private static async Task ParseUSGSOgcApiPages(HttpClient client, string initial /// /// Optional inclusive end date. If null, defaults to DateTime.Today (or Now for instantaneous). /// + /// Token used to cancel the download. /// TimeSeries of values. public static async Task FromCHMN( string stationNumber, @@ -689,12 +856,9 @@ public static async Task FromCHMN( DischargeUnit dischargeUnit = DischargeUnit.CubicMetersPerSecond, HeightUnit heightUnit = HeightUnit.Meters, DateTime? startDate = null, - DateTime? endDate = null) + DateTime? endDate = null, + CancellationToken cancellationToken = default) { - // Connectivity - if (!await IsConnectedToInternet()) - throw new InvalidOperationException("No internet connection."); - // Basic validation (WSC station numbers are 7 chars including letters) if (string.IsNullOrWhiteSpace(stationNumber) || stationNumber.Length != 7) throw new ArgumentException("The WSC station number must be 7 characters, e.g., 08LG010.", nameof(stationNumber)); @@ -762,7 +926,7 @@ public static async Task FromCHMN( var client = _decompressClient; // The endpoint returns CSV (automatically decompressed by HttpClientHandler). - var (status, csv) = await GetWithRetryAsync(client, url); + var (status, csv) = await GetWithRetryAsync(client, url, cancellationToken: cancellationToken); if ((int)status >= 400) { string snippet = csv.Length > 500 ? csv.Substring(0, 500) + "…" : csv; @@ -968,6 +1132,7 @@ private static double ConvertCHMNValue(double raw, bool isDischarge, DischargeUn /// Desired depth unit for precipitation types. /// Optional start date. If null, attempts to retrieve full period of record. /// Optional end date. If null, defaults to today. + /// Token used to cancel the download. /// TimeSeries of values. public static async Task FromABOM( string stationNumber, @@ -976,7 +1141,8 @@ public static async Task FromABOM( HeightUnit heightUnit = HeightUnit.Meters, DepthUnit depthUnit = DepthUnit.Millimeters, DateTime? startDate = null, - DateTime? endDate = null) + DateTime? endDate = null, + CancellationToken cancellationToken = default) { @@ -995,10 +1161,6 @@ public static async Task FromABOM( "BOM API supports DailyDischarge, DailyStage, InstantaneousDischarge, InstantaneousStage, and DailyPrecipitation.", nameof(timeSeriesType)); - // Check connectivity - if (!await IsConnectedToInternet()) - throw new InvalidOperationException("No internet connection."); - // Set default dates DateTime sd = startDate ?? new DateTime(1800, 1, 1); DateTime ed = endDate ?? DateTime.Today; @@ -1044,7 +1206,7 @@ public static async Task FromABOM( string listResponse; try { - (listStatus, listResponse) = await GetWithRetryAsync(client, tsListUrl); + (listStatus, listResponse) = await GetWithRetryAsync(client, tsListUrl, cancellationToken: cancellationToken); } catch (HttpRequestException ex) { @@ -1178,7 +1340,7 @@ public static async Task FromABOM( string valuesResponse; try { - (valuesStatus, valuesResponse) = await GetWithRetryAsync(client, valuesUrl); + (valuesStatus, valuesResponse) = await GetWithRetryAsync(client, valuesUrl, cancellationToken: cancellationToken); } catch (HttpRequestException ex) { diff --git a/Numerics/Numerics.csproj b/Numerics/Numerics.csproj index e7bc64f0..63ba65e5 100644 --- a/Numerics/Numerics.csproj +++ b/Numerics/Numerics.csproj @@ -30,9 +30,9 @@ CS1587 - 2.1.1 - Version 2.1.1 is a metadata and release-process update for the JOSS publication. It refreshes citation, CodeMeta, README, and NuGet package metadata; increases the VSTest connection timeout in the release workflow; and does not change the public API. - 2.1.1.0 + 2.1.2 + Version 2.1.2 removes the generic Google connectivity preflight from time series downloads so provider requests can run on restricted networks where the target data service is allowed but unrelated connectivity probes are blocked. + 2.1.2.0 diff --git a/Numerics/Properties/AssemblyInfo.cs b/Numerics/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..68df9216 --- /dev/null +++ b/Numerics/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Test_Numerics")] diff --git a/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs b/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs index f5f6693d..4739b5f2 100644 --- a/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs +++ b/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs @@ -1,6 +1,10 @@ using System; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; using Numerics.Data; @@ -165,6 +169,123 @@ private static async Task AvailableAsync(string serviceName, Func pr } } + /// + /// HTTP handler that serves one deterministic USGS peak-flow response and rejects connectivity preflight requests. + /// + /// + /// The handler makes the regression test independent of live internet access while still + /// verifying the downloader requests the expected USGS endpoint directly. + /// + private sealed class GuardedUsgsPeakHandler : HttpMessageHandler + { + /// + /// Gets the absolute URLs requested through this handler. + /// + /// + /// The test uses this collection to verify no hidden preflight request was issued. + /// + internal List Requests { get; } = new List(); + + /// + /// + /// Returns a small USGS peak-flow RDB body only for the expected data URL and fails + /// connectivity preflight requests explicitly. + /// + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + string url = request.RequestUri?.AbsoluteUri ?? ""; + Requests.Add(url); + + if (IsBlockedPreflight(url)) + { + return Task.FromException( + new HttpRequestException($"Unexpected connectivity preflight request: {url}")); + } + + if (!IsExpectedUsgsPeakUrl(url)) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"Unexpected URL: {url}") + }); + } + + string body = string.Join(Environment.NewLine, new[] + { + "USGS\t01614000\t2020-04-21\t1\t12345" + }); + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(body) + }); + } + + /// + /// Determines whether a URL is one of the connectivity probes this regression test forbids. + /// + /// The absolute URL requested through the handler. + /// True when the URL is a forbidden connectivity preflight; otherwise false. + /// + /// The blocked list covers the old Google probe and the older generic USGS root probe. + /// + internal static bool IsBlockedPreflight(string url) + { + return url.Contains("google", StringComparison.OrdinalIgnoreCase) || + url.Contains("generate_204", StringComparison.OrdinalIgnoreCase) || + url.StartsWith("https://waterservices.usgs.gov/nwis", StringComparison.OrdinalIgnoreCase); + } + + /// + /// Determines whether a URL matches the expected USGS peak-flow request for this test fixture. + /// + /// The absolute URL requested through the handler. + /// True when the URL targets the expected USGS peak-flow endpoint; otherwise false. + /// + /// The check keeps the test tied to the production URL builder, including site number, + /// agency code, and RDB output format. + /// + private static bool IsExpectedUsgsPeakUrl(string url) + { + return url.StartsWith("https://nwis.waterdata.usgs.gov/nwis/peak?", StringComparison.OrdinalIgnoreCase) && + url.Contains("site_no=01614000", StringComparison.OrdinalIgnoreCase) && + url.Contains("agency_cd=USGS", StringComparison.OrdinalIgnoreCase) && + url.Contains("format=rdb", StringComparison.OrdinalIgnoreCase); + } + } + + /// + /// HTTP handler that waits until the supplied cancellation token is canceled. + /// + /// + /// This handler verifies that downloader cancellation reaches the HTTP request rather than + /// only canceling caller-side waiting. + /// + private sealed class CancellableUsgsPeakHandler : HttpMessageHandler + { + /// + /// Gets the absolute URLs requested through this handler. + /// + /// + /// The test uses this collection to confirm the request was started before cancellation. + /// + internal List Requests { get; } = new List(); + + /// + /// + /// The handler deliberately waits for cancellation and never returns a successful response. + /// + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Add(request.RequestUri?.AbsoluteUri ?? ""); + await Task.Delay(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("") + }; + } + } + /// /// Probes a small windowed CHMN download to confirm Water Survey of Canada services are responsive. /// @@ -411,6 +532,75 @@ public async Task CHMN_PeakStage_Works() #region USGS Tests + /// + /// Verifies USGS downloads do not run a generic connectivity preflight before requesting the data. + /// + /// A task that completes when the regression check finishes. + /// + /// The test uses a deterministic in-memory HTTP handler so it can prove request routing + /// without relying on the live USGS service or the host machine's internet connection. + /// + [TestMethod] + public async Task USGS_PeakDischarge_DoesNotRunConnectivityPreflight() + { + var defaultHandler = new GuardedUsgsPeakHandler(); + var decompressHandler = new GuardedUsgsPeakHandler(); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var (ts, raw) = await TimeSeriesDownload.FromUSGS(USGS_3, TimeSeriesDownload.TimeSeriesType.PeakDischarge); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.AreEqual(1, ts.Count); + Assert.AreEqual(12345d, ts[0].Value); + Assert.IsFalse(string.IsNullOrWhiteSpace(raw), "Raw text should contain the fake USGS response."); + Assert.HasCount(1, defaultHandler.Requests, "Expected exactly one USGS data request."); + Assert.IsFalse(defaultHandler.Requests.Any(GuardedUsgsPeakHandler.IsBlockedPreflight)); + Assert.IsEmpty(decompressHandler.Requests, "Peak download should not use the decompression client."); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies USGS downloads honor caller cancellation while waiting on HTTP. + /// + /// A task that completes when the cancellation regression check finishes. + /// + /// BestFit relies on this behavior to stop waiting when an external provider is blocked or + /// unavailable from the user's network. + /// + [TestMethod] + public async Task USGS_PeakDischarge_HonorsCancellationToken() + { + var defaultHandler = new CancellableUsgsPeakHandler(); + var decompressHandler = new CancellableUsgsPeakHandler(); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(60) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(60) }; + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(50)); + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + await Assert.ThrowsAsync(async () => + await TimeSeriesDownload.FromUSGS(USGS_3, TimeSeriesDownload.TimeSeriesType.PeakDischarge, cts.Token)); + + Assert.HasCount(1, defaultHandler.Requests, "Expected one started USGS peak request."); + Assert.IsEmpty(decompressHandler.Requests, "Peak download should not use the decompression client."); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + /// /// Tests full-period-of-record USGS daily discharge download. /// From f522295baf1779b7044a680c846b6c77fbdf4180 Mon Sep 17 00:00:00 2001 From: HadenSmith Date: Wed, 8 Jul 2026 14:44:40 -0600 Subject: [PATCH 2/6] Handle duplicate DateTimes in time series downloads --- .../Time Series/Support/TimeSeriesDownload.cs | 38 +++++++ .../Time Series/Test_TimeSeriesDownload.cs | 102 ++++++++++++++++++ 2 files changed, 140 insertions(+) diff --git a/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs b/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs index 1528cf1f..4cdad490 100644 --- a/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs +++ b/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs @@ -253,6 +253,39 @@ private static async Task CanReachEndpoint(string url, CancellationToken c } } + /// + /// Removes duplicate date/time indices from a downloaded time series. + /// + /// The downloaded time series to normalize. + /// + /// Provider data can contain distinct source records with the same timestamp. Downloader + /// output keeps one ordinate per exact DateTime and uses the last parsed source value. + /// + private static void RemoveDuplicateDateTimes(TimeSeries timeSeries) + { + if (timeSeries.Count < 2) return; + + var orderedIndexes = new List(); + var lastByIndex = new Dictionary>(); + + foreach (var ordinate in timeSeries) + { + if (!lastByIndex.ContainsKey(ordinate.Index)) + orderedIndexes.Add(ordinate.Index); + + lastByIndex[ordinate.Index] = ordinate; + } + + if (orderedIndexes.Count == timeSeries.Count) return; + + timeSeries.Clear(); + foreach (var index in orderedIndexes) + { + var ordinate = lastByIndex[index]; + timeSeries.Add(new SeriesOrdinate(ordinate.Index, ordinate.Value)); + } + } + /// /// Enumeration of time series options. /// @@ -486,6 +519,7 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType } } + RemoveDuplicateDateTimes(timeSeries); return timeSeries; } @@ -742,10 +776,12 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType textDownload = rawText.ToString(); } + RemoveDuplicateDateTimes(timeSeries); timeSeries.SortByTime(); } } + RemoveDuplicateDateTimes(timeSeries); return (timeSeries, textDownload); } @@ -949,6 +985,7 @@ public static async Task FromCHMN( } } + RemoveDuplicateDateTimes(ts); return ts; } @@ -1438,6 +1475,7 @@ public static async Task FromABOM( } } + RemoveDuplicateDateTimes(ts); return ts; } diff --git a/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs b/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs index 4739b5f2..77c3593f 100644 --- a/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs +++ b/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs @@ -254,6 +254,72 @@ private static bool IsExpectedUsgsPeakUrl(string url) } } + /// + /// HTTP handler that serves a deterministic USGS measured-stage response with duplicate timestamps. + /// + /// + /// The fixture is based on USGS 01570500, where the live field-measurements API returns two + /// approved gage-height records for 1936-03-15 05:00:00 with values 14.51 and 14.52. + /// + private sealed class DuplicateUsgsMeasuredStageHandler : HttpMessageHandler + { + /// + /// Gets the absolute URLs requested through this handler. + /// + internal List Requests { get; } = new List(); + + /// + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + string url = request.RequestUri?.AbsoluteUri ?? ""; + Requests.Add(url); + + if (GuardedUsgsPeakHandler.IsBlockedPreflight(url)) + { + return Task.FromException( + new HttpRequestException($"Unexpected connectivity preflight request: {url}")); + } + + if (!IsExpectedUsgsMeasuredStageUrl(url)) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"Unexpected URL: {url}") + }); + } + + string body = string.Join("", new[] + { + "{", + "\"type\":\"FeatureCollection\",", + "\"features\":[", + "{\"type\":\"Feature\",\"id\":\"75f11a56-5796-4dc9-b22a-054da446a164\",\"properties\":{\"time\":\"1936-03-15T05:00:00+00:00\",\"value\":\"14.51\"}},", + "{\"type\":\"Feature\",\"id\":\"8dccaa2d-8480-4b5d-b19b-18122749b6c6\",\"properties\":{\"time\":\"1936-03-15T05:00:00+00:00\",\"value\":\"14.52\"}}", + "],", + "\"links\":[]", + "}" + }); + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(body) + }); + } + + /// + /// Determines whether a URL matches the expected USGS measured-stage request. + /// + private static bool IsExpectedUsgsMeasuredStageUrl(string url) + { + return url.StartsWith("https://api.waterdata.usgs.gov/ogcapi/v0/collections/field-measurements/items?", + StringComparison.OrdinalIgnoreCase) && + url.Contains("monitoring_location_id=USGS-01570500", StringComparison.OrdinalIgnoreCase) && + url.Contains("parameter_code=00065", StringComparison.OrdinalIgnoreCase) && + url.Contains("limit=10000", StringComparison.OrdinalIgnoreCase) && + url.Contains("f=json", StringComparison.OrdinalIgnoreCase); + } + } + /// /// HTTP handler that waits until the supplied cancellation token is canceled. /// @@ -601,6 +667,42 @@ await Assert.ThrowsAsync(async () => } } + /// + /// Verifies duplicate USGS measured-stage timestamps keep the last source value. + /// + [TestMethod] + public async Task USGS_MeasuredStage_DuplicateDateTimes_LastValueWins() + { + var defaultHandler = new DuplicateUsgsMeasuredStageHandler(); + var decompressHandler = new DuplicateUsgsMeasuredStageHandler(); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var (ts, raw) = await TimeSeriesDownload.FromUSGS(USGS_5, + TimeSeriesDownload.TimeSeriesType.MeasuredStage); + + var duplicateTime = new DateTime(1936, 3, 15, 5, 0, 0, DateTimeKind.Utc); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsFalse(string.IsNullOrWhiteSpace(raw), "Raw text should contain the fake USGS response."); + Assert.AreEqual(ts.Count, ts.Select(o => o.Index).Distinct().Count(), "Duplicate date indices detected."); + Assert.AreEqual(1, ts.Count(o => o.Index == duplicateTime), "Expected one retained record for the duplicate timestamp."); + Assert.AreEqual(14.52d, ts.Single(o => o.Index == duplicateTime).Value); + Assert.IsEmpty(defaultHandler.Requests, "Measured stage should not use the default client."); + Assert.HasCount(1, decompressHandler.Requests, "Expected exactly one USGS OGC data request."); + Assert.IsTrue(decompressHandler.Requests[0].Contains("monitoring_location_id=USGS-01570500", StringComparison.OrdinalIgnoreCase)); + Assert.IsTrue(decompressHandler.Requests[0].Contains("parameter_code=00065", StringComparison.OrdinalIgnoreCase)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + /// /// Tests full-period-of-record USGS daily discharge download. /// From bf0f28d84724934ba5e73c8ef3e8ada9c754f05c Mon Sep 17 00:00:00 2001 From: HadenSmith Date: Thu, 9 Jul 2026 13:36:37 -0600 Subject: [PATCH 3/6] Update time series download handling --- .../Time Series/Support/TimeSeriesDownload.cs | 582 ++++++++++----- .../Time Series/Test_TimeSeriesDownload.cs | 705 ++++++++++++++++++ 2 files changed, 1086 insertions(+), 201 deletions(-) diff --git a/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs b/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs index 4cdad490..b5063509 100644 --- a/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs +++ b/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs @@ -10,6 +10,7 @@ using System.Net.Http; using System.Globalization; using System.Collections.Generic; +using System.Text; namespace Numerics.Data { @@ -39,6 +40,24 @@ public class TimeSeriesDownload /// private static readonly TimeSpan ProviderRequestTimeout = TimeSpan.FromSeconds(10); + /// + /// Maximum time allowed for one instantaneous provider request attempt. + /// + /// + /// Instantaneous series can be substantially larger than daily, peak, or field-measurement + /// responses, especially when a provider accepts a long period-of-record query. + /// + private static readonly TimeSpan InstantaneousProviderRequestTimeout = TimeSpan.FromSeconds(120); + + /// + /// Maximum time allowed for one GHCN station-file request attempt. + /// + /// + /// NOAA NCEI can take more than two minutes to send response headers for some full station + /// files even though the resulting payload is valid and small enough to parse quickly. + /// + private static readonly TimeSpan GhcnProviderRequestTimeout = TimeSpan.FromSeconds(240); + /// /// Maximum time allowed for one explicit connectivity probe. /// @@ -75,13 +94,17 @@ public class TimeSeriesDownload /// /// Creates the default HTTP client used for uncompressed provider requests. /// - /// A configured HTTP client with the standard downloader timeout. + /// A configured HTTP client whose timeout is controlled by the retry helper. /// - /// The client is held as a static instance in production to avoid socket churn. + /// The client is held as a static instance in production to avoid socket churn. Request + /// timeouts are applied by + /// so each provider call can choose the right timeout for its payload size. /// private static HttpClient CreateDefaultClient() { - return new HttpClient() { Timeout = TimeSpan.FromSeconds(60) }; + var client = new HttpClient() { Timeout = Timeout.InfiniteTimeSpan }; + client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); + return client; } /// @@ -89,15 +112,18 @@ private static HttpClient CreateDefaultClient() /// /// A configured HTTP client with automatic GZip and Deflate decompression. /// - /// The client is held as a static instance in production to avoid socket churn. + /// The client is held as a static instance in production to avoid socket churn. Request + /// timeouts are applied by . /// private static HttpClient CreateDecompressClient() { - return new HttpClient(new HttpClientHandler + var client = new HttpClient(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }) - { Timeout = TimeSpan.FromSeconds(60) }; + { Timeout = Timeout.InfiniteTimeSpan }; + client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); + return client; } /// @@ -135,6 +161,7 @@ internal static void ResetHttpClientsForTesting() /// The HTTP client used to issue the request. /// The request URL. /// The maximum number of request attempts. + /// The maximum duration allowed for each request attempt. /// Token used to cancel the request and retry delay. /// The final HTTP status code and response body. /// Thrown when all attempts fail with a network error. @@ -145,17 +172,25 @@ internal static void ResetHttpClientsForTesting() /// still allowing one retry for transient service errors. /// private static async Task<(HttpStatusCode status, string body)> GetWithRetryAsync( - HttpClient client, string url, int maxAttempts = DefaultMaxDownloadAttempts, CancellationToken cancellationToken = default) + HttpClient client, + string url, + int maxAttempts = DefaultMaxDownloadAttempts, + TimeSpan? requestTimeout = null, + CancellationToken cancellationToken = default) { + TimeSpan timeout = requestTimeout ?? ProviderRequestTimeout; Exception? lastNet = null; for (int attempt = 1; attempt <= maxAttempts; attempt++) { try { using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - attemptCts.CancelAfter(ProviderRequestTimeout); - using var resp = await client.GetAsync(url, attemptCts.Token); - string body = await resp.Content.ReadAsStringAsync(); + attemptCts.CancelAfter(timeout); + using var resp = await client.GetAsync( + url, + HttpCompletionOption.ResponseHeadersRead, + attemptCts.Token).ConfigureAwait(false); + string body = await ReadResponseBodyAsync(resp, attemptCts.Token).ConfigureAwait(false); bool transient = (int)resp.StatusCode >= 500 || resp.StatusCode == HttpStatusCode.RequestTimeout || @@ -168,29 +203,80 @@ internal static void ResetHttpClientsForTesting() catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } catch (TaskCanceledException ex) { - lastNet = CreateDownloadTimeoutException(url, ex); + lastNet = CreateDownloadTimeoutException(url, timeout, ex); if (attempt == maxAttempts) throw lastNet; } - await Task.Delay(500 * (int)Math.Pow(2, attempt - 1), cancellationToken); + await Task.Delay(500 * (int)Math.Pow(2, attempt - 1), cancellationToken).ConfigureAwait(false); } throw lastNet ?? new Exception("GetWithRetryAsync exhausted retries"); } + /// + /// Reads an HTTP response body with cancellation support. + /// + /// The HTTP response whose content should be read. + /// Token used to cancel the body read. + /// The decoded response text. + /// Thrown when is canceled. + /// + /// cannot be canceled on older target + /// frameworks. Full-period instantaneous USGS responses are large enough that body reads + /// must honor the same bounded timeout as the request headers. + /// + private static async Task ReadResponseBodyAsync(HttpResponseMessage response, CancellationToken cancellationToken) + { + using var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + using var memory = new MemoryStream(); + byte[] buffer = new byte[81920]; + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false)) > 0) + { + memory.Write(buffer, 0, bytesRead); + } + + return GetResponseEncoding(response).GetString(memory.ToArray()); + } + + /// + /// Gets the text encoding declared by an HTTP response, defaulting to UTF-8. + /// + /// The HTTP response to inspect. + /// The declared response encoding, or UTF-8 when none is declared or recognized. + /// + /// USGS RDB and most provider text responses are UTF-8-compatible. The fallback keeps parsing + /// stable when a provider omits or misstates the charset. + /// + private static Encoding GetResponseEncoding(HttpResponseMessage response) + { + string charset = response.Content.Headers.ContentType?.CharSet ?? ""; + if (string.IsNullOrWhiteSpace(charset)) return Encoding.UTF8; + + try + { + return Encoding.GetEncoding(charset.Trim('"')); + } + catch (ArgumentException) + { + return Encoding.UTF8; + } + } + /// /// Creates a timeout exception for a provider request that did not complete quickly enough. /// /// The URL that timed out. + /// The timeout used for the request attempt. /// The cancellation exception raised by the HTTP stack. /// An exception with a user-facing timeout message. /// /// The message names the endpoint so application callers can distinguish a blocked provider /// request from the old generic Internet connectivity failure. /// - private static TimeoutException CreateDownloadTimeoutException(string url, Exception innerException) + private static TimeoutException CreateDownloadTimeoutException(string url, TimeSpan requestTimeout, Exception innerException) { return new TimeoutException( - $"The request to '{url}' did not complete within {ProviderRequestTimeout.TotalSeconds:0} seconds.", + $"The request to '{url}' did not complete within {requestTimeout.TotalSeconds:0} seconds.", innerException); } @@ -254,36 +340,36 @@ private static async Task CanReachEndpoint(string url, CancellationToken c } /// - /// Removes duplicate date/time indices from a downloaded time series. + /// Adds a downloaded ordinate or replaces the previously parsed ordinate at the same date/time. /// - /// The downloaded time series to normalize. + /// The time series being populated. + /// Lookup from parsed date/time values to their current series index. + /// The ordinate date/time. + /// The ordinate value. + /// Thrown when or is null. /// - /// Provider data can contain distinct source records with the same timestamp. Downloader - /// output keeps one ordinate per exact DateTime and uses the last parsed source value. + /// Provider data can contain distinct source records with the same timestamp. Handling that + /// as records are parsed preserves the historical "last value wins" behavior without a + /// second full-series pass after download completion. /// - private static void RemoveDuplicateDateTimes(TimeSeries timeSeries) + private static void AddOrReplaceByDateTime( + TimeSeries timeSeries, + Dictionary indexesByDateTime, + DateTime index, + double value) { - if (timeSeries.Count < 2) return; - - var orderedIndexes = new List(); - var lastByIndex = new Dictionary>(); + if (timeSeries == null) throw new ArgumentNullException(nameof(timeSeries)); + if (indexesByDateTime == null) throw new ArgumentNullException(nameof(indexesByDateTime)); - foreach (var ordinate in timeSeries) + var ordinate = new SeriesOrdinate(index, value); + if (indexesByDateTime.TryGetValue(index, out int existingIndex)) { - if (!lastByIndex.ContainsKey(ordinate.Index)) - orderedIndexes.Add(ordinate.Index); - - lastByIndex[ordinate.Index] = ordinate; + timeSeries[existingIndex] = ordinate; + return; } - if (orderedIndexes.Count == timeSeries.Count) return; - - timeSeries.Clear(); - foreach (var index in orderedIndexes) - { - var ordinate = lastByIndex[index]; - timeSeries.Add(new SeriesOrdinate(ordinate.Index, ordinate.Value)); - } + indexesByDateTime.Add(index, timeSeries.Count); + timeSeries.Add(ordinate); } /// @@ -429,8 +515,11 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType try { using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - attemptCts.CancelAfter(ProviderRequestTimeout); - response = await client.GetAsync(stationFileUrl, HttpCompletionOption.ResponseHeadersRead, attemptCts.Token); + attemptCts.CancelAfter(GhcnProviderRequestTimeout); + response = await client.GetAsync( + stationFileUrl, + HttpCompletionOption.ResponseHeadersRead, + attemptCts.Token).ConfigureAwait(false); if ((int)response.StatusCode < 500 && response.StatusCode != HttpStatusCode.RequestTimeout && (int)response.StatusCode != 429) break; @@ -443,18 +532,18 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType { if (attempt == DefaultMaxDownloadAttempts) { - throw CreateDownloadTimeoutException(stationFileUrl, ex); + throw CreateDownloadTimeoutException(stationFileUrl, GhcnProviderRequestTimeout, ex); } } - await Task.Delay(500 * (int)Math.Pow(2, attempt - 1), cancellationToken); + await Task.Delay(500 * (int)Math.Pow(2, attempt - 1), cancellationToken).ConfigureAwait(false); } using (response) { response!.EnsureSuccessStatusCode(); - using (Stream stream = await response.Content.ReadAsStreamAsync()) + using (Stream stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) using (FileStream fs = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 81920, useAsync: true)) { - await stream.CopyToAsync(fs, 81920, cancellationToken); + await stream.CopyToAsync(fs, 81920, cancellationToken).ConfigureAwait(false); } } } @@ -466,48 +555,58 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType } // Read and parse the file - string[] lines = File.ReadAllLines(tempFilePath); - foreach (string line in lines) + bool previousSuppressCollectionChanged = timeSeries.SuppressCollectionChanged; + var indexesByDateTime = new Dictionary(); + timeSeries.SuppressCollectionChanged = true; + try { - // Extract the type (PRCP, SNOW, etc.) - var typeString = line.Substring(17, 4); - if ((typeString == "PRCP" && timeSeriesType == TimeSeriesType.DailyPrecipitation) || - (typeString == "SNOW" && timeSeriesType == TimeSeriesType.DailySnow)) + string[] lines = File.ReadAllLines(tempFilePath); + foreach (string line in lines) { - - // Extract year, month, and parse days - int.TryParse(line.Substring(11, 4), NumberStyles.Integer, CultureInfo.InvariantCulture, out var year); - int.TryParse(line.Substring(15, 2), NumberStyles.Integer, CultureInfo.InvariantCulture, out var month); - int daysInMonth = DateTime.DaysInMonth(year, month); - - for (int i = 0; i < daysInMonth; i++) + // Extract the type (PRCP, SNOW, etc.) + var typeString = line.Substring(17, 4); + if ((typeString == "PRCP" && timeSeriesType == TimeSeriesType.DailyPrecipitation) || + (typeString == "SNOW" && timeSeriesType == TimeSeriesType.DailySnow)) { - int offset = 21 + (i * 8); - string strgValue = line.Substring(offset, 5).Trim(); - var currentDate = new DateTime(year, month, i + 1); - // Fill missing dates with NaN if there is a gap - if (previousDate.HasValue && (currentDate - previousDate.Value).Days > 1) - { - FillMissingDates(timeSeries, previousDate.Value, currentDate); - } + // Extract year, month, and parse days + int.TryParse(line.Substring(11, 4), NumberStyles.Integer, CultureInfo.InvariantCulture, out var year); + int.TryParse(line.Substring(15, 2), NumberStyles.Integer, CultureInfo.InvariantCulture, out var month); + int daysInMonth = DateTime.DaysInMonth(year, month); - // Parse the precipitation or snow value - if (int.TryParse(strgValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out int rawValue) && rawValue != -9999) // Valid precipitation value - { - double convertedValue = ConvertToDesiredUnit(rawValue, unit); - timeSeries.Add(new SeriesOrdinate(currentDate, convertedValue)); - } - else + for (int i = 0; i < daysInMonth; i++) { - // Add missing value if invalid - timeSeries.Add(new SeriesOrdinate(currentDate, double.NaN)); - } + int offset = 21 + (i * 8); + string strgValue = line.Substring(offset, 5).Trim(); + var currentDate = new DateTime(year, month, i + 1); - previousDate = currentDate; - } + // Fill missing dates with NaN if there is a gap + if (previousDate.HasValue && (currentDate - previousDate.Value).Days > 1) + { + FillMissingDates(timeSeries, previousDate.Value, currentDate, indexesByDateTime); + } + + // Parse the precipitation or snow value + if (int.TryParse(strgValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out int rawValue) && rawValue != -9999) // Valid precipitation value + { + double convertedValue = ConvertToDesiredUnit(rawValue, unit); + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, currentDate, convertedValue); + } + else + { + // Add missing value if invalid + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, currentDate, double.NaN); + } + + previousDate = currentDate; + } + } } } + finally + { + timeSeries.SuppressCollectionChanged = previousSuppressCollectionChanged; + } } finally @@ -519,7 +618,6 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType } } - RemoveDuplicateDateTimes(timeSeries); return timeSeries; } @@ -546,12 +644,30 @@ private static double ConvertToDesiredUnit(int rawValue, DepthUnit unit) /// The time series to fill. /// The last available date. /// The current date to reach. - private static void FillMissingDates(TimeSeries timeSeries, DateTime previousDate, DateTime currentDate) + /// Optional parsed-date lookup used when duplicate replacement is active. + /// + /// When a lookup is supplied, inserted gap ordinates participate in the same inline + /// duplicate handling as provider records. Without a lookup the method preserves the + /// historical append-only behavior. + /// + private static void FillMissingDates( + TimeSeries timeSeries, + DateTime previousDate, + DateTime currentDate, + Dictionary? indexesByDateTime = null) { DateTime missingDate = previousDate.AddDays(1); while (missingDate < currentDate) { - timeSeries.Add(new SeriesOrdinate(missingDate, double.NaN)); + if (indexesByDateTime == null) + { + timeSeries.Add(new SeriesOrdinate(missingDate, double.NaN)); + } + else + { + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, missingDate, double.NaN); + } + missingDate = missingDate.AddDays(1); } } @@ -636,78 +752,95 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType timeSeriesType == TimeSeriesType.InstantaneousStage; // Instantaneous RDB has an extra tz_cd column at index 3, pushing the value to index 4 int valueIndex = isInstantaneous ? 4 : 3; + bool previousSuppressCollectionChanged = timeSeries.SuppressCollectionChanged; + var indexesByDateTime = new Dictionary(); - { - var client = _decompressClient; - - var (status, body) = await GetWithRetryAsync(client, url, cancellationToken: cancellationToken); - if ((int)status >= 400) - { - string snippet = body.Length > 500 ? body.Substring(0, 500) + "…" : body; - throw new Exception( - $"Failed to retrieve USGS data. Status={(int)status} ({status}). Body={snippet}"); - } - - using var reader = new StringReader(body); - string? line; - bool isHeader = true; + timeSeries.SuppressCollectionChanged = true; - while ((line = await reader.ReadLineAsync()) != null) + try + { { - // Skip header row - if (isHeader) + var client = _decompressClient; + + TimeSpan requestTimeout = isInstantaneous ? InstantaneousProviderRequestTimeout : ProviderRequestTimeout; + var (status, body) = await GetWithRetryAsync( + client, + url, + requestTimeout: requestTimeout, + cancellationToken: cancellationToken).ConfigureAwait(false); + if ((int)status >= 400) { - isHeader = false; - continue; + string snippet = body.Length > 500 ? body.Substring(0, 500) + "…" : body; + throw new Exception( + $"Failed to retrieve USGS data. Status={(int)status} ({status}). Body={snippet}"); } + textDownload = body; - // Skip comment lines - if (line.StartsWith("#")) + using var reader = new StringReader(body); + string? line; + bool isHeader = true; + + while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) != null) { - if (line.Trim() == "# No sites found matching all criteria") + // Skip header row + if (isHeader) { - throw new Exception("No data found matching all criteria."); + isHeader = false; + continue; + } + + // Skip comment lines + if (line.StartsWith("#")) + { + if (line.Trim() == "# No sites found matching all criteria") + { + throw new Exception("No data found matching all criteria."); + } + continue; } - continue; - } - string[] fields = line.Split('\t'); - // Validate expected number of fields and record type - if (fields.Length <= valueIndex || fields[0] != "USGS") - continue; + string[] fields = line.Split('\t'); + // Validate expected number of fields and record type + if (fields.Length <= valueIndex || fields[0] != "USGS") + continue; - // USGS instantaneous timestamps are local-time strings without an offset; - // the tz_cd column at fields[3] reports the offset separately. We currently - // parse these as naive DateTime (Kind=Unspecified). Future work: synthesize - // a DateTimeOffset from fields[2] + fields[3] for full timezone fidelity. - if (!DateTime.TryParse(fields[2], CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime index)) - { - continue; - } + // USGS instantaneous timestamps are local-time strings without an offset; + // the tz_cd column at fields[3] reports the offset separately. We currently + // parse these as naive DateTime (Kind=Unspecified). Future work: synthesize + // a DateTimeOffset from fields[2] + fields[3] for full timezone fidelity. + if (!DateTime.TryParse(fields[2], CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime index)) + { + continue; + } - // Fill in missing days only for daily series (not instantaneous) - if (!isInstantaneous && timeSeries.Count > 0 && index != TimeSeries.AddTimeInterval(timeSeries.Last().Index, TimeInterval.OneDay)) - { - while (timeSeries.Last().Index < TimeSeries.SubtractTimeInterval(index, TimeInterval.OneDay)) + // Fill in missing days only for daily series (not instantaneous) + if (!isInstantaneous && timeSeries.Count > 0 && index != TimeSeries.AddTimeInterval(timeSeries.Last().Index, TimeInterval.OneDay)) { - timeSeries.Add(new SeriesOrdinate( - TimeSeries.AddTimeInterval(timeSeries.Last().Index, TimeInterval.OneDay), double.NaN)); + while (timeSeries.Last().Index < TimeSeries.SubtractTimeInterval(index, TimeInterval.OneDay)) + { + DateTime missingIndex = TimeSeries.AddTimeInterval(timeSeries.Last().Index, TimeInterval.OneDay); + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, missingIndex, double.NaN); + } } - } - // Get and parse value - string valueStr = fields[valueIndex]; - double value = string.IsNullOrWhiteSpace(valueStr) ? double.NaN - : double.TryParse(valueStr, NumberStyles.Float, CultureInfo.InvariantCulture, out double tempVal) ? tempVal : double.NaN; - timeSeries.Add(new SeriesOrdinate(index, value)); + // Get and parse value + string valueStr = fields[valueIndex]; + double value = string.IsNullOrWhiteSpace(valueStr) ? double.NaN + : double.TryParse(valueStr, NumberStyles.Float, CultureInfo.InvariantCulture, out double tempVal) ? tempVal : double.NaN; + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, index, value); + } } } + finally + { + timeSeries.SuppressCollectionChanged = previousSuppressCollectionChanged; + } } // For peak data (annual max values) else if (timeSeriesType == TimeSeriesType.PeakDischarge || timeSeriesType == TimeSeriesType.PeakStage) { { - var (peakStatus, peakBody) = await GetWithRetryAsync(_defaultClient, url, cancellationToken: cancellationToken); + var (peakStatus, peakBody) = await GetWithRetryAsync(_defaultClient, url, cancellationToken: cancellationToken).ConfigureAwait(false); if ((int)peakStatus >= 400) { string snippet = peakBody.Length > 500 ? peakBody.Substring(0, 500) + "…" : peakBody; @@ -717,6 +850,7 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType textDownload = peakBody; int idx = timeSeriesType == TimeSeriesType.PeakDischarge ? 4 : 6; + var indexesByDateTime = new Dictionary(); var lines = textDownload.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries); foreach (string line in lines) { @@ -761,7 +895,7 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType if (!string.IsNullOrWhiteSpace(segments[idx])) { double.TryParse(segments[idx], NumberStyles.Float, CultureInfo.InvariantCulture, out value); - timeSeries.Add(new SeriesOrdinate(index, value)); + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, index, value); } } } @@ -772,16 +906,14 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType { { var rawText = new System.Text.StringBuilder(); - await ParseUSGSOgcApiPages(_decompressClient, url, timeSeries, rawText, cancellationToken); + await ParseUSGSOgcApiPages(_decompressClient, url, timeSeries, rawText, cancellationToken).ConfigureAwait(false); textDownload = rawText.ToString(); } - RemoveDuplicateDateTimes(timeSeries); timeSeries.SortByTime(); } } - RemoveDuplicateDateTimes(timeSeries); return (timeSeries, textDownload); } @@ -797,10 +929,11 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType private static async Task ParseUSGSOgcApiPages(HttpClient client, string initialUrl, TimeSeries timeSeries, System.Text.StringBuilder? rawText = null, CancellationToken cancellationToken = default) { string? nextUrl = initialUrl; + var indexesByDateTime = new Dictionary(); while (nextUrl != null) { - var (status, json) = await GetWithRetryAsync(client, nextUrl, cancellationToken: cancellationToken); + var (status, json) = await GetWithRetryAsync(client, nextUrl, cancellationToken: cancellationToken).ConfigureAwait(false); if ((int)status >= 400) { string snippet = json.Length > 500 ? json.Substring(0, 500) + "…" : json; @@ -833,7 +966,7 @@ private static async Task ParseUSGSOgcApiPages(HttpClient client, string initial CultureInfo.InvariantCulture, out double val)) continue; - timeSeries.Add(new SeriesOrdinate(dt, val)); + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, dt, val); } } @@ -957,12 +1090,21 @@ public static async Task FromCHMN( } var ts = new TimeSeries(interval); + var indexesByDateTime = new Dictionary(); { var client = _decompressClient; // The endpoint returns CSV (automatically decompressed by HttpClientHandler). - var (status, csv) = await GetWithRetryAsync(client, url, cancellationToken: cancellationToken); + TimeSpan requestTimeout = timeSeriesType == TimeSeriesType.InstantaneousDischarge || + timeSeriesType == TimeSeriesType.InstantaneousStage + ? InstantaneousProviderRequestTimeout + : ProviderRequestTimeout; + var (status, csv) = await GetWithRetryAsync( + client, + url, + requestTimeout: requestTimeout, + cancellationToken: cancellationToken); if ((int)status >= 400) { string snippet = csv.Length > 500 ? csv.Substring(0, 500) + "…" : csv; @@ -971,32 +1113,49 @@ public static async Task FromCHMN( } // Parse the CSV response based on endpoint type - if (timeSeriesType == TimeSeriesType.PeakDischarge || timeSeriesType == TimeSeriesType.PeakStage) + bool previousSuppressCollectionChanged = ts.SuppressCollectionChanged; + ts.SuppressCollectionChanged = true; + try { - // Peak data CSV format: - // ID,Parameter/Paramètre,Date,Timezone/Fuseau horaire,Type/Catégorie,Value/Valeur,Symbol/Symbole - ParseCHMNPeakCsv(csv, stationNumber, isDischarge, dischargeUnit, heightUnit, ts); + if (timeSeriesType == TimeSeriesType.PeakDischarge || timeSeriesType == TimeSeriesType.PeakStage) + { + // Peak data CSV format: + // ID,Parameter/Paramètre,Date,Timezone/Fuseau horaire,Type/Catégorie,Value/Valeur,Symbol/Symbole + ParseCHMNPeakCsv(csv, stationNumber, isDischarge, dischargeUnit, heightUnit, ts, indexesByDateTime); + } + else + { + // Daily and real-time CSV share the same column layout: + // ID,Date,Parameter/Paramètre,Value/Valeur,... + ParseCHMNDailyCsv(csv, stationNumber, timeSeriesType, isDischarge, dischargeUnit, heightUnit, ts, indexesByDateTime); + } } - else + finally { - // Daily and real-time CSV share the same column layout: - // ID,Date,Parameter/Paramètre,Value/Valeur,... - ParseCHMNDailyCsv(csv, stationNumber, timeSeriesType, isDischarge, dischargeUnit, heightUnit, ts); + ts.SuppressCollectionChanged = previousSuppressCollectionChanged; } } - RemoveDuplicateDateTimes(ts); return ts; } /// /// Parse CHMN daily or real-time CSV data. /// + /// The CSV response body returned by the CHMN endpoint. + /// The CHMN station identifier to retain while parsing. + /// The requested time series type. + /// A value indicating whether discharge values are being parsed. + /// The discharge unit requested by the caller. + /// The height unit requested by the caller. + /// The time series being populated. + /// Lookup used to replace duplicate dates inline while parsing. /// /// CSV format: ID,Date,Parameter/Paramètre,Value/Valeur,... (additional columns vary by endpoint) /// private static void ParseCHMNDailyCsv(string csv, string stationNumber, TimeSeriesType timeSeriesType, - bool isDischarge, DischargeUnit dischargeUnit, HeightUnit heightUnit, TimeSeries ts) + bool isDischarge, DischargeUnit dischargeUnit, HeightUnit heightUnit, TimeSeries ts, + Dictionary indexesByDateTime) { bool isDailyType = timeSeriesType == TimeSeriesType.DailyDischarge || timeSeriesType == TimeSeriesType.DailyStage; @@ -1065,9 +1224,9 @@ private static void ParseCHMNDailyCsv(string csv, string stationNumber, TimeSeri // Fill gaps with NaN for daily series only if (isDailyType && prevDate.HasValue && (date - prevDate.Value).Days > 1) - FillMissingDates(ts, prevDate.Value, date); + FillMissingDates(ts, prevDate.Value, date, indexesByDateTime); - ts.Add(new SeriesOrdinate(date, val)); + AddOrReplaceByDateTime(ts, indexesByDateTime, date, val); prevDate = date; } } @@ -1076,12 +1235,20 @@ private static void ParseCHMNDailyCsv(string csv, string stationNumber, TimeSeri /// /// Parse CHMN peak data CSV. /// + /// The CSV response body returned by the CHMN endpoint. + /// The CHMN station identifier to retain while parsing. + /// A value indicating whether discharge values are being parsed. + /// The discharge unit requested by the caller. + /// The height unit requested by the caller. + /// The time series being populated. + /// Lookup used to replace duplicate dates inline while parsing. /// /// CSV format: ID,Parameter/Paramètre,Date,Timezone/Fuseau horaire,Type/Catégorie,Value/Valeur,Symbol/Symbole. /// Only "maximum" rows are included (minimums are skipped). /// private static void ParseCHMNPeakCsv(string csv, string stationNumber, bool isDischarge, - DischargeUnit dischargeUnit, HeightUnit heightUnit, TimeSeries ts) + DischargeUnit dischargeUnit, HeightUnit heightUnit, TimeSeries ts, + Dictionary indexesByDateTime) { using (var sr = new StringReader(csv)) { @@ -1127,7 +1294,7 @@ private static void ParseCHMNPeakCsv(string csv, string stationNumber, bool isDi continue; double val = ConvertCHMNValue(raw, isDischarge, dischargeUnit, heightUnit); - ts.Add(new SeriesOrdinate(date, val)); + AddOrReplaceByDateTime(ts, indexesByDateTime, date, val); } } } @@ -1355,6 +1522,7 @@ public static async Task FromABOM( var ts = isInstantaneous ? new TimeSeries(TimeInterval.Irregular) : new TimeSeries(TimeInterval.OneDay); DateTime? prevDate = null; + var indexesByDateTime = new Dictionary(); // Create HttpClientHandler with automatic decompression var handler2 = new HttpClientHandler @@ -1371,13 +1539,17 @@ public static async Task FromABOM( client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate"); client.DefaultRequestHeaders.Add("DNT", "1"); client.DefaultRequestHeaders.Add("Connection", "keep-alive"); - client.Timeout = TimeSpan.FromSeconds(60); + client.Timeout = Timeout.InfiniteTimeSpan; HttpStatusCode valuesStatus; string valuesResponse; try { - (valuesStatus, valuesResponse) = await GetWithRetryAsync(client, valuesUrl, cancellationToken: cancellationToken); + (valuesStatus, valuesResponse) = await GetWithRetryAsync( + client, + valuesUrl, + requestTimeout: isInstantaneous ? InstantaneousProviderRequestTimeout : ProviderRequestTimeout, + cancellationToken: cancellationToken); } catch (HttpRequestException ex) { @@ -1415,67 +1587,75 @@ public static async Task FromABOM( if (!dataObj.TryGetProperty("data", out var dataArray)) throw new Exception("Invalid response format from BOM"); - // Parse the data array - each element is [timestamp, value, ...] - foreach (var point in dataArray.EnumerateArray()) + bool previousSuppressCollectionChanged = ts.SuppressCollectionChanged; + ts.SuppressCollectionChanged = true; + try { - if (point.GetArrayLength() < 2) continue; - - // BOM returns ISO-8601 with explicit offset, e.g. "2021-11-01T00:00:00.000+10:00". - // DateTime.TryParse converts these to LOCAL time, which on a non-AEST machine - // shifts daily series labels by a calendar day. DateTimeOffset preserves the - // offset; .Date gives the wall-clock date BOM intended. - string? timestampStr = point[0].GetString(); - if (!DateTimeOffset.TryParse(timestampStr, CultureInfo.InvariantCulture, - DateTimeStyles.None, out DateTimeOffset dto)) - continue; - DateTime date = isInstantaneous ? dto.DateTime : dto.Date; - - // Parse value - double val = double.NaN; - var valueElement = point[1]; - - if (valueElement.ValueKind == System.Text.Json.JsonValueKind.Number) + // Parse the data array - each element is [timestamp, value, ...] + foreach (var point in dataArray.EnumerateArray()) { - double raw = valueElement.GetDouble(); + if (point.GetArrayLength() < 2) continue; + + // BOM returns ISO-8601 with explicit offset, e.g. "2021-11-01T00:00:00.000+10:00". + // DateTime.TryParse converts these to LOCAL time, which on a non-AEST machine + // shifts daily series labels by a calendar day. DateTimeOffset preserves the + // offset; .Date gives the wall-clock date BOM intended. + string? timestampStr = point[0].GetString(); + if (!DateTimeOffset.TryParse(timestampStr, CultureInfo.InvariantCulture, + DateTimeStyles.None, out DateTimeOffset dto)) + continue; + DateTime date = isInstantaneous ? dto.DateTime : dto.Date; - // Apply unit conversion - if (isDischarge) - { - // BOM returns discharge in m³/s - val = dischargeUnit == DischargeUnit.CubicFeetPerSecond - ? raw * 35.3146667 - : raw; - } - else if (isStage) - { - // BOM returns stage in meters - val = heightUnit == HeightUnit.Feet - ? raw * 3.280839895 - : raw; - } - else if (isPrecip) + // Parse value + double val = double.NaN; + var valueElement = point[1]; + + if (valueElement.ValueKind == System.Text.Json.JsonValueKind.Number) { - // BOM returns rainfall in millimeters - val = depthUnit switch + double raw = valueElement.GetDouble(); + + // Apply unit conversion + if (isDischarge) + { + // BOM returns discharge in m³/s + val = dischargeUnit == DischargeUnit.CubicFeetPerSecond + ? raw * 35.3146667 + : raw; + } + else if (isStage) + { + // BOM returns stage in meters + val = heightUnit == HeightUnit.Feet + ? raw * 3.280839895 + : raw; + } + else if (isPrecip) { - DepthUnit.Millimeters => raw, - DepthUnit.Centimeters => raw / 10.0, - DepthUnit.Inches => raw / 25.4, - _ => raw - }; + // BOM returns rainfall in millimeters + val = depthUnit switch + { + DepthUnit.Millimeters => raw, + DepthUnit.Centimeters => raw / 10.0, + DepthUnit.Inches => raw / 25.4, + _ => raw + }; + } } - } - // Fill gaps with NaN for daily series only - if (!isInstantaneous && prevDate.HasValue && (date - prevDate.Value).Days > 1) - FillMissingDates(ts, prevDate.Value, date); + // Fill gaps with NaN for daily series only + if (!isInstantaneous && prevDate.HasValue && (date - prevDate.Value).Days > 1) + FillMissingDates(ts, prevDate.Value, date, indexesByDateTime); - ts.Add(new SeriesOrdinate(date, val)); - prevDate = date; + AddOrReplaceByDateTime(ts, indexesByDateTime, date, val); + prevDate = date; + } + } + finally + { + ts.SuppressCollectionChanged = previousSuppressCollectionChanged; } } - RemoveDuplicateDateTimes(ts); return ts; } diff --git a/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs b/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs index 77c3593f..c084cb32 100644 --- a/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs +++ b/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Net; using System.Net.Http; @@ -254,6 +255,204 @@ private static bool IsExpectedUsgsPeakUrl(string url) } } + /// + /// HTTP handler that serves deterministic USGS instantaneous RDB responses. + /// + /// + /// The handler allows only the expected /nwis/iv request for USGS 01646500 and + /// rejects generic connectivity probes. This keeps the test deterministic and independent + /// from the live USGS service. + /// + private sealed class GuardedUsgsInstantaneousHandler : HttpMessageHandler + { + /// + /// USGS parameter code expected by this handler. + /// + private readonly string _parameterCode; + + /// + /// Whether the handler should simulate an HTTP timeout after URL validation. + /// + private readonly bool _simulateTimeout; + + /// + /// Whether successful responses should complete asynchronously. + /// + private readonly bool _completeAsynchronously; + + /// + /// Number of duplicate timestamp pairs to append to the deterministic response. + /// + private readonly int _duplicateTimestampPairs; + + /// + /// Gets the absolute URLs requested through this handler. + /// + internal List Requests { get; } = new List(); + + /// + /// Initializes a new instance of the class. + /// + /// The expected USGS parameter code. + /// Whether to throw a timeout after validating the request URL. + /// Whether to complete successful responses on the thread pool. + /// The number of duplicate timestamp pairs to include in the response. + /// Thrown when is empty. + /// Thrown when is negative. + /// + /// Parameter 00060 represents discharge and 00065 represents gage height. + /// Asynchronous completion is used by synchronization-context regression tests, while + /// duplicate timestamp pairs exercise the downloader's normalization cleanup. + /// + internal GuardedUsgsInstantaneousHandler( + string parameterCode, + bool simulateTimeout = false, + bool completeAsynchronously = false, + int duplicateTimestampPairs = 0) + { + if (string.IsNullOrWhiteSpace(parameterCode)) + throw new ArgumentException("A parameter code is required.", nameof(parameterCode)); + if (duplicateTimestampPairs < 0) + throw new ArgumentOutOfRangeException(nameof(duplicateTimestampPairs), "Duplicate timestamp pair count cannot be negative."); + + _parameterCode = parameterCode; + _simulateTimeout = simulateTimeout; + _completeAsynchronously = completeAsynchronously; + _duplicateTimestampPairs = duplicateTimestampPairs; + } + + /// + /// + /// Returns a small nonuniform instantaneous RDB body only for the expected USGS data URL. + /// + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + string url = request.RequestUri?.AbsoluteUri ?? ""; + Requests.Add(url); + + if (IsBlockedPreflight(url)) + { + return Task.FromException( + new HttpRequestException($"Unexpected connectivity preflight request: {url}")); + } + + if (!IsExpectedUsgsInstantaneousUrl(url)) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"Unexpected URL: {url}") + }); + } + + if (_simulateTimeout) + { + return Task.FromException( + new TaskCanceledException($"Simulated instantaneous timeout: {url}")); + } + + string valueOne = _parameterCode == "00060" ? "101.5" : "6.12"; + string valueTwo = _parameterCode == "00060" ? "102.0" : "6.13"; + string valueThree = _parameterCode == "00060" ? "104.5" : "6.17"; + var lines = new List + { + $"agency_cd\tsite_no\tdatetime\ttz_cd\t{_parameterCode}_00000\t{_parameterCode}_00000_cd", + "5s\t15s\t20d\t6s\t14n\t10s", + $"USGS\t01646500\t2024-01-01 00:00\tEST\t{valueOne}\tA", + $"USGS\t01646500\t2024-01-01 00:15\tEST\t{valueTwo}\tA", + $"USGS\t01646500\t2024-01-01 00:45\tEST\t{valueThree}\tA" + }; + + for (int i = 0; i < _duplicateTimestampPairs; i++) + { + string timestamp = new DateTime(2024, 1, 2).AddMinutes(i).ToString("yyyy-MM-dd HH:mm", CultureInfo.InvariantCulture); + lines.Add($"USGS\t01646500\t{timestamp}\tEST\t{1000d + i:0.0}\tP"); + lines.Add($"USGS\t01646500\t{timestamp}\tEST\t{2000d + i:0.0}\tA"); + } + + string body = string.Join(Environment.NewLine, lines); + + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(body) + }; + + return _completeAsynchronously + ? Task.Run(() => response) + : Task.FromResult(response); + } + + /// + /// Determines whether a URL is one of the connectivity probes this regression forbids. + /// + /// The absolute URL requested through the handler. + /// true when the URL is a forbidden preflight; otherwise, false. + /// + /// The instantaneous data endpoint is under /nwis/iv; only generic roots and the + /// old Google probe are treated as preflight requests. + /// + internal static bool IsBlockedPreflight(string url) + { + string normalized = url.TrimEnd('/'); + return url.Contains("google", StringComparison.OrdinalIgnoreCase) || + url.Contains("generate_204", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "https://waterservices.usgs.gov/nwis", StringComparison.OrdinalIgnoreCase); + } + + /// + /// Determines whether a URL matches the expected USGS instantaneous request. + /// + /// The absolute URL requested through the handler. + /// true when the URL targets the expected fixture endpoint; otherwise, false. + /// + /// The check pins the request to the instantaneous-value API, the 01646500 gage, the + /// requested parameter code, and RDB formatting. + /// + private bool IsExpectedUsgsInstantaneousUrl(string url) + { + return url.StartsWith("https://waterservices.usgs.gov/nwis/iv/?", StringComparison.OrdinalIgnoreCase) && + url.Contains("sites=01646500", StringComparison.OrdinalIgnoreCase) && + url.Contains($"parameterCd={_parameterCode}", StringComparison.OrdinalIgnoreCase) && + url.Contains("startDT=1900-01-01", StringComparison.OrdinalIgnoreCase) && + url.Contains("siteStatus=all", StringComparison.OrdinalIgnoreCase) && + url.Contains("format=rdb", StringComparison.OrdinalIgnoreCase); + } + } + + /// + /// Synchronization context that records posted continuations and executes them on the thread pool. + /// + /// + /// Tests use this context to prove downloader internals do not marshal long-running parsing + /// continuations back to a WPF-like caller context. + /// + private sealed class CountingSynchronizationContext : SynchronizationContext + { + /// + /// Count of posted callbacks. + /// + private int _postCount; + + /// + /// Gets the number of callbacks posted to this context. + /// + internal int PostCount => _postCount; + + /// + /// Records and dispatches an asynchronous callback. + /// + /// The callback to execute. + /// The callback state. + /// + /// The callback is still executed so tests do not deadlock if a regression posts to the + /// context; the post count then fails the assertion. + /// + public override void Post(SendOrPostCallback d, object state) + { + Interlocked.Increment(ref _postCount); + ThreadPool.QueueUserWorkItem(_ => d(state)); + } + } + /// /// HTTP handler that serves a deterministic USGS measured-stage response with duplicate timestamps. /// @@ -352,6 +551,182 @@ protected override async Task SendAsync(HttpRequestMessage } } + /// + /// HTTP handler that serves deterministic GHCN daily station-file responses. + /// + /// + /// The handler allows only the expected NOAA NCEI USC00040741.dly station-file URL + /// and rejects generic connectivity probes. This keeps GHCN regression tests deterministic. + /// + private sealed class GuardedGhcnDailyHandler : HttpMessageHandler + { + /// + /// Whether the handler should simulate an HTTP timeout after URL validation. + /// + private readonly bool _simulateTimeout; + + /// + /// Whether the handler should include a duplicate precipitation row for the same station month. + /// + private readonly bool _includeDuplicatePrecipitationRow; + + /// + /// Gets the absolute URLs requested through this handler. + /// + internal List Requests { get; } = new List(); + + /// + /// Initializes a new instance of the class. + /// + /// Whether to throw a timeout after validating the request URL. + /// Whether to include duplicate daily precipitation records for the same dates. + /// + /// Timeout simulation is used to verify that the downloader reports the GHCN-specific + /// timeout window without waiting for the full timeout duration. Duplicate-row simulation + /// verifies inline last-value-wins normalization without a post-download pass. + /// + internal GuardedGhcnDailyHandler( + bool simulateTimeout = false, + bool includeDuplicatePrecipitationRow = false) + { + _simulateTimeout = simulateTimeout; + _includeDuplicatePrecipitationRow = includeDuplicatePrecipitationRow; + } + + /// + /// + /// Returns a one-month GHCN daily file containing precipitation and snow rows only for + /// the expected station-file URL. + /// + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + string url = request.RequestUri?.AbsoluteUri ?? ""; + Requests.Add(url); + + if (IsBlockedPreflight(url)) + { + return Task.FromException( + new HttpRequestException($"Unexpected connectivity preflight request: {url}")); + } + + if (!IsExpectedGhcnStationUrl(url)) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"Unexpected URL: {url}") + }); + } + + if (_simulateTimeout) + { + return Task.FromException( + new TaskCanceledException($"Simulated GHCN timeout: {url}")); + } + + var rows = new List + { + BuildGhcnDailyLine("USC00040741", 2024, 1, "PRCP", new Dictionary + { + [1] = 254, + [2] = -9999, + [3] = 508 + }), + BuildGhcnDailyLine("USC00040741", 2024, 1, "SNOW", new Dictionary + { + [1] = 0 + }) + }; + + if (_includeDuplicatePrecipitationRow) + { + rows.Add(BuildGhcnDailyLine("USC00040741", 2024, 1, "PRCP", new Dictionary + { + [1] = 762, + [2] = 1016, + [3] = 1270 + })); + } + + string body = string.Join(Environment.NewLine, rows); + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(body) + }); + } + + /// + /// Determines whether a URL is one of the connectivity probes this regression forbids. + /// + /// The absolute URL requested through the handler. + /// true when the URL is a forbidden preflight; otherwise, false. + /// + /// GHCN daily data should request the station file directly; generic root checks are not + /// required before attempting the provider request. + /// + internal static bool IsBlockedPreflight(string url) + { + string normalized = url.TrimEnd('/'); + return url.Contains("google", StringComparison.OrdinalIgnoreCase) || + url.Contains("generate_204", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "https://www.ncei.noaa.gov", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "https://www.ncei.noaa.gov/pub/data/ghcn/daily/all", StringComparison.OrdinalIgnoreCase); + } + + /// + /// Determines whether a URL matches the expected GHCN station-file request. + /// + /// The absolute URL requested through the handler. + /// true when the URL targets the expected fixture endpoint; otherwise, false. + /// + /// The check pins the request to the NOAA NCEI daily station-file path and the exact + /// station that failed in BestFit. + /// + private static bool IsExpectedGhcnStationUrl(string url) + { + return string.Equals( + url, + "https://www.ncei.noaa.gov/pub/data/ghcn/daily/all/USC00040741.dly", + StringComparison.OrdinalIgnoreCase); + } + + /// + /// Builds one fixed-width GHCN daily station-file line. + /// + /// The 11-character GHCN station identifier. + /// The four-digit year. + /// The one-based month number. + /// The four-character GHCN element code. + /// Raw GHCN values keyed by one-based day of month. + /// A fixed-width daily line in GHCN .dly format. + /// + /// Each daily value is represented by a five-character integer followed by three flag + /// characters. Missing days are filled with -9999, matching the provider format. + /// + private static string BuildGhcnDailyLine( + string stationId, + int year, + int month, + string element, + IReadOnlyDictionary valuesByDay) + { + var builder = new System.Text.StringBuilder(); + builder.Append(stationId); + builder.Append(year.ToString("0000", CultureInfo.InvariantCulture)); + builder.Append(month.ToString("00", CultureInfo.InvariantCulture)); + builder.Append(element); + + for (int day = 1; day <= 31; day++) + { + int value = valuesByDay.TryGetValue(day, out int parsedValue) ? parsedValue : -9999; + builder.Append(value.ToString(CultureInfo.InvariantCulture).PadLeft(5)); + builder.Append(" "); + } + + return builder.ToString(); + } + } + /// /// Probes a small windowed CHMN download to confirm Water Survey of Canada services are responsive. /// @@ -703,6 +1078,215 @@ public async Task USGS_MeasuredStage_DuplicateDateTimes_LastValueWins() } } + /// + /// Verifies deterministic USGS instantaneous discharge downloads remain irregular. + /// + /// A task that completes when the regression check finishes. + /// + /// USGS instantaneous discharge uses parameter 00060 on the /nwis/iv endpoint. + /// + [TestMethod] + public async Task USGS_InstantaneousDischarge_DeterministicDownload_IsIrregular() + { + await VerifyUsgsInstantaneousDownloadIsIrregular( + TimeSeriesDownload.TimeSeriesType.InstantaneousDischarge, + "00060"); + } + + /// + /// Verifies deterministic USGS instantaneous stage downloads remain irregular. + /// + /// A task that completes when the regression check finishes. + /// + /// USGS instantaneous stage uses parameter 00065 on the /nwis/iv endpoint. + /// + [TestMethod] + public async Task USGS_InstantaneousStage_DeterministicDownload_IsIrregular() + { + await VerifyUsgsInstantaneousDownloadIsIrregular( + TimeSeriesDownload.TimeSeriesType.InstantaneousStage, + "00065"); + } + + /// + /// Verifies instantaneous USGS timeout messages use the longer instantaneous limit. + /// + /// A task that completes when the regression check finishes. + /// + /// The handler throws a timeout immediately after URL validation so the test does not wait + /// for the full timeout duration. + /// + [TestMethod] + public async Task USGS_InstantaneousDischarge_TimeoutMessage_UsesInstantaneousLimit() + { + var defaultHandler = new GuardedUsgsInstantaneousHandler("00060", simulateTimeout: true); + var decompressHandler = new GuardedUsgsInstantaneousHandler("00060", simulateTimeout: true); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var exception = await Assert.ThrowsAsync(async () => + await TimeSeriesDownload.FromUSGS(USGS_6, TimeSeriesDownload.TimeSeriesType.InstantaneousDischarge)); + + Assert.Contains("120 seconds", exception.Message); + Assert.IsEmpty(defaultHandler.Requests, "Instantaneous download should not use the default client."); + Assert.HasCount(2, decompressHandler.Requests, "Expected the retry helper to make two instantaneous attempts."); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies USGS instantaneous parsing does not resume on the caller synchronization context. + /// + /// A task that completes when the regression check finishes. + /// + /// BestFit starts downloads from the WPF UI thread. This test forces the fake HTTP response + /// to complete asynchronously while a synchronization context is installed, then verifies + /// the downloader did not post its large-response continuation back to that context. + /// + [TestMethod] + public async Task USGS_InstantaneousDischarge_DoesNotPostBackToSynchronizationContext() + { + var defaultHandler = new GuardedUsgsInstantaneousHandler("00060", completeAsynchronously: true); + var decompressHandler = new GuardedUsgsInstantaneousHandler("00060", completeAsynchronously: true); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var context = new CountingSynchronizationContext(); + SynchronizationContext previousContext = SynchronizationContext.Current; + Task<(TimeSeries TimeSeries, string RawText)> downloadTask; + + try + { + SynchronizationContext.SetSynchronizationContext(context); + downloadTask = TimeSeriesDownload.FromUSGS( + USGS_6, + TimeSeriesDownload.TimeSeriesType.InstantaneousDischarge); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previousContext); + } + + var (ts, raw) = await downloadTask; + + Assert.AreEqual(0, context.PostCount, "USGS instantaneous download should not post continuations to the caller synchronization context."); + Assert.AreEqual(TimeInterval.Irregular, ts.TimeInterval); + Assert.AreEqual(3, ts.Count); + Assert.IsFalse(string.IsNullOrWhiteSpace(raw), "Raw text should contain the fake USGS response."); + Assert.IsEmpty(defaultHandler.Requests, "Instantaneous download should not use the default client."); + Assert.HasCount(1, decompressHandler.Requests, "Expected exactly one USGS instantaneous data request."); + Assert.IsFalse(decompressHandler.Requests.Any(GuardedUsgsInstantaneousHandler.IsBlockedPreflight)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies duplicate-heavy USGS instantaneous responses are normalized without a long rebuild. + /// + /// A task that completes when the regression check finishes. + /// + /// The fixture mimics provisional/approved duplicate timestamps in instantaneous RDB data. + /// A previous cleanup path cleared the million-row series through per-item removal, which + /// made full-period BestFit downloads appear to spin indefinitely after the response arrived. + /// + [TestMethod] + [Timeout(15000, CooperativeCancellation = true)] + public async Task USGS_InstantaneousDischarge_DuplicateHeavyResponse_NormalizesQuickly() + { + const int duplicateTimestampPairs = 50000; + var defaultHandler = new GuardedUsgsInstantaneousHandler("00060", duplicateTimestampPairs: duplicateTimestampPairs); + var decompressHandler = new GuardedUsgsInstantaneousHandler("00060", duplicateTimestampPairs: duplicateTimestampPairs); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var (ts, raw) = await TimeSeriesDownload.FromUSGS( + USGS_6, + TimeSeriesDownload.TimeSeriesType.InstantaneousDischarge); + var finalDuplicateTimestamp = new DateTime(2024, 1, 2).AddMinutes(duplicateTimestampPairs - 1); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.AreEqual(TimeInterval.Irregular, ts.TimeInterval); + Assert.AreEqual(3 + duplicateTimestampPairs, ts.Count); + Assert.AreEqual(ts.Count, ts.Select(o => o.Index).Distinct().Count(), "Duplicate date indices detected."); + Assert.AreEqual(2000d + duplicateTimestampPairs - 1, ts.Single(o => o.Index == finalDuplicateTimestamp).Value); + Assert.IsFalse(string.IsNullOrWhiteSpace(raw), "Raw text should contain the fake USGS response."); + Assert.IsEmpty(defaultHandler.Requests, "Instantaneous download should not use the default client."); + Assert.HasCount(1, decompressHandler.Requests, "Expected exactly one USGS instantaneous data request."); + Assert.IsFalse(decompressHandler.Requests.Any(GuardedUsgsInstantaneousHandler.IsBlockedPreflight)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies a deterministic USGS instantaneous download preserves nonuniform timestamps. + /// + /// The instantaneous USGS series type to request. + /// The expected USGS parameter code. + /// A task that completes when the regression check finishes. + /// + /// The fixture has a 15-minute step followed by a 30-minute step. The downloader must keep + /// those timestamps exactly and must not convert the series into a regular interval. + /// + private static async Task VerifyUsgsInstantaneousDownloadIsIrregular( + TimeSeriesDownload.TimeSeriesType seriesType, + string parameterCode) + { + var defaultHandler = new GuardedUsgsInstantaneousHandler(parameterCode); + var decompressHandler = new GuardedUsgsInstantaneousHandler(parameterCode); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var (ts, raw) = await TimeSeriesDownload.FromUSGS(USGS_6, seriesType); + var expectedDates = new[] + { + new DateTime(2024, 1, 1, 0, 0, 0), + new DateTime(2024, 1, 1, 0, 15, 0), + new DateTime(2024, 1, 1, 0, 45, 0) + }; + + Assert.IsNotNull(ts, "Time series is null."); + Assert.AreEqual(TimeInterval.Irregular, ts.TimeInterval); + Assert.AreEqual(expectedDates.Length, ts.Count); + CollectionAssert.AreEqual(expectedDates, ts.Select(o => o.Index).ToArray()); + Assert.IsFalse(string.IsNullOrWhiteSpace(raw), "Raw text should contain the fake USGS response."); + Assert.IsEmpty(defaultHandler.Requests, "Instantaneous download should not use the default client."); + Assert.HasCount(1, decompressHandler.Requests, "Expected exactly one USGS instantaneous data request."); + Assert.IsTrue(decompressHandler.Requests[0].Contains("/nwis/iv/", StringComparison.OrdinalIgnoreCase)); + Assert.IsTrue(decompressHandler.Requests[0].Contains($"parameterCd={parameterCode}", StringComparison.OrdinalIgnoreCase)); + Assert.IsTrue(decompressHandler.Requests[0].Contains("startDT=1900-01-01", StringComparison.OrdinalIgnoreCase)); + Assert.IsFalse(decompressHandler.Requests.Any(GuardedUsgsInstantaneousHandler.IsBlockedPreflight)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + /// /// Tests full-period-of-record USGS daily discharge download. /// @@ -859,6 +1443,127 @@ public async Task USGS_InstantaneousStage_Works() #region GHCN Tests + /// + /// Verifies deterministic GHCN daily precipitation downloads parse station files directly. + /// + /// A task that completes when the regression check finishes. + /// + /// The fixture pins the request to NOAA NCEI station USC00040741, confirms no + /// connectivity preflight is made, and verifies GHCN tenths-of-millimeters are converted to + /// inches. + /// + [TestMethod] + public async Task GHCN_DailyPrecipitation_DeterministicDownload_ParsesAndNoPreflight() + { + var defaultHandler = new GuardedGhcnDailyHandler(); + var decompressHandler = new GuardedGhcnDailyHandler(); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var ts = await TimeSeriesDownload.FromGHCN( + GHCN_1, + TimeSeriesDownload.TimeSeriesType.DailyPrecipitation, + TimeSeriesDownload.DepthUnit.Inches); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.AreEqual(TimeInterval.OneDay, ts.TimeInterval); + Assert.AreEqual(31, ts.Count); + Assert.AreEqual(new DateTime(2024, 1, 1), ts.StartDate); + Assert.AreEqual(new DateTime(2024, 1, 31), ts.EndDate); + Assert.AreEqual(1d, ts[0].Value); + Assert.IsTrue(double.IsNaN(ts[1].Value)); + Assert.AreEqual(2d, ts[2].Value); + Assert.HasCount(1, defaultHandler.Requests, "Expected exactly one GHCN station-file request."); + Assert.IsEmpty(decompressHandler.Requests, "GHCN should not use the compressed client."); + Assert.IsFalse(defaultHandler.Requests.Any(GuardedGhcnDailyHandler.IsBlockedPreflight)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies GHCN duplicate daily timestamps are replaced while station rows are parsed. + /// + /// A task that completes when the regression check finishes. + /// + /// This pins the intended single-pass behavior: the later GHCN row for a duplicate date wins + /// at ordinate creation time, avoiding the previous post-download duplicate scan. + /// + [TestMethod] + public async Task GHCN_DailyPrecipitation_DuplicateDateTimes_LastValueWinsInline() + { + var defaultHandler = new GuardedGhcnDailyHandler(includeDuplicatePrecipitationRow: true); + var decompressHandler = new GuardedGhcnDailyHandler(includeDuplicatePrecipitationRow: true); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var ts = await TimeSeriesDownload.FromGHCN( + GHCN_1, + TimeSeriesDownload.TimeSeriesType.DailyPrecipitation, + TimeSeriesDownload.DepthUnit.Inches); + + Assert.AreEqual(31, ts.Count); + Assert.AreEqual(ts.Count, ts.Select(o => o.Index).Distinct().Count(), "Duplicate date indices detected."); + Assert.AreEqual(3d, ts[0].Value); + Assert.AreEqual(4d, ts[1].Value); + Assert.AreEqual(5d, ts[2].Value); + Assert.HasCount(1, defaultHandler.Requests, "Expected exactly one GHCN station-file request."); + Assert.IsEmpty(decompressHandler.Requests, "GHCN should not use the compressed client."); + Assert.IsFalse(defaultHandler.Requests.Any(GuardedGhcnDailyHandler.IsBlockedPreflight)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies GHCN timeout messages use the longer GHCN provider limit. + /// + /// A task that completes when the regression check finishes. + /// + /// NOAA NCEI can take longer than the generic provider timeout to return station-file + /// headers. The handler throws a timeout immediately after URL validation so this test does + /// not wait for the full provider timeout. + /// + [TestMethod] + public async Task GHCN_DailyPrecipitation_TimeoutMessage_UsesGhcnLimit() + { + var defaultHandler = new GuardedGhcnDailyHandler(simulateTimeout: true); + var decompressHandler = new GuardedGhcnDailyHandler(simulateTimeout: true); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var exception = await Assert.ThrowsAsync(async () => + await TimeSeriesDownload.FromGHCN( + GHCN_1, + TimeSeriesDownload.TimeSeriesType.DailyPrecipitation)); + + Assert.Contains("240 seconds", exception.Message); + Assert.HasCount(2, defaultHandler.Requests, "Expected the retry helper to make two GHCN attempts."); + Assert.IsEmpty(decompressHandler.Requests, "GHCN should not use the compressed client."); + Assert.IsFalse(defaultHandler.Requests.Any(GuardedGhcnDailyHandler.IsBlockedPreflight)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + /// /// Tests full-period-of-record GHCN daily precipitation download. /// From 41f78b473678f541b0c01dfc32e3a2832f8cd1f2 Mon Sep 17 00:00:00 2001 From: HadenSmith Date: Thu, 9 Jul 2026 13:46:20 -0600 Subject: [PATCH 4/6] Add RunningStatistics.Clone() and fix Combine aliasing with empty operands Combine(a, b) returned the non-empty operand instance directly when the other operand was empty, so the result shared state with its input. Combine and the + operator now always return a new independent instance. Added unit tests proving Clone independence and that Combine no longer aliases its operands. --- Numerics/Data/Statistics/RunningStatistics.cs | 25 +++++- .../Data/Statistics/Test_RunningStatistics.cs | 82 +++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/Numerics/Data/Statistics/RunningStatistics.cs b/Numerics/Data/Statistics/RunningStatistics.cs index 4c82bc3f..bed45556 100644 --- a/Numerics/Data/Statistics/RunningStatistics.cs +++ b/Numerics/Data/Statistics/RunningStatistics.cs @@ -273,18 +273,39 @@ public void Push(IList values) Push(value); } + /// + /// Creates a copy of the running statistics. + /// + /// A new running statistics instance with the same accumulated state, independent of the original. + public RunningStatistics Clone() + { + return new RunningStatistics() + { + _n = _n, + _m1 = _m1, + _m2 = _m2, + _m3 = _m3, + _m4 = _m4, + _min = _min, + _max = _max + }; + } + /// /// Create a new running statistics over the combined samples of two existing running statistics. /// + /// The first running statistics. + /// The second running statistics. + /// A new running statistics instance, independent of both inputs. public static RunningStatistics Combine(RunningStatistics a, RunningStatistics b) { if (a._n == 0L) { - return b; + return b.Clone(); } else if (b._n == 0L) { - return a; + return a.Clone(); } long n = a._n + b._n; diff --git a/Test_Numerics/Data/Statistics/Test_RunningStatistics.cs b/Test_Numerics/Data/Statistics/Test_RunningStatistics.cs index 61735579..5ba52f6f 100644 --- a/Test_Numerics/Data/Statistics/Test_RunningStatistics.cs +++ b/Test_Numerics/Data/Statistics/Test_RunningStatistics.cs @@ -285,5 +285,87 @@ public void Test_Add() var kurtosis = complete.Kurtosis; Assert.AreEqual(1.3434868130194, kurtosis, 1E-10); } + + /// + /// Test the clone method of the running statistics class. The clone should match the original statistics + /// and remain independent of the original after either instance is updated. + /// + [TestMethod] + public void Test_Clone() + { + var values = new double[] { 1, 2, 3, 4, 5 }; + var runningstat = new RunningStatistics(values); + var clone = runningstat.Clone(); + + // The clone should be a new instance that matches the original statistics. + Assert.AreNotSame(runningstat, clone); + Assert.AreEqual(5, clone.Count); + Assert.AreEqual(1, clone.Minimum); + Assert.AreEqual(5, clone.Maximum); + Assert.AreEqual(3, clone.Mean, 1E-10); + Assert.AreEqual(2.5, clone.Variance, 1E-10); + + // Pushing to the original should not change the clone. + runningstat.Push(new double[] { -10, 30 }); + Assert.AreEqual(5, clone.Count); + Assert.AreEqual(1, clone.Minimum); + Assert.AreEqual(5, clone.Maximum); + Assert.AreEqual(3, clone.Mean, 1E-10); + Assert.AreEqual(2.5, clone.Variance, 1E-10); + + // Pushing to the clone should not change the original. + var originalCount = runningstat.Count; + var originalMean = runningstat.Mean; + clone.Push(100); + Assert.AreEqual(originalCount, runningstat.Count); + Assert.AreEqual(originalMean, runningstat.Mean, 1E-10); + } + + /// + /// Test that combining with an empty running statistics returns a new instance + /// rather than aliasing the non-empty operand. + /// + [TestMethod] + public void Test_Combine_WithEmpty() + { + var values = new double[] { 1, 2, 3, 4, 5 }; + var stats = new RunningStatistics(values); + + // Empty first operand. The result should match the non-empty operand. + var combined = RunningStatistics.Combine(new RunningStatistics(), stats); + Assert.AreNotSame(stats, combined); + Assert.AreEqual(5, combined.Count); + Assert.AreEqual(1, combined.Minimum); + Assert.AreEqual(5, combined.Maximum); + Assert.AreEqual(3, combined.Mean, 1E-10); + Assert.AreEqual(2.5, combined.Variance, 1E-10); + + // Mutating the result should not mutate the operand. + combined.Push(1000); + Assert.AreEqual(5, stats.Count); + Assert.AreEqual(3, stats.Mean, 1E-10); + + // Empty second operand. + var combined2 = RunningStatistics.Combine(stats, new RunningStatistics()); + Assert.AreNotSame(stats, combined2); + combined2.Push(1000); + Assert.AreEqual(5, stats.Count); + Assert.AreEqual(3, stats.Mean, 1E-10); + + // The + operator should behave the same way. + var combined3 = new RunningStatistics() + stats; + Assert.AreNotSame(stats, combined3); + combined3.Push(1000); + Assert.AreEqual(5, stats.Count); + Assert.AreEqual(3, stats.Mean, 1E-10); + + // Combining two empty instances should return a new empty instance. + var empty1 = new RunningStatistics(); + var empty2 = new RunningStatistics(); + var combinedEmpty = RunningStatistics.Combine(empty1, empty2); + Assert.AreNotSame(empty1, combinedEmpty); + Assert.AreNotSame(empty2, combinedEmpty); + Assert.AreEqual(0, combinedEmpty.Count); + } } } From eabcce115055243546dcb6fbf9365441fc924964 Mon Sep 17 00:00:00 2001 From: HadenSmith Date: Thu, 9 Jul 2026 14:34:00 -0600 Subject: [PATCH 5/6] Prefer IPv4 connections to fix 10-100x time series download slowdown GHCN downloads had slowed from about 2 seconds to over 2 minutes, and commit bf0f28d had raised the GHCN per-attempt timeout to 240 seconds on the belief that NOAA NCEI takes that long to send response headers. Measurement showed the server is not slow: www.ncei.noaa.gov now publishes six IPv6 (AAAA) DNS records, and on networks that silently drop IPv6 traffic every HttpClient connection walked all six dead addresses sequentially (about 21 seconds of TCP timeout each, 126 seconds measured on both net481 and net10) before reaching a working IPv4 address, while curl fetched the same station file in under 2 seconds. api.waterdata.usgs.gov carries two AAAA records (about 42 seconds); the other providers have none, which is why GHCN was hit worst. The fix makes every downloader connection prefer IPv4 while keeping IPv6 as a fallback. On modern targets a SocketsHttpHandler.ConnectCallback resolves DNS, orders IPv4 first, and bounds each address attempt at 5 seconds. On .NET Framework, where the handler offers no connect hook, a ServicePoint BindIPEndPointDelegate throws for IPv6 remote endpoints so the service point skips immediately to IPv4; this is guarded by Socket.OSSupportsIPv4 so IPv6-only machines keep working. Cold-connection time to first byte drops from 126 seconds to under half a second on both stacks. With connects fixed, the GHCN timeout returns to 60 seconds and its remark no longer blames NCEI. Also included: BOM reuses one static browser-headered client instead of building two clients per call, so the station lookup and values request share pooled connections; Series.Clear() no longer removes elements one at a time through two full equality scans each (O(n squared) on a 50k-point download, the parse-side cost behind f522295's dedup pass); and Series.RemoveAt(index) now removes the requested index instead of the first equal element, which silently deleted the wrong ordinate when duplicate values existed. Verified with 1789 deterministic tests and 31 live integration tests passing on both net481 and net10.0, plus new unit tests for the address ordering, the bind delegate, and the Series contracts. The GHCN full-period integration tests now carry 60-second ceilings so a connect regression fails loudly instead of quietly taking minutes. --- Numerics/Data/Time Series/Support/Series.cs | 31 +-- .../Time Series/Support/TimeSeriesDownload.cs | 215 ++++++++++++++---- .../Data/Time Series/Test_TimeSeries.cs | 77 +++++++ .../Time Series/Test_TimeSeriesDownload.cs | 101 +++++++- 4 files changed, 367 insertions(+), 57 deletions(-) diff --git a/Numerics/Data/Time Series/Support/Series.cs b/Numerics/Data/Time Series/Support/Series.cs index 8bdf9d75..0065b407 100644 --- a/Numerics/Data/Time Series/Support/Series.cs +++ b/Numerics/Data/Time Series/Support/Series.cs @@ -125,12 +125,10 @@ public void Insert(int index, object? item) public virtual bool Remove(SeriesOrdinate item) { var index = IndexOf(item); - if (_seriesOrdinates.Remove(item) == true) - { - RaiseCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item, index)); - return true; - } - return false; + if (index < 0) return false; + _seriesOrdinates.RemoveAt(index); + RaiseCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item, index)); + return true; } /// @@ -144,23 +142,30 @@ public void Remove(object? item) } /// + /// + /// Removes the ordinate at the requested position directly, so when equal-valued + /// ordinates exist elsewhere in the series the element at is + /// the one removed and reported. + /// public virtual void RemoveAt(int index) { - Remove(_seriesOrdinates[index]); + var item = _seriesOrdinates[index]; + _seriesOrdinates.RemoveAt(index); + RaiseCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item, index)); } /// /// Remove all elements from the collection. /// + /// + /// Clears the backing list in one operation and raises a single reset event. The former + /// element-by-element removal cost two full equality scans per element, which made + /// clearing a large series O(n²). + /// public void Clear() { - bool suppress = SuppressCollectionChanged; - SuppressCollectionChanged = true; - for (int i = _seriesOrdinates.Count - 1; i >= 0; i--) - Remove(_seriesOrdinates[i]); - SuppressCollectionChanged = false; + _seriesOrdinates.Clear(); RaiseCollectionChangedReset(); - SuppressCollectionChanged = suppress; } /// diff --git a/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs b/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs index b5063509..f5f4f503 100644 --- a/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs +++ b/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using System.Net; +using System.Net.Sockets; using System.Xml.Linq; using System; using System.Text.RegularExpressions; @@ -28,6 +29,7 @@ public class TimeSeriesDownload { private static HttpClient _defaultClient = CreateDefaultClient(); private static HttpClient _decompressClient = CreateDecompressClient(); + private static readonly HttpClient _bomClient = CreateBomClient(); private const string UserAgent = "USACE-Numerics/2.0"; @@ -53,10 +55,12 @@ public class TimeSeriesDownload /// Maximum time allowed for one GHCN station-file request attempt. /// /// - /// NOAA NCEI can take more than two minutes to send response headers for some full station - /// files even though the resulting payload is valid and small enough to parse quickly. + /// NCEI serves full station files in seconds over IPv4. The historical multi-minute + /// header waits were sequential TCP timeouts against NCEI's six IPv6 addresses on + /// networks that drop IPv6 traffic, which the downloader now avoids by preferring IPv4. + /// This margin only covers genuinely slow links and large station files. /// - private static readonly TimeSpan GhcnProviderRequestTimeout = TimeSpan.FromSeconds(240); + private static readonly TimeSpan GhcnProviderRequestTimeout = TimeSpan.FromSeconds(60); /// /// Maximum time allowed for one explicit connectivity probe. @@ -91,6 +95,142 @@ public class TimeSeriesDownload "https://www.bom.gov.au/waterdata/" }; +#if !NETFRAMEWORK + /// + /// Maximum time allowed for one TCP connect attempt to a single resolved address. + /// + /// + /// Bounding each address separately keeps one unreachable address family from consuming + /// the whole request timeout while still letting later addresses in the list be tried. + /// + private static readonly TimeSpan ConnectAttemptTimeout = TimeSpan.FromSeconds(5); +#endif + + /// + /// Creates the HTTP handler used by the downloader's shared clients. + /// + /// True to enable automatic GZip and Deflate response decompression. + /// A configured HTTP handler. + /// + /// Some providers publish IPv6 (AAAA) DNS records while many restricted networks silently + /// drop IPv6 traffic. A default handler tries every unreachable IPv6 address before + /// falling back to IPv4 (~21 s of TCP timeout per address; measured at ~126 s to first + /// byte for NOAA NCEI's six AAAA records). On modern targets the handler connects with + /// ConnectPreferringIPv4Async so IPv4 is tried first and IPv6 remains a + /// fallback. On .NET Framework the equivalent ordering is applied per request through + /// . + /// + private static HttpMessageHandler CreateHandler(bool automaticDecompression) + { +#if NETFRAMEWORK + var handler = new HttpClientHandler(); +#else + var handler = new SocketsHttpHandler { ConnectCallback = ConnectPreferringIPv4Async }; +#endif + if (automaticDecompression) + { + handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; + } + return handler; + } + +#if !NETFRAMEWORK + /// + /// Opens a TCP connection for an HTTP request, preferring IPv4 addresses over IPv6. + /// + /// The connection context supplied by the HTTP handler. + /// Token used to cancel the connection attempt. + /// A connected network stream. + /// Thrown when no resolved address accepts a connection. + /// Thrown when is canceled. + /// + /// The default connect logic tries DNS results in returned order, which lists IPv6 first. + /// On networks that silently drop IPv6 traffic every dead address costs a full TCP + /// timeout before IPv4 is reached. Trying IPv4 first keeps downloads fast there while the + /// IPv6 fallback keeps IPv6-only networks working. + /// + private static async ValueTask ConnectPreferringIPv4Async(SocketsHttpConnectionContext context, CancellationToken cancellationToken) + { + IPAddress[] addresses = await Dns.GetHostAddressesAsync(context.DnsEndPoint.Host, cancellationToken).ConfigureAwait(false); + IPAddress[] ordered = OrderAddressesForConnect(addresses); + Exception? lastFailure = null; + foreach (IPAddress address in ordered) + { + var socket = new Socket(address.AddressFamily, SocketType.Stream, ProtocolType.Tcp) { NoDelay = true }; + try + { + using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + attemptCts.CancelAfter(ConnectAttemptTimeout); + await socket.ConnectAsync(new IPEndPoint(address, context.DnsEndPoint.Port), attemptCts.Token).ConfigureAwait(false); + return new NetworkStream(socket, ownsSocket: true); + } + catch (Exception ex) + { + socket.Dispose(); + cancellationToken.ThrowIfCancellationRequested(); + lastFailure = ex; + } + } + + throw new HttpRequestException( + $"Could not connect to '{context.DnsEndPoint.Host}:{context.DnsEndPoint.Port}' on any resolved address.", + lastFailure); + } +#endif + + /// + /// Orders resolved addresses so IPv4 is attempted before IPv6. + /// + /// The addresses returned by DNS resolution. + /// The addresses with IPv4 first; relative order within each family is preserved. + /// Thrown when is null. + internal static IPAddress[] OrderAddressesForConnect(IPAddress[] addresses) + { + if (addresses == null) throw new ArgumentNullException(nameof(addresses)); + return addresses.OrderBy(address => address.AddressFamily == AddressFamily.InterNetworkV6 ? 1 : 0).ToArray(); + } + + /// + /// Local-endpoint binder that rejects IPv6 remote endpoints so connects fall through to IPv4. + /// + /// The remote endpoint the connection is being bound for. + /// Null for IPv4 remote endpoints, letting the OS choose the local endpoint. + /// Thrown when is null. + /// Thrown for IPv6 remote endpoints to make the connection attempt skip to the next resolved address. + /// + /// On .NET Framework a tries resolved addresses in DNS order + /// (IPv6 first) with no per-address bound, so blocked IPv6 routes cost a full TCP timeout + /// per address. Throwing from the bind delegate fails an IPv6 attempt immediately and the + /// service point moves on to the next (IPv4) address. + /// + internal static IPEndPoint? RejectIPv6BindEndPoint(IPEndPoint remoteEndPoint) + { + if (remoteEndPoint == null) throw new ArgumentNullException(nameof(remoteEndPoint)); + if (remoteEndPoint.AddressFamily == AddressFamily.InterNetworkV6) + { + throw new SocketException((int)SocketError.AddressFamilyNotSupported); + } + return null; + } + + /// + /// Configures the .NET Framework service point for a request URI to prefer IPv4 connections. + /// + /// The absolute request URI about to be downloaded. + /// + /// No-op on modern targets, where installs an IPv4-preferring + /// connect callback instead. Skipped when the OS has no IPv4 stack so IPv6-only machines + /// keep working. + /// + private static void PreferIPv4ForNetFramework(Uri requestUri) + { +#if NETFRAMEWORK + if (!Socket.OSSupportsIPv4) return; + ServicePoint servicePoint = ServicePointManager.FindServicePoint(requestUri); + servicePoint.BindIPEndPointDelegate = (servicePointArg, remoteEndPoint, retryCount) => RejectIPv6BindEndPoint(remoteEndPoint); +#endif + } + /// /// Creates the default HTTP client used for uncompressed provider requests. /// @@ -102,7 +242,7 @@ public class TimeSeriesDownload /// private static HttpClient CreateDefaultClient() { - var client = new HttpClient() { Timeout = Timeout.InfiniteTimeSpan }; + var client = new HttpClient(CreateHandler(automaticDecompression: false)) { Timeout = Timeout.InfiniteTimeSpan }; client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); return client; } @@ -117,15 +257,33 @@ private static HttpClient CreateDefaultClient() /// private static HttpClient CreateDecompressClient() { - var client = new HttpClient(new HttpClientHandler - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate - }) - { Timeout = Timeout.InfiniteTimeSpan }; + var client = new HttpClient(CreateHandler(automaticDecompression: true)) { Timeout = Timeout.InfiniteTimeSpan }; client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); return client; } + /// + /// Creates the HTTP client used for BOM KiWIS requests. + /// + /// A configured HTTP client with browser-like default headers. + /// + /// BOM sits behind a security appliance that rejects bare programmatic requests, so the + /// client sends browser-like headers. The client is held as a static instance so the + /// station lookup and value download in one call, and repeated calls, reuse pooled + /// connections instead of paying a fresh TCP and TLS handshake each time. The + /// Accept-Encoding header is supplied by the handler's automatic decompression. + /// + private static HttpClient CreateBomClient() + { + var client = new HttpClient(CreateHandler(automaticDecompression: true)) { Timeout = Timeout.InfiniteTimeSpan }; + client.DefaultRequestHeaders.Add("User-Agent", UserAgent); + client.DefaultRequestHeaders.Add("Accept", "application/json,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); + client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9"); + client.DefaultRequestHeaders.Add("DNT", "1"); + client.DefaultRequestHeaders.Add("Connection", "keep-alive"); + return client; + } + /// /// Replaces the static HTTP clients used by the downloader for deterministic tests. /// @@ -179,6 +337,7 @@ internal static void ResetHttpClientsForTesting() CancellationToken cancellationToken = default) { TimeSpan timeout = requestTimeout ?? ProviderRequestTimeout; + PreferIPv4ForNetFramework(new Uri(url)); Exception? lastNet = null; for (int attempt = 1; attempt <= maxAttempts; attempt++) { @@ -322,6 +481,7 @@ private static async Task CanReachEndpoint(string url, CancellationToken c { try { + PreferIPv4ForNetFramework(new Uri(url)); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); cts.CancelAfter(ConnectivityProbeTimeout); using (await _defaultClient.GetAsync( @@ -506,6 +666,7 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType string stationFileUrl = $"{ghcnBaseUrl}{Uri.EscapeDataString(siteNumber)}.dly"; { var client = _defaultClient; + PreferIPv4ForNetFramework(new Uri(stationFileUrl)); // GHCN is large enough to want streaming, so we can't use GetWithRetryAsync // directly (it buffers the whole body). Inline the same retry shape on the // headers-only GetAsync, then stream from the successful response. @@ -1389,22 +1550,10 @@ public static async Task FromABOM( string? tsId = null; - // Create HttpClientHandler with automatic decompression - var handler = new HttpClientHandler - { - AutomaticDecompression = System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate - }; - - using (var client = new HttpClient(handler)) + // The shared BOM client carries the browser-like headers the BOM security + // appliance expects and reuses pooled connections across both KiWIS requests. { - // Add browser-like headers to avoid security proxy issues - client.DefaultRequestHeaders.Add("User-Agent", UserAgent); - client.DefaultRequestHeaders.Add("Accept", "application/json,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); - client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9"); - client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate"); - client.DefaultRequestHeaders.Add("DNT", "1"); - client.DefaultRequestHeaders.Add("Connection", "keep-alive"); - client.Timeout = TimeSpan.FromSeconds(30); + var client = _bomClient; HttpStatusCode listStatus; string listResponse; @@ -1524,22 +1673,10 @@ public static async Task FromABOM( DateTime? prevDate = null; var indexesByDateTime = new Dictionary(); - // Create HttpClientHandler with automatic decompression - var handler2 = new HttpClientHandler - { - AutomaticDecompression = System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate - }; - - using (var client = new HttpClient(handler2)) + // Reuse the shared BOM client so the values request rides the connection the + // station lookup already established. { - // Add browser-like headers to avoid security proxy issues - client.DefaultRequestHeaders.Add("User-Agent", UserAgent); - client.DefaultRequestHeaders.Add("Accept", "application/json,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); - client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9"); - client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate"); - client.DefaultRequestHeaders.Add("DNT", "1"); - client.DefaultRequestHeaders.Add("Connection", "keep-alive"); - client.Timeout = Timeout.InfiniteTimeSpan; + var client = _bomClient; HttpStatusCode valuesStatus; string valuesResponse; diff --git a/Test_Numerics/Data/Time Series/Test_TimeSeries.cs b/Test_Numerics/Data/Time Series/Test_TimeSeries.cs index 7aced5d8..9a0e7b73 100644 --- a/Test_Numerics/Data/Time Series/Test_TimeSeries.cs +++ b/Test_Numerics/Data/Time Series/Test_TimeSeries.cs @@ -1474,5 +1474,82 @@ public void Test_ResampleWithBlockBootstrap_PreservesMarginalVariance() #endregion + #region Collection Behavior + + /// + /// Clear should empty the series and raise a single reset event. + /// + /// + /// Clear used to remove elements one at a time, costing two full equality scans per + /// element (O(n²) on large downloads). This locks in the single-reset contract of the + /// rewritten implementation. + /// + [TestMethod] + public void Test_Clear_EmptiesSeriesAndRaisesSingleReset() + { + var ts = new TimeSeries(TimeInterval.OneDay, new DateTime(2024, 1, 1), new double[] { 1, 2, 3, 4, 5 }); + var actions = new List(); + ts.CollectionChanged += (s, e) => actions.Add(e.Action); + + ts.Clear(); + + Assert.AreEqual(0, ts.Count); + Assert.HasCount(1, actions); + Assert.AreEqual(System.Collections.Specialized.NotifyCollectionChangedAction.Reset, actions[0]); + } + + /// + /// Clearing a large series should complete quickly. + /// + /// + /// Guards against reintroducing the element-by-element removal that made clearing a + /// century of daily data take seconds to minutes. + /// + [TestMethod] + [Timeout(5000, CooperativeCancellation = true)] + public void Test_Clear_LargeSeries_CompletesQuickly() + { + var ts = new TimeSeries(TimeInterval.OneDay, new DateTime(1900, 1, 1), new double[200000]); + + ts.Clear(); + + Assert.AreEqual(0, ts.Count); + } + + /// + /// RemoveAt should remove the ordinate at the requested index even when an equal-valued + /// ordinate appears earlier in the series. + /// + /// + /// RemoveAt used to delegate to Remove(item), which removed the first equal element, so + /// removing a duplicate by index silently deleted the wrong ordinate. + /// + [TestMethod] + public void Test_RemoveAt_WithDuplicateOrdinates_RemovesRequestedIndex() + { + var first = new SeriesOrdinate(new DateTime(2024, 1, 1), 1.0); + var middle = new SeriesOrdinate(new DateTime(2024, 1, 2), 2.0); + var duplicateOfFirst = new SeriesOrdinate(new DateTime(2024, 1, 1), 1.0); + var ts = new TimeSeries(TimeInterval.Irregular) { first, middle, duplicateOfFirst }; + + var removals = new List<(object item, int index)>(); + ts.CollectionChanged += (s, e) => + { + if (e.Action == System.Collections.Specialized.NotifyCollectionChangedAction.Remove) + removals.Add((e.OldItems?[0], e.OldStartingIndex)); + }; + + ts.RemoveAt(2); + + Assert.AreEqual(2, ts.Count); + Assert.IsTrue(ReferenceEquals(first, ts[0]), "The first ordinate should remain at index 0."); + Assert.IsTrue(ReferenceEquals(middle, ts[1]), "The middle ordinate should remain at index 1."); + Assert.HasCount(1, removals); + Assert.IsTrue(ReferenceEquals(duplicateOfFirst, removals[0].item), "The event should carry the removed ordinate."); + Assert.AreEqual(2, removals[0].index, "The event should report the requested index."); + } + + #endregion + } } diff --git a/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs b/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs index c084cb32..c66b49e1 100644 --- a/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs +++ b/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Net; using System.Net.Http; +using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -801,6 +802,86 @@ private static void AssertRoughlyEqual(double a, double b, double relTol = 1e-6, #endregion + #region Connection Handling Tests + + /// + /// Verifies connect-address ordering puts IPv4 before IPv6 while preserving relative order. + /// + /// + /// Providers such as NOAA NCEI publish several AAAA records. On networks that silently + /// drop IPv6, connecting in DNS order costs a full TCP timeout per dead address, so the + /// downloader must try IPv4 first. + /// + [TestMethod] + public void OrderAddressesForConnect_PutsIPv4First_PreservingRelativeOrder() + { + var v6First = IPAddress.Parse("2610:20:8040:2::178"); + var v4First = IPAddress.Parse("205.167.25.177"); + var v6Second = IPAddress.Parse("2610:20:8040:2::177"); + var v4Second = IPAddress.Parse("205.167.25.178"); + + var ordered = TimeSeriesDownload.OrderAddressesForConnect( + new[] { v6First, v4First, v6Second, v4Second }); + + CollectionAssert.AreEqual(new[] { v4First, v4Second, v6First, v6Second }, ordered); + } + + /// + /// Verifies connect-address ordering passes an all-IPv6 result through unchanged. + /// + /// + /// IPv6-only hosts must remain reachable; preferring IPv4 must never drop IPv6 addresses. + /// + [TestMethod] + public void OrderAddressesForConnect_AllIPv6_PassesThroughUnchanged() + { + var addresses = new[] + { + IPAddress.Parse("2610:20:8040:2::178"), + IPAddress.Parse("2610:20:8040:2::177") + }; + + var ordered = TimeSeriesDownload.OrderAddressesForConnect(addresses); + + CollectionAssert.AreEqual(addresses, ordered); + } + + /// + /// Verifies connect-address ordering tolerates an empty DNS result. + /// + [TestMethod] + public void OrderAddressesForConnect_Empty_ReturnsEmpty() + { + Assert.IsEmpty(TimeSeriesDownload.OrderAddressesForConnect(Array.Empty())); + } + + /// + /// Verifies the .NET Framework bind delegate rejects IPv6 remote endpoints. + /// + /// + /// On .NET Framework the service point tries resolved addresses in DNS order with no + /// per-address bound. Throwing a socket exception from the bind delegate makes a blocked + /// IPv6 attempt fail immediately so the service point moves on to an IPv4 address. + /// + [TestMethod] + public void RejectIPv6BindEndPoint_ThrowsSocketExceptionForIPv6() + { + var remote = new IPEndPoint(IPAddress.Parse("2610:20:8040:2::178"), 443); + Assert.Throws(() => TimeSeriesDownload.RejectIPv6BindEndPoint(remote)); + } + + /// + /// Verifies the .NET Framework bind delegate lets IPv4 remote endpoints bind normally. + /// + [TestMethod] + public void RejectIPv6BindEndPoint_ReturnsNullForIPv4() + { + var remote = new IPEndPoint(IPAddress.Parse("205.167.25.177"), 443); + Assert.IsNull(TimeSeriesDownload.RejectIPv6BindEndPoint(remote)); + } + + #endregion + #region CHMN (Canada) Tests /// @@ -1528,13 +1609,13 @@ public async Task GHCN_DailyPrecipitation_DuplicateDateTimes_LastValueWinsInline } /// - /// Verifies GHCN timeout messages use the longer GHCN provider limit. + /// Verifies GHCN timeout messages use the dedicated GHCN provider limit. /// /// A task that completes when the regression check finishes. /// - /// NOAA NCEI can take longer than the generic provider timeout to return station-file - /// headers. The handler throws a timeout immediately after URL validation so this test does - /// not wait for the full provider timeout. + /// Full station files can be larger than other provider payloads, so GHCN uses a limit + /// above the generic provider timeout. The handler throws a timeout immediately after URL + /// validation so this test does not wait for the full provider timeout. /// [TestMethod] public async Task GHCN_DailyPrecipitation_TimeoutMessage_UsesGhcnLimit() @@ -1553,7 +1634,7 @@ await TimeSeriesDownload.FromGHCN( GHCN_1, TimeSeriesDownload.TimeSeriesType.DailyPrecipitation)); - Assert.Contains("240 seconds", exception.Message); + Assert.Contains("60 seconds", exception.Message); Assert.HasCount(2, defaultHandler.Requests, "Expected the retry helper to make two GHCN attempts."); Assert.IsEmpty(decompressHandler.Requests, "GHCN should not use the compressed client."); Assert.IsFalse(defaultHandler.Requests.Any(GuardedGhcnDailyHandler.IsBlockedPreflight)); @@ -1567,7 +1648,12 @@ await TimeSeriesDownload.FromGHCN( /// /// Tests full-period-of-record GHCN daily precipitation download. /// + /// + /// The ceiling guards against connection-establishment regressions: walking dead IPv6 + /// addresses before IPv4 once made this download take minutes instead of seconds. + /// [TestMethod, TestCategory("Integration")] + [Timeout(60000, CooperativeCancellation = true)] public async Task GHCN_FullPor_Precipitation() { if (!await GhcnAvailable()) return; @@ -1578,7 +1664,12 @@ public async Task GHCN_FullPor_Precipitation() /// /// Tests full-period-of-record GHCN daily snow download. /// + /// + /// The ceiling guards against connection-establishment regressions: walking dead IPv6 + /// addresses before IPv4 once made this download take minutes instead of seconds. + /// [TestMethod, TestCategory("Integration")] + [Timeout(60000, CooperativeCancellation = true)] public async Task GHCN_FullPor_Snow() { if (!await GhcnAvailable()) return; From 861aa018f8fc164daa7527c8af9b985caad62ef2 Mon Sep 17 00:00:00 2001 From: HadenSmith Date: Thu, 9 Jul 2026 15:10:46 -0600 Subject: [PATCH 6/6] Prepare v2.1.2 release --- .github/workflows/Snapshot.yml | 4 ++-- CITATION.cff | 4 ++-- Numerics/Numerics.csproj | 2 +- codemeta.json | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/Snapshot.yml b/.github/workflows/Snapshot.yml index f7894141..ea5276b1 100644 --- a/.github/workflows/Snapshot.yml +++ b/.github/workflows/Snapshot.yml @@ -10,8 +10,8 @@ jobs: with: dotnet-version: '10.0.x' project-names: 'Numerics' - # Snapshot workflow appends .-dev, so this tracks the next development patch after v2.1.1. - version: '2.1.2' + # Snapshot workflow appends .-dev, so this tracks the next development patch after v2.1.2. + version: '2.1.3' # PR integration already runs the full multi-target test suite. run-tests: false nuget-source: 'https://www.hec.usace.army.mil/nexus/repository/consequences-nuget-public/' diff --git a/CITATION.cff b/CITATION.cff index 7759cdf3..c3bf0a23 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,8 +2,8 @@ cff-version: 1.2.0 message: "If you use this software, please cite our article in the Journal of Open Source Software." type: software title: "Numerics: A .NET Library for Numerical Computing, Statistical Analysis, and Risk Assessment" -version: "2.1.1" -date-released: "2026-06-17" +version: "2.1.2" +date-released: "2026-07-09" license: 0BSD repository-code: "https://github.com/USACE-RMC/Numerics" url: "https://github.com/USACE-RMC/Numerics" diff --git a/Numerics/Numerics.csproj b/Numerics/Numerics.csproj index 63ba65e5..23427c6c 100644 --- a/Numerics/Numerics.csproj +++ b/Numerics/Numerics.csproj @@ -31,7 +31,7 @@ CS1587 2.1.2 - Version 2.1.2 removes the generic Google connectivity preflight from time series downloads so provider requests can run on restricted networks where the target data service is allowed but unrelated connectivity probes are blocked. + Version 2.1.2 improves time-series download reliability and statistics utilities. Time-series downloads now prefer IPv4 while preserving IPv6 fallback, avoid unrelated connectivity preflights on restricted networks, normalize duplicate USGS/GHCN timestamps deterministically, and fix Series.Clear()/RemoveAt behavior with large or duplicate-valued series. This release also adds RunningStatistics.Clone() and fixes Combine()/+ so combined statistics do not alias input instances when one operand is empty. 2.1.2.0 diff --git a/codemeta.json b/codemeta.json index 72b1eb33..046bf4b6 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,9 +4,9 @@ "name": "Numerics: A .NET Library for Numerical Computing, Statistical Analysis, and Risk Assessment", "alternateName": "Numerics", "description": "A free and open-source .NET library providing numerical methods, probability distributions, statistical analysis, and Bayesian inference tools for quantitative risk assessment in water resources engineering.", - "version": "2.1.1", + "version": "2.1.2", "dateCreated": "2023-09-28", - "dateModified": "2026-06-17", + "dateModified": "2026-07-09", "license": "https://spdx.org/licenses/0BSD", "codeRepository": "https://github.com/USACE-RMC/Numerics", "issueTracker": "https://github.com/USACE-RMC/Numerics/issues",