diff --git a/.github/workflows/Snapshot.yml b/.github/workflows/Snapshot.yml index f7894141..ea5276b1 100644 --- a/.github/workflows/Snapshot.yml +++ b/.github/workflows/Snapshot.yml @@ -10,8 +10,8 @@ jobs: with: dotnet-version: '10.0.x' project-names: 'Numerics' - # Snapshot workflow appends .-dev, so this tracks the next development patch after v2.1.1. - version: '2.1.2' + # Snapshot workflow appends .-dev, so this tracks the next development patch after v2.1.2. + version: '2.1.3' # PR integration already runs the full multi-target test suite. run-tests: false nuget-source: 'https://www.hec.usace.army.mil/nexus/repository/consequences-nuget-public/' diff --git a/CITATION.cff b/CITATION.cff index 7759cdf3..c3bf0a23 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,8 +2,8 @@ cff-version: 1.2.0 message: "If you use this software, please cite our article in the Journal of Open Source Software." type: software title: "Numerics: A .NET Library for Numerical Computing, Statistical Analysis, and Risk Assessment" -version: "2.1.1" -date-released: "2026-06-17" +version: "2.1.2" +date-released: "2026-07-09" license: 0BSD repository-code: "https://github.com/USACE-RMC/Numerics" url: "https://github.com/USACE-RMC/Numerics" diff --git a/Numerics/Data/Statistics/RunningStatistics.cs b/Numerics/Data/Statistics/RunningStatistics.cs index 4c82bc3f..bed45556 100644 --- a/Numerics/Data/Statistics/RunningStatistics.cs +++ b/Numerics/Data/Statistics/RunningStatistics.cs @@ -273,18 +273,39 @@ public void Push(IList values) Push(value); } + /// + /// Creates a copy of the running statistics. + /// + /// A new running statistics instance with the same accumulated state, independent of the original. + public RunningStatistics Clone() + { + return new RunningStatistics() + { + _n = _n, + _m1 = _m1, + _m2 = _m2, + _m3 = _m3, + _m4 = _m4, + _min = _min, + _max = _max + }; + } + /// /// Create a new running statistics over the combined samples of two existing running statistics. /// + /// The first running statistics. + /// The second running statistics. + /// A new running statistics instance, independent of both inputs. public static RunningStatistics Combine(RunningStatistics a, RunningStatistics b) { if (a._n == 0L) { - return b; + return b.Clone(); } else if (b._n == 0L) { - return a; + return a.Clone(); } long n = a._n + b._n; diff --git a/Numerics/Data/Time Series/Support/Series.cs b/Numerics/Data/Time Series/Support/Series.cs index 8bdf9d75..0065b407 100644 --- a/Numerics/Data/Time Series/Support/Series.cs +++ b/Numerics/Data/Time Series/Support/Series.cs @@ -125,12 +125,10 @@ public void Insert(int index, object? item) public virtual bool Remove(SeriesOrdinate item) { var index = IndexOf(item); - if (_seriesOrdinates.Remove(item) == true) - { - RaiseCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item, index)); - return true; - } - return false; + if (index < 0) return false; + _seriesOrdinates.RemoveAt(index); + RaiseCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item, index)); + return true; } /// @@ -144,23 +142,30 @@ public void Remove(object? item) } /// + /// + /// Removes the ordinate at the requested position directly, so when equal-valued + /// ordinates exist elsewhere in the series the element at is + /// the one removed and reported. + /// public virtual void RemoveAt(int index) { - Remove(_seriesOrdinates[index]); + var item = _seriesOrdinates[index]; + _seriesOrdinates.RemoveAt(index); + RaiseCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item, index)); } /// /// Remove all elements from the collection. /// + /// + /// Clears the backing list in one operation and raises a single reset event. The former + /// element-by-element removal cost two full equality scans per element, which made + /// clearing a large series O(n²). + /// public void Clear() { - bool suppress = SuppressCollectionChanged; - SuppressCollectionChanged = true; - for (int i = _seriesOrdinates.Count - 1; i >= 0; i--) - Remove(_seriesOrdinates[i]); - SuppressCollectionChanged = false; + _seriesOrdinates.Clear(); RaiseCollectionChangedReset(); - SuppressCollectionChanged = suppress; } /// diff --git a/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs b/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs index 6c294aa1..f5f4f503 100644 --- a/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs +++ b/Numerics/Data/Time Series/Support/TimeSeriesDownload.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using System.Net; +using System.Net.Sockets; using System.Xml.Linq; using System; using System.Text.RegularExpressions; @@ -10,6 +11,7 @@ using System.Net.Http; using System.Globalization; using System.Collections.Generic; +using System.Text; namespace Numerics.Data { @@ -25,31 +27,329 @@ namespace Numerics.Data /// public class TimeSeriesDownload { - private static readonly HttpClient _defaultClient = new HttpClient() { Timeout = TimeSpan.FromSeconds(60) }; - private static readonly HttpClient _decompressClient = new HttpClient(new HttpClientHandler - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate - }) - { Timeout = TimeSpan.FromSeconds(60) }; + private static HttpClient _defaultClient = CreateDefaultClient(); + private static HttpClient _decompressClient = CreateDecompressClient(); + private static readonly HttpClient _bomClient = CreateBomClient(); private const string UserAgent = "USACE-Numerics/2.0"; /// - /// GET with up to three attempts and exponential backoff (0.5s, 1s, 2s - /// + jitter) on transient failures. Retries 5xx, 408, 429, network + /// Maximum time allowed for one provider request attempt before retry/failure handling runs. + /// + /// + /// A short per-attempt timeout prevents blocked government-network routes from making UI + /// callers wait for the default minute-long HTTP timeout on each retry. + /// + private static readonly TimeSpan ProviderRequestTimeout = TimeSpan.FromSeconds(10); + + /// + /// Maximum time allowed for one instantaneous provider request attempt. + /// + /// + /// Instantaneous series can be substantially larger than daily, peak, or field-measurement + /// responses, especially when a provider accepts a long period-of-record query. + /// + private static readonly TimeSpan InstantaneousProviderRequestTimeout = TimeSpan.FromSeconds(120); + + /// + /// Maximum time allowed for one GHCN station-file request attempt. + /// + /// + /// NCEI serves full station files in seconds over IPv4. The historical multi-minute + /// header waits were sequential TCP timeouts against NCEI's six IPv6 addresses on + /// networks that drop IPv6 traffic, which the downloader now avoids by preferring IPv4. + /// This margin only covers genuinely slow links and large station files. + /// + private static readonly TimeSpan GhcnProviderRequestTimeout = TimeSpan.FromSeconds(60); + + /// + /// Maximum time allowed for one explicit connectivity probe. + /// + /// + /// Connectivity probes are advisory only and must remain faster than real provider downloads. + /// + private static readonly TimeSpan ConnectivityProbeTimeout = TimeSpan.FromSeconds(2); + + /// + /// Default number of attempts for provider requests made through the retry helper. + /// + /// + /// Two attempts are enough to smooth over one transient outage without turning blocked + /// network paths into multi-minute waits. + /// + private const int DefaultMaxDownloadAttempts = 2; + + /// + /// Provider root endpoints used for the public Internet connectivity check. + /// + /// + /// The downloader no longer uses these URLs as a precondition for data requests. They are + /// retained only for callers that explicitly ask whether at least one supported provider + /// endpoint is reachable. + /// + private static readonly string[] InternetProbeUrls = + { + "https://waterservices.usgs.gov/nwis/", + "https://www.ncei.noaa.gov/", + "https://wateroffice.ec.gc.ca/", + "https://www.bom.gov.au/waterdata/" + }; + +#if !NETFRAMEWORK + /// + /// Maximum time allowed for one TCP connect attempt to a single resolved address. + /// + /// + /// Bounding each address separately keeps one unreachable address family from consuming + /// the whole request timeout while still letting later addresses in the list be tried. + /// + private static readonly TimeSpan ConnectAttemptTimeout = TimeSpan.FromSeconds(5); +#endif + + /// + /// Creates the HTTP handler used by the downloader's shared clients. + /// + /// True to enable automatic GZip and Deflate response decompression. + /// A configured HTTP handler. + /// + /// Some providers publish IPv6 (AAAA) DNS records while many restricted networks silently + /// drop IPv6 traffic. A default handler tries every unreachable IPv6 address before + /// falling back to IPv4 (~21 s of TCP timeout per address; measured at ~126 s to first + /// byte for NOAA NCEI's six AAAA records). On modern targets the handler connects with + /// ConnectPreferringIPv4Async so IPv4 is tried first and IPv6 remains a + /// fallback. On .NET Framework the equivalent ordering is applied per request through + /// . + /// + private static HttpMessageHandler CreateHandler(bool automaticDecompression) + { +#if NETFRAMEWORK + var handler = new HttpClientHandler(); +#else + var handler = new SocketsHttpHandler { ConnectCallback = ConnectPreferringIPv4Async }; +#endif + if (automaticDecompression) + { + handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; + } + return handler; + } + +#if !NETFRAMEWORK + /// + /// Opens a TCP connection for an HTTP request, preferring IPv4 addresses over IPv6. + /// + /// The connection context supplied by the HTTP handler. + /// Token used to cancel the connection attempt. + /// A connected network stream. + /// Thrown when no resolved address accepts a connection. + /// Thrown when is canceled. + /// + /// The default connect logic tries DNS results in returned order, which lists IPv6 first. + /// On networks that silently drop IPv6 traffic every dead address costs a full TCP + /// timeout before IPv4 is reached. Trying IPv4 first keeps downloads fast there while the + /// IPv6 fallback keeps IPv6-only networks working. + /// + private static async ValueTask ConnectPreferringIPv4Async(SocketsHttpConnectionContext context, CancellationToken cancellationToken) + { + IPAddress[] addresses = await Dns.GetHostAddressesAsync(context.DnsEndPoint.Host, cancellationToken).ConfigureAwait(false); + IPAddress[] ordered = OrderAddressesForConnect(addresses); + Exception? lastFailure = null; + foreach (IPAddress address in ordered) + { + var socket = new Socket(address.AddressFamily, SocketType.Stream, ProtocolType.Tcp) { NoDelay = true }; + try + { + using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + attemptCts.CancelAfter(ConnectAttemptTimeout); + await socket.ConnectAsync(new IPEndPoint(address, context.DnsEndPoint.Port), attemptCts.Token).ConfigureAwait(false); + return new NetworkStream(socket, ownsSocket: true); + } + catch (Exception ex) + { + socket.Dispose(); + cancellationToken.ThrowIfCancellationRequested(); + lastFailure = ex; + } + } + + throw new HttpRequestException( + $"Could not connect to '{context.DnsEndPoint.Host}:{context.DnsEndPoint.Port}' on any resolved address.", + lastFailure); + } +#endif + + /// + /// Orders resolved addresses so IPv4 is attempted before IPv6. + /// + /// The addresses returned by DNS resolution. + /// The addresses with IPv4 first; relative order within each family is preserved. + /// Thrown when is null. + internal static IPAddress[] OrderAddressesForConnect(IPAddress[] addresses) + { + if (addresses == null) throw new ArgumentNullException(nameof(addresses)); + return addresses.OrderBy(address => address.AddressFamily == AddressFamily.InterNetworkV6 ? 1 : 0).ToArray(); + } + + /// + /// Local-endpoint binder that rejects IPv6 remote endpoints so connects fall through to IPv4. + /// + /// The remote endpoint the connection is being bound for. + /// Null for IPv4 remote endpoints, letting the OS choose the local endpoint. + /// Thrown when is null. + /// Thrown for IPv6 remote endpoints to make the connection attempt skip to the next resolved address. + /// + /// On .NET Framework a tries resolved addresses in DNS order + /// (IPv6 first) with no per-address bound, so blocked IPv6 routes cost a full TCP timeout + /// per address. Throwing from the bind delegate fails an IPv6 attempt immediately and the + /// service point moves on to the next (IPv4) address. + /// + internal static IPEndPoint? RejectIPv6BindEndPoint(IPEndPoint remoteEndPoint) + { + if (remoteEndPoint == null) throw new ArgumentNullException(nameof(remoteEndPoint)); + if (remoteEndPoint.AddressFamily == AddressFamily.InterNetworkV6) + { + throw new SocketException((int)SocketError.AddressFamilyNotSupported); + } + return null; + } + + /// + /// Configures the .NET Framework service point for a request URI to prefer IPv4 connections. + /// + /// The absolute request URI about to be downloaded. + /// + /// No-op on modern targets, where installs an IPv4-preferring + /// connect callback instead. Skipped when the OS has no IPv4 stack so IPv6-only machines + /// keep working. + /// + private static void PreferIPv4ForNetFramework(Uri requestUri) + { +#if NETFRAMEWORK + if (!Socket.OSSupportsIPv4) return; + ServicePoint servicePoint = ServicePointManager.FindServicePoint(requestUri); + servicePoint.BindIPEndPointDelegate = (servicePointArg, remoteEndPoint, retryCount) => RejectIPv6BindEndPoint(remoteEndPoint); +#endif + } + + /// + /// Creates the default HTTP client used for uncompressed provider requests. + /// + /// A configured HTTP client whose timeout is controlled by the retry helper. + /// + /// The client is held as a static instance in production to avoid socket churn. Request + /// timeouts are applied by + /// so each provider call can choose the right timeout for its payload size. + /// + private static HttpClient CreateDefaultClient() + { + var client = new HttpClient(CreateHandler(automaticDecompression: false)) { Timeout = Timeout.InfiniteTimeSpan }; + client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); + return client; + } + + /// + /// Creates the HTTP client used for provider requests that may return compressed payloads. + /// + /// A configured HTTP client with automatic GZip and Deflate decompression. + /// + /// The client is held as a static instance in production to avoid socket churn. Request + /// timeouts are applied by . + /// + private static HttpClient CreateDecompressClient() + { + var client = new HttpClient(CreateHandler(automaticDecompression: true)) { Timeout = Timeout.InfiniteTimeSpan }; + client.DefaultRequestHeaders.UserAgent.ParseAdd(UserAgent); + return client; + } + + /// + /// Creates the HTTP client used for BOM KiWIS requests. + /// + /// A configured HTTP client with browser-like default headers. + /// + /// BOM sits behind a security appliance that rejects bare programmatic requests, so the + /// client sends browser-like headers. The client is held as a static instance so the + /// station lookup and value download in one call, and repeated calls, reuse pooled + /// connections instead of paying a fresh TCP and TLS handshake each time. The + /// Accept-Encoding header is supplied by the handler's automatic decompression. + /// + private static HttpClient CreateBomClient() + { + var client = new HttpClient(CreateHandler(automaticDecompression: true)) { Timeout = Timeout.InfiniteTimeSpan }; + client.DefaultRequestHeaders.Add("User-Agent", UserAgent); + client.DefaultRequestHeaders.Add("Accept", "application/json,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); + client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9"); + client.DefaultRequestHeaders.Add("DNT", "1"); + client.DefaultRequestHeaders.Add("Connection", "keep-alive"); + return client; + } + + /// + /// Replaces the static HTTP clients used by the downloader for deterministic tests. + /// + /// The client to use for ordinary provider requests. + /// The client to use for compressed provider requests. + /// Thrown when either client is null. + /// + /// This method is internal so tests can verify request routing without adding public API. + /// + internal static void SetHttpClientsForTesting(HttpClient defaultClient, HttpClient decompressClient) + { + _defaultClient = defaultClient ?? throw new ArgumentNullException(nameof(defaultClient)); + _decompressClient = decompressClient ?? throw new ArgumentNullException(nameof(decompressClient)); + } + + /// + /// Restores the production HTTP clients after a deterministic test. + /// + /// + /// Tests call this method in a finally block so later tests do not inherit fake handlers. + /// + internal static void ResetHttpClientsForTesting() + { + _defaultClient = CreateDefaultClient(); + _decompressClient = CreateDecompressClient(); + } + + /// + /// GET with bounded retry and exponential backoff on transient failures. Retries 5xx, 408, 429, network /// exceptions, and KiWIS DatasourceError bodies. Does NOT retry 4xx — /// those mean the request itself is wrong. /// + /// The HTTP client used to issue the request. + /// The request URL. + /// The maximum number of request attempts. + /// The maximum duration allowed for each request attempt. + /// Token used to cancel the request and retry delay. + /// The final HTTP status code and response body. + /// Thrown when all attempts fail with a network error. + /// Thrown when is canceled. + /// Thrown when all attempts exceed the per-attempt timeout. + /// + /// The helper bounds each attempt separately so blocked network paths fail quickly while + /// still allowing one retry for transient service errors. + /// private static async Task<(HttpStatusCode status, string body)> GetWithRetryAsync( - HttpClient client, string url, int maxAttempts = 3) + HttpClient client, + string url, + int maxAttempts = DefaultMaxDownloadAttempts, + TimeSpan? requestTimeout = null, + CancellationToken cancellationToken = default) { + TimeSpan timeout = requestTimeout ?? ProviderRequestTimeout; + PreferIPv4ForNetFramework(new Uri(url)); Exception? lastNet = null; for (int attempt = 1; attempt <= maxAttempts; attempt++) { try { - using var resp = await client.GetAsync(url); - string body = await resp.Content.ReadAsStringAsync(); + using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + attemptCts.CancelAfter(timeout); + using var resp = await client.GetAsync( + url, + HttpCompletionOption.ResponseHeadersRead, + attemptCts.Token).ConfigureAwait(false); + string body = await ReadResponseBodyAsync(resp, attemptCts.Token).ConfigureAwait(false); bool transient = (int)resp.StatusCode >= 500 || resp.StatusCode == HttpStatusCode.RequestTimeout || @@ -59,27 +359,139 @@ public class TimeSeriesDownload if (!transient || attempt == maxAttempts) return (resp.StatusCode, body); } catch (HttpRequestException ex) { lastNet = ex; if (attempt == maxAttempts) throw; } - catch (TaskCanceledException ex) { lastNet = ex; if (attempt == maxAttempts) throw; } + catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } + catch (TaskCanceledException ex) + { + lastNet = CreateDownloadTimeoutException(url, timeout, ex); + if (attempt == maxAttempts) throw lastNet; + } - await Task.Delay(500 * (int)Math.Pow(2, attempt - 1)); + await Task.Delay(500 * (int)Math.Pow(2, attempt - 1), cancellationToken).ConfigureAwait(false); } throw lastNet ?? new Exception("GetWithRetryAsync exhausted retries"); } /// - /// Checks if there is an Internet connection by probing Google's - /// connectivity-check endpoint (HTTP 204 No Content). + /// Reads an HTTP response body with cancellation support. + /// + /// The HTTP response whose content should be read. + /// Token used to cancel the body read. + /// The decoded response text. + /// Thrown when is canceled. + /// + /// cannot be canceled on older target + /// frameworks. Full-period instantaneous USGS responses are large enough that body reads + /// must honor the same bounded timeout as the request headers. + /// + private static async Task ReadResponseBodyAsync(HttpResponseMessage response, CancellationToken cancellationToken) + { + using var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + using var memory = new MemoryStream(); + byte[] buffer = new byte[81920]; + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false)) > 0) + { + memory.Write(buffer, 0, bytesRead); + } + + return GetResponseEncoding(response).GetString(memory.ToArray()); + } + + /// + /// Gets the text encoding declared by an HTTP response, defaulting to UTF-8. + /// + /// The HTTP response to inspect. + /// The declared response encoding, or UTF-8 when none is declared or recognized. + /// + /// USGS RDB and most provider text responses are UTF-8-compatible. The fallback keeps parsing + /// stable when a provider omits or misstates the charset. + /// + private static Encoding GetResponseEncoding(HttpResponseMessage response) + { + string charset = response.Content.Headers.ContentType?.CharSet ?? ""; + if (string.IsNullOrWhiteSpace(charset)) return Encoding.UTF8; + + try + { + return Encoding.GetEncoding(charset.Trim('"')); + } + catch (ArgumentException) + { + return Encoding.UTF8; + } + } + + /// + /// Creates a timeout exception for a provider request that did not complete quickly enough. + /// + /// The URL that timed out. + /// The timeout used for the request attempt. + /// The cancellation exception raised by the HTTP stack. + /// An exception with a user-facing timeout message. + /// + /// The message names the endpoint so application callers can distinguish a blocked provider + /// request from the old generic Internet connectivity failure. + /// + private static TimeoutException CreateDownloadTimeoutException(string url, TimeSpan requestTimeout, Exception innerException) + { + return new TimeoutException( + $"The request to '{url}' did not complete within {requestTimeout.TotalSeconds:0} seconds.", + innerException); + } + + /// + /// Checks if there is an Internet connection by probing provider endpoints used by the downloader. /// + /// True when at least one supported provider endpoint completes an HTTP request; otherwise false. + /// + /// This method is not used as a preflight gate for provider downloads because restricted + /// networks may allow one data provider while blocking unrelated endpoints. + /// public static async Task IsConnectedToInternet() + { + using var cts = new CancellationTokenSource(); + var probes = InternetProbeUrls.Select(url => CanReachEndpoint(url, cts.Token)).ToList(); + + while (probes.Count > 0) + { + Task completed = await Task.WhenAny(probes); + probes.Remove(completed); + if (await completed) + { + cts.Cancel(); + return true; + } + } + + return false; + } + + /// + /// Determines whether an endpoint can complete an HTTP request. + /// + /// The endpoint URL to probe. + /// Token used to cancel the probe. + /// True when the request completes with any HTTP response; otherwise false. + /// + /// Any HTTP response means the network path exists. HTTP status codes are intentionally not + /// interpreted because security appliances and provider roots may return redirects or errors + /// while still proving transport reachability. + /// + private static async Task CanReachEndpoint(string url, CancellationToken cancellationToken = default) { try { - using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + PreferIPv4ForNetFramework(new Uri(url)); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(ConnectivityProbeTimeout); + using (await _defaultClient.GetAsync( + url, + HttpCompletionOption.ResponseHeadersRead, + cts.Token)) { - var resp = await _defaultClient.GetAsync( - "https://www.google.com/generate_204", cts.Token); - return resp.IsSuccessStatusCode; } + + return true; } catch { @@ -87,6 +499,39 @@ public static async Task IsConnectedToInternet() } } + /// + /// Adds a downloaded ordinate or replaces the previously parsed ordinate at the same date/time. + /// + /// The time series being populated. + /// Lookup from parsed date/time values to their current series index. + /// The ordinate date/time. + /// The ordinate value. + /// Thrown when or is null. + /// + /// Provider data can contain distinct source records with the same timestamp. Handling that + /// as records are parsed preserves the historical "last value wins" behavior without a + /// second full-series pass after download completion. + /// + private static void AddOrReplaceByDateTime( + TimeSeries timeSeries, + Dictionary indexesByDateTime, + DateTime index, + double value) + { + if (timeSeries == null) throw new ArgumentNullException(nameof(timeSeries)); + if (indexesByDateTime == null) throw new ArgumentNullException(nameof(indexesByDateTime)); + + var ordinate = new SeriesOrdinate(index, value); + if (indexesByDateTime.TryGetValue(index, out int existingIndex)) + { + timeSeries[existingIndex] = ordinate; + return; + } + + indexesByDateTime.Add(index, timeSeries.Count); + timeSeries.Add(ordinate); + } + /// /// Enumeration of time series options. /// @@ -191,8 +636,9 @@ public enum HeightUnit /// The station identification code. /// The time series type. Default = Daily precipitation. /// The depth unit. Default = inches. + /// Token used to cancel the download. /// A downloaded time series. - public static async Task FromGHCN(string siteNumber, TimeSeriesType timeSeriesType = TimeSeriesType.DailyPrecipitation, DepthUnit unit = DepthUnit.Inches) + public static async Task FromGHCN(string siteNumber, TimeSeriesType timeSeriesType = TimeSeriesType.DailyPrecipitation, DepthUnit unit = DepthUnit.Inches, CancellationToken cancellationToken = default) { @@ -212,12 +658,6 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType DateTime? previousDate = null; string tempFilePath = Path.Combine(Path.GetTempPath(), $"{Path.GetRandomFileName()}.dly"); - // Check internet connection - if (!await IsConnectedToInternet()) - { - throw new InvalidOperationException("No internet connection."); - } - try { @@ -226,32 +666,45 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType string stationFileUrl = $"{ghcnBaseUrl}{Uri.EscapeDataString(siteNumber)}.dly"; { var client = _defaultClient; + PreferIPv4ForNetFramework(new Uri(stationFileUrl)); // GHCN is large enough to want streaming, so we can't use GetWithRetryAsync // directly (it buffers the whole body). Inline the same retry shape on the // headers-only GetAsync, then stream from the successful response. HttpResponseMessage? response = null; - for (int attempt = 1; attempt <= 3; attempt++) + for (int attempt = 1; attempt <= DefaultMaxDownloadAttempts; attempt++) { try { - response = await client.GetAsync(stationFileUrl, HttpCompletionOption.ResponseHeadersRead); + using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + attemptCts.CancelAfter(GhcnProviderRequestTimeout); + response = await client.GetAsync( + stationFileUrl, + HttpCompletionOption.ResponseHeadersRead, + attemptCts.Token).ConfigureAwait(false); if ((int)response.StatusCode < 500 && response.StatusCode != HttpStatusCode.RequestTimeout && (int)response.StatusCode != 429) break; - if (attempt == 3) break; + if (attempt == DefaultMaxDownloadAttempts) break; response.Dispose(); } - catch (HttpRequestException) when (attempt < 3) { /* retry */ } - catch (TaskCanceledException) when (attempt < 3) { /* retry */ } - await Task.Delay(500 * (int)Math.Pow(2, attempt - 1)); + catch (HttpRequestException) when (attempt < DefaultMaxDownloadAttempts) { /* retry */ } + catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } + catch (TaskCanceledException ex) + { + if (attempt == DefaultMaxDownloadAttempts) + { + throw CreateDownloadTimeoutException(stationFileUrl, GhcnProviderRequestTimeout, ex); + } + } + await Task.Delay(500 * (int)Math.Pow(2, attempt - 1), cancellationToken).ConfigureAwait(false); } using (response) { response!.EnsureSuccessStatusCode(); - using (Stream stream = await response.Content.ReadAsStreamAsync()) + using (Stream stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) using (FileStream fs = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 81920, useAsync: true)) { - await stream.CopyToAsync(fs); + await stream.CopyToAsync(fs, 81920, cancellationToken).ConfigureAwait(false); } } } @@ -263,48 +716,58 @@ public static async Task FromGHCN(string siteNumber, TimeSeriesType } // Read and parse the file - string[] lines = File.ReadAllLines(tempFilePath); - foreach (string line in lines) + bool previousSuppressCollectionChanged = timeSeries.SuppressCollectionChanged; + var indexesByDateTime = new Dictionary(); + timeSeries.SuppressCollectionChanged = true; + try { - // Extract the type (PRCP, SNOW, etc.) - var typeString = line.Substring(17, 4); - if ((typeString == "PRCP" && timeSeriesType == TimeSeriesType.DailyPrecipitation) || - (typeString == "SNOW" && timeSeriesType == TimeSeriesType.DailySnow)) + string[] lines = File.ReadAllLines(tempFilePath); + foreach (string line in lines) { - - // Extract year, month, and parse days - int.TryParse(line.Substring(11, 4), NumberStyles.Integer, CultureInfo.InvariantCulture, out var year); - int.TryParse(line.Substring(15, 2), NumberStyles.Integer, CultureInfo.InvariantCulture, out var month); - int daysInMonth = DateTime.DaysInMonth(year, month); - - for (int i = 0; i < daysInMonth; i++) + // Extract the type (PRCP, SNOW, etc.) + var typeString = line.Substring(17, 4); + if ((typeString == "PRCP" && timeSeriesType == TimeSeriesType.DailyPrecipitation) || + (typeString == "SNOW" && timeSeriesType == TimeSeriesType.DailySnow)) { - int offset = 21 + (i * 8); - string strgValue = line.Substring(offset, 5).Trim(); - var currentDate = new DateTime(year, month, i + 1); - // Fill missing dates with NaN if there is a gap - if (previousDate.HasValue && (currentDate - previousDate.Value).Days > 1) - { - FillMissingDates(timeSeries, previousDate.Value, currentDate); - } + // Extract year, month, and parse days + int.TryParse(line.Substring(11, 4), NumberStyles.Integer, CultureInfo.InvariantCulture, out var year); + int.TryParse(line.Substring(15, 2), NumberStyles.Integer, CultureInfo.InvariantCulture, out var month); + int daysInMonth = DateTime.DaysInMonth(year, month); - // Parse the precipitation or snow value - if (int.TryParse(strgValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out int rawValue) && rawValue != -9999) // Valid precipitation value + for (int i = 0; i < daysInMonth; i++) { - double convertedValue = ConvertToDesiredUnit(rawValue, unit); - timeSeries.Add(new SeriesOrdinate(currentDate, convertedValue)); - } - else - { - // Add missing value if invalid - timeSeries.Add(new SeriesOrdinate(currentDate, double.NaN)); - } + int offset = 21 + (i * 8); + string strgValue = line.Substring(offset, 5).Trim(); + var currentDate = new DateTime(year, month, i + 1); - previousDate = currentDate; - } + // Fill missing dates with NaN if there is a gap + if (previousDate.HasValue && (currentDate - previousDate.Value).Days > 1) + { + FillMissingDates(timeSeries, previousDate.Value, currentDate, indexesByDateTime); + } + + // Parse the precipitation or snow value + if (int.TryParse(strgValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out int rawValue) && rawValue != -9999) // Valid precipitation value + { + double convertedValue = ConvertToDesiredUnit(rawValue, unit); + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, currentDate, convertedValue); + } + else + { + // Add missing value if invalid + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, currentDate, double.NaN); + } + + previousDate = currentDate; + } + } } } + finally + { + timeSeries.SuppressCollectionChanged = previousSuppressCollectionChanged; + } } finally @@ -342,12 +805,30 @@ private static double ConvertToDesiredUnit(int rawValue, DepthUnit unit) /// The time series to fill. /// The last available date. /// The current date to reach. - private static void FillMissingDates(TimeSeries timeSeries, DateTime previousDate, DateTime currentDate) + /// Optional parsed-date lookup used when duplicate replacement is active. + /// + /// When a lookup is supplied, inserted gap ordinates participate in the same inline + /// duplicate handling as provider records. Without a lookup the method preserves the + /// historical append-only behavior. + /// + private static void FillMissingDates( + TimeSeries timeSeries, + DateTime previousDate, + DateTime currentDate, + Dictionary? indexesByDateTime = null) { DateTime missingDate = previousDate.AddDays(1); while (missingDate < currentDate) { - timeSeries.Add(new SeriesOrdinate(missingDate, double.NaN)); + if (indexesByDateTime == null) + { + timeSeries.Add(new SeriesOrdinate(missingDate, double.NaN)); + } + else + { + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, missingDate, double.NaN); + } + missingDate = missingDate.AddDays(1); } } @@ -395,7 +876,8 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType /// /// USGS site number. /// The time series type. - public static async Task<(TimeSeries TimeSeries, string RawText)> FromUSGS(string siteNumber, TimeSeriesType timeSeriesType = TimeSeriesType.DailyDischarge) + /// Token used to cancel the download. + public static async Task<(TimeSeries TimeSeries, string RawText)> FromUSGS(string siteNumber, TimeSeriesType timeSeriesType = TimeSeriesType.DailyDischarge, CancellationToken cancellationToken = default) { @@ -411,12 +893,6 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType throw new ArgumentException("The time series type cannot be daily precipitation or daily snow.", nameof(timeSeriesType)); } - // Check internet connection - if (!await IsConnectedToInternet()) - { - throw new InvalidOperationException("No internet connection."); - } - var timeSeries = (timeSeriesType == TimeSeriesType.MeasuredDischarge || timeSeriesType == TimeSeriesType.MeasuredStage || timeSeriesType == TimeSeriesType.InstantaneousDischarge || timeSeriesType == TimeSeriesType.InstantaneousStage) ? new TimeSeries(TimeInterval.Irregular) @@ -437,78 +913,95 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType timeSeriesType == TimeSeriesType.InstantaneousStage; // Instantaneous RDB has an extra tz_cd column at index 3, pushing the value to index 4 int valueIndex = isInstantaneous ? 4 : 3; + bool previousSuppressCollectionChanged = timeSeries.SuppressCollectionChanged; + var indexesByDateTime = new Dictionary(); - { - var client = _decompressClient; - - var (status, body) = await GetWithRetryAsync(client, url); - if ((int)status >= 400) - { - string snippet = body.Length > 500 ? body.Substring(0, 500) + "…" : body; - throw new Exception( - $"Failed to retrieve USGS data. Status={(int)status} ({status}). Body={snippet}"); - } - - using var reader = new StringReader(body); - string? line; - bool isHeader = true; + timeSeries.SuppressCollectionChanged = true; - while ((line = await reader.ReadLineAsync()) != null) + try + { { - // Skip header row - if (isHeader) + var client = _decompressClient; + + TimeSpan requestTimeout = isInstantaneous ? InstantaneousProviderRequestTimeout : ProviderRequestTimeout; + var (status, body) = await GetWithRetryAsync( + client, + url, + requestTimeout: requestTimeout, + cancellationToken: cancellationToken).ConfigureAwait(false); + if ((int)status >= 400) { - isHeader = false; - continue; + string snippet = body.Length > 500 ? body.Substring(0, 500) + "…" : body; + throw new Exception( + $"Failed to retrieve USGS data. Status={(int)status} ({status}). Body={snippet}"); } + textDownload = body; - // Skip comment lines - if (line.StartsWith("#")) + using var reader = new StringReader(body); + string? line; + bool isHeader = true; + + while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) != null) { - if (line.Trim() == "# No sites found matching all criteria") + // Skip header row + if (isHeader) { - throw new Exception("No data found matching all criteria."); + isHeader = false; + continue; } - continue; - } - string[] fields = line.Split('\t'); - // Validate expected number of fields and record type - if (fields.Length <= valueIndex || fields[0] != "USGS") - continue; + // Skip comment lines + if (line.StartsWith("#")) + { + if (line.Trim() == "# No sites found matching all criteria") + { + throw new Exception("No data found matching all criteria."); + } + continue; + } - // USGS instantaneous timestamps are local-time strings without an offset; - // the tz_cd column at fields[3] reports the offset separately. We currently - // parse these as naive DateTime (Kind=Unspecified). Future work: synthesize - // a DateTimeOffset from fields[2] + fields[3] for full timezone fidelity. - if (!DateTime.TryParse(fields[2], CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime index)) - { - continue; - } + string[] fields = line.Split('\t'); + // Validate expected number of fields and record type + if (fields.Length <= valueIndex || fields[0] != "USGS") + continue; - // Fill in missing days only for daily series (not instantaneous) - if (!isInstantaneous && timeSeries.Count > 0 && index != TimeSeries.AddTimeInterval(timeSeries.Last().Index, TimeInterval.OneDay)) - { - while (timeSeries.Last().Index < TimeSeries.SubtractTimeInterval(index, TimeInterval.OneDay)) + // USGS instantaneous timestamps are local-time strings without an offset; + // the tz_cd column at fields[3] reports the offset separately. We currently + // parse these as naive DateTime (Kind=Unspecified). Future work: synthesize + // a DateTimeOffset from fields[2] + fields[3] for full timezone fidelity. + if (!DateTime.TryParse(fields[2], CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime index)) { - timeSeries.Add(new SeriesOrdinate( - TimeSeries.AddTimeInterval(timeSeries.Last().Index, TimeInterval.OneDay), double.NaN)); + continue; + } + + // Fill in missing days only for daily series (not instantaneous) + if (!isInstantaneous && timeSeries.Count > 0 && index != TimeSeries.AddTimeInterval(timeSeries.Last().Index, TimeInterval.OneDay)) + { + while (timeSeries.Last().Index < TimeSeries.SubtractTimeInterval(index, TimeInterval.OneDay)) + { + DateTime missingIndex = TimeSeries.AddTimeInterval(timeSeries.Last().Index, TimeInterval.OneDay); + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, missingIndex, double.NaN); + } } - } - // Get and parse value - string valueStr = fields[valueIndex]; - double value = string.IsNullOrWhiteSpace(valueStr) ? double.NaN - : double.TryParse(valueStr, NumberStyles.Float, CultureInfo.InvariantCulture, out double tempVal) ? tempVal : double.NaN; - timeSeries.Add(new SeriesOrdinate(index, value)); + // Get and parse value + string valueStr = fields[valueIndex]; + double value = string.IsNullOrWhiteSpace(valueStr) ? double.NaN + : double.TryParse(valueStr, NumberStyles.Float, CultureInfo.InvariantCulture, out double tempVal) ? tempVal : double.NaN; + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, index, value); + } } } + finally + { + timeSeries.SuppressCollectionChanged = previousSuppressCollectionChanged; + } } // For peak data (annual max values) else if (timeSeriesType == TimeSeriesType.PeakDischarge || timeSeriesType == TimeSeriesType.PeakStage) { { - var (peakStatus, peakBody) = await GetWithRetryAsync(_defaultClient, url); + var (peakStatus, peakBody) = await GetWithRetryAsync(_defaultClient, url, cancellationToken: cancellationToken).ConfigureAwait(false); if ((int)peakStatus >= 400) { string snippet = peakBody.Length > 500 ? peakBody.Substring(0, 500) + "…" : peakBody; @@ -518,6 +1011,7 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType textDownload = peakBody; int idx = timeSeriesType == TimeSeriesType.PeakDischarge ? 4 : 6; + var indexesByDateTime = new Dictionary(); var lines = textDownload.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries); foreach (string line in lines) { @@ -562,7 +1056,7 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType if (!string.IsNullOrWhiteSpace(segments[idx])) { double.TryParse(segments[idx], NumberStyles.Float, CultureInfo.InvariantCulture, out value); - timeSeries.Add(new SeriesOrdinate(index, value)); + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, index, value); } } } @@ -573,7 +1067,7 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType { { var rawText = new System.Text.StringBuilder(); - await ParseUSGSOgcApiPages(_decompressClient, url, timeSeries, rawText); + await ParseUSGSOgcApiPages(_decompressClient, url, timeSeries, rawText, cancellationToken).ConfigureAwait(false); textDownload = rawText.ToString(); } @@ -592,13 +1086,15 @@ private static string CreateURLForUSGSDownload(string siteNumber, TimeSeriesType /// The initial API URL. /// TimeSeries to populate with parsed data. /// Optional StringBuilder to accumulate raw JSON responses. - private static async Task ParseUSGSOgcApiPages(HttpClient client, string initialUrl, TimeSeries timeSeries, System.Text.StringBuilder? rawText = null) + /// Token used to cancel pagination requests. + private static async Task ParseUSGSOgcApiPages(HttpClient client, string initialUrl, TimeSeries timeSeries, System.Text.StringBuilder? rawText = null, CancellationToken cancellationToken = default) { string? nextUrl = initialUrl; + var indexesByDateTime = new Dictionary(); while (nextUrl != null) { - var (status, json) = await GetWithRetryAsync(client, nextUrl); + var (status, json) = await GetWithRetryAsync(client, nextUrl, cancellationToken: cancellationToken).ConfigureAwait(false); if ((int)status >= 400) { string snippet = json.Length > 500 ? json.Substring(0, 500) + "…" : json; @@ -631,7 +1127,7 @@ private static async Task ParseUSGSOgcApiPages(HttpClient client, string initial CultureInfo.InvariantCulture, out double val)) continue; - timeSeries.Add(new SeriesOrdinate(dt, val)); + AddOrReplaceByDateTime(timeSeries, indexesByDateTime, dt, val); } } @@ -651,7 +1147,7 @@ private static async Task ParseUSGSOgcApiPages(HttpClient client, string initial nextUrl = candidateUrl; } // Small delay between pages to avoid rate limiting - await System.Threading.Tasks.Task.Delay(200); + await System.Threading.Tasks.Task.Delay(200, cancellationToken); break; } } @@ -682,6 +1178,7 @@ private static async Task ParseUSGSOgcApiPages(HttpClient client, string initial /// /// Optional inclusive end date. If null, defaults to DateTime.Today (or Now for instantaneous). /// + /// Token used to cancel the download. /// TimeSeries of values. public static async Task FromCHMN( string stationNumber, @@ -689,12 +1186,9 @@ public static async Task FromCHMN( DischargeUnit dischargeUnit = DischargeUnit.CubicMetersPerSecond, HeightUnit heightUnit = HeightUnit.Meters, DateTime? startDate = null, - DateTime? endDate = null) + DateTime? endDate = null, + CancellationToken cancellationToken = default) { - // Connectivity - if (!await IsConnectedToInternet()) - throw new InvalidOperationException("No internet connection."); - // Basic validation (WSC station numbers are 7 chars including letters) if (string.IsNullOrWhiteSpace(stationNumber) || stationNumber.Length != 7) throw new ArgumentException("The WSC station number must be 7 characters, e.g., 08LG010.", nameof(stationNumber)); @@ -757,12 +1251,21 @@ public static async Task FromCHMN( } var ts = new TimeSeries(interval); + var indexesByDateTime = new Dictionary(); { var client = _decompressClient; // The endpoint returns CSV (automatically decompressed by HttpClientHandler). - var (status, csv) = await GetWithRetryAsync(client, url); + TimeSpan requestTimeout = timeSeriesType == TimeSeriesType.InstantaneousDischarge || + timeSeriesType == TimeSeriesType.InstantaneousStage + ? InstantaneousProviderRequestTimeout + : ProviderRequestTimeout; + var (status, csv) = await GetWithRetryAsync( + client, + url, + requestTimeout: requestTimeout, + cancellationToken: cancellationToken); if ((int)status >= 400) { string snippet = csv.Length > 500 ? csv.Substring(0, 500) + "…" : csv; @@ -771,17 +1274,26 @@ public static async Task FromCHMN( } // Parse the CSV response based on endpoint type - if (timeSeriesType == TimeSeriesType.PeakDischarge || timeSeriesType == TimeSeriesType.PeakStage) + bool previousSuppressCollectionChanged = ts.SuppressCollectionChanged; + ts.SuppressCollectionChanged = true; + try { - // Peak data CSV format: - // ID,Parameter/Paramètre,Date,Timezone/Fuseau horaire,Type/Catégorie,Value/Valeur,Symbol/Symbole - ParseCHMNPeakCsv(csv, stationNumber, isDischarge, dischargeUnit, heightUnit, ts); + if (timeSeriesType == TimeSeriesType.PeakDischarge || timeSeriesType == TimeSeriesType.PeakStage) + { + // Peak data CSV format: + // ID,Parameter/Paramètre,Date,Timezone/Fuseau horaire,Type/Catégorie,Value/Valeur,Symbol/Symbole + ParseCHMNPeakCsv(csv, stationNumber, isDischarge, dischargeUnit, heightUnit, ts, indexesByDateTime); + } + else + { + // Daily and real-time CSV share the same column layout: + // ID,Date,Parameter/Paramètre,Value/Valeur,... + ParseCHMNDailyCsv(csv, stationNumber, timeSeriesType, isDischarge, dischargeUnit, heightUnit, ts, indexesByDateTime); + } } - else + finally { - // Daily and real-time CSV share the same column layout: - // ID,Date,Parameter/Paramètre,Value/Valeur,... - ParseCHMNDailyCsv(csv, stationNumber, timeSeriesType, isDischarge, dischargeUnit, heightUnit, ts); + ts.SuppressCollectionChanged = previousSuppressCollectionChanged; } } @@ -791,11 +1303,20 @@ public static async Task FromCHMN( /// /// Parse CHMN daily or real-time CSV data. /// + /// The CSV response body returned by the CHMN endpoint. + /// The CHMN station identifier to retain while parsing. + /// The requested time series type. + /// A value indicating whether discharge values are being parsed. + /// The discharge unit requested by the caller. + /// The height unit requested by the caller. + /// The time series being populated. + /// Lookup used to replace duplicate dates inline while parsing. /// /// CSV format: ID,Date,Parameter/Paramètre,Value/Valeur,... (additional columns vary by endpoint) /// private static void ParseCHMNDailyCsv(string csv, string stationNumber, TimeSeriesType timeSeriesType, - bool isDischarge, DischargeUnit dischargeUnit, HeightUnit heightUnit, TimeSeries ts) + bool isDischarge, DischargeUnit dischargeUnit, HeightUnit heightUnit, TimeSeries ts, + Dictionary indexesByDateTime) { bool isDailyType = timeSeriesType == TimeSeriesType.DailyDischarge || timeSeriesType == TimeSeriesType.DailyStage; @@ -864,9 +1385,9 @@ private static void ParseCHMNDailyCsv(string csv, string stationNumber, TimeSeri // Fill gaps with NaN for daily series only if (isDailyType && prevDate.HasValue && (date - prevDate.Value).Days > 1) - FillMissingDates(ts, prevDate.Value, date); + FillMissingDates(ts, prevDate.Value, date, indexesByDateTime); - ts.Add(new SeriesOrdinate(date, val)); + AddOrReplaceByDateTime(ts, indexesByDateTime, date, val); prevDate = date; } } @@ -875,12 +1396,20 @@ private static void ParseCHMNDailyCsv(string csv, string stationNumber, TimeSeri /// /// Parse CHMN peak data CSV. /// + /// The CSV response body returned by the CHMN endpoint. + /// The CHMN station identifier to retain while parsing. + /// A value indicating whether discharge values are being parsed. + /// The discharge unit requested by the caller. + /// The height unit requested by the caller. + /// The time series being populated. + /// Lookup used to replace duplicate dates inline while parsing. /// /// CSV format: ID,Parameter/Paramètre,Date,Timezone/Fuseau horaire,Type/Catégorie,Value/Valeur,Symbol/Symbole. /// Only "maximum" rows are included (minimums are skipped). /// private static void ParseCHMNPeakCsv(string csv, string stationNumber, bool isDischarge, - DischargeUnit dischargeUnit, HeightUnit heightUnit, TimeSeries ts) + DischargeUnit dischargeUnit, HeightUnit heightUnit, TimeSeries ts, + Dictionary indexesByDateTime) { using (var sr = new StringReader(csv)) { @@ -926,7 +1455,7 @@ private static void ParseCHMNPeakCsv(string csv, string stationNumber, bool isDi continue; double val = ConvertCHMNValue(raw, isDischarge, dischargeUnit, heightUnit); - ts.Add(new SeriesOrdinate(date, val)); + AddOrReplaceByDateTime(ts, indexesByDateTime, date, val); } } } @@ -968,6 +1497,7 @@ private static double ConvertCHMNValue(double raw, bool isDischarge, DischargeUn /// Desired depth unit for precipitation types. /// Optional start date. If null, attempts to retrieve full period of record. /// Optional end date. If null, defaults to today. + /// Token used to cancel the download. /// TimeSeries of values. public static async Task FromABOM( string stationNumber, @@ -976,7 +1506,8 @@ public static async Task FromABOM( HeightUnit heightUnit = HeightUnit.Meters, DepthUnit depthUnit = DepthUnit.Millimeters, DateTime? startDate = null, - DateTime? endDate = null) + DateTime? endDate = null, + CancellationToken cancellationToken = default) { @@ -995,10 +1526,6 @@ public static async Task FromABOM( "BOM API supports DailyDischarge, DailyStage, InstantaneousDischarge, InstantaneousStage, and DailyPrecipitation.", nameof(timeSeriesType)); - // Check connectivity - if (!await IsConnectedToInternet()) - throw new InvalidOperationException("No internet connection."); - // Set default dates DateTime sd = startDate ?? new DateTime(1800, 1, 1); DateTime ed = endDate ?? DateTime.Today; @@ -1023,28 +1550,16 @@ public static async Task FromABOM( string? tsId = null; - // Create HttpClientHandler with automatic decompression - var handler = new HttpClientHandler + // The shared BOM client carries the browser-like headers the BOM security + // appliance expects and reuses pooled connections across both KiWIS requests. { - AutomaticDecompression = System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate - }; - - using (var client = new HttpClient(handler)) - { - // Add browser-like headers to avoid security proxy issues - client.DefaultRequestHeaders.Add("User-Agent", UserAgent); - client.DefaultRequestHeaders.Add("Accept", "application/json,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); - client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9"); - client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate"); - client.DefaultRequestHeaders.Add("DNT", "1"); - client.DefaultRequestHeaders.Add("Connection", "keep-alive"); - client.Timeout = TimeSpan.FromSeconds(30); + var client = _bomClient; HttpStatusCode listStatus; string listResponse; try { - (listStatus, listResponse) = await GetWithRetryAsync(client, tsListUrl); + (listStatus, listResponse) = await GetWithRetryAsync(client, tsListUrl, cancellationToken: cancellationToken); } catch (HttpRequestException ex) { @@ -1156,29 +1671,22 @@ public static async Task FromABOM( var ts = isInstantaneous ? new TimeSeries(TimeInterval.Irregular) : new TimeSeries(TimeInterval.OneDay); DateTime? prevDate = null; + var indexesByDateTime = new Dictionary(); - // Create HttpClientHandler with automatic decompression - var handler2 = new HttpClientHandler - { - AutomaticDecompression = System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate - }; - - using (var client = new HttpClient(handler2)) + // Reuse the shared BOM client so the values request rides the connection the + // station lookup already established. { - // Add browser-like headers to avoid security proxy issues - client.DefaultRequestHeaders.Add("User-Agent", UserAgent); - client.DefaultRequestHeaders.Add("Accept", "application/json,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); - client.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9"); - client.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate"); - client.DefaultRequestHeaders.Add("DNT", "1"); - client.DefaultRequestHeaders.Add("Connection", "keep-alive"); - client.Timeout = TimeSpan.FromSeconds(60); + var client = _bomClient; HttpStatusCode valuesStatus; string valuesResponse; try { - (valuesStatus, valuesResponse) = await GetWithRetryAsync(client, valuesUrl); + (valuesStatus, valuesResponse) = await GetWithRetryAsync( + client, + valuesUrl, + requestTimeout: isInstantaneous ? InstantaneousProviderRequestTimeout : ProviderRequestTimeout, + cancellationToken: cancellationToken); } catch (HttpRequestException ex) { @@ -1216,63 +1724,72 @@ public static async Task FromABOM( if (!dataObj.TryGetProperty("data", out var dataArray)) throw new Exception("Invalid response format from BOM"); - // Parse the data array - each element is [timestamp, value, ...] - foreach (var point in dataArray.EnumerateArray()) + bool previousSuppressCollectionChanged = ts.SuppressCollectionChanged; + ts.SuppressCollectionChanged = true; + try { - if (point.GetArrayLength() < 2) continue; - - // BOM returns ISO-8601 with explicit offset, e.g. "2021-11-01T00:00:00.000+10:00". - // DateTime.TryParse converts these to LOCAL time, which on a non-AEST machine - // shifts daily series labels by a calendar day. DateTimeOffset preserves the - // offset; .Date gives the wall-clock date BOM intended. - string? timestampStr = point[0].GetString(); - if (!DateTimeOffset.TryParse(timestampStr, CultureInfo.InvariantCulture, - DateTimeStyles.None, out DateTimeOffset dto)) - continue; - DateTime date = isInstantaneous ? dto.DateTime : dto.Date; - - // Parse value - double val = double.NaN; - var valueElement = point[1]; - - if (valueElement.ValueKind == System.Text.Json.JsonValueKind.Number) + // Parse the data array - each element is [timestamp, value, ...] + foreach (var point in dataArray.EnumerateArray()) { - double raw = valueElement.GetDouble(); + if (point.GetArrayLength() < 2) continue; + + // BOM returns ISO-8601 with explicit offset, e.g. "2021-11-01T00:00:00.000+10:00". + // DateTime.TryParse converts these to LOCAL time, which on a non-AEST machine + // shifts daily series labels by a calendar day. DateTimeOffset preserves the + // offset; .Date gives the wall-clock date BOM intended. + string? timestampStr = point[0].GetString(); + if (!DateTimeOffset.TryParse(timestampStr, CultureInfo.InvariantCulture, + DateTimeStyles.None, out DateTimeOffset dto)) + continue; + DateTime date = isInstantaneous ? dto.DateTime : dto.Date; - // Apply unit conversion - if (isDischarge) - { - // BOM returns discharge in m³/s - val = dischargeUnit == DischargeUnit.CubicFeetPerSecond - ? raw * 35.3146667 - : raw; - } - else if (isStage) - { - // BOM returns stage in meters - val = heightUnit == HeightUnit.Feet - ? raw * 3.280839895 - : raw; - } - else if (isPrecip) + // Parse value + double val = double.NaN; + var valueElement = point[1]; + + if (valueElement.ValueKind == System.Text.Json.JsonValueKind.Number) { - // BOM returns rainfall in millimeters - val = depthUnit switch + double raw = valueElement.GetDouble(); + + // Apply unit conversion + if (isDischarge) + { + // BOM returns discharge in m³/s + val = dischargeUnit == DischargeUnit.CubicFeetPerSecond + ? raw * 35.3146667 + : raw; + } + else if (isStage) + { + // BOM returns stage in meters + val = heightUnit == HeightUnit.Feet + ? raw * 3.280839895 + : raw; + } + else if (isPrecip) { - DepthUnit.Millimeters => raw, - DepthUnit.Centimeters => raw / 10.0, - DepthUnit.Inches => raw / 25.4, - _ => raw - }; + // BOM returns rainfall in millimeters + val = depthUnit switch + { + DepthUnit.Millimeters => raw, + DepthUnit.Centimeters => raw / 10.0, + DepthUnit.Inches => raw / 25.4, + _ => raw + }; + } } - } - // Fill gaps with NaN for daily series only - if (!isInstantaneous && prevDate.HasValue && (date - prevDate.Value).Days > 1) - FillMissingDates(ts, prevDate.Value, date); + // Fill gaps with NaN for daily series only + if (!isInstantaneous && prevDate.HasValue && (date - prevDate.Value).Days > 1) + FillMissingDates(ts, prevDate.Value, date, indexesByDateTime); - ts.Add(new SeriesOrdinate(date, val)); - prevDate = date; + AddOrReplaceByDateTime(ts, indexesByDateTime, date, val); + prevDate = date; + } + } + finally + { + ts.SuppressCollectionChanged = previousSuppressCollectionChanged; } } diff --git a/Numerics/Numerics.csproj b/Numerics/Numerics.csproj index e7bc64f0..23427c6c 100644 --- a/Numerics/Numerics.csproj +++ b/Numerics/Numerics.csproj @@ -30,9 +30,9 @@ CS1587 - 2.1.1 - Version 2.1.1 is a metadata and release-process update for the JOSS publication. It refreshes citation, CodeMeta, README, and NuGet package metadata; increases the VSTest connection timeout in the release workflow; and does not change the public API. - 2.1.1.0 + 2.1.2 + Version 2.1.2 improves time-series download reliability and statistics utilities. Time-series downloads now prefer IPv4 while preserving IPv6 fallback, avoid unrelated connectivity preflights on restricted networks, normalize duplicate USGS/GHCN timestamps deterministically, and fix Series.Clear()/RemoveAt behavior with large or duplicate-valued series. This release also adds RunningStatistics.Clone() and fixes Combine()/+ so combined statistics do not alias input instances when one operand is empty. + 2.1.2.0 diff --git a/Numerics/Properties/AssemblyInfo.cs b/Numerics/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..68df9216 --- /dev/null +++ b/Numerics/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Test_Numerics")] diff --git a/Test_Numerics/Data/Statistics/Test_RunningStatistics.cs b/Test_Numerics/Data/Statistics/Test_RunningStatistics.cs index 61735579..5ba52f6f 100644 --- a/Test_Numerics/Data/Statistics/Test_RunningStatistics.cs +++ b/Test_Numerics/Data/Statistics/Test_RunningStatistics.cs @@ -285,5 +285,87 @@ public void Test_Add() var kurtosis = complete.Kurtosis; Assert.AreEqual(1.3434868130194, kurtosis, 1E-10); } + + /// + /// Test the clone method of the running statistics class. The clone should match the original statistics + /// and remain independent of the original after either instance is updated. + /// + [TestMethod] + public void Test_Clone() + { + var values = new double[] { 1, 2, 3, 4, 5 }; + var runningstat = new RunningStatistics(values); + var clone = runningstat.Clone(); + + // The clone should be a new instance that matches the original statistics. + Assert.AreNotSame(runningstat, clone); + Assert.AreEqual(5, clone.Count); + Assert.AreEqual(1, clone.Minimum); + Assert.AreEqual(5, clone.Maximum); + Assert.AreEqual(3, clone.Mean, 1E-10); + Assert.AreEqual(2.5, clone.Variance, 1E-10); + + // Pushing to the original should not change the clone. + runningstat.Push(new double[] { -10, 30 }); + Assert.AreEqual(5, clone.Count); + Assert.AreEqual(1, clone.Minimum); + Assert.AreEqual(5, clone.Maximum); + Assert.AreEqual(3, clone.Mean, 1E-10); + Assert.AreEqual(2.5, clone.Variance, 1E-10); + + // Pushing to the clone should not change the original. + var originalCount = runningstat.Count; + var originalMean = runningstat.Mean; + clone.Push(100); + Assert.AreEqual(originalCount, runningstat.Count); + Assert.AreEqual(originalMean, runningstat.Mean, 1E-10); + } + + /// + /// Test that combining with an empty running statistics returns a new instance + /// rather than aliasing the non-empty operand. + /// + [TestMethod] + public void Test_Combine_WithEmpty() + { + var values = new double[] { 1, 2, 3, 4, 5 }; + var stats = new RunningStatistics(values); + + // Empty first operand. The result should match the non-empty operand. + var combined = RunningStatistics.Combine(new RunningStatistics(), stats); + Assert.AreNotSame(stats, combined); + Assert.AreEqual(5, combined.Count); + Assert.AreEqual(1, combined.Minimum); + Assert.AreEqual(5, combined.Maximum); + Assert.AreEqual(3, combined.Mean, 1E-10); + Assert.AreEqual(2.5, combined.Variance, 1E-10); + + // Mutating the result should not mutate the operand. + combined.Push(1000); + Assert.AreEqual(5, stats.Count); + Assert.AreEqual(3, stats.Mean, 1E-10); + + // Empty second operand. + var combined2 = RunningStatistics.Combine(stats, new RunningStatistics()); + Assert.AreNotSame(stats, combined2); + combined2.Push(1000); + Assert.AreEqual(5, stats.Count); + Assert.AreEqual(3, stats.Mean, 1E-10); + + // The + operator should behave the same way. + var combined3 = new RunningStatistics() + stats; + Assert.AreNotSame(stats, combined3); + combined3.Push(1000); + Assert.AreEqual(5, stats.Count); + Assert.AreEqual(3, stats.Mean, 1E-10); + + // Combining two empty instances should return a new empty instance. + var empty1 = new RunningStatistics(); + var empty2 = new RunningStatistics(); + var combinedEmpty = RunningStatistics.Combine(empty1, empty2); + Assert.AreNotSame(empty1, combinedEmpty); + Assert.AreNotSame(empty2, combinedEmpty); + Assert.AreEqual(0, combinedEmpty.Count); + } } } diff --git a/Test_Numerics/Data/Time Series/Test_TimeSeries.cs b/Test_Numerics/Data/Time Series/Test_TimeSeries.cs index 7aced5d8..9a0e7b73 100644 --- a/Test_Numerics/Data/Time Series/Test_TimeSeries.cs +++ b/Test_Numerics/Data/Time Series/Test_TimeSeries.cs @@ -1474,5 +1474,82 @@ public void Test_ResampleWithBlockBootstrap_PreservesMarginalVariance() #endregion + #region Collection Behavior + + /// + /// Clear should empty the series and raise a single reset event. + /// + /// + /// Clear used to remove elements one at a time, costing two full equality scans per + /// element (O(n²) on large downloads). This locks in the single-reset contract of the + /// rewritten implementation. + /// + [TestMethod] + public void Test_Clear_EmptiesSeriesAndRaisesSingleReset() + { + var ts = new TimeSeries(TimeInterval.OneDay, new DateTime(2024, 1, 1), new double[] { 1, 2, 3, 4, 5 }); + var actions = new List(); + ts.CollectionChanged += (s, e) => actions.Add(e.Action); + + ts.Clear(); + + Assert.AreEqual(0, ts.Count); + Assert.HasCount(1, actions); + Assert.AreEqual(System.Collections.Specialized.NotifyCollectionChangedAction.Reset, actions[0]); + } + + /// + /// Clearing a large series should complete quickly. + /// + /// + /// Guards against reintroducing the element-by-element removal that made clearing a + /// century of daily data take seconds to minutes. + /// + [TestMethod] + [Timeout(5000, CooperativeCancellation = true)] + public void Test_Clear_LargeSeries_CompletesQuickly() + { + var ts = new TimeSeries(TimeInterval.OneDay, new DateTime(1900, 1, 1), new double[200000]); + + ts.Clear(); + + Assert.AreEqual(0, ts.Count); + } + + /// + /// RemoveAt should remove the ordinate at the requested index even when an equal-valued + /// ordinate appears earlier in the series. + /// + /// + /// RemoveAt used to delegate to Remove(item), which removed the first equal element, so + /// removing a duplicate by index silently deleted the wrong ordinate. + /// + [TestMethod] + public void Test_RemoveAt_WithDuplicateOrdinates_RemovesRequestedIndex() + { + var first = new SeriesOrdinate(new DateTime(2024, 1, 1), 1.0); + var middle = new SeriesOrdinate(new DateTime(2024, 1, 2), 2.0); + var duplicateOfFirst = new SeriesOrdinate(new DateTime(2024, 1, 1), 1.0); + var ts = new TimeSeries(TimeInterval.Irregular) { first, middle, duplicateOfFirst }; + + var removals = new List<(object item, int index)>(); + ts.CollectionChanged += (s, e) => + { + if (e.Action == System.Collections.Specialized.NotifyCollectionChangedAction.Remove) + removals.Add((e.OldItems?[0], e.OldStartingIndex)); + }; + + ts.RemoveAt(2); + + Assert.AreEqual(2, ts.Count); + Assert.IsTrue(ReferenceEquals(first, ts[0]), "The first ordinate should remain at index 0."); + Assert.IsTrue(ReferenceEquals(middle, ts[1]), "The middle ordinate should remain at index 1."); + Assert.HasCount(1, removals); + Assert.IsTrue(ReferenceEquals(duplicateOfFirst, removals[0].item), "The event should carry the removed ordinate."); + Assert.AreEqual(2, removals[0].index, "The event should report the requested index."); + } + + #endregion + } } diff --git a/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs b/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs index f5f6693d..c66b49e1 100644 --- a/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs +++ b/Test_Numerics/Data/Time Series/Test_TimeSeriesDownload.cs @@ -1,6 +1,12 @@ using System; using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Sockets; +using System.Threading; using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; using Numerics.Data; @@ -165,6 +171,563 @@ private static async Task AvailableAsync(string serviceName, Func pr } } + /// + /// HTTP handler that serves one deterministic USGS peak-flow response and rejects connectivity preflight requests. + /// + /// + /// The handler makes the regression test independent of live internet access while still + /// verifying the downloader requests the expected USGS endpoint directly. + /// + private sealed class GuardedUsgsPeakHandler : HttpMessageHandler + { + /// + /// Gets the absolute URLs requested through this handler. + /// + /// + /// The test uses this collection to verify no hidden preflight request was issued. + /// + internal List Requests { get; } = new List(); + + /// + /// + /// Returns a small USGS peak-flow RDB body only for the expected data URL and fails + /// connectivity preflight requests explicitly. + /// + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + string url = request.RequestUri?.AbsoluteUri ?? ""; + Requests.Add(url); + + if (IsBlockedPreflight(url)) + { + return Task.FromException( + new HttpRequestException($"Unexpected connectivity preflight request: {url}")); + } + + if (!IsExpectedUsgsPeakUrl(url)) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"Unexpected URL: {url}") + }); + } + + string body = string.Join(Environment.NewLine, new[] + { + "USGS\t01614000\t2020-04-21\t1\t12345" + }); + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(body) + }); + } + + /// + /// Determines whether a URL is one of the connectivity probes this regression test forbids. + /// + /// The absolute URL requested through the handler. + /// True when the URL is a forbidden connectivity preflight; otherwise false. + /// + /// The blocked list covers the old Google probe and the older generic USGS root probe. + /// + internal static bool IsBlockedPreflight(string url) + { + return url.Contains("google", StringComparison.OrdinalIgnoreCase) || + url.Contains("generate_204", StringComparison.OrdinalIgnoreCase) || + url.StartsWith("https://waterservices.usgs.gov/nwis", StringComparison.OrdinalIgnoreCase); + } + + /// + /// Determines whether a URL matches the expected USGS peak-flow request for this test fixture. + /// + /// The absolute URL requested through the handler. + /// True when the URL targets the expected USGS peak-flow endpoint; otherwise false. + /// + /// The check keeps the test tied to the production URL builder, including site number, + /// agency code, and RDB output format. + /// + private static bool IsExpectedUsgsPeakUrl(string url) + { + return url.StartsWith("https://nwis.waterdata.usgs.gov/nwis/peak?", StringComparison.OrdinalIgnoreCase) && + url.Contains("site_no=01614000", StringComparison.OrdinalIgnoreCase) && + url.Contains("agency_cd=USGS", StringComparison.OrdinalIgnoreCase) && + url.Contains("format=rdb", StringComparison.OrdinalIgnoreCase); + } + } + + /// + /// HTTP handler that serves deterministic USGS instantaneous RDB responses. + /// + /// + /// The handler allows only the expected /nwis/iv request for USGS 01646500 and + /// rejects generic connectivity probes. This keeps the test deterministic and independent + /// from the live USGS service. + /// + private sealed class GuardedUsgsInstantaneousHandler : HttpMessageHandler + { + /// + /// USGS parameter code expected by this handler. + /// + private readonly string _parameterCode; + + /// + /// Whether the handler should simulate an HTTP timeout after URL validation. + /// + private readonly bool _simulateTimeout; + + /// + /// Whether successful responses should complete asynchronously. + /// + private readonly bool _completeAsynchronously; + + /// + /// Number of duplicate timestamp pairs to append to the deterministic response. + /// + private readonly int _duplicateTimestampPairs; + + /// + /// Gets the absolute URLs requested through this handler. + /// + internal List Requests { get; } = new List(); + + /// + /// Initializes a new instance of the class. + /// + /// The expected USGS parameter code. + /// Whether to throw a timeout after validating the request URL. + /// Whether to complete successful responses on the thread pool. + /// The number of duplicate timestamp pairs to include in the response. + /// Thrown when is empty. + /// Thrown when is negative. + /// + /// Parameter 00060 represents discharge and 00065 represents gage height. + /// Asynchronous completion is used by synchronization-context regression tests, while + /// duplicate timestamp pairs exercise the downloader's normalization cleanup. + /// + internal GuardedUsgsInstantaneousHandler( + string parameterCode, + bool simulateTimeout = false, + bool completeAsynchronously = false, + int duplicateTimestampPairs = 0) + { + if (string.IsNullOrWhiteSpace(parameterCode)) + throw new ArgumentException("A parameter code is required.", nameof(parameterCode)); + if (duplicateTimestampPairs < 0) + throw new ArgumentOutOfRangeException(nameof(duplicateTimestampPairs), "Duplicate timestamp pair count cannot be negative."); + + _parameterCode = parameterCode; + _simulateTimeout = simulateTimeout; + _completeAsynchronously = completeAsynchronously; + _duplicateTimestampPairs = duplicateTimestampPairs; + } + + /// + /// + /// Returns a small nonuniform instantaneous RDB body only for the expected USGS data URL. + /// + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + string url = request.RequestUri?.AbsoluteUri ?? ""; + Requests.Add(url); + + if (IsBlockedPreflight(url)) + { + return Task.FromException( + new HttpRequestException($"Unexpected connectivity preflight request: {url}")); + } + + if (!IsExpectedUsgsInstantaneousUrl(url)) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"Unexpected URL: {url}") + }); + } + + if (_simulateTimeout) + { + return Task.FromException( + new TaskCanceledException($"Simulated instantaneous timeout: {url}")); + } + + string valueOne = _parameterCode == "00060" ? "101.5" : "6.12"; + string valueTwo = _parameterCode == "00060" ? "102.0" : "6.13"; + string valueThree = _parameterCode == "00060" ? "104.5" : "6.17"; + var lines = new List + { + $"agency_cd\tsite_no\tdatetime\ttz_cd\t{_parameterCode}_00000\t{_parameterCode}_00000_cd", + "5s\t15s\t20d\t6s\t14n\t10s", + $"USGS\t01646500\t2024-01-01 00:00\tEST\t{valueOne}\tA", + $"USGS\t01646500\t2024-01-01 00:15\tEST\t{valueTwo}\tA", + $"USGS\t01646500\t2024-01-01 00:45\tEST\t{valueThree}\tA" + }; + + for (int i = 0; i < _duplicateTimestampPairs; i++) + { + string timestamp = new DateTime(2024, 1, 2).AddMinutes(i).ToString("yyyy-MM-dd HH:mm", CultureInfo.InvariantCulture); + lines.Add($"USGS\t01646500\t{timestamp}\tEST\t{1000d + i:0.0}\tP"); + lines.Add($"USGS\t01646500\t{timestamp}\tEST\t{2000d + i:0.0}\tA"); + } + + string body = string.Join(Environment.NewLine, lines); + + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(body) + }; + + return _completeAsynchronously + ? Task.Run(() => response) + : Task.FromResult(response); + } + + /// + /// Determines whether a URL is one of the connectivity probes this regression forbids. + /// + /// The absolute URL requested through the handler. + /// true when the URL is a forbidden preflight; otherwise, false. + /// + /// The instantaneous data endpoint is under /nwis/iv; only generic roots and the + /// old Google probe are treated as preflight requests. + /// + internal static bool IsBlockedPreflight(string url) + { + string normalized = url.TrimEnd('/'); + return url.Contains("google", StringComparison.OrdinalIgnoreCase) || + url.Contains("generate_204", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "https://waterservices.usgs.gov/nwis", StringComparison.OrdinalIgnoreCase); + } + + /// + /// Determines whether a URL matches the expected USGS instantaneous request. + /// + /// The absolute URL requested through the handler. + /// true when the URL targets the expected fixture endpoint; otherwise, false. + /// + /// The check pins the request to the instantaneous-value API, the 01646500 gage, the + /// requested parameter code, and RDB formatting. + /// + private bool IsExpectedUsgsInstantaneousUrl(string url) + { + return url.StartsWith("https://waterservices.usgs.gov/nwis/iv/?", StringComparison.OrdinalIgnoreCase) && + url.Contains("sites=01646500", StringComparison.OrdinalIgnoreCase) && + url.Contains($"parameterCd={_parameterCode}", StringComparison.OrdinalIgnoreCase) && + url.Contains("startDT=1900-01-01", StringComparison.OrdinalIgnoreCase) && + url.Contains("siteStatus=all", StringComparison.OrdinalIgnoreCase) && + url.Contains("format=rdb", StringComparison.OrdinalIgnoreCase); + } + } + + /// + /// Synchronization context that records posted continuations and executes them on the thread pool. + /// + /// + /// Tests use this context to prove downloader internals do not marshal long-running parsing + /// continuations back to a WPF-like caller context. + /// + private sealed class CountingSynchronizationContext : SynchronizationContext + { + /// + /// Count of posted callbacks. + /// + private int _postCount; + + /// + /// Gets the number of callbacks posted to this context. + /// + internal int PostCount => _postCount; + + /// + /// Records and dispatches an asynchronous callback. + /// + /// The callback to execute. + /// The callback state. + /// + /// The callback is still executed so tests do not deadlock if a regression posts to the + /// context; the post count then fails the assertion. + /// + public override void Post(SendOrPostCallback d, object state) + { + Interlocked.Increment(ref _postCount); + ThreadPool.QueueUserWorkItem(_ => d(state)); + } + } + + /// + /// HTTP handler that serves a deterministic USGS measured-stage response with duplicate timestamps. + /// + /// + /// The fixture is based on USGS 01570500, where the live field-measurements API returns two + /// approved gage-height records for 1936-03-15 05:00:00 with values 14.51 and 14.52. + /// + private sealed class DuplicateUsgsMeasuredStageHandler : HttpMessageHandler + { + /// + /// Gets the absolute URLs requested through this handler. + /// + internal List Requests { get; } = new List(); + + /// + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + string url = request.RequestUri?.AbsoluteUri ?? ""; + Requests.Add(url); + + if (GuardedUsgsPeakHandler.IsBlockedPreflight(url)) + { + return Task.FromException( + new HttpRequestException($"Unexpected connectivity preflight request: {url}")); + } + + if (!IsExpectedUsgsMeasuredStageUrl(url)) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"Unexpected URL: {url}") + }); + } + + string body = string.Join("", new[] + { + "{", + "\"type\":\"FeatureCollection\",", + "\"features\":[", + "{\"type\":\"Feature\",\"id\":\"75f11a56-5796-4dc9-b22a-054da446a164\",\"properties\":{\"time\":\"1936-03-15T05:00:00+00:00\",\"value\":\"14.51\"}},", + "{\"type\":\"Feature\",\"id\":\"8dccaa2d-8480-4b5d-b19b-18122749b6c6\",\"properties\":{\"time\":\"1936-03-15T05:00:00+00:00\",\"value\":\"14.52\"}}", + "],", + "\"links\":[]", + "}" + }); + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(body) + }); + } + + /// + /// Determines whether a URL matches the expected USGS measured-stage request. + /// + private static bool IsExpectedUsgsMeasuredStageUrl(string url) + { + return url.StartsWith("https://api.waterdata.usgs.gov/ogcapi/v0/collections/field-measurements/items?", + StringComparison.OrdinalIgnoreCase) && + url.Contains("monitoring_location_id=USGS-01570500", StringComparison.OrdinalIgnoreCase) && + url.Contains("parameter_code=00065", StringComparison.OrdinalIgnoreCase) && + url.Contains("limit=10000", StringComparison.OrdinalIgnoreCase) && + url.Contains("f=json", StringComparison.OrdinalIgnoreCase); + } + } + + /// + /// HTTP handler that waits until the supplied cancellation token is canceled. + /// + /// + /// This handler verifies that downloader cancellation reaches the HTTP request rather than + /// only canceling caller-side waiting. + /// + private sealed class CancellableUsgsPeakHandler : HttpMessageHandler + { + /// + /// Gets the absolute URLs requested through this handler. + /// + /// + /// The test uses this collection to confirm the request was started before cancellation. + /// + internal List Requests { get; } = new List(); + + /// + /// + /// The handler deliberately waits for cancellation and never returns a successful response. + /// + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Add(request.RequestUri?.AbsoluteUri ?? ""); + await Task.Delay(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("") + }; + } + } + + /// + /// HTTP handler that serves deterministic GHCN daily station-file responses. + /// + /// + /// The handler allows only the expected NOAA NCEI USC00040741.dly station-file URL + /// and rejects generic connectivity probes. This keeps GHCN regression tests deterministic. + /// + private sealed class GuardedGhcnDailyHandler : HttpMessageHandler + { + /// + /// Whether the handler should simulate an HTTP timeout after URL validation. + /// + private readonly bool _simulateTimeout; + + /// + /// Whether the handler should include a duplicate precipitation row for the same station month. + /// + private readonly bool _includeDuplicatePrecipitationRow; + + /// + /// Gets the absolute URLs requested through this handler. + /// + internal List Requests { get; } = new List(); + + /// + /// Initializes a new instance of the class. + /// + /// Whether to throw a timeout after validating the request URL. + /// Whether to include duplicate daily precipitation records for the same dates. + /// + /// Timeout simulation is used to verify that the downloader reports the GHCN-specific + /// timeout window without waiting for the full timeout duration. Duplicate-row simulation + /// verifies inline last-value-wins normalization without a post-download pass. + /// + internal GuardedGhcnDailyHandler( + bool simulateTimeout = false, + bool includeDuplicatePrecipitationRow = false) + { + _simulateTimeout = simulateTimeout; + _includeDuplicatePrecipitationRow = includeDuplicatePrecipitationRow; + } + + /// + /// + /// Returns a one-month GHCN daily file containing precipitation and snow rows only for + /// the expected station-file URL. + /// + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + string url = request.RequestUri?.AbsoluteUri ?? ""; + Requests.Add(url); + + if (IsBlockedPreflight(url)) + { + return Task.FromException( + new HttpRequestException($"Unexpected connectivity preflight request: {url}")); + } + + if (!IsExpectedGhcnStationUrl(url)) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"Unexpected URL: {url}") + }); + } + + if (_simulateTimeout) + { + return Task.FromException( + new TaskCanceledException($"Simulated GHCN timeout: {url}")); + } + + var rows = new List + { + BuildGhcnDailyLine("USC00040741", 2024, 1, "PRCP", new Dictionary + { + [1] = 254, + [2] = -9999, + [3] = 508 + }), + BuildGhcnDailyLine("USC00040741", 2024, 1, "SNOW", new Dictionary + { + [1] = 0 + }) + }; + + if (_includeDuplicatePrecipitationRow) + { + rows.Add(BuildGhcnDailyLine("USC00040741", 2024, 1, "PRCP", new Dictionary + { + [1] = 762, + [2] = 1016, + [3] = 1270 + })); + } + + string body = string.Join(Environment.NewLine, rows); + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(body) + }); + } + + /// + /// Determines whether a URL is one of the connectivity probes this regression forbids. + /// + /// The absolute URL requested through the handler. + /// true when the URL is a forbidden preflight; otherwise, false. + /// + /// GHCN daily data should request the station file directly; generic root checks are not + /// required before attempting the provider request. + /// + internal static bool IsBlockedPreflight(string url) + { + string normalized = url.TrimEnd('/'); + return url.Contains("google", StringComparison.OrdinalIgnoreCase) || + url.Contains("generate_204", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "https://www.ncei.noaa.gov", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "https://www.ncei.noaa.gov/pub/data/ghcn/daily/all", StringComparison.OrdinalIgnoreCase); + } + + /// + /// Determines whether a URL matches the expected GHCN station-file request. + /// + /// The absolute URL requested through the handler. + /// true when the URL targets the expected fixture endpoint; otherwise, false. + /// + /// The check pins the request to the NOAA NCEI daily station-file path and the exact + /// station that failed in BestFit. + /// + private static bool IsExpectedGhcnStationUrl(string url) + { + return string.Equals( + url, + "https://www.ncei.noaa.gov/pub/data/ghcn/daily/all/USC00040741.dly", + StringComparison.OrdinalIgnoreCase); + } + + /// + /// Builds one fixed-width GHCN daily station-file line. + /// + /// The 11-character GHCN station identifier. + /// The four-digit year. + /// The one-based month number. + /// The four-character GHCN element code. + /// Raw GHCN values keyed by one-based day of month. + /// A fixed-width daily line in GHCN .dly format. + /// + /// Each daily value is represented by a five-character integer followed by three flag + /// characters. Missing days are filled with -9999, matching the provider format. + /// + private static string BuildGhcnDailyLine( + string stationId, + int year, + int month, + string element, + IReadOnlyDictionary valuesByDay) + { + var builder = new System.Text.StringBuilder(); + builder.Append(stationId); + builder.Append(year.ToString("0000", CultureInfo.InvariantCulture)); + builder.Append(month.ToString("00", CultureInfo.InvariantCulture)); + builder.Append(element); + + for (int day = 1; day <= 31; day++) + { + int value = valuesByDay.TryGetValue(day, out int parsedValue) ? parsedValue : -9999; + builder.Append(value.ToString(CultureInfo.InvariantCulture).PadLeft(5)); + builder.Append(" "); + } + + return builder.ToString(); + } + } + /// /// Probes a small windowed CHMN download to confirm Water Survey of Canada services are responsive. /// @@ -239,6 +802,86 @@ private static void AssertRoughlyEqual(double a, double b, double relTol = 1e-6, #endregion + #region Connection Handling Tests + + /// + /// Verifies connect-address ordering puts IPv4 before IPv6 while preserving relative order. + /// + /// + /// Providers such as NOAA NCEI publish several AAAA records. On networks that silently + /// drop IPv6, connecting in DNS order costs a full TCP timeout per dead address, so the + /// downloader must try IPv4 first. + /// + [TestMethod] + public void OrderAddressesForConnect_PutsIPv4First_PreservingRelativeOrder() + { + var v6First = IPAddress.Parse("2610:20:8040:2::178"); + var v4First = IPAddress.Parse("205.167.25.177"); + var v6Second = IPAddress.Parse("2610:20:8040:2::177"); + var v4Second = IPAddress.Parse("205.167.25.178"); + + var ordered = TimeSeriesDownload.OrderAddressesForConnect( + new[] { v6First, v4First, v6Second, v4Second }); + + CollectionAssert.AreEqual(new[] { v4First, v4Second, v6First, v6Second }, ordered); + } + + /// + /// Verifies connect-address ordering passes an all-IPv6 result through unchanged. + /// + /// + /// IPv6-only hosts must remain reachable; preferring IPv4 must never drop IPv6 addresses. + /// + [TestMethod] + public void OrderAddressesForConnect_AllIPv6_PassesThroughUnchanged() + { + var addresses = new[] + { + IPAddress.Parse("2610:20:8040:2::178"), + IPAddress.Parse("2610:20:8040:2::177") + }; + + var ordered = TimeSeriesDownload.OrderAddressesForConnect(addresses); + + CollectionAssert.AreEqual(addresses, ordered); + } + + /// + /// Verifies connect-address ordering tolerates an empty DNS result. + /// + [TestMethod] + public void OrderAddressesForConnect_Empty_ReturnsEmpty() + { + Assert.IsEmpty(TimeSeriesDownload.OrderAddressesForConnect(Array.Empty())); + } + + /// + /// Verifies the .NET Framework bind delegate rejects IPv6 remote endpoints. + /// + /// + /// On .NET Framework the service point tries resolved addresses in DNS order with no + /// per-address bound. Throwing a socket exception from the bind delegate makes a blocked + /// IPv6 attempt fail immediately so the service point moves on to an IPv4 address. + /// + [TestMethod] + public void RejectIPv6BindEndPoint_ThrowsSocketExceptionForIPv6() + { + var remote = new IPEndPoint(IPAddress.Parse("2610:20:8040:2::178"), 443); + Assert.Throws(() => TimeSeriesDownload.RejectIPv6BindEndPoint(remote)); + } + + /// + /// Verifies the .NET Framework bind delegate lets IPv4 remote endpoints bind normally. + /// + [TestMethod] + public void RejectIPv6BindEndPoint_ReturnsNullForIPv4() + { + var remote = new IPEndPoint(IPAddress.Parse("205.167.25.177"), 443); + Assert.IsNull(TimeSeriesDownload.RejectIPv6BindEndPoint(remote)); + } + + #endregion + #region CHMN (Canada) Tests /// @@ -411,6 +1054,320 @@ public async Task CHMN_PeakStage_Works() #region USGS Tests + /// + /// Verifies USGS downloads do not run a generic connectivity preflight before requesting the data. + /// + /// A task that completes when the regression check finishes. + /// + /// The test uses a deterministic in-memory HTTP handler so it can prove request routing + /// without relying on the live USGS service or the host machine's internet connection. + /// + [TestMethod] + public async Task USGS_PeakDischarge_DoesNotRunConnectivityPreflight() + { + var defaultHandler = new GuardedUsgsPeakHandler(); + var decompressHandler = new GuardedUsgsPeakHandler(); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var (ts, raw) = await TimeSeriesDownload.FromUSGS(USGS_3, TimeSeriesDownload.TimeSeriesType.PeakDischarge); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.AreEqual(1, ts.Count); + Assert.AreEqual(12345d, ts[0].Value); + Assert.IsFalse(string.IsNullOrWhiteSpace(raw), "Raw text should contain the fake USGS response."); + Assert.HasCount(1, defaultHandler.Requests, "Expected exactly one USGS data request."); + Assert.IsFalse(defaultHandler.Requests.Any(GuardedUsgsPeakHandler.IsBlockedPreflight)); + Assert.IsEmpty(decompressHandler.Requests, "Peak download should not use the decompression client."); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies USGS downloads honor caller cancellation while waiting on HTTP. + /// + /// A task that completes when the cancellation regression check finishes. + /// + /// BestFit relies on this behavior to stop waiting when an external provider is blocked or + /// unavailable from the user's network. + /// + [TestMethod] + public async Task USGS_PeakDischarge_HonorsCancellationToken() + { + var defaultHandler = new CancellableUsgsPeakHandler(); + var decompressHandler = new CancellableUsgsPeakHandler(); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(60) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(60) }; + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(50)); + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + await Assert.ThrowsAsync(async () => + await TimeSeriesDownload.FromUSGS(USGS_3, TimeSeriesDownload.TimeSeriesType.PeakDischarge, cts.Token)); + + Assert.HasCount(1, defaultHandler.Requests, "Expected one started USGS peak request."); + Assert.IsEmpty(decompressHandler.Requests, "Peak download should not use the decompression client."); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies duplicate USGS measured-stage timestamps keep the last source value. + /// + [TestMethod] + public async Task USGS_MeasuredStage_DuplicateDateTimes_LastValueWins() + { + var defaultHandler = new DuplicateUsgsMeasuredStageHandler(); + var decompressHandler = new DuplicateUsgsMeasuredStageHandler(); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var (ts, raw) = await TimeSeriesDownload.FromUSGS(USGS_5, + TimeSeriesDownload.TimeSeriesType.MeasuredStage); + + var duplicateTime = new DateTime(1936, 3, 15, 5, 0, 0, DateTimeKind.Utc); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.IsFalse(string.IsNullOrWhiteSpace(raw), "Raw text should contain the fake USGS response."); + Assert.AreEqual(ts.Count, ts.Select(o => o.Index).Distinct().Count(), "Duplicate date indices detected."); + Assert.AreEqual(1, ts.Count(o => o.Index == duplicateTime), "Expected one retained record for the duplicate timestamp."); + Assert.AreEqual(14.52d, ts.Single(o => o.Index == duplicateTime).Value); + Assert.IsEmpty(defaultHandler.Requests, "Measured stage should not use the default client."); + Assert.HasCount(1, decompressHandler.Requests, "Expected exactly one USGS OGC data request."); + Assert.IsTrue(decompressHandler.Requests[0].Contains("monitoring_location_id=USGS-01570500", StringComparison.OrdinalIgnoreCase)); + Assert.IsTrue(decompressHandler.Requests[0].Contains("parameter_code=00065", StringComparison.OrdinalIgnoreCase)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies deterministic USGS instantaneous discharge downloads remain irregular. + /// + /// A task that completes when the regression check finishes. + /// + /// USGS instantaneous discharge uses parameter 00060 on the /nwis/iv endpoint. + /// + [TestMethod] + public async Task USGS_InstantaneousDischarge_DeterministicDownload_IsIrregular() + { + await VerifyUsgsInstantaneousDownloadIsIrregular( + TimeSeriesDownload.TimeSeriesType.InstantaneousDischarge, + "00060"); + } + + /// + /// Verifies deterministic USGS instantaneous stage downloads remain irregular. + /// + /// A task that completes when the regression check finishes. + /// + /// USGS instantaneous stage uses parameter 00065 on the /nwis/iv endpoint. + /// + [TestMethod] + public async Task USGS_InstantaneousStage_DeterministicDownload_IsIrregular() + { + await VerifyUsgsInstantaneousDownloadIsIrregular( + TimeSeriesDownload.TimeSeriesType.InstantaneousStage, + "00065"); + } + + /// + /// Verifies instantaneous USGS timeout messages use the longer instantaneous limit. + /// + /// A task that completes when the regression check finishes. + /// + /// The handler throws a timeout immediately after URL validation so the test does not wait + /// for the full timeout duration. + /// + [TestMethod] + public async Task USGS_InstantaneousDischarge_TimeoutMessage_UsesInstantaneousLimit() + { + var defaultHandler = new GuardedUsgsInstantaneousHandler("00060", simulateTimeout: true); + var decompressHandler = new GuardedUsgsInstantaneousHandler("00060", simulateTimeout: true); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var exception = await Assert.ThrowsAsync(async () => + await TimeSeriesDownload.FromUSGS(USGS_6, TimeSeriesDownload.TimeSeriesType.InstantaneousDischarge)); + + Assert.Contains("120 seconds", exception.Message); + Assert.IsEmpty(defaultHandler.Requests, "Instantaneous download should not use the default client."); + Assert.HasCount(2, decompressHandler.Requests, "Expected the retry helper to make two instantaneous attempts."); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies USGS instantaneous parsing does not resume on the caller synchronization context. + /// + /// A task that completes when the regression check finishes. + /// + /// BestFit starts downloads from the WPF UI thread. This test forces the fake HTTP response + /// to complete asynchronously while a synchronization context is installed, then verifies + /// the downloader did not post its large-response continuation back to that context. + /// + [TestMethod] + public async Task USGS_InstantaneousDischarge_DoesNotPostBackToSynchronizationContext() + { + var defaultHandler = new GuardedUsgsInstantaneousHandler("00060", completeAsynchronously: true); + var decompressHandler = new GuardedUsgsInstantaneousHandler("00060", completeAsynchronously: true); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var context = new CountingSynchronizationContext(); + SynchronizationContext previousContext = SynchronizationContext.Current; + Task<(TimeSeries TimeSeries, string RawText)> downloadTask; + + try + { + SynchronizationContext.SetSynchronizationContext(context); + downloadTask = TimeSeriesDownload.FromUSGS( + USGS_6, + TimeSeriesDownload.TimeSeriesType.InstantaneousDischarge); + } + finally + { + SynchronizationContext.SetSynchronizationContext(previousContext); + } + + var (ts, raw) = await downloadTask; + + Assert.AreEqual(0, context.PostCount, "USGS instantaneous download should not post continuations to the caller synchronization context."); + Assert.AreEqual(TimeInterval.Irregular, ts.TimeInterval); + Assert.AreEqual(3, ts.Count); + Assert.IsFalse(string.IsNullOrWhiteSpace(raw), "Raw text should contain the fake USGS response."); + Assert.IsEmpty(defaultHandler.Requests, "Instantaneous download should not use the default client."); + Assert.HasCount(1, decompressHandler.Requests, "Expected exactly one USGS instantaneous data request."); + Assert.IsFalse(decompressHandler.Requests.Any(GuardedUsgsInstantaneousHandler.IsBlockedPreflight)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies duplicate-heavy USGS instantaneous responses are normalized without a long rebuild. + /// + /// A task that completes when the regression check finishes. + /// + /// The fixture mimics provisional/approved duplicate timestamps in instantaneous RDB data. + /// A previous cleanup path cleared the million-row series through per-item removal, which + /// made full-period BestFit downloads appear to spin indefinitely after the response arrived. + /// + [TestMethod] + [Timeout(15000, CooperativeCancellation = true)] + public async Task USGS_InstantaneousDischarge_DuplicateHeavyResponse_NormalizesQuickly() + { + const int duplicateTimestampPairs = 50000; + var defaultHandler = new GuardedUsgsInstantaneousHandler("00060", duplicateTimestampPairs: duplicateTimestampPairs); + var decompressHandler = new GuardedUsgsInstantaneousHandler("00060", duplicateTimestampPairs: duplicateTimestampPairs); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var (ts, raw) = await TimeSeriesDownload.FromUSGS( + USGS_6, + TimeSeriesDownload.TimeSeriesType.InstantaneousDischarge); + var finalDuplicateTimestamp = new DateTime(2024, 1, 2).AddMinutes(duplicateTimestampPairs - 1); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.AreEqual(TimeInterval.Irregular, ts.TimeInterval); + Assert.AreEqual(3 + duplicateTimestampPairs, ts.Count); + Assert.AreEqual(ts.Count, ts.Select(o => o.Index).Distinct().Count(), "Duplicate date indices detected."); + Assert.AreEqual(2000d + duplicateTimestampPairs - 1, ts.Single(o => o.Index == finalDuplicateTimestamp).Value); + Assert.IsFalse(string.IsNullOrWhiteSpace(raw), "Raw text should contain the fake USGS response."); + Assert.IsEmpty(defaultHandler.Requests, "Instantaneous download should not use the default client."); + Assert.HasCount(1, decompressHandler.Requests, "Expected exactly one USGS instantaneous data request."); + Assert.IsFalse(decompressHandler.Requests.Any(GuardedUsgsInstantaneousHandler.IsBlockedPreflight)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies a deterministic USGS instantaneous download preserves nonuniform timestamps. + /// + /// The instantaneous USGS series type to request. + /// The expected USGS parameter code. + /// A task that completes when the regression check finishes. + /// + /// The fixture has a 15-minute step followed by a 30-minute step. The downloader must keep + /// those timestamps exactly and must not convert the series into a regular interval. + /// + private static async Task VerifyUsgsInstantaneousDownloadIsIrregular( + TimeSeriesDownload.TimeSeriesType seriesType, + string parameterCode) + { + var defaultHandler = new GuardedUsgsInstantaneousHandler(parameterCode); + var decompressHandler = new GuardedUsgsInstantaneousHandler(parameterCode); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var (ts, raw) = await TimeSeriesDownload.FromUSGS(USGS_6, seriesType); + var expectedDates = new[] + { + new DateTime(2024, 1, 1, 0, 0, 0), + new DateTime(2024, 1, 1, 0, 15, 0), + new DateTime(2024, 1, 1, 0, 45, 0) + }; + + Assert.IsNotNull(ts, "Time series is null."); + Assert.AreEqual(TimeInterval.Irregular, ts.TimeInterval); + Assert.AreEqual(expectedDates.Length, ts.Count); + CollectionAssert.AreEqual(expectedDates, ts.Select(o => o.Index).ToArray()); + Assert.IsFalse(string.IsNullOrWhiteSpace(raw), "Raw text should contain the fake USGS response."); + Assert.IsEmpty(defaultHandler.Requests, "Instantaneous download should not use the default client."); + Assert.HasCount(1, decompressHandler.Requests, "Expected exactly one USGS instantaneous data request."); + Assert.IsTrue(decompressHandler.Requests[0].Contains("/nwis/iv/", StringComparison.OrdinalIgnoreCase)); + Assert.IsTrue(decompressHandler.Requests[0].Contains($"parameterCd={parameterCode}", StringComparison.OrdinalIgnoreCase)); + Assert.IsTrue(decompressHandler.Requests[0].Contains("startDT=1900-01-01", StringComparison.OrdinalIgnoreCase)); + Assert.IsFalse(decompressHandler.Requests.Any(GuardedUsgsInstantaneousHandler.IsBlockedPreflight)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + /// /// Tests full-period-of-record USGS daily discharge download. /// @@ -567,10 +1524,136 @@ public async Task USGS_InstantaneousStage_Works() #region GHCN Tests + /// + /// Verifies deterministic GHCN daily precipitation downloads parse station files directly. + /// + /// A task that completes when the regression check finishes. + /// + /// The fixture pins the request to NOAA NCEI station USC00040741, confirms no + /// connectivity preflight is made, and verifies GHCN tenths-of-millimeters are converted to + /// inches. + /// + [TestMethod] + public async Task GHCN_DailyPrecipitation_DeterministicDownload_ParsesAndNoPreflight() + { + var defaultHandler = new GuardedGhcnDailyHandler(); + var decompressHandler = new GuardedGhcnDailyHandler(); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var ts = await TimeSeriesDownload.FromGHCN( + GHCN_1, + TimeSeriesDownload.TimeSeriesType.DailyPrecipitation, + TimeSeriesDownload.DepthUnit.Inches); + + Assert.IsNotNull(ts, "Time series is null."); + Assert.AreEqual(TimeInterval.OneDay, ts.TimeInterval); + Assert.AreEqual(31, ts.Count); + Assert.AreEqual(new DateTime(2024, 1, 1), ts.StartDate); + Assert.AreEqual(new DateTime(2024, 1, 31), ts.EndDate); + Assert.AreEqual(1d, ts[0].Value); + Assert.IsTrue(double.IsNaN(ts[1].Value)); + Assert.AreEqual(2d, ts[2].Value); + Assert.HasCount(1, defaultHandler.Requests, "Expected exactly one GHCN station-file request."); + Assert.IsEmpty(decompressHandler.Requests, "GHCN should not use the compressed client."); + Assert.IsFalse(defaultHandler.Requests.Any(GuardedGhcnDailyHandler.IsBlockedPreflight)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies GHCN duplicate daily timestamps are replaced while station rows are parsed. + /// + /// A task that completes when the regression check finishes. + /// + /// This pins the intended single-pass behavior: the later GHCN row for a duplicate date wins + /// at ordinate creation time, avoiding the previous post-download duplicate scan. + /// + [TestMethod] + public async Task GHCN_DailyPrecipitation_DuplicateDateTimes_LastValueWinsInline() + { + var defaultHandler = new GuardedGhcnDailyHandler(includeDuplicatePrecipitationRow: true); + var decompressHandler = new GuardedGhcnDailyHandler(includeDuplicatePrecipitationRow: true); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var ts = await TimeSeriesDownload.FromGHCN( + GHCN_1, + TimeSeriesDownload.TimeSeriesType.DailyPrecipitation, + TimeSeriesDownload.DepthUnit.Inches); + + Assert.AreEqual(31, ts.Count); + Assert.AreEqual(ts.Count, ts.Select(o => o.Index).Distinct().Count(), "Duplicate date indices detected."); + Assert.AreEqual(3d, ts[0].Value); + Assert.AreEqual(4d, ts[1].Value); + Assert.AreEqual(5d, ts[2].Value); + Assert.HasCount(1, defaultHandler.Requests, "Expected exactly one GHCN station-file request."); + Assert.IsEmpty(decompressHandler.Requests, "GHCN should not use the compressed client."); + Assert.IsFalse(defaultHandler.Requests.Any(GuardedGhcnDailyHandler.IsBlockedPreflight)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + + /// + /// Verifies GHCN timeout messages use the dedicated GHCN provider limit. + /// + /// A task that completes when the regression check finishes. + /// + /// Full station files can be larger than other provider payloads, so GHCN uses a limit + /// above the generic provider timeout. The handler throws a timeout immediately after URL + /// validation so this test does not wait for the full provider timeout. + /// + [TestMethod] + public async Task GHCN_DailyPrecipitation_TimeoutMessage_UsesGhcnLimit() + { + var defaultHandler = new GuardedGhcnDailyHandler(simulateTimeout: true); + var decompressHandler = new GuardedGhcnDailyHandler(simulateTimeout: true); + + using var defaultClient = new HttpClient(defaultHandler) { Timeout = TimeSpan.FromSeconds(5) }; + using var decompressClient = new HttpClient(decompressHandler) { Timeout = TimeSpan.FromSeconds(5) }; + + TimeSeriesDownload.SetHttpClientsForTesting(defaultClient, decompressClient); + try + { + var exception = await Assert.ThrowsAsync(async () => + await TimeSeriesDownload.FromGHCN( + GHCN_1, + TimeSeriesDownload.TimeSeriesType.DailyPrecipitation)); + + Assert.Contains("60 seconds", exception.Message); + Assert.HasCount(2, defaultHandler.Requests, "Expected the retry helper to make two GHCN attempts."); + Assert.IsEmpty(decompressHandler.Requests, "GHCN should not use the compressed client."); + Assert.IsFalse(defaultHandler.Requests.Any(GuardedGhcnDailyHandler.IsBlockedPreflight)); + } + finally + { + TimeSeriesDownload.ResetHttpClientsForTesting(); + } + } + /// /// Tests full-period-of-record GHCN daily precipitation download. /// + /// + /// The ceiling guards against connection-establishment regressions: walking dead IPv6 + /// addresses before IPv4 once made this download take minutes instead of seconds. + /// [TestMethod, TestCategory("Integration")] + [Timeout(60000, CooperativeCancellation = true)] public async Task GHCN_FullPor_Precipitation() { if (!await GhcnAvailable()) return; @@ -581,7 +1664,12 @@ public async Task GHCN_FullPor_Precipitation() /// /// Tests full-period-of-record GHCN daily snow download. /// + /// + /// The ceiling guards against connection-establishment regressions: walking dead IPv6 + /// addresses before IPv4 once made this download take minutes instead of seconds. + /// [TestMethod, TestCategory("Integration")] + [Timeout(60000, CooperativeCancellation = true)] public async Task GHCN_FullPor_Snow() { if (!await GhcnAvailable()) return; diff --git a/codemeta.json b/codemeta.json index 72b1eb33..046bf4b6 100644 --- a/codemeta.json +++ b/codemeta.json @@ -4,9 +4,9 @@ "name": "Numerics: A .NET Library for Numerical Computing, Statistical Analysis, and Risk Assessment", "alternateName": "Numerics", "description": "A free and open-source .NET library providing numerical methods, probability distributions, statistical analysis, and Bayesian inference tools for quantitative risk assessment in water resources engineering.", - "version": "2.1.1", + "version": "2.1.2", "dateCreated": "2023-09-28", - "dateModified": "2026-06-17", + "dateModified": "2026-07-09", "license": "https://spdx.org/licenses/0BSD", "codeRepository": "https://github.com/USACE-RMC/Numerics", "issueTracker": "https://github.com/USACE-RMC/Numerics/issues",