diff --git a/.claude/reference/architecture.md b/.claude/reference/architecture.md index c59600e..6e0e4aa 100644 --- a/.claude/reference/architecture.md +++ b/.claude/reference/architecture.md @@ -68,7 +68,7 @@ Runtime (`src/Eftdb/`): - `Generators/SqlBuilderHelper` — `Regclass()`, `QualifiedIdentifier()`, `QuoteIdentifier()`, `EscapeStringLiteral`, `FormatTimestamp`, command grouping, SELECT→PERFORM rewriting for idempotent scripts - `Generators/PolicyJobSqlBuilder` — shared `alter_job` clause builder - `Generators/CompressionSettingsSqlHelper` — `SET (timescaledb.enable_columnstore = ...)` vs legacy `timescaledb.compress` clause, changed-settings diff -- `Query/` — `EF.Functions.TimeBucket()` overloads + `Internal/` translator plugin mapping to `time_bucket(...)`; runtime-only, throw outside LINQ +- `Query/` — `EF.Functions.TimeBucket()` overloads + `Internal/` translator plugin mapping to `time_bucket(...)`; `EF.Functions.First()`/`Last()` over a `ValueTuple.Create(value, time)` selector + an `IAggregateMethodCallTranslatorPlugin` mapping to `first(...)`/`last(...)` inside a `GroupBy`; runtime-only, throw outside LINQ - `Diagnostics/` — **single place that defines and emits provider warnings**, mirroring EF's own layering: `TimescaleDbEventId` (public) exposes stable `EventId`s (base `63000`, clear of EF/relational/Npgsql ranges) usable with `ConfigureWarnings`; `TimescaleDbLoggingDefinitions : NpgsqlLoggingDefinitions` (public, replaces the `LoggingDefinitions` service via the same replace-and-derive pattern as the validator/SQL generator) holds one cached `EventDefinitionBase` field per event; `TimescaleDbLoggerExtensions` (internal) provides one strongly-typed emitter per event, lazy-initializing definitions from `logger.Definitions` (fail-loud cast — a foreign instance means the Replace registration was lost) and dispatching through EF's pipeline (ShouldLog+Log for `ILogger`, NeedsEventData+DispatchEventData for `LogTo`/`DiagnosticSource`) so warnings reach every sink and honour Ignore/Throw. New warning = EventId + definitions field + emitter method. Never emit raw `Logger.LogWarning` — `LogTo` sinks never receive it. Design-time scaffolding warnings use `IOperationReporter` instead (correct channel for the dotnet-ef console). Design (`src/Eftdb.Design/`): diff --git a/docs/query-functions/first-last.md b/docs/query-functions/first-last.md new file mode 100644 index 0000000..94d8c84 --- /dev/null +++ b/docs/query-functions/first-last.md @@ -0,0 +1,91 @@ +# First and Last + +The `first` and `last` aggregates in TimescaleDB return the value of a column at the smallest or largest value of another column, typically the time column. They answer questions like "what was the opening and closing price of each day" without a window function or a sub-query. The `CmdScale.EntityFrameworkCore.TimescaleDB` package translates `EF.Functions.First(...)` and `EF.Functions.Last(...)` LINQ calls inside a `GroupBy` directly to `first(value, time)` and `last(value, time)` SQL. + +[See also: first](https://docs.timescale.com/api/latest/hyperfunctions/first/) and [last](https://docs.timescale.com/api/latest/hyperfunctions/last/) + +## Available Overloads + +Both methods are generic over the value type and the ordering type, and are available on `EF.Functions`. + +| C# Method | SQL Translation | +|---|---| +| `First(IEnumerable<(TValue Value, TTime Time)> values)` | `first(value, time)` | +| `Last(IEnumerable<(TValue Value, TTime Time)> values)` | `last(value, time)` | + +The value and the ordering column are passed together as a `ValueTuple.Create(value, time)` pair, projected from the group: + +```csharp +EF.Functions.First(g.Select(r => ValueTuple.Create(r.Value, r.Timestamp))) +``` + +The result has the type of the value. For a nullable column use a nullable value type (for example `double?`): the result is `null` when the value of the first or last row is `null`, and for a group whose values are all `null`. A `Where` on the group that matches no row of a group also gives `null`, so use a nullable result type (for example `double?`) together with `Where`; with a non-nullable type such as `double`, materializing that group throws. + +Any value type with a database mapping is supported. The ordering type can be any type PostgreSQL can order, such as `DateTime`, `DateTimeOffset`, `int` or `long`. + +## Usage Patterns + +### GROUP BY + +Open and close values per sensor: + +```csharp +using CmdScale.EntityFrameworkCore.TimescaleDB.Query; +using Microsoft.EntityFrameworkCore; + +var results = await context.SensorReadings + .GroupBy(r => r.SensorId) + .Select(g => new + { + SensorId = g.Key, + Opening = EF.Functions.First(g.Select(r => ValueTuple.Create(r.Temperature, r.Timestamp))), + Closing = EF.Functions.Last(g.Select(r => ValueTuple.Create(r.Temperature, r.Timestamp))) + }) + .ToListAsync(); +``` + +### With TimeBucket + +Combine with [TimeBucket](time-bucket.md) to get the first and last value per interval: + +```csharp +using CmdScale.EntityFrameworkCore.TimescaleDB.Query; +using Microsoft.EntityFrameworkCore; + +var results = await context.SensorReadings + .GroupBy(r => EF.Functions.TimeBucket(TimeSpan.FromHours(1), r.Timestamp)) + .Select(g => new + { + Bucket = g.Key, + Opening = EF.Functions.First(g.Select(r => ValueTuple.Create(r.Temperature, r.Timestamp))), + Closing = EF.Functions.Last(g.Select(r => ValueTuple.Create(r.Temperature, r.Timestamp))), + Count = g.Count() + }) + .OrderBy(r => r.Bucket) + .ToListAsync(); +``` + +### With a Filter + +A `Where` on the group becomes a `FILTER (WHERE ...)` clause on the aggregate, so rows can be excluded from `first` and `last` without affecting other aggregates in the same query: + +```csharp +using CmdScale.EntityFrameworkCore.TimescaleDB.Query; +using Microsoft.EntityFrameworkCore; + +var results = await context.SensorReadings + .GroupBy(r => r.SensorId) + .Select(g => new + { + SensorId = g.Key, + FirstHot = EF.Functions.First(g.Where(r => r.Temperature > 30).Select(r => ValueTuple.Create(r.Temperature, r.Timestamp))), + Count = g.Count() + }) + .ToListAsync(); +``` + +## Unsupported Shapes + +`first` and `last` are defined by the ordering column, so `Distinct()` and `OrderBy(...)` inside the grouping have no meaning for them. The translator refuses them with an `InvalidOperationException` ("`EF.Functions.First does not support Distinct or OrderBy inside the grouping; the time argument already orders the rows.`") while the query is compiled, before anything is sent to the database, instead of silently ignoring the call. On an entity with a primary key EF Core drops a plain `Distinct()` as a no-op before the translator sees it, so it has no effect there. The same applies to a selector that is not a `ValueTuple.Create(value, time)` pair. + +The only valid source is the grouping of a `GroupBy` (`g` in the examples above). Calling the methods on any other sequence, or on the client, is not translated and throws an `InvalidOperationException`. diff --git a/src/Eftdb/Query/Internal/TimescaleDbAggregateMethodCallTranslatorPlugin.cs b/src/Eftdb/Query/Internal/TimescaleDbAggregateMethodCallTranslatorPlugin.cs new file mode 100644 index 0000000..c1c3e66 --- /dev/null +++ b/src/Eftdb/Query/Internal/TimescaleDbAggregateMethodCallTranslatorPlugin.cs @@ -0,0 +1,15 @@ +using Microsoft.EntityFrameworkCore.Query; +using Npgsql.EntityFrameworkCore.PostgreSQL.Query; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Query.Internal; + +/// +/// Registers TimescaleDB aggregate method call translators with the EF Core query pipeline. +/// +internal sealed class TimescaleDbAggregateMethodCallTranslatorPlugin(ISqlExpressionFactory sqlExpressionFactory) : IAggregateMethodCallTranslatorPlugin +{ + public IEnumerable Translators { get; } = + [ + new TimescaleDbFirstLastTranslator((NpgsqlSqlExpressionFactory)sqlExpressionFactory), + ]; +} diff --git a/src/Eftdb/Query/Internal/TimescaleDbFirstLastTranslator.cs b/src/Eftdb/Query/Internal/TimescaleDbFirstLastTranslator.cs new file mode 100644 index 0000000..a90678a --- /dev/null +++ b/src/Eftdb/Query/Internal/TimescaleDbFirstLastTranslator.cs @@ -0,0 +1,63 @@ +using System.Reflection; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Query; +using Microsoft.EntityFrameworkCore.Query.SqlExpressions; +using Npgsql.EntityFrameworkCore.PostgreSQL.Query; +using Npgsql.EntityFrameworkCore.PostgreSQL.Query.Expressions.Internal; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Query.Internal; + +#pragma warning disable EF1001 // Npgsql hands a ValueTuple selector to aggregate translators only as a PgRowValueExpression. + +/// +/// Translates and +/// calls to the first and last SQL aggregates. +/// +internal sealed class TimescaleDbFirstLastTranslator(NpgsqlSqlExpressionFactory sqlExpressionFactory) : IAggregateMethodCallTranslator +{ + private static readonly MethodInfo FirstDefinition = typeof(TimescaleDbFunctionsExtensions) + .GetMethod(nameof(TimescaleDbFunctionsExtensions.First))!.GetGenericMethodDefinition(); + + private static readonly MethodInfo LastDefinition = typeof(TimescaleDbFunctionsExtensions) + .GetMethod(nameof(TimescaleDbFunctionsExtensions.Last))!.GetGenericMethodDefinition(); + + public SqlExpression? Translate( + MethodInfo method, + EnumerableExpression source, + IReadOnlyList arguments, + IDiagnosticsLogger logger) + { + if (!method.IsGenericMethod || method.DeclaringType != typeof(TimescaleDbFunctionsExtensions)) + { + return null; + } + + MethodInfo definition = method.GetGenericMethodDefinition(); + string? functionName = definition == FirstDefinition ? "first" : definition == LastDefinition ? "last" : null; + + if (functionName is null || source.Selector is not PgRowValueExpression { Values.Count: 2 } row) + { + return null; + } + + // Returning null here would make EF fall back to a collection subquery and fail with an unrelated error. + if (source.IsDistinct || source.Orderings.Count > 0) + { + throw new InvalidOperationException( + $"EF.Functions.{method.Name} does not support Distinct or OrderBy inside the grouping; the time argument already orders the rows."); + } + + SqlExpression value = sqlExpressionFactory.ApplyDefaultTypeMapping(row.Values[0]); + SqlExpression time = sqlExpressionFactory.ApplyDefaultTypeMapping(row.Values[1]); + + return sqlExpressionFactory.AggregateFunction( + functionName, + [value, time], + source, + nullable: true, + argumentsPropagateNullability: [false, false], + returnType: method.ReturnType, + typeMapping: value.TypeMapping); + } +} diff --git a/src/Eftdb/Query/TimescaleDbFunctionsExtensions.FirstLast.cs b/src/Eftdb/Query/TimescaleDbFunctionsExtensions.FirstLast.cs new file mode 100644 index 0000000..49447b4 --- /dev/null +++ b/src/Eftdb/Query/TimescaleDbFunctionsExtensions.FirstLast.cs @@ -0,0 +1,22 @@ +using Microsoft.EntityFrameworkCore; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Query; + +public static partial class TimescaleDbFunctionsExtensions +{ + /// + /// Returns the value of the row with the smallest time in the group, or when that value is null. + /// Pass the value and the ordering column as a pair, for example + /// EF.Functions.First(g.Select(x => ValueTuple.Create(x.Value, x.Timestamp))). + /// Translates to first(value, time). + /// + public static TValue First(this DbFunctions _, IEnumerable<(TValue Value, TTime Time)> values) => Throw(); + + /// + /// Returns the value of the row with the largest time in the group, or when that value is null. + /// Pass the value and the ordering column as a pair, for example + /// EF.Functions.Last(g.Select(x => ValueTuple.Create(x.Value, x.Timestamp))). + /// Translates to last(value, time). + /// + public static TValue Last(this DbFunctions _, IEnumerable<(TValue Value, TTime Time)> values) => Throw(); +} diff --git a/src/Eftdb/TimescaleDbContextOptionsBuilderExtensions.cs b/src/Eftdb/TimescaleDbContextOptionsBuilderExtensions.cs index e348c8c..b079cad 100644 --- a/src/Eftdb/TimescaleDbContextOptionsBuilderExtensions.cs +++ b/src/Eftdb/TimescaleDbContextOptionsBuilderExtensions.cs @@ -110,6 +110,8 @@ public void ApplyServices(IServiceCollection services) services.Replace(ServiceDescriptor.Singleton()); services.TryAddEnumerable( ServiceDescriptor.Scoped()); + services.TryAddEnumerable( + ServiceDescriptor.Scoped()); } public void Validate(IDbContextOptions options) { } diff --git a/src/Eftdb/TimescaleDbServiceCollectionExtensions.cs b/src/Eftdb/TimescaleDbServiceCollectionExtensions.cs index f761c82..63dcd78 100644 --- a/src/Eftdb/TimescaleDbServiceCollectionExtensions.cs +++ b/src/Eftdb/TimescaleDbServiceCollectionExtensions.cs @@ -18,7 +18,8 @@ public static IServiceCollection AddEntityFrameworkTimescaleDb(this IServiceColl new EntityFrameworkRelationalServicesBuilder(services) .TryAdd() .TryAdd() - .TryAdd(); + .TryAdd() + .TryAdd(); return services; } diff --git a/tests/Eftdb.FunctionalTests/FirstLastQueryTests.cs b/tests/Eftdb.FunctionalTests/FirstLastQueryTests.cs new file mode 100644 index 0000000..801baa5 --- /dev/null +++ b/tests/Eftdb.FunctionalTests/FirstLastQueryTests.cs @@ -0,0 +1,126 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests.Utils; +using CmdScale.EntityFrameworkCore.TimescaleDB.Query; +using Microsoft.EntityFrameworkCore; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.FunctionalTests; + +public class FirstLastQueryTests(TimescaleQueryFixture fixture) + : FirstLastQueryTestsBase(fixture), IClassFixture; + +public class FirstLastApacheQueryTests(TimescaleApacheQueryFixture fixture) + : FirstLastQueryTestsBase(fixture), IClassFixture; + +public abstract class FirstLastQueryTestsBase(TFixture fixture) + where TFixture : TimescaleQueryFixtureBase +{ + #region Should_Generate_First_And_Last_In_GroupBy + + [Fact] + public async Task Should_Generate_First_And_Last_In_GroupBy() + { + await using TimescaleQueryFixtureBase.QueryTestContext context = fixture.CreateContext(); + + var results = await context.Metrics + .GroupBy(m => m.SequenceNumber / 6) + .Select(g => new + { + Bucket = g.Key, + First = EF.Functions.First(g.Select(m => ValueTuple.Create(m.Value, m.Timestamp))), + Last = EF.Functions.Last(g.Select(m => ValueTuple.Create(m.Value, m.Timestamp))) + }) + .OrderBy(r => r.Bucket) + .ToListAsync(); + + Assert.Equal(new[] { 10.0, 70.0 }, results.Select(r => r.First)); + Assert.Equal(new[] { 60.0, 120.0 }, results.Select(r => r.Last)); + + AssertSql( + """ + SELECT q0."Key" AS "Bucket", first(q0."Value", q0."Timestamp") AS "First", last(q0."Value", q0."Timestamp") AS "Last" + FROM ( + SELECT q."Timestamp", q."Value", q."SequenceNumber" / 6 AS "Key" + FROM query_metrics AS q + ) AS q0 + GROUP BY q0."Key" + ORDER BY q0."Key" + """); + } + + #endregion + + #region Should_Generate_First_With_Filter + + [Fact] + public async Task Should_Generate_First_With_Filter() + { + await using TimescaleQueryFixtureBase.QueryTestContext context = fixture.CreateContext(); + + var results = await context.Metrics + .GroupBy(m => m.SequenceNumber / 6) + .Select(g => new + { + Bucket = g.Key, + First = EF.Functions.First(g.Where(m => m.Value > 30).Select(m => ValueTuple.Create(m.Value, m.Timestamp))), + Count = g.Count() + }) + .OrderBy(r => r.Bucket) + .ToListAsync(); + + Assert.Equal(new[] { 40.0, 70.0 }, results.Select(r => r.First)); + Assert.Equal(new[] { 6, 6 }, results.Select(r => r.Count)); + + AssertSql( + """ + SELECT q0."Key" AS "Bucket", first(q0."Value", q0."Timestamp") FILTER (WHERE q0."Value" > 30.0) AS "First", count(*)::int AS "Count" + FROM ( + SELECT q."Timestamp", q."Value", q."SequenceNumber" / 6 AS "Key" + FROM query_metrics AS q + ) AS q0 + GROUP BY q0."Key" + ORDER BY q0."Key" + """); + } + + #endregion + + #region Should_Generate_First_And_Last_With_TimeBucket + + [Fact] + public async Task Should_Generate_First_And_Last_With_TimeBucket() + { + await using TimescaleQueryFixtureBase.QueryTestContext context = fixture.CreateContext(); + + TimeSpan bucket = TimeSpan.FromMinutes(5); + var results = await context.Metrics + .GroupBy(m => EF.Functions.TimeBucket(bucket, m.Timestamp)) + .Select(g => new + { + Bucket = g.Key, + First = EF.Functions.First(g.Select(m => ValueTuple.Create(m.Value, m.Timestamp))), + Last = EF.Functions.Last(g.Select(m => ValueTuple.Create(m.Value, m.Timestamp))) + }) + .OrderBy(r => r.Bucket) + .ToListAsync(); + + Assert.Equal(new[] { 10.0, 60.0, 110.0 }, results.Select(r => r.First)); + Assert.Equal(new[] { 50.0, 100.0, 120.0 }, results.Select(r => r.Last)); + + AssertSql( + """ + @bucket='00:05:00' (DbType = Object) + + SELECT q0."Key" AS "Bucket", first(q0."Value", q0."Timestamp") AS "First", last(q0."Value", q0."Timestamp") AS "Last" + FROM ( + SELECT q."Timestamp", q."Value", time_bucket(@bucket, q."Timestamp") AS "Key" + FROM query_metrics AS q + ) AS q0 + GROUP BY q0."Key" + ORDER BY q0."Key" + """); + } + + #endregion + + private void AssertSql(params string[] expected) + => fixture.TestSqlLoggerFactory.AssertBaseline(expected); +} diff --git a/tests/Eftdb.Tests/Integration/FirstLastIntegrationTests.cs b/tests/Eftdb.Tests/Integration/FirstLastIntegrationTests.cs new file mode 100644 index 0000000..b5fdd4b --- /dev/null +++ b/tests/Eftdb.Tests/Integration/FirstLastIntegrationTests.cs @@ -0,0 +1,585 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Configuration.Hypertable; +using CmdScale.EntityFrameworkCore.TimescaleDB.Query; +using Microsoft.EntityFrameworkCore; +using Testcontainers.PostgreSql; +using CmdScale.EntityFrameworkCore.TimescaleDB.Tests.Utils; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Tests.Integration; + +public class FirstLastIntegrationTests : MigrationTestBase, IAsyncLifetime +{ + private static readonly DateTime BaseTime = new(2025, 1, 6, 10, 0, 0, DateTimeKind.Utc); + + private PostgreSqlContainer? _container; + private string? _connectionString; + + public async ValueTask InitializeAsync() + { + _container = new PostgreSqlBuilder(TimescaleImages.Community) + .WithDatabase("test_db") + .WithUsername("test_user") + .WithPassword("test_password") + .Build(); + + await _container.StartAsync(); + _connectionString = _container.GetConnectionString(); + } + + public async ValueTask DisposeAsync() + { + if (_container != null) + { + await _container.DisposeAsync(); + } + } + + #region Should_Translate_First_And_Last_In_GroupBy + + private class FirstLastGroupByReading + { + public string Sensor { get; set; } = string.Empty; + public DateTime Timestamp { get; set; } + public double? Value { get; set; } + } + + private class FirstLastGroupByContext(string connectionString) : DbContext + { + public DbSet Readings => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(connectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("fl_groupby_readings"); + entity.HasNoKey(); + entity.IsHypertable(x => x.Timestamp); + }); + } + } + + [Fact] + public async Task Should_Translate_First_And_Last_In_GroupBy() + { + // Arrange + await using FirstLastGroupByContext context = new(_connectionString!); + await CreateDatabaseViaMigrationAsync(context); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_groupby_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {BaseTime.AddMinutes(2)}, {30.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_groupby_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {BaseTime.AddMinutes(0)}, {10.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_groupby_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {BaseTime.AddMinutes(1)}, {20.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_groupby_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('b', {BaseTime.AddMinutes(5)}, {7.0})", + TestContext.Current.CancellationToken); + + // Act + var results = await context.Readings + .GroupBy(r => r.Sensor) + .Select(g => new + { + Sensor = g.Key, + First = EF.Functions.First(g.Select(r => ValueTuple.Create(r.Value, r.Timestamp))), + Last = EF.Functions.Last(g.Select(r => ValueTuple.Create(r.Value, r.Timestamp))) + }) + .OrderBy(r => r.Sensor) + .ToListAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(2, results.Count); + + Assert.Equal("a", results[0].Sensor); + Assert.Equal(10.0, results[0].First); + Assert.Equal(30.0, results[0].Last); + + Assert.Equal("b", results[1].Sensor); + Assert.Equal(7.0, results[1].First); + Assert.Equal(7.0, results[1].Last); + } + + #endregion + + #region Should_Translate_First_And_Last_With_DateTimeOffset_Time_Column + + private class FirstLastOffsetReading + { + public string Sensor { get; set; } = string.Empty; + public DateTimeOffset Timestamp { get; set; } + public double? Value { get; set; } + } + + private class FirstLastOffsetContext(string connectionString) : DbContext + { + public DbSet Readings => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(connectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("fl_offset_readings"); + entity.HasNoKey(); + entity.IsHypertable(x => x.Timestamp); + }); + } + } + + [Fact] + public async Task Should_Translate_First_And_Last_With_DateTimeOffset_Time_Column() + { + // Arrange + DateTimeOffset baseOffset = new(BaseTime); + await using FirstLastOffsetContext context = new(_connectionString!); + await CreateDatabaseViaMigrationAsync(context); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_offset_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {baseOffset.AddMinutes(2)}, {30.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_offset_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {baseOffset.AddMinutes(0)}, {10.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_offset_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {baseOffset.AddMinutes(1)}, {20.0})", + TestContext.Current.CancellationToken); + + // Act + var result = await context.Readings + .GroupBy(r => r.Sensor) + .Select(g => new + { + First = EF.Functions.First(g.Select(r => ValueTuple.Create(r.Value, r.Timestamp))), + Last = EF.Functions.Last(g.Select(r => ValueTuple.Create(r.Value, r.Timestamp))) + }) + .SingleAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(10.0, result.First); + Assert.Equal(30.0, result.Last); + } + + #endregion + + #region Should_Return_Null_When_First_Or_Last_Value_Is_Null + + private class FirstLastNullableReading + { + public string Sensor { get; set; } = string.Empty; + public DateTime Timestamp { get; set; } + public double? Value { get; set; } + } + + private class FirstLastNullableContext(string connectionString) : DbContext + { + public DbSet Readings => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(connectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("fl_nullable_readings"); + entity.HasNoKey(); + entity.IsHypertable(x => x.Timestamp); + }); + } + } + + [Fact] + public async Task Should_Return_Null_When_First_Or_Last_Value_Is_Null() + { + // Arrange + await using FirstLastNullableContext context = new(_connectionString!); + await CreateDatabaseViaMigrationAsync(context); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_nullable_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('x', {BaseTime.AddMinutes(0)}, NULL)", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_nullable_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('x', {BaseTime.AddMinutes(1)}, {5.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_nullable_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('x', {BaseTime.AddMinutes(2)}, {9.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_nullable_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('y', {BaseTime.AddMinutes(0)}, {4.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_nullable_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('y', {BaseTime.AddMinutes(1)}, {6.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_nullable_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('y', {BaseTime.AddMinutes(2)}, NULL)", + TestContext.Current.CancellationToken); + + // Act + var results = await context.Readings + .GroupBy(r => r.Sensor) + .Select(g => new + { + Sensor = g.Key, + First = EF.Functions.First(g.Select(r => ValueTuple.Create(r.Value, r.Timestamp))), + Last = EF.Functions.Last(g.Select(r => ValueTuple.Create(r.Value, r.Timestamp))) + }) + .OrderBy(r => r.Sensor) + .ToListAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(2, results.Count); + + Assert.Null(results[0].First); + Assert.Equal(9.0, results[0].Last); + + Assert.Equal(4.0, results[1].First); + Assert.Null(results[1].Last); + } + + #endregion + + #region Should_Return_Null_For_All_Null_Group + + private class FirstLastAllNullReading + { + public string Sensor { get; set; } = string.Empty; + public DateTime Timestamp { get; set; } + public double? Value { get; set; } + } + + private class FirstLastAllNullContext(string connectionString) : DbContext + { + public DbSet Readings => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(connectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("fl_allnull_readings"); + entity.HasNoKey(); + entity.IsHypertable(x => x.Timestamp); + }); + } + } + + [Fact] + public async Task Should_Return_Null_For_All_Null_Group() + { + // Arrange + await using FirstLastAllNullContext context = new(_connectionString!); + await CreateDatabaseViaMigrationAsync(context); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_allnull_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('z', {BaseTime.AddMinutes(0)}, NULL)", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_allnull_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('z', {BaseTime.AddMinutes(1)}, NULL)", + TestContext.Current.CancellationToken); + + // Act + var result = await context.Readings + .GroupBy(r => r.Sensor) + .Select(g => new + { + First = EF.Functions.First(g.Select(r => ValueTuple.Create(r.Value, r.Timestamp))), + Last = EF.Functions.Last(g.Select(r => ValueTuple.Create(r.Value, r.Timestamp))) + }) + .SingleAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.Null(result.First); + Assert.Null(result.Last); + } + + #endregion + + #region Should_Apply_Where_Filter_Inside_Grouping + + private class FirstLastWhereReading + { + public string Sensor { get; set; } = string.Empty; + public DateTime Timestamp { get; set; } + public double? Value { get; set; } + } + + private class FirstLastWhereContext(string connectionString) : DbContext + { + public DbSet Readings => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(connectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("fl_where_readings"); + entity.HasNoKey(); + entity.IsHypertable(x => x.Timestamp); + }); + } + } + + [Fact] + public async Task Should_Apply_Where_Filter_Inside_Grouping() + { + // Arrange + await using FirstLastWhereContext context = new(_connectionString!); + await CreateDatabaseViaMigrationAsync(context); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_where_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {BaseTime.AddMinutes(0)}, {10.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_where_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {BaseTime.AddMinutes(1)}, {20.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_where_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {BaseTime.AddMinutes(2)}, {30.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_where_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('c', {BaseTime.AddMinutes(0)}, {1.0})", + TestContext.Current.CancellationToken); + + // Act + var results = await context.Readings + .GroupBy(r => r.Sensor) + .Select(g => new + { + Sensor = g.Key, + First = EF.Functions.First(g.Where(r => r.Value > 10).Select(r => ValueTuple.Create(r.Value, r.Timestamp))), + Last = EF.Functions.Last(g.Where(r => r.Value > 10).Select(r => ValueTuple.Create(r.Value, r.Timestamp))) + }) + .OrderBy(r => r.Sensor) + .ToListAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(2, results.Count); + + Assert.Equal(20.0, results[0].First); + Assert.Equal(30.0, results[0].Last); + + Assert.Null(results[1].First); + Assert.Null(results[1].Last); + } + + #endregion + + #region Should_Translate_First_And_Last_With_TimeBucket_GroupBy + + private class FirstLastBucketReading + { + public string Sensor { get; set; } = string.Empty; + public DateTime Timestamp { get; set; } + public double? Value { get; set; } + } + + private class FirstLastBucketContext(string connectionString) : DbContext + { + public DbSet Readings => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(connectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("fl_bucket_readings"); + entity.HasNoKey(); + entity.IsHypertable(x => x.Timestamp); + }); + } + } + + [Fact] + public async Task Should_Translate_First_And_Last_With_TimeBucket_GroupBy() + { + // Arrange + await using FirstLastBucketContext context = new(_connectionString!); + await CreateDatabaseViaMigrationAsync(context); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_bucket_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {BaseTime.AddMinutes(3)}, {30.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_bucket_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {BaseTime.AddMinutes(0)}, {10.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_bucket_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {BaseTime.AddMinutes(1)}, {20.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_bucket_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {BaseTime.AddMinutes(7)}, {70.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_bucket_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {BaseTime.AddMinutes(5)}, {50.0})", + TestContext.Current.CancellationToken); + await context.Database.ExecuteSqlInterpolatedAsync( + $"INSERT INTO \"fl_bucket_readings\" (\"Sensor\", \"Timestamp\", \"Value\") VALUES ('a', {BaseTime.AddMinutes(6)}, {60.0})", + TestContext.Current.CancellationToken); + + // Act + var results = await context.Readings + .GroupBy(r => EF.Functions.TimeBucket(TimeSpan.FromMinutes(5), r.Timestamp)) + .Select(g => new + { + Bucket = g.Key, + First = EF.Functions.First(g.Select(r => ValueTuple.Create(r.Value, r.Timestamp))), + Last = EF.Functions.Last(g.Select(r => ValueTuple.Create(r.Value, r.Timestamp))), + Count = g.Count() + }) + .OrderBy(r => r.Bucket) + .ToListAsync(TestContext.Current.CancellationToken); + + // Assert + Assert.Equal(2, results.Count); + + Assert.Equal(10.0, results[0].First); + Assert.Equal(30.0, results[0].Last); + Assert.Equal(3, results[0].Count); + + Assert.Equal(50.0, results[1].First); + Assert.Equal(70.0, results[1].Last); + Assert.Equal(3, results[1].Count); + } + + #endregion + + #region Should_Refuse_Distinct_Inside_Grouping + + private class FirstLastDistinctReading + { + public string Sensor { get; set; } = string.Empty; + public DateTime Timestamp { get; set; } + public double? Value { get; set; } + } + + private class FirstLastDistinctContext(string connectionString) : DbContext + { + public DbSet Readings => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(connectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("fl_distinct_readings"); + entity.HasNoKey(); + entity.IsHypertable(x => x.Timestamp); + }); + } + } + + [Fact] + public async Task Should_Refuse_Distinct_Inside_Grouping() + { + // Arrange + await using FirstLastDistinctContext context = new(_connectionString!); + + // Act + InvalidOperationException exception = await Assert.ThrowsAsync(() => context.Readings + .GroupBy(r => r.Sensor) + .Select(g => EF.Functions.First(g.Distinct().Select(r => ValueTuple.Create(r.Value, r.Timestamp)))) + .ToListAsync(TestContext.Current.CancellationToken)); + + // Assert + Assert.Contains("EF.Functions.First does not support Distinct or OrderBy inside the grouping", exception.Message); + } + + #endregion + + #region Should_Refuse_OrderBy_Inside_Grouping + + private class FirstLastOrderByReading + { + public string Sensor { get; set; } = string.Empty; + public DateTime Timestamp { get; set; } + public double? Value { get; set; } + } + + private class FirstLastOrderByContext(string connectionString) : DbContext + { + public DbSet Readings => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(connectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("fl_orderby_readings"); + entity.HasNoKey(); + entity.IsHypertable(x => x.Timestamp); + }); + } + } + + [Fact] + public async Task Should_Refuse_OrderBy_Inside_Grouping() + { + // Arrange + await using FirstLastOrderByContext context = new(_connectionString!); + + // Act + InvalidOperationException exception = await Assert.ThrowsAsync(() => context.Readings + .GroupBy(r => r.Sensor) + .Select(g => EF.Functions.Last(g.OrderBy(r => r.Value).Select(r => ValueTuple.Create(r.Value, r.Timestamp)))) + .ToListAsync(TestContext.Current.CancellationToken)); + + // Assert + Assert.Contains("EF.Functions.Last does not support Distinct or OrderBy inside the grouping", exception.Message); + } + + #endregion + + #region Should_Refuse_OrderBy_Inside_Grouping_On_Keyed_Entity + + private class FirstLastKeyedOrderByReading + { + public int Id { get; set; } + public string Sensor { get; set; } = string.Empty; + public DateTime Timestamp { get; set; } + public double? Value { get; set; } + } + + private class FirstLastKeyedOrderByContext(string connectionString) : DbContext + { + public DbSet Readings => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(connectionString).UseTimescaleDb(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("fl_keyed_orderby_readings"); + entity.HasKey(x => x.Id); + }); + } + } + + [Fact] + public async Task Should_Refuse_OrderBy_Inside_Grouping_On_Keyed_Entity() + { + // Arrange + await using FirstLastKeyedOrderByContext context = new(_connectionString!); + + // Act + InvalidOperationException exception = await Assert.ThrowsAsync(() => context.Readings + .GroupBy(r => r.Sensor) + .Select(g => EF.Functions.Last(g.OrderBy(r => r.Value).Select(r => ValueTuple.Create(r.Value, r.Timestamp)))) + .ToListAsync(TestContext.Current.CancellationToken)); + + // Assert + Assert.Contains("EF.Functions.Last does not support Distinct or OrderBy inside the grouping", exception.Message); + } + + #endregion +} diff --git a/tests/Eftdb.Tests/Query/TimescaleDbFirstLastTranslatorTests.cs b/tests/Eftdb.Tests/Query/TimescaleDbFirstLastTranslatorTests.cs new file mode 100644 index 0000000..027f994 --- /dev/null +++ b/tests/Eftdb.Tests/Query/TimescaleDbFirstLastTranslatorTests.cs @@ -0,0 +1,156 @@ +using System.Reflection; +using CmdScale.EntityFrameworkCore.TimescaleDB.Query; +using CmdScale.EntityFrameworkCore.TimescaleDB.Query.Internal; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Query; +using Microsoft.EntityFrameworkCore.Query.SqlExpressions; +using Moq; +using Npgsql.EntityFrameworkCore.PostgreSQL.Query.Expressions.Internal; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Tests.Query; + +#pragma warning disable EF1001 // Internal EF Core API usage. + +public class TimescaleDbFirstLastTranslatorTests +{ + private readonly Mock> _loggerMock = new(); + private readonly TimescaleDbFirstLastTranslator _translator = new(null!); + + private static SqlExpression CreateMockArgument(Type type) + { + Mock mock = new(type, null!); + return mock.Object; + } + + private static EnumerableExpression CreateRowSource(int valueCount = 2) + { + List values = []; + for (int i = 0; i < valueCount; i++) + { + values.Add(CreateMockArgument(i == 0 ? typeof(double) : typeof(DateTime))); + } + + return new EnumerableExpression(new PgRowValueExpression(values, typeof(ValueTuple))); + } + + private static MethodInfo GetMethod(string name) + => typeof(TimescaleDbFunctionsExtensions).GetMethod(name)!.MakeGenericMethod(typeof(double), typeof(DateTime)); + + #region Should_Return_Null_For_Non_Matching_Method + + [Fact] + public void Should_Return_Null_For_Non_Matching_Method() + { + // Arrange + MethodInfo nonMatchingMethod = typeof(Math).GetMethod(nameof(Math.Abs), [typeof(int)])!; + + // Act + SqlExpression? result = _translator.Translate(nonMatchingMethod, CreateRowSource(), [], _loggerMock.Object); + + // Assert + Assert.Null(result); + } + + #endregion + + #region Should_Return_Null_For_Generic_Method_Of_Another_Type + + [Fact] + public void Should_Return_Null_For_Generic_Method_Of_Another_Type() + { + // Arrange + MethodInfo otherGenericMethod = typeof(Enumerable) + .GetMethod(nameof(Enumerable.First), 1, [typeof(IEnumerable<>).MakeGenericType(Type.MakeGenericMethodParameter(0))])! + .MakeGenericMethod(typeof(double)); + + // Act + SqlExpression? result = _translator.Translate(otherGenericMethod, CreateRowSource(), [], _loggerMock.Object); + + // Assert + Assert.Null(result); + } + + #endregion + + #region Should_Throw_When_Source_Is_Distinct + + [Theory] + [InlineData(nameof(TimescaleDbFunctionsExtensions.First))] + [InlineData(nameof(TimescaleDbFunctionsExtensions.Last))] + public void Should_Throw_When_Source_Is_Distinct(string methodName) + { + // Arrange + EnumerableExpression source = CreateRowSource().SetDistinct(true); + + // Act + InvalidOperationException exception = Assert.Throws( + () => _translator.Translate(GetMethod(methodName), source, [], _loggerMock.Object)); + + // Assert + Assert.Contains($"EF.Functions.{methodName} does not support Distinct or OrderBy", exception.Message); + } + + #endregion + + #region Should_Throw_When_Source_Is_Ordered + + [Theory] + [InlineData(nameof(TimescaleDbFunctionsExtensions.First))] + [InlineData(nameof(TimescaleDbFunctionsExtensions.Last))] + public void Should_Throw_When_Source_Is_Ordered(string methodName) + { + // Arrange + EnumerableExpression source = CreateRowSource() + .ApplyOrdering(new OrderingExpression(CreateMockArgument(typeof(double)), ascending: true)); + + // Act + InvalidOperationException exception = Assert.Throws( + () => _translator.Translate(GetMethod(methodName), source, [], _loggerMock.Object)); + + // Assert + Assert.Contains($"EF.Functions.{methodName} does not support Distinct or OrderBy", exception.Message); + } + + #endregion + + #region Should_Return_Null_For_Other_Generic_Method_Of_The_Same_Type + + [Fact] + public void Should_Return_Null_For_Other_Generic_Method_Of_The_Same_Type() + { + // Arrange + MethodInfo otherMethod = typeof(TimescaleDbFunctionsExtensions) + .GetMethod("Throw", BindingFlags.NonPublic | BindingFlags.Static)! + .MakeGenericMethod(typeof(double)); + + // Act + SqlExpression? result = _translator.Translate(otherMethod, CreateRowSource(), [], _loggerMock.Object); + + // Assert + Assert.Null(result); + } + + #endregion + + #region Should_Return_Null_When_Selector_Is_Not_A_Row_Of_Two_Values + + [Fact] + public void Should_Return_Null_When_Selector_Is_Not_A_Row_Of_Two_Values() + { + // Arrange + EnumerableExpression tripleSource = CreateRowSource(valueCount: 3); + EnumerableExpression scalarSource = new(CreateMockArgument(typeof(double))); + MethodInfo method = GetMethod(nameof(TimescaleDbFunctionsExtensions.First)); + + // Act + SqlExpression? tripleResult = _translator.Translate(method, tripleSource, [], _loggerMock.Object); + SqlExpression? scalarResult = _translator.Translate(method, scalarSource, [], _loggerMock.Object); + + // Assert + Assert.Null(tripleResult); + Assert.Null(scalarResult); + } + + #endregion +} diff --git a/tests/Eftdb.Tests/Query/TimescaleDbFunctionsExtensionsFirstLastTests.cs b/tests/Eftdb.Tests/Query/TimescaleDbFunctionsExtensionsFirstLastTests.cs new file mode 100644 index 0000000..53ed8f0 --- /dev/null +++ b/tests/Eftdb.Tests/Query/TimescaleDbFunctionsExtensionsFirstLastTests.cs @@ -0,0 +1,37 @@ +using CmdScale.EntityFrameworkCore.TimescaleDB.Query; +using Microsoft.EntityFrameworkCore; + +namespace CmdScale.EntityFrameworkCore.TimescaleDB.Tests.Query; + +public class TimescaleDbFunctionsExtensionsFirstLastTests +{ + private static readonly IEnumerable<(double Value, DateTime Time)> Values = [(1.0, DateTime.UnixEpoch)]; + + #region Should_Throw_When_First_Is_Called_Outside_A_Query + + [Fact] + public void Should_Throw_When_First_Is_Called_Outside_A_Query() + { + // Act + InvalidOperationException exception = Assert.Throws(() => EF.Functions.First(Values)); + + // Assert + Assert.Contains("no in-memory implementation", exception.Message); + } + + #endregion + + #region Should_Throw_When_Last_Is_Called_Outside_A_Query + + [Fact] + public void Should_Throw_When_Last_Is_Called_Outside_A_Query() + { + // Act + InvalidOperationException exception = Assert.Throws(() => EF.Functions.Last(Values)); + + // Assert + Assert.Contains("no in-memory implementation", exception.Message); + } + + #endregion +}