Skip to content

Commit 52bff7f

Browse files
slorello89slorello89
authored
Bugfix/comprehensive index equality (#496)
* fixing issues with index equality * fixing hash vector corner case
1 parent 1956366 commit 52bff7f

File tree

3 files changed

+345
-7
lines changed

3 files changed

+345
-7
lines changed

‎src/Redis.OM/Modeling/RedisIndex.cs

Lines changed: 148 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,29 +20,88 @@ public static class RedisIndex
2020
public static bool IndexDefinitionEquals(this RedisIndexInfo redisIndexInfo, Type type)
2121
{
2222
var serialisedDefinition = SerializeIndex(type);
23-
var existingSet = redisIndexInfo.Attributes?.Select(a => (Property: a.Attribute!, a.Type!)).OrderBy(a => a.Property);
2423
var isJson = redisIndexInfo.IndexDefinition?.Identifier == "JSON";
2524

25+
var currentOffset = 0;
2626
if (serialisedDefinition.Length < 5)
2727
{
2828
throw new ArgumentException($"Could not parse the index definition for type: {type.Name}.");
2929
}
3030

31-
if (redisIndexInfo.IndexName != serialisedDefinition[0])
31+
if (redisIndexInfo.IndexDefinition is null)
3232
{
3333
return false;
3434
}
3535

36-
if (redisIndexInfo.IndexDefinition?.Identifier?.Equals(serialisedDefinition[2], StringComparison.OrdinalIgnoreCase) == false)
36+
// these are properties we cannot process because FT.INFO does not respond with them
37+
var unprocessableProperties = new string[] { "EPSILON", "EF_RUNTIME", "PHONETIC", "STOPWORDS" };
38+
39+
foreach (var property in unprocessableProperties)
40+
{
41+
if (serialisedDefinition.Any(x => x.Equals(property)))
42+
{
43+
throw new ArgumentException($"Could not validate index definition that contains {property}");
44+
}
45+
}
46+
47+
if (redisIndexInfo.IndexName != serialisedDefinition[currentOffset])
48+
{
49+
return false;
50+
}
51+
52+
currentOffset += 2; // skip to the index type at index 2
53+
54+
if (redisIndexInfo.IndexDefinition?.Identifier?.Equals(serialisedDefinition[currentOffset], StringComparison.OrdinalIgnoreCase) == false)
55+
{
56+
return false;
57+
}
58+
59+
currentOffset += 2; // skip to prefix count
60+
61+
if (!int.TryParse(serialisedDefinition[currentOffset], out var numPrefixes))
62+
{
63+
throw new ArgumentException("Could not parse index with unknown number of prefixes");
64+
}
65+
66+
currentOffset += 2; // skip to first prefix
67+
68+
if (redisIndexInfo.IndexDefinition?.Prefixes is null || redisIndexInfo.IndexDefinition.Prefixes.Length != numPrefixes || serialisedDefinition.Skip(currentOffset).Take(numPrefixes).SequenceEqual(redisIndexInfo.IndexDefinition.Prefixes))
69+
{
70+
return false;
71+
}
72+
73+
currentOffset += numPrefixes;
74+
75+
if (redisIndexInfo.IndexDefinition?.Filter is not null && !redisIndexInfo.IndexDefinition.Filter.Equals(serialisedDefinition.ElementAt(currentOffset)))
76+
{
77+
return false;
78+
}
79+
80+
if (redisIndexInfo.IndexDefinition?.Filter is not null)
81+
{
82+
currentOffset += 2;
83+
}
84+
85+
if (redisIndexInfo.IndexDefinition?.DefaultLanguage is not null && !redisIndexInfo.IndexDefinition.DefaultLanguage.Equals(serialisedDefinition.ElementAt(currentOffset)))
3786
{
3887
return false;
3988
}
4089

41-
if (redisIndexInfo.IndexDefinition?.Prefixes.FirstOrDefault().Equals(serialisedDefinition[5]) == false)
90+
if (redisIndexInfo.IndexDefinition?.DefaultLanguage is not null)
91+
{
92+
currentOffset += 2;
93+
}
94+
95+
if (redisIndexInfo.IndexDefinition?.LanguageField is not null && !redisIndexInfo.IndexDefinition.LanguageField.Equals(serialisedDefinition.ElementAt(currentOffset)))
4296
{
4397
return false;
4498
}
4599

100+
if (redisIndexInfo.IndexDefinition?.LanguageField is not null)
101+
{
102+
currentOffset += 2;
103+
}
104+
46105
var target = redisIndexInfo.Attributes?.SelectMany(a =>
47106
{
48107
var attr = new List<string>();
@@ -58,11 +117,81 @@ public static bool IndexDefinitionEquals(this RedisIndexInfo redisIndexInfo, Typ
58117
attr.Add("AS");
59118
}
60119

120+
if (!isJson && a.Type is not null && a.Type == "VECTOR")
121+
{
122+
attr.Add($"{a.Attribute!}.Vector");
123+
attr.Add("AS");
124+
}
125+
61126
attr.Add(a.Attribute!);
62127

63128
if (a.Type != null)
64129
{
65130
attr.Add(a.Type);
131+
if (a.Type == "TAG")
132+
{
133+
attr.Add("SEPARATOR");
134+
attr.Add(a.Separator ?? "|");
135+
}
136+
137+
if (a.Type == "TEXT")
138+
{
139+
if (a.NoStem == true)
140+
{
141+
attr.Add("NOSTEM");
142+
}
143+
144+
if (a.Weight is not null && a.Weight != "1")
145+
{
146+
attr.Add("WEIGHT");
147+
attr.Add(a.Weight);
148+
}
149+
}
150+
151+
if (a.Type == "VECTOR")
152+
{
153+
if (a.Algorithm is null)
154+
{
155+
throw new InvalidOperationException("Encountered Vector field with no algorithm");
156+
}
157+
158+
attr.Add(a.Algorithm);
159+
if (a.VectorType is null)
160+
{
161+
throw new InvalidOperationException("Encountered vector field with no Vector Type");
162+
}
163+
164+
attr.Add(NumVectorArgs(a).ToString());
165+
166+
attr.Add("TYPE");
167+
attr.Add(a.VectorType);
168+
169+
if (a.Dimension is null)
170+
{
171+
throw new InvalidOperationException("Encountered vector field with no dimension");
172+
}
173+
174+
attr.Add("DIM");
175+
attr.Add(a.Dimension);
176+
177+
if (a.DistanceMetric is not null)
178+
{
179+
attr.Add("DISTANCE_METRIC");
180+
attr.Add(a.DistanceMetric);
181+
}
182+
183+
if (a.M is not null)
184+
{
185+
attr.Add("M");
186+
attr.Add(a.M);
187+
}
188+
189+
if (a.EfConstruction is not null)
190+
{
191+
attr.Add("EF_CONSTRUCTION");
192+
attr.Add(a.EfConstruction);
193+
}
194+
}
66195
}
67196

68197
if (a.Sortable == true)
@@ -73,7 +202,21 @@ public static bool IndexDefinitionEquals(this RedisIndexInfo redisIndexInfo, Typ
73202
return attr.ToArray();
74203
});
75204

76-
return target.SequenceEqual(serialisedDefinition.Skip(7));
205+
return target.SequenceEqual(serialisedDefinition.Skip(currentOffset));
206+
}
207+
208+
/// <summary>
209+
/// calculates the number of arguments that would be required based to reverse engineer the index based off what
210+
/// is in the Info attribute.
211+
/// </summary>
212+
/// <param name="attr">The attribute.</param>
213+
/// <returns>The number of arguments.</returns>
214+
internal static int NumVectorArgs(this RedisIndexInfo.RedisIndexInfoAttribute attr)
215+
{
216+
var numArgs = 6;
217+
numArgs += attr.M is not null ? 2 : 0;
218+
numArgs += attr.EfConstruction is not null ? 2 : 0;
219+
return numArgs;
77220
}
78221

79222
/// <summary>

‎src/Redis.OM/RedisIndexInfo.cs

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Collections.Generic;
33
using System.Globalization;
44
using System.Linq;
5+
using Redis.OM.Modeling;
56

67
namespace Redis.OM
78
{
@@ -220,6 +221,9 @@ public RedisIndexInfoIndexDefinition(RedisReply redisReply)
220221
case "key_type": Identifier = value.ToString(CultureInfo.InvariantCulture); break;
221222
case "prefixes": Prefixes = value.ToArray().Select(x => x.ToString(CultureInfo.InvariantCulture)).ToArray(); break;
222223
case "default_score": DefaultScore = value.ToString(CultureInfo.InvariantCulture); break;
224+
case "default_language": DefaultLanguage = value.ToString(CultureInfo.InvariantCulture); break;
225+
case "filter": Filter = value.ToString(CultureInfo.InvariantCulture); break;
226+
case "language_field": LanguageField = value.ToString(CultureInfo.InvariantCulture); break;
223227
}
224228
}
225229
}
@@ -238,6 +242,21 @@ public RedisIndexInfoIndexDefinition(RedisReply redisReply)
238242
/// Gets default_score.
239243
/// </summary>
240244
public string? DefaultScore { get; }
245+
246+
/// <summary>
247+
/// Gets Filter.
248+
/// </summary>
249+
public string? Filter { get; }
250+
251+
/// <summary>
252+
/// Gets language.
253+
/// </summary>
254+
public string? DefaultLanguage { get; }
255+
256+
/// <summary>
257+
/// Gets LanguageField.
258+
/// </summary>
259+
public string? LanguageField { get; }
241260
}
242261

243262
/// <summary>
@@ -266,9 +285,21 @@ public RedisIndexInfoAttribute(RedisReply redisReply)
266285
case "attribute": Attribute = value; break;
267286
case "type": Type = value; break;
268287
case "SEPARATOR": Separator = value; break;
288+
case "algorithm": Algorithm = value; break;
289+
case "data_type": VectorType = value; break;
290+
case "dim": Dimension = value; break;
291+
case "distance_metric": DistanceMetric = value; break;
292+
case "M": M = value; break;
293+
case "ef_construction": EfConstruction = value; break;
294+
case "WEIGHT": Weight = value; break;
269295
}
270296
}
271297

298+
if (responseArray.Any(x => ((string)x).Equals("NOSTEM", StringComparison.InvariantCultureIgnoreCase)))
299+
{
300+
NoStem = true;
301+
}
302+
272303
if (responseArray.Select(x => x.ToString())
273304
.Any(x => x.Equals("SORTABLE", StringComparison.InvariantCultureIgnoreCase)))
274305
{
@@ -300,6 +331,46 @@ public RedisIndexInfoAttribute(RedisReply redisReply)
300331
/// Gets SORTABLE.
301332
/// </summary>
302333
public bool? Sortable { get; }
334+
335+
/// <summary>
336+
/// Gets NOSTEM.
337+
/// </summary>
338+
public bool? NoStem { get; }
339+
340+
/// <summary>
341+
/// Gets weight.
342+
/// </summary>
343+
public string? Weight { get; }
344+
345+
/// <summary>
346+
/// Gets Algorithm.
347+
/// </summary>
348+
public string? Algorithm { get; }
349+
350+
/// <summary>
351+
/// Gets the VectorType.
352+
/// </summary>
353+
public string? VectorType { get; }
354+
355+
/// <summary>
356+
/// Gets Dimension.
357+
/// </summary>
358+
public string? Dimension { get; }
359+
360+
/// <summary>
361+
/// Gets DistanceMetric.
362+
/// </summary>
363+
public string? DistanceMetric { get; }
364+
365+
/// <summary>
366+
/// Gets M.
367+
/// </summary>
368+
public string? M { get; }
369+
370+
/// <summary>
371+
/// Gets EF constructor.
372+
/// </summary>
373+
public string? EfConstruction { get; }
303374
}
304375

305376
/// <summary>

0 commit comments

Comments
 (0)