Fix #17509: Continue instead of return with unfinished generics (#22563)

Close #17509

Current knowledge:
- delaying cache fixes the issue
- changing return of `if inst.len < key.len:` in `searchInstTypes` to
`continue` fixes the issue. With return the broken types are also cached
over and over

Related issues are completely unaffected as of now, so there must be
something deeper.

I am also still trying to find the true cause, so feel free to ignore
for now

---------

Co-authored-by: SirOlaf <>

compiler/semtypinst.nim

@@ -45,7 +45,8 @@ proc searchInstTypes*(g: ModuleGraph; key: PType): PType =
  # XXX: This happens for prematurely cached
  # types such as Channel[empty]. Why?
  # See the notes for PActor in handleGenericInvocation
- return
+ # if this is return the same type gets cached more than it needs to
+ continue
  if not sameFlags(inst, key):
  continue
 

tests/generics/t17509.nim

@@ -0,0 +1,25 @@
+type List[O] = object
+ next: ptr List[O]
+
+proc initList[O](l: ptr List[O]) =
+ l[].next = l
+
+type
+ PolytopeVertex[R] = object
+ list: List[PolytopeVertex[R]]
+
+ PolytopeEdge[R] = object
+ list: List[PolytopeEdge[R]]
+
+ Polytope[R] = object
+ vertices: List[PolytopeVertex[R]]
+ edges: List[PolytopeEdge[R]]
+
+var pt: Polytope[float]
+
+static:
+ doAssert pt.vertices.next is (ptr List[PolytopeVertex[float]])
+ doAssert pt.edges.next is (ptr List[PolytopeEdge[float]])
+
+initList(addr pt.vertices)
+initList(addr pt.edges)