Merge pull request #83 from ultraleap/element.net/feature/better-pars…

…e-trace

Element.net/feature/better parse trace

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "dependencies/toml11"]
  path = dependencies/toml11
  url = https://github.com/ToruNiina/toml11.git
+[submodule "dependencies/Lexico"]
+ path = dependencies/Lexico
+ url = https://github.com/hamish-milne/Lexico.git

Element.NET/!0-SourcesAndPackages/CompilerOptions.cs

@@ -3,7 +3,7 @@
 namespace Element
 {
  /// <summary>
- /// Options which change how the compiler operates.
+ /// Flags which change how the compiler operates.
  /// </summary>
  public class CompilerOptions // TODO: Change to record type when available
  {

Element.NET/!0-SourcesAndPackages/SourceContext.cs

@@ -13,6 +13,7 @@ public class SourceContext
  public SourceContext(CompilerOptions? compilerOptions) => CompilerOptions = compilerOptions;
 
  public GlobalScope GlobalScope { get; } = new GlobalScope();
+ public List<StructuralTuple> GeneratedStructuralTuples { get; } = new List<StructuralTuple>();
  public CompilerOptions? CompilerOptions { get; }
 
  /// <summary>

Element.NET/!1-AST/Declarations/ConstraintDeclaration.cs

@@ -1,7 +1,9 @@
 using System;
+using Lexico;
 
 namespace Element.AST
 {
+ [Sequence(ParserFlags = ParserFlags.TraceHeader)]
  public class IntrinsicConstraintDeclaration : Declaration
  {
  protected override string IntrinsicQualifier { get; } = "intrinsic ";
@@ -19,6 +21,7 @@ protected override void ValidateDeclaration(ResultBuilder builder, Context conte
  }
  }
 
+ [Sequence(ParserFlags = ParserFlags.TraceHeader)]
  public class CustomConstraintDeclaration : Declaration
  {
  protected override string IntrinsicQualifier { get; } = string.Empty;

Element.NET/!1-AST/Declarations/Declaration.cs

@@ -3,17 +3,17 @@
 
 namespace Element.AST
 {
- [WhitespaceSurrounded, MultiLine]
+ [WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace), MultiLine]
  public abstract class Declaration : AstNode
  {
 #pragma warning disable 649, 8618, 169
  // ReSharper disable UnassignedField.Global UnusedAutoPropertyAccessor.Local UnusedAutoPropertyAccessor.Global
  [IndirectLiteral(nameof(IntrinsicQualifier))] protected Unnamed _;
- [IndirectLiteral(nameof(Qualifier)), WhitespaceSurrounded] protected Unnamed __;
+ [IndirectLiteral(nameof(Qualifier)), WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace)] protected Unnamed __;
  [Term] public Identifier Identifier;
  [Optional] public PortList? PortList;
  [Optional] public PortConstraint? ReturnConstraint;
- [IndirectAlternative(nameof(BodyAlternatives)), WhitespaceSurrounded, MultiLine] public object Body;
+ [IndirectAlternative(nameof(BodyAlternatives)), WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace), MultiLine] public object Body;
  // ReSharper restore UnassignedField.Global UnusedAutoPropertyAccessor.Local UnusedAutoPropertyAccessor.Global
 #pragma warning restore 649, 8618, 169
 

Element.NET/!1-AST/Declarations/DeclarationComponents/Block.cs

@@ -5,7 +5,7 @@
 
 namespace Element.AST
 {
- [WhitespaceSurrounded, MultiLine]
+ [WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace), MultiLine]
  public abstract class Block : AstNode
  {
 #pragma warning disable 649, 169
@@ -58,12 +58,21 @@ protected override void ValidateImpl(ResultBuilder builder, Context context)
 
  public class FreeformBlock : Block
  {
- [field: SurroundBy("{", "}"), WhitespaceSurrounded, Optional] protected override List<Declaration>? _items { get; set; }
+ [
+ SurroundBy("{", "}"),
+ WhitespaceSurrounded,
+ Optional
+ ] protected override List<Declaration>? _items { get; set; }
  }
 
  public class CommaSeparatedBlock : Block
  {
- [field: SurroundBy("{", "}"), WhitespaceSurrounded, Optional, SeparatedBy(typeof(ListSeparator))] protected override List<Declaration>? _items { get; set; }
+ [
+ SurroundBy("{", "}"),
+ WhitespaceSurrounded,
+ Optional,
+ SeparatedBy(typeof(ListSeparator))
+ ] protected override List<Declaration>? _items { get; set; }
  }
 
  public class DeclarationBlock : FreeformBlock, IDeclarationScope

Element.NET/!1-AST/Declarations/DeclarationComponents/ExpressionBody.cs

@@ -2,7 +2,7 @@
 
 namespace Element.AST
 {
- [WhitespaceSurrounded]
+ [WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace)]
  public class ExpressionBody
  {
 #pragma warning disable 169, 8618

Element.NET/!1-AST/Declarations/DeclarationComponents/Identifier.cs

@@ -8,7 +8,7 @@ public struct Identifier : IEquatable<Identifier>, IExpressionChainStart
  public Identifier(string value) => String = value;
 
  // ReSharper disable once AutoPropertyCanBeMadeGetOnly.Local
- [field: Regex(@"_?[a-zA-Z\u0080-\uFFFF][_a-zA-Z0-9\u0080-\uFFFF]*")]
+ [field: Regex(@"_?[a-zA-Z\u0080-\uFFFF][_a-zA-Z0-9\u0080-\uFFFF]*", ParserFlags = ParserFlags.IgnoreInTrace)]
  public string String { get; private set; }
  public override string ToString() => String;
  // ReSharper disable once NonReadonlyMemberInGetHashCode

Element.NET/!1-AST/Declarations/DeclarationComponents/ListOf.cs

@@ -5,18 +5,22 @@
 
 namespace Element.AST
 {
- [WhitespaceSurrounded, MultiLine]
+ [WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace), MultiLine]
  // ReSharper disable once ClassNeverInstantiated.Global
  public class ListOf<T>
  {
  // ReSharper disable once UnusedAutoPropertyAccessor.Local
  // ReSharper disable once CollectionNeverUpdated.Global
- [field: SurroundBy("(", ")"), SeparatedBy(typeof(ListSeparator))] public List<T> List { get; private set; }
+ [
+ field:
+ SurroundBy("(", ")"),
+ SeparatedBy(typeof(ListSeparator))
+ ] public List<T> List { get; private set; }
 
  public override string ToString() => $"({string.Join(", ", List)})";
  }
 
- [WhitespaceSurrounded, MultiLine]
+ [WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace), MultiLine]
  internal struct ListSeparator
  {
  [Literal(",")] private Unnamed _;

Element.NET/!1-AST/Declarations/DeclarationComponents/Port.cs

@@ -3,7 +3,7 @@
 
 namespace Element.AST
 {
- [WhitespaceSurrounded, MultiLine]
+ [WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace), MultiLine]
  public class Port : AstNode
  {
 #pragma warning disable 649, 8618

Element.NET/!1-AST/Declarations/DeclarationComponents/PortConstraint.cs

@@ -5,7 +5,7 @@ namespace Element.AST
  public class PortConstraint : AstNode
  {
 #pragma warning disable 169, 8618
- [Literal(":"), WhitespaceSurrounded, MultiLine] private Unnamed _;
+ [Literal(":"), WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace), MultiLine] private Unnamed _;
 
  // ReSharper disable once UnusedAutoPropertyAccessor.Local
  [field: Term] public Expression Expression { get; private set; }

Element.NET/!1-AST/Declarations/DeclarationComponents/PortList.cs

@@ -9,7 +9,7 @@ public class PortList : AstNode
  {
 #pragma warning disable 8618
  // ReSharper disable once UnusedAutoPropertyAccessor.Local
- [field: Term] public ListOf<Port> Ports { get; private set; }
+ [field: Sequence, WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace)] public ListOf<Port> Ports { get; private set; }
 #pragma warning restore 8618
 
  public override string ToString() => Ports.ToString();

Element.NET/!1-AST/Declarations/FunctionDeclaration.cs

@@ -1,9 +1,11 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using Lexico;
 
 namespace Element.AST
 {
+ [Sequence(ParserFlags = ParserFlags.TraceHeader)]
  public sealed class IntrinsicFunctionDeclaration : Declaration
  {
  protected override string IntrinsicQualifier => "intrinsic ";
@@ -35,6 +37,7 @@ protected override void ValidateDeclaration(ResultBuilder builder, Context conte
  }
  }
 
+ [Sequence(ParserFlags = ParserFlags.TraceHeader)]
  public sealed class ExpressionBodiedFunctionDeclaration : Declaration
  {
  protected override string IntrinsicQualifier => string.Empty;
@@ -60,6 +63,7 @@ protected override void ValidateDeclaration(ResultBuilder builder, Context conte
  }
  }
 
+ [Sequence(ParserFlags = ParserFlags.TraceHeader)]
  public sealed class ScopeBodiedFunctionDeclaration : Declaration
  {
  protected override string IntrinsicQualifier => string.Empty;

Element.NET/!1-AST/Declarations/NamespaceDeclaration.cs

@@ -1,8 +1,9 @@
 using System;
-using System.Linq;
+using Lexico;
 
 namespace Element.AST
 {
+ [Sequence(ParserFlags = ParserFlags.TraceHeader)]
  public class NamespaceDeclaration : Declaration
  {
  protected override string IntrinsicQualifier => string.Empty;

Element.NET/!1-AST/Declarations/StructDeclaration.cs

@@ -1,8 +1,10 @@
 using System;
 using System.Linq;
+using Lexico;
 
 namespace Element.AST
 {
+ [Sequence(ParserFlags = ParserFlags.TraceHeader)]
  public class IntrinsicStructDeclaration : Declaration
  {
  protected override string IntrinsicQualifier { get; } = "intrinsic ";
@@ -45,6 +47,7 @@ protected override void ValidateDeclaration(ResultBuilder builder, Context conte
  }
  }
 
+ [Sequence(ParserFlags = ParserFlags.TraceHeader)]
  public class CustomStructDeclaration : Declaration
  {
  protected override string IntrinsicQualifier { get; } = string.Empty;

Element.NET/!1-AST/Expressions/AnonymousBlock.cs

@@ -2,14 +2,15 @@
 
 namespace Element.AST
 {
+ [Sequence(ParserFlags = ParserFlags.TraceHeader)]
  public class AnonymousBlock : Expression
  {
 #pragma warning disable 8618
  // ReSharper disable once UnusedAutoPropertyAccessor.Local
- [Term] public CommaSeparatedBlock Block { get; private set; }
+ [Sequence] public CommaSeparatedBlock Block { get; private set; }
 #pragma warning restore 8618
 
  protected override void ValidateImpl(ResultBuilder builder, Context context) => Block.Validate(builder, context);
- protected override Result<IValue> ExpressionImpl(IScope parentScope, Context context) => Block.ResolveBlock(parentScope, context).Cast<IValue>();
+ protected override Result<IValue> ExpressionImpl(IScope parentScope, Context context) => Block.ResolveBlock(parentScope, context).Bind(block => StructuralTuple.CreateInstance(block, context));
  }
 }

Element.NET/!1-AST/Expressions/Expression.cs

@@ -2,7 +2,7 @@
 
 namespace Element.AST
 {
- [WhitespaceSurrounded, MultiLine]
+ [WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace), MultiLine]
  // ReSharper disable once ClassNeverInstantiated.Global
  public abstract class Expression : AstNode
  {

Element.NET/!1-AST/Expressions/ExpressionChain.cs

@@ -12,12 +12,13 @@ public interface IExpressionChainStart
  }
 
  // ReSharper disable once ClassNeverInstantiated.Global
+ [Sequence(ParserFlags = ParserFlags.TraceHeader)]
  public class ExpressionChain : Expression
  {
  // ReSharper disable UnusedAutoPropertyAccessor.Local
 #pragma warning disable 8618
  [field: Alternative(typeof(Identifier), typeof(Constant))] public IExpressionChainStart ExpressionChainStart { get; private set; }
- [field: Optional] public List<SubExpression>? SubExpressions { get; private set; }
+ [field: Repeat, Optional] public List<SubExpression>? SubExpressions { get; private set; }
 #pragma warning restore 8618
  // ReSharper restore UnusedAutoPropertyAccessor.Local
 
@@ -48,6 +49,7 @@ protected override Result<IValue> ExpressionImpl(IScope parentScope, Context con
  : ExpressionChainStart.Resolve(this, parentScope, context);
  }
 
+ [WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace)]
  public abstract class SubExpression : AstNode
  {
  public Result<IValue> ResolveSubExpression(ExpressionChain chain, IValue previous, IScope parentScope, Context context)
@@ -66,7 +68,7 @@ public class CallExpression : SubExpression
  {
 #pragma warning disable 169, 8618
  // ReSharper disable once UnusedAutoPropertyAccessor.Local
- [field: Term] public ListOf<Expression> Expressions { get; private set; }
+ [field: Sequence] public ListOf<Expression> Expressions { get; private set; }
 #pragma warning restore 169, 8618
 
  public override string ToString() => Expressions.ToString();

Element.NET/!1-AST/Expressions/Lambda.cs

@@ -2,14 +2,15 @@
 
 namespace Element.AST
 {
+ [Sequence(ParserFlags = ParserFlags.TraceHeader)]
  public class Lambda : Expression
  {
 #pragma warning disable 649, 169, 8618
  // ReSharper disable UnassignedField.Global UnusedAutoPropertyAccessor.Local
  [Term] private Unidentifier _;
  [Term] public PortList PortList;
  [Optional] public PortConstraint? ReturnConstraint;
- [Alternative(typeof(ExpressionBody), typeof(FunctionBlock)), WhitespaceSurrounded, MultiLine] public object Body;
+ [Alternative(typeof(ExpressionBody), typeof(FunctionBlock)), WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace), MultiLine] public object Body;
  // ReSharper restore UnassignedField.Global UnusedAutoPropertyAccessor.Local
 #pragma warning restore 649, 169, 8618
 

Element.NET/!1-AST/Parser.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -24,6 +25,11 @@ public static class Parser
 
  public static string Preprocess(string text) => Regex.Replace(text, @"#.*", string.Empty, RegexOptions.Multiline | RegexOptions.Compiled);
 
+ private class ParserTrace : UserTrace
+ {
+ public ParserTrace(Action<string> onError) : base(onError) { }
+ }
+
  public static Result<T> Parse<T>(SourceInfo source, Context context, bool noParseTrace = false) where T : notnull
  {
  if (Lexico.Lexico.TryParse(source.PreprocessedText, out T output, userObject: source))
@@ -32,7 +38,7 @@ public static class Parser
 
  // Using StringBuilder as there's potentially a lot of trace lines
  var sb = new StringBuilder();
- Lexico.Lexico.TryParse<T>(source.PreprocessedText, out _, new DelegateTextTrace(msg => { if (!string.IsNullOrEmpty(msg)) sb.AppendLine(msg); }), source);
+ Lexico.Lexico.TryParse<T>(source.PreprocessedText, out _, new ParserTrace(msg => { if (!string.IsNullOrEmpty(msg)) sb.AppendLine(msg); }), source);
  return context.Trace(EleMessageCode.ParseError, $"Parsing failed within '{source.DisplayName}' - see parse trace below for details.\n{sb}");
  }
 
@@ -51,7 +57,7 @@ public static void Validate(this Identifier identifier, ResultBuilder builder, I
  }
  }
 
- [TopLevel]
+ [TopLevel(ParserFlags = ParserFlags.IgnoreInTrace)]
  // ReSharper disable once ClassNeverInstantiated.Global
  public class TopLevel<T>
  {

Element.NET/!1-AST/SourceBlob.cs

@@ -8,13 +8,16 @@ namespace Element.AST
  /// <summary>
  /// Represents a top-level blob of Element source code.
  /// </summary>
- [WhitespaceSurrounded, MultiLine, TopLevel]
+ [WhitespaceSurrounded(ParserFlags = ParserFlags.IgnoreInTrace), MultiLine, TopLevel(ParserFlags = ParserFlags.IgnoreInTrace)]
  // ReSharper disable once ClassNeverInstantiated.Global
  public class SourceBlob : AstNode, IEnumerable<Declaration>
  {
 #pragma warning disable 649
  // ReSharper disable once CollectionNeverUpdated.Local
- [Optional] private List<Declaration>? _items;
+ [
+ Repeat,
+ Optional
+ ] private List<Declaration>? _items;
 #pragma warning restore 649
 
  public IEnumerator<Declaration> GetEnumerator() => _items?.GetEnumerator() ?? Enumerable.Empty<Declaration>().GetEnumerator();

Element.NET/!2-ValueObjectModel/Intrinsics/Functions/If.cs

@@ -15,7 +15,7 @@ private If()
  public bool IsVariadic => false;
 
  public override Result<IValue> Call(IReadOnlyList<IValue> arguments, Context context) =>
- // Make a list out of the true and false options
+ // Make a list out of the true and false flags
  List.Instance.Call(arguments.Skip(1).Reverse().ToArray(), context)
  .CastInner<StructInstance>()
  // Get the option lists indexer (field 0)