/*
 * RiskScape™ Copyright New Zealand Institute for Earth Science Limited
 * (Earth Sciences New Zealand) is distributed for research purposes only
 * under the terms of AGPLv3.
 *
 * RiskScape™ Copyright 2026 New Zealand Institute for Earth Science
 * Limited (Earth Sciences New Zealand). All rights reserved. Source code
 * available under the AGPLv3.
 * 
 * This program is free software: you can redistribute it and/or modify it under
 *  the terms of the GNU Affero General Public License as published by the Free
 *  Software Foundation, either version 3 of the License, or (at your option) any
 *  later version.
 * 
 * This program is distributed for RESEARCH PURPOSES ONLY, in the hope that it will
 * be useful for research and education initiatives.
 * 
 * If you are not a researcher, or you are a researcher who wishes to use this
 * program on terms other than AGPLv3 (including those who wish to restrict the
 * distribution of any source code created using this program), please contact:
 * https://riskscape.org.nz
 * 
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Affero General Public License for more details.  You should have received a copy
 * of the GNU Affero General Public License along with this program.  If not, see
 * <http://www.gnu.org/licenses/>.
 * 
 * By way of summary only, under the AGPLv3:
 *     • Permissions of this strongest copyleft license are conditioned
 *       on making available complete source code of licensed works and
 *       modifications, which include larger works using a licensed work,
 *       under the same license.
 *     • Copyright and license notices must be preserved.
 *     • Contributors provide an express grant of patent rights.
 *     • When a modified version is used to provide a service over a
 *       network, the complete source code of the modified version must be made
 *       available.
 */
package nz.org.riskscape.engine.rl;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Optional;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.stream.Collectors;

import com.google.common.base.CaseFormat;
import com.google.common.collect.Lists;

import lombok.Getter;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import nz.org.riskscape.dsl.Token;
import nz.org.riskscape.engine.FailedObjectException;
import nz.org.riskscape.engine.NoSuchObjectException;
import nz.org.riskscape.engine.RiskscapeException;
import nz.org.riskscape.engine.Tuple;
import nz.org.riskscape.engine.expr.StructAccessExpression;
import nz.org.riskscape.engine.expr.StructMemberAccessExpression;
import nz.org.riskscape.engine.function.ArgumentList;
import nz.org.riskscape.engine.function.FunctionArgument;
import nz.org.riskscape.engine.function.IdentifiedFunction;
import nz.org.riskscape.engine.function.RiskscapeFunction;
import nz.org.riskscape.engine.rl.agg.AggregateExpressionRealizer;
import nz.org.riskscape.engine.rl.agg.RealizedAggregateExpression;
import nz.org.riskscape.engine.types.EmptyList;
import nz.org.riskscape.engine.types.LambdaType;
import nz.org.riskscape.engine.types.Nullable;
import nz.org.riskscape.engine.types.RSList;
import nz.org.riskscape.engine.types.Struct;
import nz.org.riskscape.engine.types.Type;
import nz.org.riskscape.engine.types.TypeProblems;
import nz.org.riskscape.engine.types.Types;
import nz.org.riskscape.engine.types.ancestor.AncestorTypeList;
import nz.org.riskscape.problem.Problem;
import nz.org.riskscape.problem.Problems;
import nz.org.riskscape.problem.ResultOrProblems;
import nz.org.riskscape.rl.ExpressionParser;
import nz.org.riskscape.rl.TokenTypes;
import nz.org.riskscape.rl.ast.BinaryOperation;
import nz.org.riskscape.rl.ast.BracketedExpression;
import nz.org.riskscape.rl.ast.Constant;
import nz.org.riskscape.rl.ast.Expression;
import nz.org.riskscape.rl.ast.ExpressionProblems;
import nz.org.riskscape.rl.ast.FunctionCall;
import nz.org.riskscape.rl.ast.Lambda;
import nz.org.riskscape.rl.ast.ListDeclaration;
import nz.org.riskscape.rl.ast.ParameterToken;
import nz.org.riskscape.rl.ast.PropertyAccess;
import nz.org.riskscape.rl.ast.SelectAllExpression;
import nz.org.riskscape.rl.ast.StructDeclaration;

/**
 * Realizes riskscape language expressions in to reified/realized objects that can be evaluated.
 */
@RequiredArgsConstructor
@Slf4j
public class DefaultExpressionRealizer implements ExpressionRealizer {

  static boolean anyFailed(RealizedExpression... realizedArgs) {
    return anyFailed(List.of(realizedArgs));
  }

  static boolean anyFailed(Collection<RealizedExpression> realizedArgs) {
    for (RealizedExpression child : realizedArgs) {
      if (child instanceof Realized realized) {
        if (realized.isFailed()) {
          return true;
        }
      }
    }

    return false;
  }

  /**
   * Implements {@link RealizedExpression} for the results that come from this realizer.
   */
  @RequiredArgsConstructor
  static final class Realized implements RealizedExpression {

    /**
     * @return a failed {@link RealizedExpression} - used for tracking failed functional dependencies so that
     * we can accumulate as many problems as possible from a single realization without causing lots of noise due to
     * failed dependencies.
     */
    static Realized failed(Type inputType, Expression failedOn) {
      return new Realized(
          inputType,
          failedOn,
          Types.ANYTHING,
          (r, in) -> new RiskscapeException("Can not evaluate a failed expression"),
          true,
          Collections.emptyList(),
          null
      );
    }

    Realized(
        Type inputType,
        Expression expr,
        Type resultType,
        BiFunction<Realized, Object, Object> function,
        List<? extends RealizedExpression> deps,
        Runnable closeFunction
    ) {
      this.inputType = inputType;
      this.expression = expr;
      this.resultType = resultType;
      this.function = function;
      this.failed = false;
      this.dependencies = Lists.newArrayList(deps);
      this.closeFunction = closeFunction;
    }

    @Getter
    private final Type inputType;

    @Getter
    private final Expression expression;

    @Getter
    private final Type resultType;

    private final BiFunction<Realized, Object, Object> function;

    @Getter
    private final boolean failed;

    @Getter
    private final List<RealizedExpression> dependencies;

    private final Runnable closeFunction;

    @Override
    public Object evaluate(Object input) {
      try {
        return function.apply(this, input);
      } catch (RuntimeException e) {
        throw new EvalException(e, this, input);
      }
    }

    @Override
    public void close() {
      RealizedExpression.super.close();
      if (closeFunction != null) {
        closeFunction.run();
      }
    }

    @Override
    public String toString() {
      return String.format("Realized(expr=[%s] returns=%s)", expression.toSource(), resultType);
    }

    @Override
    public boolean isConstant() {
      return false;
    }

    @Override
    public Object evaluateConstant() {
      throw new IllegalStateException("Not constant");
    }
  }

  /**
   * Wraps an expression and makes it constant.  The result of evaluate and evaluateConstant will always return the same
   * value.
   */

  static final class RealizedConstant extends RealizedExpressionWrapper {

    static RealizedExpression create(RealizedExpression wrapped) {
      try {
        return new RealizedConstant(wrapped,
            wrapped.evaluate(new ConstantTupleSentinel(wrapped.getInputType().asStruct())));
      } catch (EvalException e) {
        return new ExceptionalConstant(wrapped, e);
      }
    }

    private final Object value;

    RealizedConstant(RealizedExpression wrapped, Object value) {
      super(wrapped);
      this.value = value;
    }

    @Override
    public Object evaluate(Object input) {
      return value;
    }

    @Override
    public Object evaluateConstant() {
      return value;
    }

    @Override
    public boolean isConstant() {
      return true;
    }
  }

  private static final class ExceptionalConstant extends RealizedExpressionWrapper {

    private final EvalException exception;

    ExceptionalConstant(RealizedExpression wrapped, EvalException exception) {
      super(wrapped);
      this.exception = exception;
    }

    @Override
    public Object evaluate(Object input) {
      throw exception;
    }

    @Override
    public Object evaluateConstant() {
      throw exception;
    }

    @Override
    public boolean isConstant() {
      return true;
    }
  }

  /**
   * Wraps a {@link BinaryOperation} adding links to the previous and next {@link BinaryOperation}s
   * that may exist in a chain of binary operations.
   *
   * This class is to assist with realizing of chained binary operations in precedence order.
   */
  @RequiredArgsConstructor
  static final class LinkedBinaryOperation implements Comparable<LinkedBinaryOperation> {
    @Getter
    private final BinaryOperation operation;

    @Getter
    private final LinkedBinaryOperation previous;

    @Getter @Setter
    private LinkedBinaryOperation next;

    /**
     * Get the effective {@link Realized} expression if available.
     *
     * This {@link Realized} is potentially updated during the realization process. For example the
     * expression `3 * 10 / 5 + 2`
     *
     * 1) `10 / 5` is realized
     * 2) `3 * (10/5)` is realized which involves getting the RHS from the previous expression. The expression
     *                 realized here will be set as realized for both of these expression. This then allows
     *                 the last part `xxx + 2` to obtain the effective realized expression from its previous
     *                 link.
     */
    @Getter
    private RealizedExpression realized;

    LinkedBinaryOperation(BinaryOperation operation) {
      this.operation = operation;
      this.previous = null;
    }


    @Override
    public int compareTo(LinkedBinaryOperation o) {
      for (EnumSet level: BinaryOperation.PRIORITY) {
        if (level.contains(operation.getNormalizedOperator())) {
          return level.contains(o.operation.getNormalizedOperator()) ? 0 : -1;
        }
        if (level.contains(o.operation.getNormalizedOperator())) {
          return 1;
        }
      }
      //This exception will only be thrown if a binary operator is missing from the BinaryOperation.PRIORITY
      //collection.
      //Hopefully this will only happen whilst writing test cases for any new binary operators.
      throw new RiskscapeException(String.format("Could not prioritise %s and %s",
          operation.getNormalizedOperator().toString(), o.operation.getNormalizedOperator().toString()));
    }

    public void setRealized(RealizedExpression realized) {
      this.realized = realized;
      //We need to push this realized to any realized ascendant/descendant.
      LinkedBinaryOperation ascendant = previous;
      while (ascendant != null && ascendant.getRealized() != null) {
        //If our next has already been realized, we push this realized to it as well. This is so that
        //if there is a 'next' further down the chain yet to be realized it will have access to the
        //composite expression.
        ascendant.realized = realized;
        ascendant = ascendant.previous;
      }
      LinkedBinaryOperation descendant = next;
      while (descendant != null && descendant.getRealized() != null) {
        //If our next has already been realized, we push this realized to it as well. This is so that
        //if there is a 'next' further down the chain yet to be realized it will have access to the
        //composite expression.
        descendant.realized = realized;
        descendant = descendant.next;
      }
    }

  }

  /**
   * A {@link Realized} that can be used as a placeholder for an argument that has not been provided.
   *
   * This realized uses {@link Types#NOTHING} for both the input and result type and when evaluated will
   * return a null value.
   */
  static final Realized MISSING_ARGUMENT = new Realized(
      Types.NOTHING,
      ExpressionParser.INSTANCE.parse("[]"),
      Types.NOTHING,
      (r, in) -> null,
      false,
      Collections.emptyList(),
      null
  );

  private final ExpressionParser parser;
  private final RealizationContext realizationContext;

  public DefaultExpressionRealizer(@NonNull RealizationContext context) {
    this.realizationContext = context;
    this.parser = new ExpressionParser();
  }

  @Override
  public RealizedExpression asStruct(RealizationContext context, RealizedExpression expression) {
    if (expression.getResultType() instanceof Struct) {
      // NB there are cases here where we might still be able to return a struct value, but it probably
      // relies on us tapping in to coercion rules, and that needs some more thought w.r.t to a general approach.
      // This method is more here to make it easier for users to give an expression like:
      // `foo` instead of `{foo}` as a pipeline parameter
      return expression;
    } else {
      String memberName = ExpressionRealizer.getImplicitName(context, expression, Collections.emptyList());
      Struct struct = realizationContext.normalizeStruct(Struct.of(memberName, expression.getResultType()));
      return new StructConverter(struct, expression);
    }
  }


  /**
   * Wraps a {@link RealizedExpression} to put its result in a single-member struct.  Used with
   * {@link DefaultExpressionRealizer#asStruct(RealizedExpression)}
   */
  private static class StructConverter extends RealizedExpressionWrapper {

    private final Struct struct;

    StructConverter(Struct struct, RealizedExpression wrapping) {
      super(wrapping);
      this.struct = struct;
    }

    @Override
    public Type getResultType() {
      return struct;
    }

    @Override
    public List<RealizedExpression> getDependencies() {
      return Arrays.asList(wrapped);
    }

    @Override
    public Object evaluate(Object input) {
      return Tuple.ofValues(struct, wrapped.evaluate(input));
    }

    @Override
    public Object evaluateConstant() {
      return Tuple.ofValues(struct, wrapped.evaluateConstant());
    }
  }

  @Override
  public ResultOrProblems<RealizedExpression> realize(Type inputType, String source) {
    return parser.parseOr(source).flatMap(expression ->
      new Instance(inputType, expression, realizationContext).realize());
  }

  @Override
  public ResultOrProblems<RealizedExpression> realize(Type inputType, Expression expression) {
    return new Instance(inputType, expression, realizationContext).realize();
  }

  @Override
  public Expression parse(String source) {
    return parser.parse(source);
  }

  /**
   * Encapsulates all the per-realization state for the realize method
   */
  private class Instance {

    // list of problems we accrue during realization
    private List<Problem> problems = new ArrayList<>();
    private Type inputType;

    // the root of the ast we are realizing
    private Expression root;

    RealizationContext context;

    StructDeclarationRealizer structDeclRealizer;

    Instance(@NonNull Type inputType, @NonNull Expression root, @NonNull RealizationContext context) {
      this.inputType = inputType;
      this.root = root;
      this.context = context;
      this.structDeclRealizer =
        new StructDeclarationRealizer(problems, inputType, context, (e1, e2) -> realizeRaw(e1, e2));
    }

    // do the realization
    public ResultOrProblems<RealizedExpression> realize() {
      normalizeStruct(inputType);

      RealizedExpression realized = realizeRaw(root, root);

      if (Problem.hasErrors(problems)) {
        return ResultOrProblems.failed(problems);
      } else {
        return ResultOrProblems.of(realized);
      }
    }

    private void normalizeStruct(Type maybeAStruct) {
      maybeAStruct.findAllowNull(Struct.class).ifPresent(s -> {
        Struct normalized = context.normalizeStruct(s);

        if (normalized != s) {
          // this used to be fairly important, but then we relaxed the requirement for struct identity to matter
          // - See Tuple#checkOwner
          log.debug("Input type for realization of {} did not normalize to itself.  This can lead to "
              + "proliferation of identical struct types.", root);
        }
      });
    }

    private RealizedExpression wrapIfLiteral(RealizedExpression realized) {
      if (realized.getExpression().isLiteral()) {
        return RealizedConstant.create(realized);
      } else {
        return realized;
      }
    }

    private RealizedExpression realizeRaw(Expression expr, Expression parent) {
      RealizedExpression realized = tryThese(expr, Arrays.asList(
          realizeIf(expr, BinaryOperation.class, fc -> realizeRaw(fc, parent)),
          realizeIf(expr, BracketedExpression.class, fc -> realizeRaw(fc, parent)),
          realizeIf(expr, FunctionCall.class, fc -> realizeRaw(fc, parent)),
          realizeIf(expr, PropertyAccess.class, pa-> realizeRaw(pa, parent)),
          realizeIf(expr, Constant.class, c-> realizeRaw(c, parent)),
          realizeIf(expr, ListDeclaration.class, ld -> realizeRaw(ld, parent)),
          realizeIf(expr, StructDeclaration.class, sd -> realizeRaw(sd, parent)),
          realizeIf(expr, SelectAllExpression.class, se -> realizeRaw(se, parent)),
          realizeIf(expr, Lambda.class, le -> realizeRaw(le)),
          realizeIf(expr, ParameterToken.class, pt -> realizeRaw(pt))
      ));

      return wrapIfLiteral(realized);
    }

    private RealizedExpression realizeRaw(ListDeclaration ld, Expression parent) {
      List<RealizedExpression> realizedElements = new ArrayList<>(ld.size());

      for (Expression element : ld.getElements()) {
        realizedElements.add(realizeRaw(element, ld));
      }

      if (anyFailed(realizedElements)) {
        return Realized.failed(inputType, ld);
      }

      List<Type> listElementTypes = realizedElements.stream()
          .map(realized -> realized.getResultType())
          .collect(Collectors.toList());

      // work out the common type for all the list items. This may involve some conversion
      // of data for convenience, e.g. convert a mix of ints and floats to all floats
      AncestorTypeList ancestorType = context.getProject().getTypeSet().computeAncestors(listElementTypes);

      Type listType =
        ancestorType.getType() == Types.NOTHING ? EmptyList.INSTANCE : RSList.create(ancestorType.getType());

      return new Realized(inputType, ld, listType, (re, input) -> {
        List<Object> evaluated = new ArrayList<>(realizedElements.size());
        for (RealizedExpression element : realizedElements) {
          evaluated.add(element.evaluate(input));
        }
        return ancestorType.getConverter().apply(evaluated);
      }, realizedElements, null);
    }

    private RealizedExpression realizeRaw(Constant c, Expression parent) {
      Token token = c.getToken();
      Object javaObject;
      Type simpleType;

      switch ((TokenTypes) token.type) {
      case INTEGER:
        simpleType = Types.INTEGER;
        javaObject = Long.parseLong(token.value);
        break;
      case DECIMAL:
        simpleType = Types.FLOATING;
        javaObject = Double.parseDouble(token.value);
        break;
      case SCIENTIFIC_NOTATION:
        simpleType = Types.FLOATING;
        javaObject = Double.valueOf(token.value);
        break;
      case STRING:
        simpleType = Types.TEXT;
        javaObject = token.value;
        break;
      case KEYWORD_TRUE:
        simpleType = Types.BOOLEAN;
        javaObject = Boolean.TRUE;
        break;
      case KEYWORD_FALSE:
        simpleType = Types.BOOLEAN;
        javaObject = Boolean.FALSE;
        break;
      case KEYWORD_NULL:
        simpleType = Types.NOTHING;
        javaObject = null;
        break;
      default:
        return failed(c, "unexpected constant token: " + token);
      }

      return new Realized(inputType, c, simpleType, (x, y) -> javaObject, Collections.emptyList(), null);
    }

    private Realized failed(Expression failedOn, String message) {
      return failed(failedOn, message, Collections.emptyList());
    }

    private Realized failed(Expression failedOn, String message, List<Problem> causedBy) {
      if (causedBy.isEmpty()) {
        problems.add(Problem.error("%s", message));
      } else {
        problems.add(Problem.composite(causedBy, "%s", message));
      }
      return Realized.failed(inputType, failedOn);
    }

    private Realized realizeRaw(Lambda lambda) {
      // when we realize a lambda expression, we don't actually realize the body of the expression - this is deferred
      // to the user of the expression (and at the moment the only users are functions)
      return new Realized(inputType, lambda, LambdaType.create(lambda).scoped(inputType.asStruct()),
          (realized, input) -> new ScopedLambdaExpression(lambda, (Tuple) input), Collections.emptyList(), null);
    }

    private RealizedExpression realizeRaw(PropertyAccess access, Expression parent) {

      // make sure it's inside a struct expression
      if (access.isTrailingSelectAll()) {
        if (!parent.isA(StructDeclaration.class).isPresent()) {
          problems.add(ExpressionProblems.get().pointlessSelectAll(access));
          return Realized.failed(inputType, access);
        }
      }

      Type toBeEvaluated;
      BiFunction<Realized, Object, Object> toUse;
      RealizedExpression receiver;
      if (access.getReceiver().isPresent()) {
        receiver = realizeRaw(access.getReceiver().get(), access);

        if (anyFailed(receiver)) {
          return Realized.failed(inputType, access);
        }

        toBeEvaluated = receiver.getResultType();
        toUse = (re, input) -> receiver.evaluate(input);
      } else {
        receiver = null;
        toBeEvaluated = inputType;
        toUse = (re, v) -> v;
      }

      boolean isNullable = Nullable.is(toBeEvaluated);
      Struct structInput = toBeEvaluated.findAllowNull(Struct.class).orElse(null);
      if (structInput == null) {
        problems.add(Problems.get(TypeProblems.class).notStruct(access, toBeEvaluated));
        return Realized.failed(inputType, access);
      }

      // this is just a select-all on the receiver, so proxy through to the receiver
      // and remember that it came from a select-all expression
      if (access.isReceiverSelectAll()) {
        return new Realized(
            inputType,
            access,
            receiver.getResultType(),
            (re, input) -> receiver.evaluate(input),
            Collections.singletonList(receiver),
            null
        );
      }

      StructAccessExpression sae = new StructAccessExpression(access.getIdentifiers().stream()
          // drop any asterisk tokens representing a select all - the parser should ensure they are only at the end of
          // the path, the select-all operator is currently used by struct access expressions as an instruction to splat
          // the members of this thing in to the struct being declared
          .filter(t -> t.type != TokenTypes.MULTIPLY)
          .map(t -> t.value)
          .collect(Collectors.toList()));

      ResultOrProblems<StructMemberAccessExpression> smae = sae.getExpressionFor(structInput);

      if (smae.hasProblems()) {
        problems.addAll(smae.getProblems());
        return Realized.failed(inputType, access);
      } else {
        // TODO move null handling to smae?
        toUse = toUse.andThen(input -> input == null ? null : smae.get().evaluate(input));
        return new Realized(
            inputType,
            access,
            Nullable.ifTrue(isNullable, smae.get().getType()),
            toUse,
            receiver == null ? Collections.emptyList() : Collections.singletonList(receiver),
            null
        );
      }
    }

    private RealizedExpression realizeRaw(FunctionCall fc, Expression parent) {
      List<RealizedExpression> unsortedArgs = fc.getArguments().stream()
          .map(arg -> realizeRaw(arg.getExpression(), fc))
          .collect(Collectors.toList());

      String functionId = fc.getIdentifier().value;
      IdentifiedFunction byId;

      try {
        byId = context.getProject().getFunctionSet().get(functionId, context.getProblemSink());
      } catch (FailedObjectException ex) {
        problems.add(Problems.caught(ex));
        return Realized.failed(inputType, fc);
      } catch (NoSuchObjectException ex) {
        // If the function wasn't found we attempt to mung the name a little as a user convenience for
        // function renames. This should catch:
        // createPoint -> create_point
        // geomFromWKT -> geom_from_wkt
        // isNull -> is_null
        if ("geomFromWKT".equals(functionId)) {
          // geomFromWKT is a special case because case formation conversion results in `geom_from_w_k_t`
          functionId = "geom_from_wkt";
        } else {
          functionId = CaseFormat.LOWER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, functionId);
        }
        try {
          byId = context.getProject().getFunctionSet().get(functionId, context.getProblemSink());
        } catch (FailedObjectException | NoSuchObjectException ex2) {
          problems.add(Problems.caught(new MissingFunctionException(fc)));
          if (ex2 instanceof FailedObjectException) {
            log.warn("Likely internal function with id {} (after snake case conversion) was failed - pretending "
                + "it does not exist", functionId, ex2);
          }
          return Realized.failed(inputType, fc);
        }
      }

      if (byId == null) {
        problems.add(Problems.caught(new MissingFunctionException(fc)));
        return Realized.failed(inputType, fc);
      }

      List<RealizedExpression> realizedArgs = sort(unsortedArgs, fc, byId);

      if (realizedArgs == null) {
        // don't pollute problems with failures caused by dependencies
        return Realized.failed(inputType, fc);
      }

      List<Type> argTypes = realizedArgs.stream()
          .map(RealizedExpression::getResultType)
          .collect(Collectors.toList());

      ResultOrProblems<RiskscapeFunction> functionOr =
          context.getFunctionResolver().resolve(context, fc, inputType, argTypes, byId);
      if (functionOr.hasErrors()) {
        problems.addAll(functionOr.getProblems());
        return Realized.failed(inputType, fc);
      }
      RiskscapeFunction function = functionOr.get();

      return new Realized(inputType, fc, function.getReturnType(), (re, input) -> {
        Object[] argValues = new Object[function.getArgumentTypes().size()];
        for (int idx = 0; idx < realizedArgs.size(); idx++) {
          try {
            argValues[idx] = realizedArgs.get(idx).evaluate(input);
          } catch (RuntimeException e) {
            throw new EvalException("Failed to evaluate arg " + idx + " `" + re.getExpression().toSource() + "`",
                e, re, input);
          }
        }

        return function.call(Arrays.asList(argValues));
      }, realizedArgs, () -> function.close());
    }

    private RealizedExpression realizeRaw(BracketedExpression fc, Expression parent) {
      return realizeRaw(fc.getExpression(), fc);
    }

    /**
     * Realizes a {@link BinaryOperation} so that BEDMAS precedence is applied to any nested {@link BinaryOperation}s.
     * Eg `3 * 10 + 5` is effectively `(3 * 10) + 5`.
     *
     * @param op binary operation to realize
     * @return realized expression
     */
    private RealizedExpression realizeRaw(BinaryOperation op, Expression parent) {
      //Start to converting the possibly nested binary op to a list of LinkedBinaryOperation
      List<LinkedBinaryOperation> binaryExpressions = new ArrayList<>();
      unnestBinaryOperationChains(op, binaryExpressions, null);

      //Sort the binary expressions into precedence based order, highest first.
      Collections.sort(binaryExpressions);

      //Now the binary operation chain can be realized in precedence order.
      RealizedExpression realized = null;
      for (LinkedBinaryOperation be: binaryExpressions) {

        RealizedExpression lhs;
        if (be.getPrevious() != null && be.getPrevious().getRealized() != null) {
          //If the previous expression has already been realized then it becomes our LHS.
          //This is because it will have lifed our LHS to realize itself.
          lhs = be.getPrevious().getRealized();
        } else {
          //Otherwise we have to use our own LHS
          lhs = realizeRaw(be.getOperation().getLhs(), op);
        }

        RealizedExpression rhs = null;
        if (be.getNext() == null) {
          rhs = realizeRaw(be.getOperation().getRhs(), op);
        } else {
          //If there is a next then it is the source of the RHS
          if (be.getNext().getRealized() != null) {
            //If the next has already been realized it has precedence, so we use it.
            rhs = be.getNext().getRealized();
          } else {
            //Otherwise we lift the next/lhs to become our RHS.
            //When next is realized it will use this realized as its lhs
            rhs = realizeRaw(be.getNext().getOperation().getLhs(), op);
          }
        }
        realized = realizeRaw(
            new BinaryOperation(lhs.getExpression(), be.getOperation().getOperator(), rhs.getExpression()), lhs, rhs);

        if (anyFailed(realized)) {
          //bail on first failure
          return realized;
        }
        be.setRealized(realized);
      }
      return realized;
    }

    private RealizedExpression realizeRaw(BinaryOperation op, RealizedExpression lhs, RealizedExpression rhs) {
      if (anyFailed(Arrays.asList(lhs, rhs))) {
        return Realized.failed(inputType, op);
      }

      List<Type> argTypes = Arrays.asList(lhs.getResultType(), rhs.getResultType());

      RiskscapeFunction picked = context.getProject().getFunctionSet()
          .resolve(context, op, inputType, lhs.getResultType(), rhs.getResultType())
          .orElse(null);

      if (picked == null) {
        problems.add(Problems.get(ExpressionProblems.class).noSuchOperatorFunction(op.getOperator().value, argTypes));
        return Realized.failed(inputType, op);
      }


      return new Realized(inputType, op, picked.getReturnType(), (re, input) -> {
        List<Object> args = Arrays.asList(
            lhs.evaluate(input),
            rhs.evaluate(input)
        );

        return picked.call(args);
      }, Arrays.asList(lhs, rhs), null);
    }

    /**
     * Wraps a {@link BinaryOperation} in a {@link LinkedBinaryOperation} and recurses into the right hand side
     * expression if that is also a {@link BinaryOperation}.
     *
     * The produced {@link LinkedBinaryOperation} is added to the linkedBinaryOperators list so that the list is
     * ordered left to right relative to the original source expression.
     *
     * @param target binary operation to link up
     * @param linkedBinaryOperators list to added links to
     * @param previous the LinkedBinaryOperation to the left of this one, if any
     * @return the created link
     */
    private LinkedBinaryOperation unnestBinaryOperationChains(BinaryOperation target,
        List<LinkedBinaryOperation> linkedBinaryOperators, LinkedBinaryOperation previous) {

      LinkedBinaryOperation self = new LinkedBinaryOperation(target, previous);
      linkedBinaryOperators.add(self);

      if (target.getRhs() instanceof BinaryOperation) {
        self.setNext(
            unnestBinaryOperationChains((BinaryOperation)target.getRhs(), linkedBinaryOperators, self));
      }
      return self;
    }

    private RealizedExpression realizeRaw(StructDeclaration sd, Expression parent) {
      return structDeclRealizer.realize(sd, parent);
    }

    private RealizedExpression realizeRaw(SelectAllExpression expression, Expression parent) {
      return new Realized(inputType, expression, inputType, (r, i) -> i, Collections.emptyList(), null);
    }

    private RealizedExpression realizeRaw(ParameterToken expression) {
      throw new UnresolvedExpressionParameterException(Collections.singletonList(expression));
    }

    /**
     * Sorts the given arguments returning them in the positions expected by {@link IdentifiedFunction}.
     *
     * To obtain this ordering null returning {@link Realized} may be added to the list as placeholders for
     * missing arguments. Note that these null returning placeholders will only exist if a user specified
     * argument exist later in the list.
     *
     * @return the sorted args or null if there are any errors at all.
     */
    private List<RealizedExpression> sort(
        List<RealizedExpression> unsortedArgs,
        FunctionCall fc,
        IdentifiedFunction byId
    ) {
      if (anyFailed(unsortedArgs)) {
        // if there are any failures we can bail straight away. no need for sorting in that case
        return null;
      }

      boolean hasErrors = false;

      if (!fc.hasKeywordArguments()) {
        return unsortedArgs;
      }

      ArgumentList al = byId.getArguments();

      final int offset = fc.keywordArgumentsOffset();
      // we make sorted args the larger of what we have or what the function can accept.
      // this is to make room for null values for any optional arguments that are omitted.
      RealizedExpression[] sortedArgs =
          new RealizedExpression[Math.max(unsortedArgs.size(), byId.getArguments().size())];

      for (int i = 0; i < unsortedArgs.size(); i++) {
        if (i < offset) {
          // copy any unnamed args verbatim
          sortedArgs[i] = unsortedArgs.get(i);
        } else if (i < fc.getArguments().size()) {
          FunctionCall.Argument arg = fc.getArguments().get(i);
          String keyword = arg.getName().orElse(null);

          if (keyword == null) {
            problems.add(Problem.error(ProblemCodes.KEYWORD_REQUIRED, i + 1));
            hasErrors = true;
            continue;
          }

          if (!al.hasArgument(keyword)) {
            problems.add(Problem.error(ProblemCodes.KEYWORD_DOES_NOT_EXIST, keyword,
                al.getKeywords()));
            hasErrors = true;
            continue;
          }

          FunctionArgument declared = al.get(keyword);

          if (declared.getIndex() >= sortedArgs.length) {
            // currently we only support omitting optional args at the end of the function,
            // e.g. if you have args a,b,c then you can omit c but not a
            problems.add(Problem.error(ProblemCodes.MISSING_KEYWORD_ARGUMENTS,
                declared.getKeyword(), declared.getIndex() + 1, sortedArgs.length));
            hasErrors = true;
          } else if (sortedArgs[declared.getIndex()] != null) {
            // argument has already been specified (either positionally or by name)
            int specifiedAs = unsortedArgs.indexOf(sortedArgs[declared.getIndex()]) + 1;
            problems.add(Problem.error(ProblemCodes.KEYWORD_OUT_OF_ORDER, i + 1,
                declared.getKeyword(), specifiedAs));
            hasErrors = true;
          } else {
            sortedArgs[declared.getIndex()] = unsortedArgs.get(i);
          }
        }
      }

      if (hasErrors) {
        // return null to signal there are errors
        return null;
      }

      // prune trailing null values. most of our functions use the number of given types to switch behaviour.
      // so we only want to have holes where they are really needed.
      int idx = -1;
      for (int i = sortedArgs.length - 1; i >=0; i--) {
        if (sortedArgs[i] != null) {
          break;
        }
        idx = i;
      }
      if (idx > -1) {
        sortedArgs = Arrays.copyOf(sortedArgs, idx);
      }

      for (int i = 0; i < sortedArgs.length; i++) {
        if (sortedArgs[i] == null) {
          // fill any holes with the nothing argument.
          sortedArgs[i] = MISSING_ARGUMENT;
        }
      }

      return Arrays.asList(sortedArgs);
    }

    private <T extends Expression> Optional<RealizedExpression> realizeIf(
        Expression expression,
        Class<T> ifType,
        Function<T, RealizedExpression> callback
    ) {
      return expression.isA(ifType).map(callback);
    }

    public RealizedExpression tryThese(Expression expr, List<Optional<RealizedExpression>> optionals) {
      return optionals.stream()
          .filter(Optional::isPresent)
          .findFirst()
          .map(Optional::get)
          .orElseThrow(() ->  new RuntimeException("Unrecognized expression type - " + expr));
    }
  }

  @Override
  public ResultOrProblems<RealizedAggregateExpression> realizeAggregate(Type inputType, Expression expression) {
    return new AggregateExpressionRealizer(realizationContext, inputType, expression).realize();
  }

  /**
   * Attempts to realize the given expression against an empty struct, returning a meaningful problem if it
   * looks like it failed to realize because it had property access problems (meaning it wasn't constant as it relies
   * on the input scope)
   * @return a {@link RealizedExpression} that is constant, i.e should succeed when called like
   * `expr.evaluate(Tuple.EMPTY_TUPLE)` and should always return the same result.
   */
  @Override
  public ResultOrProblems<RealizedExpression> realizeConstant(Expression expression) {
    ResultOrProblems<RealizedExpression> realizedOr = realize(Struct.EMPTY_STRUCT, expression);
    if (realizedOr.hasErrors()) {
      // generally, if it's not a constant expression then it'll have missing attribute errors,
      // e.g. the expression is trying to access foo.bar, which wasn't found
      if (realizedOr.getProblems().stream().allMatch(prob -> prob.getCode().equals(NO_SUCH_MEMBER))) {
        return ResultOrProblems.failed(ExpressionProblems.get().constantRequired(expression));
      }
    }

    return realizedOr.map(RealizedConstant::create);
  }

}
