/*
 * RiskScape™ Copyright New Zealand Institute for Earth Science Limited
 * (Earth Sciences New Zealand) is distributed for research purposes only
 * under the terms of AGPLv3.
 *
 * RiskScape™ Copyright 2026 New Zealand Institute for Earth Science
 * Limited (Earth Sciences New Zealand). All rights reserved. Source code
 * available under the AGPLv3.
 * 
 * This program is free software: you can redistribute it and/or modify it under
 *  the terms of the GNU Affero General Public License as published by the Free
 *  Software Foundation, either version 3 of the License, or (at your option) any
 *  later version.
 * 
 * This program is distributed for RESEARCH PURPOSES ONLY, in the hope that it will
 * be useful for research and education initiatives.
 * 
 * If you are not a researcher, or you are a researcher who wishes to use this
 * program on terms other than AGPLv3 (including those who wish to restrict the
 * distribution of any source code created using this program), please contact:
 * https://riskscape.org.nz
 * 
 * This program is distributed WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Affero General Public License for more details.  You should have received a copy
 * of the GNU Affero General Public License along with this program.  If not, see
 * <http://www.gnu.org/licenses/>.
 * 
 * By way of summary only, under the AGPLv3:
 *     • Permissions of this strongest copyleft license are conditioned
 *       on making available complete source code of licensed works and
 *       modifications, which include larger works using a licensed work,
 *       under the same license.
 *     • Copyright and license notices must be preserved.
 *     • Contributors provide an express grant of patent rights.
 *     • When a modified version is used to provide a service over a
 *       network, the complete source code of the modified version must be made
 *       available.
 */
package nz.org.riskscape.engine.cli.tests;

import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;

import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

import org.junit.Test;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;

import nz.org.riskscape.engine.Assert;
import nz.org.riskscape.engine.FileSystemMatchers;
import nz.org.riskscape.engine.Matchers;
import nz.org.riskscape.engine.bind.ParamProblems;
import nz.org.riskscape.engine.bind.Parameter;
import nz.org.riskscape.engine.cli.ExitException;
import nz.org.riskscape.engine.model.Model;
import nz.org.riskscape.engine.pipeline.PipelineProblems;
import nz.org.riskscape.engine.problem.GeneralProblems;
import nz.org.riskscape.engine.problem.ProblemMatchers;
import nz.org.riskscape.problem.Problems;

public class ModelRunPipelineCommandTest extends BaseModelRunCommandTest {

  @Test
  public void canRunADefinedPipelineModel() throws Exception {
    runCommand.modelId = "inline";
    runCommand.run();

    List<String> collectedAnimals = openCsv("sort.csv", "name").stream().map(list -> list.get(0))
        .collect(Collectors.toList());

    // check animals come out in sorted order
    List<String> expectedAnimals = Arrays.asList(
      "cat",
      "dog",
      "elephant",
      "parakeet",
      "pig",
      "rabbit",
      "rat"
    );
    assertEquals(expectedAnimals, collectedAnimals);

    // check that the pipeline file has some content. we're just checking it isn't empty here
    assertThat(openFile("pipeline.txt"), containsString("input"));
  }

  @Test
  public void canRunAModelPassingInputAsADataEncodedURI_GL1092() throws Exception {
    List<String> names = Arrays.asList(
        "name,age",
        "Jane,6",
        "Tāne,66",
        "Sébastien,42"
    );

    byte[] csvData = names.stream().collect(Collectors.joining("\n")).getBytes("UTF-8");

    runCommand.modelId = "bookmark-as-arg";
    String bookmarkUri = "'data:;base64," + Base64.getUrlEncoder().encodeToString(csvData) + "'";

    runCommand.parameters = Arrays.asList("bookmark=" + bookmarkUri);
    runCommand.run();

    List<List<String>> collectedPeople = openCsv("sort.csv", "name", "age");

    // check animals come out in sorted order
    List<List<String>> expectedPeople= Arrays.asList(
      Arrays.asList("Jane", "6"),
      Arrays.asList("Sébastien", "42"),
      Arrays.asList("Tāne", "66")
    );

    assertThat(expectedPeople, equalTo(collectedPeople));
  }

  @Test
  public void warningForUnknownParameters() throws Exception {
    runCommand.modelId = "inline";
    runCommand.parameters = Lists.newArrayList("unknown=20");
    runCommand.run();

    assertThat(terminal.getCollectedProblems(), contains(
        ProblemMatchers.isProblem(ParamProblems.class, (r, problems) ->
            problems.ignoredWithHints(r.eq(Sets.newHashSet("unknown")), r.eq(Sets.newHashSet("inputOffset"))))
    ));
  }

  @Test
  public void pipelineParametersMayBeMissingFromModelButAreRequiredToRun() throws Exception {
    runCommand.modelId = "external-with-missing-param";

    // we haven't set the required inputLimit parameter, shouldn't run
    ExitException ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());
    assertThat(
      ex.getProblem(),
      Matchers.hasAncestorProblem(
        ProblemMatchers.isProblem(GeneralProblems.class, (r, p) ->
          p.required(r.match(ProblemMatchers.namedArg(Parameter.class, "inputLimit")))
        )
      )
    );

    runCommand.parameters.add("inputLimit=2");
    runCommand.run();


    List<String> collectedAnimals = openCsv("sort.csv", "name").stream().map(list -> list.get(0))
        .collect(Collectors.toList());

    List<String> expectedAnimals = Arrays.asList(
      "cat",
      "dog"
    );
    assertEquals(expectedAnimals, collectedAnimals);
  }


  @Test
  public void pipelineParametersMayBeEmptyModelButAreRequiredToRun() throws Exception {
    runCommand.modelId = "external-with-empty-param";

    // we haven't set the required inputLimit parameter, shouldn't run
    ExitException ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());
    assertThat(
      ex.getProblem(),
      Matchers.hasAncestorProblem(
        ProblemMatchers.isProblem(GeneralProblems.class, (r, p) ->
          p.required(r.match(ProblemMatchers.namedArg(Parameter.class, "inputLimit")))
        )
      )
    );

    runCommand.parameters.add("inputLimit=2");
    runCommand.run();


    List<String> collectedAnimals = openCsv("sort.csv", "name").stream().map(list -> list.get(0))
        .collect(Collectors.toList());

    List<String> expectedAnimals = Arrays.asList(
      "cat",
      "dog"
    );
    assertEquals(expectedAnimals, collectedAnimals);
  }

  @Test
  public void canOverrideInlinePipelineParametersOnTheCommandLine() throws Exception {
    doOverrideInlinePipelineParametersOnTheCommandLineTest("inputOffset=2");
  }

  @Test
  public void canOverrideInlinePipelineParametersOnTheCommandLineWithWhiteSpace() throws Exception {
    doOverrideInlinePipelineParametersOnTheCommandLineTest("inputOffset = 2");
  }

  private void doOverrideInlinePipelineParametersOnTheCommandLineTest(String inputOffsetParam) throws Exception {
    runCommand.modelId = "inline";
    runCommand.parameters.add(inputOffsetParam);
    runCommand.run();

    List<String> collectedAnimals = openCsv("sort.csv", "name").stream().map(list -> list.get(0))
        .collect(Collectors.toList());

    List<String> expectedAnimals = Arrays.asList(
      // cat and dog are missing - offset by 2
      "elephant",
      "parakeet",
      "pig",
      "rabbit",
      "rat"
    );
    assertEquals(expectedAnimals, collectedAnimals);
  }

  @Test
  public void canOverrideExternalPipelineParametersOnTheCommandLine() throws Exception {
    // override parameters again, but this time we can override the input file
    // for an externally defined pipeline (with a file that's not a bookmark)
    runCommand.modelId = "external";
    runCommand.parameters.add("inputRelation='insects'");
    runCommand.run();

    checkSortedNames(Arrays.asList(
        "ant",
        "bee",
        "cricket",
        "dung beetle"));
  }

  @Test
  public void canOverrideModelParametersFromIniFileWithSingleSection() throws Exception {
    runCommand.modelId = "external";
    // parameters-external-only.ini only has one INI section (named 'my-parameters'). The single section
    // is always the winner, regardless of it's name.
    // This parameter file changes the relation to insects (model default is animals)
    runCommand.parametersFile = stdhome().resolve("parameters-single-section-only.ini");
    runCommand.run();

    checkSortedNames(Arrays.asList(
        "ant",
        "bee",
        "cricket",
        "dung beetle"));
  }

  @Test
  public void canOverrideModelParametersFromIniFileWithSectionNamedForModelId() throws Exception {
    runCommand.modelId = "external";
    // parameters.ini has many INI sections so the test will use the one named 'external'. that section
    // will change relation to 'insects' and set the inputLimit to 2
    runCommand.parametersFile = stdhome().resolve("parameters.ini");
    runCommand.run();

    checkSortedNames(Arrays.asList("bee", "dung beetle"));
  }

  @Test
  public void canOverrideModelParametersFromIniFileAndOverrideWithParameters() throws Exception {
    runCommand.modelId = "external";
    // parameters.ini has many INI sections so the test will use the one named 'external'. that section
    // will change relation to 'insects' and set the inputLimit to 2
    runCommand.parametersFile = stdhome().resolve("parameters.ini");
    // but setting the inputLimit with --parameter should take precedence.
    runCommand.parameters.add("inputLimit=3");
    runCommand.run();

    checkSortedNames(Arrays.asList("bee", "cricket", "dung beetle"));
  }

  private void checkSortedNames(List<String> expectedItems) throws Exception {

    List<String> collectedItems = openCsv("sort.csv", "name").stream().map(list -> list.get(0))
        .collect(Collectors.toList());

    assertEquals(expectedItems, collectedItems);
  }

  @Test
  public void commandLineParametersTakePrecedenceOverModelDefinition() throws Exception {
    runCommand.modelId = "inline-with-args";
    runCommand.parameters.add("inputOffset=2");
    runCommand.parameters.add("inputLimit=1");
    runCommand.run();

    Set<String> collectedAnimals =
        openCsv("sort.csv", "name").stream().map(list -> list.get(0)).collect(Collectors.toSet());

    Set<String> expectedAnimals = Sets.newHashSet(
      "pig"
    );
    assertEquals(expectedAnimals, collectedAnimals);
  }


  @Test
  public void canOverridePipelineBuildParameterItselfViaCli() throws Exception {
    // we can override the pipeline DSL itself on the CLI (although it's probably
    // not a great use case for something users would typically do). This also
    // requires an existing pipeline to override (there's no anonymous default)
    runCommand.modelId = "inline";
    runCommand.parameters.add("pipeline=input('animals') -> filter(starts_with(name, 'p'))");
    runCommand.run();

    List<String> collectedAnimals = openCsv("filter.csv", "name").stream().map(list -> list.get(0))
        .collect(Collectors.toList());

    List<String> expectedAnimals = Arrays.asList(
      // note: not alphabetized because we're only filtering
      "pig",
      "parakeet"
    );
    assertEquals(expectedAnimals, collectedAnimals);
  }

  @Test
  public void canRunModelDefinedInSubdirectoryWithRelativeParameters() throws Exception {
    runCommand.modelId = "external-dir1";
    runCommand.run();

    Set<String> collectedAnimals =
        openCsv("sort.csv", "name").stream().map(list -> list.get(0)).collect(Collectors.toSet());

    Set<String> expectedAnimals = Sets.newHashSet(
      "cat",
      "dog",
      "elephant",
      "parakeet",
      "pig",
      "rabbit",
      "rat"
    );
    assertEquals(expectedAnimals, collectedAnimals);
  }

  @Test
  public void parameterErrorsAreHandledNicely() throws Exception {
    runCommand.modelId = "inline";
    // set a single parameter twice and watch it die
    runCommand.parameters.add("inputOffset=2");
    runCommand.parameters.add("inputOffset=3");

    ExitException ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());
    assertThat(
        ex.getProblem(),
        Matchers.hasAncestorProblem(equalTo(ParamProblems.get().wrongNumberGiven("inputOffset", "1", 2)))
    );
  }

  @Test
  public void pipelineRealizationErrorsAreHandledNicely() throws Exception {
    runCommand.modelId = "err-realization";

    ExitException ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());
    assertThat(ex.getProblem(), Matchers.equalIgnoringChildren(
        PipelineProblems.get().cannotRealize(Model.class)));
  }

  @Test
  public void canStillPrintAPipelineWithRealizationErrors() throws Exception {
    // we can *try* to print it at least...
    runCommand.modelId = "err-realization";
    runCommand.runnerOptions.printPipeline = true;

    runCommand.run();
    assertThat(outBytes.toString(), allOf(
        containsString("Step: sort"),
        containsString("Produces:"),
        containsString("name => Text"),
        containsString("Result: failed")
    ));
  }

  @Test
  public void canPrintAPipelineStep() throws Exception {
    // we can *try* to print it at least...
    runCommand.modelId = "err-realization";
    runCommand.runnerOptions.printStep = "sort";

    runCommand.run();

    assertThat(outBytes.toString(), allOf(
        containsString("Step: sort"),
        not(containsString("Step: input"))
    ));
  }

  @Test
  public void badPipelineFileErrorIsHandledNicely() throws Exception {
    runCommand.modelId = "err-bad-location";

    ExitException ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());
    assertThat(ex.getProblem(), Matchers.hasAncestorProblem(Matchers.equalIgnoringChildren(
        Problems.foundWith(Parameter.class, "location")
        )));
  }

  @Test
  public void cannotSpecifyPipelineAndLocation() throws Exception {
    runCommand.modelId = "err-both-pipeline-and-location";

    ExitException ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());
    assertThat(ex.getProblem(), Matchers.hasAncestorProblem(Matchers.equalIgnoringChildren(
        ParamProblems.get().mutuallyExclusive("pipeline", "location")
        )));
  }

  @Test
  public void problemsFromExpressionArgsAreIncludedInProblemOutput() {
    runCommand.modelId = "inline-with-args";
    runCommand.parameters = Arrays.asList("bookmark='does-not-exist.csv'");
    ExitException ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());
    assertThat(render(ex.getProblem()), allOf(
        containsString("Problems found with 'does-not-exist.csv' bookmark"),
        containsString("does-not-exist.csv' does not exist")
    ));
  }

  @Test
  public void mustSpecifyPipelineOrLocation() throws Exception {
    runCommand.modelId = "empty";

    ExitException ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());
    assertThat(ex.getProblem(), Matchers.hasAncestorProblem(Matchers.equalIgnoringChildren(
        ParamProblems.oneOfTheseRequired("pipeline", "location"))));
  }

  @Test
  public void aCycleInAPipelineGivesADecentErrorMessage() throws Exception {
    runCommand.modelId = "cyclic";

    ExitException ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());

    assertThat(
      render(ex.getProblem()),
      containsString("You have a cycle in your pipeline introduced by "
          + "`filter(len(name) > 2) -> select` at line 4, column 3. " +
          "Step `select` is already part of this branch of the pipeline (first seen at line 2, column 6). " +
          "Make sure you are using the correct step name, then change or remove `-> select` from " +
          "line 4, column 3 and try again.")
    );
  }

  @Test
  public void canSampleGeometryAndGetCorrectGeometryFamily() throws Exception {
    // this model highlights the bug in GL817 - the project data lives off in its own separate dir
    populateProjectAndSetupCommands(homes().resolve("GL817_sampling"));
    runCommand.modelId = "sample";
    runCommand.run();

    assertThat(openCsv("exposed.csv", "exposed_ratio"), contains(
        Arrays.asList("1.0"),
        Arrays.asList("1.0")
    ));

    assertThat(getTempDirectory(), FileSystemMatchers.hasFile(FileSystemMatchers.fileWithName("hazard_geom.shp")));
  }

  @Test
  public void canSegmentGeometryAndGetCorrectGeometryFamily() throws Exception {
    // this model lives off in its own separate dir
    populateProjectAndSetupCommands(homes().resolve("GL817_sampling"));
    runCommand.modelId = "segment-by-grid";
    runCommand.run();

    assertThat(openCsv("exposed.csv", "exposed_ratio"), contains(
        Arrays.asList("1.0"),
        Arrays.asList("1.0")
    ));

    assertThat(getTempDirectory(), FileSystemMatchers.hasFile(FileSystemMatchers.fileWithName("segmented_geom.shp")));
  }

  @Test
  public void canSampleOverlappingPolygons() throws Exception {
    // we use the kaiju data here, just because it was easy to create overlapping polygons
    populateProjectAndSetupCommands(Paths.get("..", "..", "examples", "kaiju"));
    runCommand.modelId = "sample_overlapping_polygons";
    runCommand.run();

    // these buildings overlap *two* polygons in the layer being sampled (Trump Tower 100% overlaps both
    // and the Plaza partially overlaps). So sampling should match both polygons and we end up with two
    // different results. Microsites rely on this behaviour to apply mitigation polygons (aka adaptation layers)
    assertThat(openCsv("results.csv", "name", "num", "min", "max"), contains(
        Arrays.asList("The Plaza Hotel", "2", "9.0", "11.0"),
        Arrays.asList("Trump Tower", "2", "9.0", "11.0")
    ));
  }

  @Test
  public void canRunModelWithParameterTemplates() throws Exception {
    runCommand.modelId = "param-templates";
    // NB: note the lack of quotes around the bookmark ID. This shows that the
    // parameter is binding to a ResolvedBookmark, rather than a plain ol' expression
    runCommand.parameters.add("relation=animals");
    runCommand.parameters.add("number=1");
    runCommand.parameters.add("list=['foo', 'bar']");
    runCommand.run();

    assertThat(openCsv("results.csv", "name", "list"),
        contains(Arrays.asList("dog", "foo"), Arrays.asList("dog", "bar")));
  }

  @Test
  public void cannotRunModelWithInvalidTemplateValues() throws Exception {
    runCommand.modelId = "param-templates";

    // the parameter binding becomes more obvious when we try to bind to the
    // wrong type of value. Without the template, any old expression would be accepted.
    // E.g. here the 'list' parameter is not a list
    runCommand.parameters = Arrays.asList("list='foo'", "relation=animals", "number=1");
    ExitException ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());

    assertThat(render(ex.getProblem()), allOf(
      containsString("Problems found with 'list' parameter"),
      containsString("Type mismatch for expression ''foo''. Expected 'list declaration' but found 'constant'")
    ));

    // 'number' -1 is less than the min of zero
    runCommand.parameters = Arrays.asList("list=['foo', 'bar']", "relation=animals", "number=-1");
    ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());

    assertThat(render(ex.getProblem()), allOf(
        containsString("Problems found with 'number' parameter"),
        containsString("Value '-1' is invalid for 'min' parameter property, it must be >= 0.0")
    ));

    // 'relation' is not a valid bookmark
    runCommand.parameters = Arrays.asList("list=['foo', 'bar']", "relation=dinosaurs", "number=1");
    ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());

    assertThat(render(ex.getProblem()), allOf(
        containsString("Problems found with 'relation' parameter"),
        containsString("No bookmark with id 'dinosaurs' exists")
    ));
  }

  @Test
  public void canOverrideParameterValidationForExperts() throws Exception {
    runCommand.modelId = "param-templates";

    // 'number' -1 is less than the min of zero so would normally result in a model run error
    runCommand.parameters = Arrays.asList("list=['foo']", "relation=animals", "number=-1",
        // ...but you can skip parameter validation with this extremely clunky CLI override
        "param.number.properties=expression");

    runCommand.run();

    // -1 doesn't limit the input rows, so we get all the animals
    assertThat(openCsv("results.csv", "name", "list"), contains(
        Arrays.asList("dog", "foo"),
        Arrays.asList("cat", "foo"),
        Arrays.asList("pig", "foo"),
        Arrays.asList("rabbit", "foo"),
        Arrays.asList("elephant", "foo"),
        Arrays.asList("rat", "foo"),
        Arrays.asList("parakeet", "foo")
    ));
  }

  @Test
  public void canRunModelWithBookmarkTemplateParameter() throws Exception {
    // we use the kaiju data here because it was convenient to test with
    populateProjectAndSetupCommands(Paths.get("..", "..", "examples", "kaiju"));
    runCommand.modelId = "bookmark_params";
    runCommand.runnerOptions.replace = true;

    // run with default building dataset
    runCommand.run();
    assertThat(openCsv("results.csv", "Num_buildings", "Average_stories"), contains(Arrays.asList("10", "29.4")));

    // swap out the building data CSV file. The model relies on the nyc_buildings bookmark being applied
    // to the CSV file (otherwise we won't read in the geom/stories properly)
    runCommand.parameters = Arrays.asList("buildings=data/more_landmarks.csv");
    runCommand.run();
    assertThat(openCsv("results.csv", "Num_buildings", "Average_stories"), contains(Arrays.asList("3", "4.33")));

    // we should get a parameter error if we try to use a bookmark that doesn't
    // conform to the template, e.g. it's missing attributes
    runCommand.parameters = Arrays.asList("buildings=data/new-york.csv");
    ExitException ex = Assert.assertThrows(ExitException.class, () -> runCommand.run());
    assertThat(render(ex.getProblem()), allOf(
        containsString("Problems found with 'buildings' parameter"),
        containsString("Problems found with 'nyc_buildings' bookmark"),
        containsString("Source data is missing 'stories' attribute")
    ));
  }

  @Test
  public void canRunModelWithBookmarkInQuotes() throws Exception {
    // we use the kaiju data here because it was convenient to test with
    populateProjectAndSetupCommands(Paths.get("..", "..", "examples", "kaiju"));
    runCommand.modelId = "bookmark_params";
    runCommand.runnerOptions.replace = true;

    runCommand.parameters = Arrays.asList("buildings='data/more_landmarks.csv'");
    runCommand.run();
    assertThat(openCsv("results.csv", "Num_buildings", "Average_stories"), contains(Arrays.asList("3", "4.33")));
  }

  @Test
  public void canRunModelWithBookmarkExpressionParameters() throws Exception {
    populateProjectAndSetupCommands(Paths.get("..", "..", "examples", "kaiju"));
    runCommand.modelId = "bookmark_params";
    runCommand.runnerOptions.replace = true;

    // bookmark expressions just get used by the model as is, i.e. it doesn't try to apply the bookmark template
    // This is more to mimic params coming from the Platform, which should build up a bookmark expression correctly
    runCommand.parameters = Arrays.asList("buildings=bookmark('nyc_buildings')");
    runCommand.run();
    assertThat(openCsv("results.csv", "Num_buildings", "Average_stories"), contains(Arrays.asList("10", "29.4")));

    runCommand.parameters =
        Arrays.asList("buildings=bookmark('nyc_buildings', { location: 'data/more_landmarks.csv' })");
    runCommand.run();
    assertThat(openCsv("results.csv", "Num_buildings", "Average_stories"), contains(Arrays.asList("3", "4.33")));

    // this won't work because the bookmark template isn't applied at all
    // (when the engine gets a bookmark() expression, it's hard to tell if the bookmark template has
    // already been applied on the platform/UI side, so we currently just use the expression as is)
    runCommand.parameters = Arrays.asList("buildings=bookmark('data/more_landmarks.csv')");
    Assert.assertThrows(ExitException.class, () -> runCommand.run());
  }

  @Test
  public void canAutomaticallyUseParameterTemplate() throws Exception {
    runCommand.modelId = "external-parameter";

    // Automatically use the parameter template with the same name
    // We don't specify a value for the parameter, so it uses the default from the template
    // It has the property 'file' and starts with ./, so is resolved relative to dir1
    runCommand.run();
    assertThat(openCsv("input.csv", "name"), hasSize(7));
  }
}
